Skip to content

Commit

Permalink
Add LEAKY_RELU
Browse files Browse the repository at this point in the history 
This added LEAKY_RELU op. Still need to add that as cnnp model.
  • Loading branch information
@liuliu
liuliu committed Nov 12, 2022
1 parent 0e8c2c9 commit 79170de
Show file tree
Hide file tree
Showing 14 changed files with 1,624 additions and 1,099 deletions.
3 changes: 3 additions & 0 deletions lib/nnc/ccv_nnc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,9 @@ typedef struct {
float max; /**< [histogram.min] The maximal number, for even or logarithmic. */
float rate; /**< [histogram.ratio] The rate from min to max, only applied to logarithmic. */
} histogram;
struct {
float negative_slope; /**< [leaky_relu.negative_slop] The negative slope to be applied when activation < 0. */
} leaky_relu;
void* userdata;
};
} ccv_nnc_cmd_param_t;
Expand Down
2 changes: 1 addition & 1 deletion lib/nnc/cmd/ccv_nnc_backend.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ enum {
CCV_NNC_BACKEND_GPU_REF = 0x5f19790a,
CCV_NNC_BACKEND_MPS = 0xb2f325e2,
CCV_NNC_BACKEND_CPU_OPT = 0x46deb194,
CCV_NNC_BACKEND_GPU_CUBLAS = 0x9b8cfed,
CCV_NNC_BACKEND_GPU_CUDNN = 0x854b679a,
CCV_NNC_BACKEND_GPU_CUBLAS = 0x9b8cfed,
CCV_NNC_BACKEND_GPU_NCCL = 0x7afed9c7,
CCV_NNC_BACKEND_COUNT = 7,
};
Expand Down
156 changes: 79 additions & 77 deletions lib/nnc/cmd/ccv_nnc_cmd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,58 @@ enum {
CCV_NNC_CUSTOM_BACKWARD,
CCV_NNC_GRAPH_FORWARD,
CCV_NNC_GRAPH_BACKWARD,
CCV_NNC_LAMB_FORWARD = 0x450edb1a,
CCV_NNC_LAMB_BACKWARD = 0x450edb1b,
CCV_NNC_ROI_ALIGN_FORWARD = 0xfef55168,
CCV_NNC_ROI_ALIGN_BACKWARD = 0xfef55169,
CCV_NNC_RANDOM_UNIFORM_FORWARD = 0xa0cd1d5e,
CCV_NNC_RANDOM_UNIFORM_BACKWARD = 0xa0cd1d5f,
CCV_NNC_RANDOM_NORMAL_FORWARD = 0x7062c8b4,
CCV_NNC_RANDOM_NORMAL_BACKWARD = 0x7062c8b5,
CCV_NNC_CONVOLUTION_FORWARD = 0x254d05f4,
CCV_NNC_CONVOLUTION_BACKWARD = 0x254d05f5,
CCV_NNC_SWISH_FORWARD = 0x583d90c2,
CCV_NNC_SWISH_BACKWARD = 0x583d90c3,
CCV_NNC_LSTM_FORWARD = 0xc5cb998c,
CCV_NNC_LSTM_BACKWARD = 0xc5cb998d,
CCV_NNC_DROPOUT_FORWARD = 0x7f2dc3e4,
CCV_NNC_DROPOUT_BACKWARD = 0x7f2dc3e5,
CCV_NNC_SOFTMAX_CROSSENTROPY_FORWARD = 0xc26b7b5e,
CCV_NNC_SOFTMAX_CROSSENTROPY_BACKWARD = 0xc26b7b5f,
CCV_NNC_TANH_FORWARD = 0x6a62be30,
CCV_NNC_TANH_BACKWARD = 0x6a62be31,
CCV_NNC_LEAKY_RELU_FORWARD = 0x507144e0,
CCV_NNC_LEAKY_RELU_BACKWARD = 0x507144e1,
CCV_NNC_SGD_FORWARD = 0xe650ad26,
CCV_NNC_SGD_BACKWARD = 0xe650ad27,
CCV_NNC_MAX_POOL_FORWARD = 0x7bec9360,
CCV_NNC_MAX_POOL_BACKWARD = 0x7bec9361,
CCV_NNC_AVERAGE_POOL_FORWARD = 0x51267ab8,
CCV_NNC_AVERAGE_POOL_BACKWARD = 0x51267ab9,
CCV_NNC_GELU_FORWARD = 0xb1527ab8,
CCV_NNC_GELU_BACKWARD = 0xb1527ab9,
CCV_NNC_UPSAMPLE_FORWARD = 0x73875556,
CCV_NNC_UPSAMPLE_BACKWARD = 0x73875557,
CCV_NNC_SIGMOID_BINARY_CROSSENTROPY_FORWARD = 0xd9e0e4a,
CCV_NNC_SIGMOID_BINARY_CROSSENTROPY_BACKWARD = 0xd9e0e4b,
CCV_NNC_COMPRESSION_LSSC_FORWARD = 0x17ea8f72,
CCV_NNC_COMPRESSION_LSSC_BACKWARD = 0x17ea8f73,
CCV_NNC_MIN_FORWARD = 0x972fbd26,
CCV_NNC_MIN_BACKWARD = 0x972fbd27,
CCV_NNC_MAX_FORWARD = 0xdf6f014c,
CCV_NNC_MAX_BACKWARD = 0xdf6f014d,
CCV_NNC_HISTOGRAM_FORWARD = 0xc5473e44,
CCV_NNC_HISTOGRAM_BACKWARD = 0xc5473e45,
CCV_NNC_SOFTMAX_FORWARD = 0xc969a252,
CCV_NNC_SOFTMAX_BACKWARD = 0xc969a253,
CCV_NNC_BINARY_CROSSENTROPY_FORWARD = 0xcd2107ec,
CCV_NNC_BINARY_CROSSENTROPY_BACKWARD = 0xcd2107ed,
CCV_NNC_CATEGORICAL_CROSSENTROPY_FORWARD = 0x1eb327a2,
CCV_NNC_CATEGORICAL_CROSSENTROPY_BACKWARD = 0x1eb327a3,
CCV_NNC_MSE_FORWARD = 0x6904a9a2,
CCV_NNC_MSE_BACKWARD = 0x6904a9a3,
CCV_NNC_SMOOTH_L1_FORWARD = 0x4e428e,
CCV_NNC_SMOOTH_L1_BACKWARD = 0x4e428f,
CCV_NNC_RELU_FORWARD = 0xc51eaa80,
CCV_NNC_RELU_BACKWARD = 0xc51eaa81,
CCV_NNC_ADAM_FORWARD = 0xe30099dc,
CCV_NNC_ADAM_BACKWARD = 0xe30099dd,
CCV_NNC_NMS_FORWARD = 0xdba26106,
CCV_NNC_NMS_BACKWARD = 0xdba26107,
CCV_NNC_GEMM_FORWARD = 0x7e87d00c,
CCV_NNC_GEMM_BACKWARD = 0x7e87d00d,
CCV_NNC_ADD_FORWARD = 0x58fb3664,
Expand All @@ -24,8 +68,14 @@ enum {
CCV_NNC_MUL_BACKWARD = 0x24721a47,
CCV_NNC_SCALAR_MUL_FORWARD = 0x8b4d86aa,
CCV_NNC_SCALAR_MUL_BACKWARD = 0x8b4d86ab,
CCV_NNC_RMSPROP_FORWARD = 0x9c886b1c,
CCV_NNC_RMSPROP_BACKWARD = 0x9c886b1d,
CCV_NNC_UPSAMPLE_FORWARD = 0x73875556,
CCV_NNC_UPSAMPLE_BACKWARD = 0x73875557,
CCV_NNC_COMM_ALLREDUCE_FORWARD = 0x75c8d340,
CCV_NNC_COMM_ALLREDUCE_BACKWARD = 0x75c8d341,
CCV_NNC_COMM_BROADCAST_FORWARD = 0x830eee,
CCV_NNC_COMM_BROADCAST_BACKWARD = 0x830eef,
CCV_NNC_COMM_REDUCE_FORWARD = 0x3434ead8,
CCV_NNC_COMM_REDUCE_BACKWARD = 0x3434ead9,
CCV_NNC_SET_FORWARD = 0x2b070804,
CCV_NNC_SET_BACKWARD = 0x2b070805,
CCV_NNC_MASKED_FILL_FORWARD = 0x7f992d84,
Expand All @@ -38,24 +88,16 @@ enum {
CCV_NNC_TRANSPOSE_BACKWARD = 0xb4d506e1,
CCV_NNC_DATATYPE_CONVERSION_FORWARD = 0xd873e38c,
CCV_NNC_DATATYPE_CONVERSION_BACKWARD = 0xd873e38d,
CCV_NNC_SMOOTH_L1_FORWARD = 0x4e428e,
CCV_NNC_SMOOTH_L1_BACKWARD = 0x4e428f,
CCV_NNC_MSE_FORWARD = 0x6904a9a2,
CCV_NNC_MSE_BACKWARD = 0x6904a9a3,
CCV_NNC_BINARY_CROSSENTROPY_FORWARD = 0xcd2107ec,
CCV_NNC_BINARY_CROSSENTROPY_BACKWARD = 0xcd2107ed,
CCV_NNC_CATEGORICAL_CROSSENTROPY_FORWARD = 0x1eb327a2,
CCV_NNC_CATEGORICAL_CROSSENTROPY_BACKWARD = 0x1eb327a3,
CCV_NNC_REDUCE_SUM_FORWARD = 0x52970f06,
CCV_NNC_REDUCE_SUM_BACKWARD = 0x52970f07,
CCV_NNC_REDUCE_MEAN_FORWARD = 0xf23556c6,
CCV_NNC_REDUCE_MEAN_BACKWARD = 0xf23556c7,
CCV_NNC_REDUCE_MAX_FORWARD = 0x80f1a506,
CCV_NNC_REDUCE_MAX_BACKWARD = 0x80f1a507,
CCV_NNC_REDUCE_NORM2_FORWARD = 0xb3034e16,
CCV_NNC_REDUCE_NORM2_BACKWARD = 0xb3034e17,
CCV_NNC_ARGMAX_FORWARD = 0x68af2804,
CCV_NNC_ARGMAX_BACKWARD = 0x68af2805,
CCV_NNC_ROI_ALIGN_FORWARD = 0xfef55168,
CCV_NNC_ROI_ALIGN_BACKWARD = 0xfef55169,
CCV_NNC_SIGMOID_FORWARD = 0xf2f69650,
CCV_NNC_SIGMOID_BACKWARD = 0xf2f69651,
CCV_NNC_INDEX_SELECT_FORWARD = 0x7ee7771e,
CCV_NNC_INDEX_SELECT_BACKWARD = 0x7ee7771f,
CCV_NNC_RMSPROP_FORWARD = 0x9c886b1c,
CCV_NNC_RMSPROP_BACKWARD = 0x9c886b1d,
CCV_NNC_LAMB_FORWARD = 0x450edb1a,
CCV_NNC_LAMB_BACKWARD = 0x450edb1b,
CCV_NNC_EWSUM_FORWARD = 0xe21a2c4c,
CCV_NNC_EWSUM_BACKWARD = 0xe21a2c4d,
CCV_NNC_EWPROD_FORWARD = 0xee07e8fe,
Expand All @@ -70,62 +112,22 @@ enum {
CCV_NNC_EWSQRT_BACKWARD = 0x8870a61f,
CCV_NNC_CLAMP_FORWARD = 0x2640d854,
CCV_NNC_CLAMP_BACKWARD = 0x2640d855,
CCV_NNC_SOFTMAX_CROSSENTROPY_FORWARD = 0xc26b7b5e,
CCV_NNC_SOFTMAX_CROSSENTROPY_BACKWARD = 0xc26b7b5f,
CCV_NNC_COMM_ALLREDUCE_FORWARD = 0x75c8d340,
CCV_NNC_COMM_ALLREDUCE_BACKWARD = 0x75c8d341,
CCV_NNC_COMM_BROADCAST_FORWARD = 0x830eee,
CCV_NNC_COMM_BROADCAST_BACKWARD = 0x830eef,
CCV_NNC_COMM_REDUCE_FORWARD = 0x3434ead8,
CCV_NNC_COMM_REDUCE_BACKWARD = 0x3434ead9,
CCV_NNC_REDUCE_SUM_FORWARD = 0x52970f06,
CCV_NNC_REDUCE_SUM_BACKWARD = 0x52970f07,
CCV_NNC_REDUCE_MEAN_FORWARD = 0xf23556c6,
CCV_NNC_REDUCE_MEAN_BACKWARD = 0xf23556c7,
CCV_NNC_REDUCE_MAX_FORWARD = 0x80f1a506,
CCV_NNC_REDUCE_MAX_BACKWARD = 0x80f1a507,
CCV_NNC_REDUCE_NORM2_FORWARD = 0xb3034e16,
CCV_NNC_REDUCE_NORM2_BACKWARD = 0xb3034e17,
CCV_NNC_ARGMAX_FORWARD = 0x68af2804,
CCV_NNC_ARGMAX_BACKWARD = 0x68af2805,
CCV_NNC_BATCH_NORM_FORWARD = 0x5419819c,
CCV_NNC_BATCH_NORM_BACKWARD = 0x5419819d,
CCV_NNC_LAYER_NORM_FORWARD = 0xbed3c264,
CCV_NNC_LAYER_NORM_BACKWARD = 0xbed3c265,
CCV_NNC_GROUP_NORM_FORWARD = 0x17deb074,
CCV_NNC_GROUP_NORM_BACKWARD = 0x17deb075,
CCV_NNC_TANH_FORWARD = 0x6a62be30,
CCV_NNC_TANH_BACKWARD = 0x6a62be31,
CCV_NNC_SGD_FORWARD = 0xe650ad26,
CCV_NNC_SGD_BACKWARD = 0xe650ad27,
CCV_NNC_DROPOUT_FORWARD = 0x7f2dc3e4,
CCV_NNC_DROPOUT_BACKWARD = 0x7f2dc3e5,
CCV_NNC_COMPRESSION_LSSC_FORWARD = 0x17ea8f72,
CCV_NNC_COMPRESSION_LSSC_BACKWARD = 0x17ea8f73,
CCV_NNC_SOFTMAX_FORWARD = 0xc969a252,
CCV_NNC_SOFTMAX_BACKWARD = 0xc969a253,
CCV_NNC_MIN_FORWARD = 0x972fbd26,
CCV_NNC_MIN_BACKWARD = 0x972fbd27,
CCV_NNC_MAX_FORWARD = 0xdf6f014c,
CCV_NNC_MAX_BACKWARD = 0xdf6f014d,
CCV_NNC_RELU_FORWARD = 0xc51eaa80,
CCV_NNC_RELU_BACKWARD = 0xc51eaa81,
CCV_NNC_SIGMOID_FORWARD = 0xf2f69650,
CCV_NNC_SIGMOID_BACKWARD = 0xf2f69651,
CCV_NNC_INDEX_SELECT_FORWARD = 0x7ee7771e,
CCV_NNC_INDEX_SELECT_BACKWARD = 0x7ee7771f,
CCV_NNC_SIGMOID_BINARY_CROSSENTROPY_FORWARD = 0xd9e0e4a,
CCV_NNC_SIGMOID_BINARY_CROSSENTROPY_BACKWARD = 0xd9e0e4b,
CCV_NNC_NMS_FORWARD = 0xdba26106,
CCV_NNC_NMS_BACKWARD = 0xdba26107,
CCV_NNC_RANDOM_UNIFORM_FORWARD = 0xa0cd1d5e,
CCV_NNC_RANDOM_UNIFORM_BACKWARD = 0xa0cd1d5f,
CCV_NNC_RANDOM_NORMAL_FORWARD = 0x7062c8b4,
CCV_NNC_RANDOM_NORMAL_BACKWARD = 0x7062c8b5,
CCV_NNC_ADAM_FORWARD = 0xe30099dc,
CCV_NNC_ADAM_BACKWARD = 0xe30099dd,
CCV_NNC_MAX_POOL_FORWARD = 0x7bec9360,
CCV_NNC_MAX_POOL_BACKWARD = 0x7bec9361,
CCV_NNC_AVERAGE_POOL_FORWARD = 0x51267ab8,
CCV_NNC_AVERAGE_POOL_BACKWARD = 0x51267ab9,
CCV_NNC_CONVOLUTION_FORWARD = 0x254d05f4,
CCV_NNC_CONVOLUTION_BACKWARD = 0x254d05f5,
CCV_NNC_LSTM_FORWARD = 0xc5cb998c,
CCV_NNC_LSTM_BACKWARD = 0xc5cb998d,
CCV_NNC_SWISH_FORWARD = 0x583d90c2,
CCV_NNC_SWISH_BACKWARD = 0x583d90c3,
CCV_NNC_HISTOGRAM_FORWARD = 0xc5473e44,
CCV_NNC_HISTOGRAM_BACKWARD = 0xc5473e45,
CCV_NNC_COUNT = 121,
CCV_NNC_COUNT = 123,
};
/** @} */
Loading

0 comments on commit 79170de

Please sign in to comment.