code cleaning

LightCTR/common/avx.h

@@ -206,7 +206,7 @@ inline void avx_vecRcp(const float* x, float *res, size_t len) {
     }
 }
 
-inline void avx_vecScale(const float* x, float *res, size_t len, float scalar) {
+inline void avx_vecScale(const float* x, float *res, size_t len, const float scalar) {
     const __m256 _scalar = _mm256_broadcast_ss(&scalar);
     if (len > 7) {
         for (; len > 7; len -= 8) {

LightCTR/dl_algo_abst.h

@@ -64,7 +64,7 @@ class DL_Algo_Abst {
                     vector<float> pred = Predict(rid, dataSet);
 
                     assert(pred.size() == multiclass_output_cnt);
-                    outputActivFun.forward(&pred);
+                    outputActivFun.forward(pred.data(), pred.size());
 
                     // init threadLocal var
                     vector<float>& grad = *tl_grad;
@@ -84,11 +84,12 @@ class DL_Algo_Abst {
                     } else {
                         onehot[label[rid]] = 1; // label should begin from 0
                     }
-                    lossFun.gradient(&pred, &onehot, &grad);
+                    lossFun.gradient(pred.data(), onehot.data(), grad.data(), pred.size());
                     if (multiclass_output_cnt > 1) {
                         // Notice when LossFunction is Logistic annotation next line,
                         // otherwise run this line like square + softmax
-                        outputActivFun.backward(&grad, &pred, &grad);
+                        outputActivFun.backward(grad.data(), pred.data(),
+                                                grad.data(), grad.size());
                     }
                     grad_Matrix->loadDataPtr(&grad);
                     wrapper[0] = grad_Matrix;
@@ -112,13 +113,13 @@ class DL_Algo_Abst {
                 GradientUpdater::__global_bTraining = false;
 
                 // Validate Loss
-                std::atomic<float> loss(0.0f);
-                std::atomic<int> correct(0);
+                float loss = 0.0f;
+                int correct = 0;
                 for (size_t rid = 0; rid < dataRow_cnt; rid++) {
                     auto task = [&, rid]() {
                         vector<float> pred = Predict(rid, dataSet);
 
-                        outputActivFun.forward(&pred);
+                        outputActivFun.forward(pred.data(), pred.size());
 
                         // init threadLocal var
                         vector<float>& grad = *tl_grad;
@@ -137,7 +138,7 @@ class DL_Algo_Abst {
                         } else {
                             onehot[label[rid]] = 1; // label should begin from 0
                         }
-                        loss = loss + lossFun.loss(&pred, &onehot);
+                        loss += lossFun.loss(pred.data(), onehot.data(), pred.size());
                     };
                     if (dl_algo == RNN) {
                         task();
@@ -147,7 +148,7 @@ class DL_Algo_Abst {
                 }
                 threadpool->wait();
                 printf("Epoch %zu Loss = %f correct = %.3f\n",
-                       p, loss.load(), 1.0f * correct / dataRow_cnt);
+                       p, loss, 1.0f * correct / dataRow_cnt);
             }
         }
     }

LightCTR/predict/gbm_predict.cpp

@@ -39,8 +39,8 @@ void GBM_Predict::Predict(string savePath) {
             pCTR = sigmoid.forward(tmp[0]);
             pLabel.emplace_back(pCTR > 0.5 ? 1 : 0);
         } else {
-            softmax.forward(&tmp);
-            size_t idx = softmax.forward_max(&tmp);
+            softmax.forward(tmp.data(), tmp.size());
+            size_t idx = softmax.forward_max(tmp.data(), tmp.size());
             pCTR = tmp[idx];
             pLabel.emplace_back(idx);
         }

LightCTR/train/layer/convLayer.h

@@ -155,7 +155,7 @@ class Conv_Layer : public Layer_Base {
             // apply Activation Function
             m_ptr->operate([this](vector<float>* matrix) {
                 assert(matrix);
-                this->getActiveFun().forward(matrix);
+                this->getActiveFun().forward(matrix->data(), matrix->size());
             });
             output_act[filid] = m_ptr;
         }
@@ -196,8 +196,9 @@ class Conv_Layer : public Layer_Base {
                     }
                 }
                 m_ptr->operate([&, i](vector<float>* matrix) {
-                    this->prevLayer->getActiveFun().backward(matrix,
-                            this->prevLayer->output()[i]->pointer(), matrix);
+                    this->prevLayer->getActiveFun().backward(matrix->data(),
+                                            this->prevLayer->output()[i]->pointer()->data(),
+                                            matrix->data(), matrix->size());
                 });
                 input_delta[i] = m_ptr;
             }

LightCTR/train/layer/fullyconnLayer.h

@@ -108,7 +108,7 @@ class Fully_Conn_Layer : public Layer_Base {
         wrapper.resize(1);
 
         if (this->nextLayer) {
-            this->getActiveFun().forward(output_act.pointer());
+            this->getActiveFun().forward(output_act.pointer()->data(), output_act.size());
             wrapper[0] = &output_act;
             return this->nextLayer->forward(wrapper);
         } else {
@@ -150,7 +150,10 @@ class Fully_Conn_Layer : public Layer_Base {
                 prev_output_act = this->prevLayer->output()[0]->pointer();
                 assert(prev_output_act && prev_output_act->size() == this->input_dimension);
 
-                this->prevLayer->getActiveFun().backward(input_delta.pointer(), prev_output_act, input_delta.pointer());
+                this->prevLayer->getActiveFun().backward(input_delta.pointer()->data(),
+                                                         prev_output_act->data(),
+                                                         input_delta.pointer()->data(),
+                                                         input_delta.size());
                 assert(input_delta.pointer()->size() == this->input_dimension);
 
                 vector<Matrix*>& wrapper = *tl_wrapper;

LightCTR/train/layer/sampleLayer.h

@@ -99,7 +99,10 @@ class Sample_Layer : public Layer_Base {
 
             assert(!isinf(*sigmaPtr));
         }
-        this->prevLayer->getActiveFun().backward(input_delta.pointer(), prev_output_act, input_delta.pointer());
+        this->prevLayer->getActiveFun().backward(input_delta.pointer()->data(),
+                                                 prev_output_act->data(),
+                                                 input_delta.pointer()->data(),
+                                                 input_delta.size());
 
         vector<Matrix*>& wrapper = *tl_wrapper;
         wrapper[0] = &input_delta;

LightCTR/train/train_gbm_algo.cpp

@@ -77,7 +77,7 @@ void Train_GBM_Algo::flash(RegTreeNode *root, size_t inClass) { // run per gbm t
             tmp->assign(&dataSet_Pred[rid * multiclass],
                         &dataSet_Pred[rid * multiclass + multiclass]);
             assert(tmp->size() == multiclass);
-            softmax.forward(tmp);
+            softmax.forward(tmp->data(), tmp->size());
             for (size_t c = 0; c < multiclass; c++) {
                 float grad_t = tmp->at(c);
                 float hess_t = grad_t * (1.0 - grad_t) * 2.0;

LightCTR/train/train_rnn_algo.h

@@ -47,29 +47,27 @@ class Train_RNN_Algo : public DL_Algo_Abst<LossFunction, ActivationFunction,
     vector<float>& Predict(size_t rid, vector<vector<float> >& dataRow) {
         static Matrix* dataRow_Matrix = new Matrix(1, 28);
         static Matrix* dataRow_Matrix_fc = new Matrix(1, hidden_size, 0);
-        static vector<Matrix*> *tmp = new vector<Matrix*>();
-        tmp->resize(1);
-
-        vector<float> *pred = NULL;
-        tmp->at(0) = dataRow_Matrix;
+        static vector<Matrix*> tmp;
+        tmp.resize(1);
+        tmp[0] = dataRow_Matrix;
 
         auto begin = dataRow[rid].begin();
         auto end = begin;
         FOR(i, batch_size) {
             begin = dataRow[rid].begin() + i * 28;
             end = dataRow[rid].begin() + (i + 1) * 28;
             dataRow_Matrix->pointer()->assign(begin, end);
-            pred = this->inputLayer->forward(*tmp);
+            this->inputLayer->forward(tmp);
         }
         assert(end == dataRow[rid].end());
 
         // Attention Unit
-        pred = attentionLayer->forward(*inputLayer->seq_output());
+        vector<float> pred = attentionLayer->forward(inputLayer->seq_output());
 
-        assert(pred && pred->size() == hidden_size);
-        dataRow_Matrix_fc->loadDataPtr(pred);
-        tmp->at(0) = dataRow_Matrix_fc;
-        return this->fcLayer->forward(*tmp);
+        assert(pred.size() == hidden_size);
+        dataRow_Matrix_fc->loadDataPtr(&pred);
+        tmp[0] = dataRow_Matrix_fc;
+        return this->fcLayer->forward(tmp);
     }
 
     void BP(size_t rid, const vector<Matrix*>& grad) {

LightCTR/train/train_vae_algo.h

@@ -68,11 +68,11 @@ class Train_VAE_Algo {
                 dataRow_Matrix->loadDataPtr(&dataSet[rid]);
                 tmp[0] = dataRow_Matrix;
                 vector<float>& pred = this->encodeLayer->forward(tmp);
-                outputActivFun.forward(&pred);
+                outputActivFun.forward(pred.data(), pred.size());
                 assert(pred.size() == feature_cnt);
                 grad.resize(pred.size());
-                lossFun.gradient(&pred, &dataSet[rid], &grad);
-                outputActivFun.backward(&grad, &pred, &grad);
+                lossFun.gradient(pred.data(), dataSet[rid].data(), grad.data(), grad.size());
+                outputActivFun.backward(grad.data(), pred.data(), grad.data(), grad.size());
                 // if LossFunction is Logistic, annotation last line
                 grad_Matrix->loadDataPtr(&grad);
                 tmp[0] = grad_Matrix;
@@ -91,19 +91,10 @@ class Train_VAE_Algo {
                     dataRow_Matrix->loadDataPtr(&dataSet[rid]);
                     tmp[0] = dataRow_Matrix;
                     vector<float> pred = this->encodeLayer->forward(tmp);
-                    outputActivFun.forward(&pred);
-                    loss += lossFun.loss(&pred, &dataSet[rid]);
-                    if (rid == 4 || rid == 8) { // look like number 4 or 5
-                        for (size_t i = 0; i < feature_cnt; i++) {
-                            cout.width(3);
-                            cout << int(pred[i] * 255) << ",";
-                            if ((i + 1) % 28 == 0) {
-                                cout << endl;
-                            }
-                        }
-                    }
+                    outputActivFun.forward(pred.data(), pred.size());
+                    loss += lossFun.loss(pred.data(), dataSet[rid].data(), pred.size());
                 }
-                printf("\nepoch %zu Loss = %f\n", p, loss);
+                printf("Epoch %zu Loss = %f\n", p, loss);
             }
         }
     }
@@ -171,21 +162,6 @@ class Train_VAE_Algo {
         }
         this->dataRow_cnt = this->dataSet.size();
         assert(this->dataRow_cnt > 0);
-
-        for (size_t i = 0; i < feature_cnt; i++) {
-            cout.width(3);
-            cout << int(dataSet[4][i] * 255) << ",";
-            if ((i + 1) % 28 == 0) {
-                cout << endl;
-            }
-        }
-        for (size_t i = 0; i < feature_cnt; i++) {
-            cout.width(3);
-            cout << int(dataSet[8][i] * 255) << ",";
-            if ((i + 1) % 28 == 0) {
-                cout << endl;
-            }
-        }
     }
 
 private:

LightCTR/train/unit/attention_unit.h

@@ -65,7 +65,7 @@ class Attention_Unit : public Layer_Base {
             *fc_output_act->getEle(0, idx) = res[0];
         }
         // Softmax normalization
-        softmax.forward(fc_output_act->pointer());
+        softmax.forward(fc_output_act->pointer()->data(), fc_output_act->size());
 
         attentionOutput->zeroInit();
         FOR(idx, prevLOutputMatrix.size()) {
@@ -103,7 +103,8 @@ class Attention_Unit : public Layer_Base {
             assert(res->size() == 1);
             scaleDelta[idx] = *cache_bp->getEle(0, 0);
         }
-        softmax.backward(&scaleDelta, fc_output_act->pointer(), &scaleDelta);
+        softmax.backward(scaleDelta.data(), fc_output_act->pointer()->data(),
+                         scaleDelta.data(), scaleDelta.size());
         // update transformFunc
         FOR(idx, input.size()) {
             *cache_bp->getEle(0, 0) = scaleDelta[idx];

LightCTR/train/unit/lstm_unit.h

@@ -33,9 +33,9 @@ using namespace std;
                        x##_h_grad_w->zeroInit();   \
                        x##_grad_b->zeroInit();
 
-#define UPDATE(x) updater_##x##_b.update(0, hidden_size, x##_b->reference(), x##_grad_b->reference()); \
-                  updater_##x##_w.update(0, dimension * hidden_size, x##_w->reference(), x##_grad_w->reference()); \
-                  updater_##x##_h_w.update(0, hidden_size * hidden_size, x##_h_w->reference(), x##_h_grad_w->reference());
+#define UPDATE(x) updater_##x##_b.update(0, hidden_size, x##_b->pointer()->data(), x##_grad_b->pointer()->data()); \
+                  updater_##x##_w.update(0, dimension * hidden_size, x##_w->pointer()->data(), x##_grad_w->pointer()->data()); \
+                  updater_##x##_h_w.update(0, hidden_size * hidden_size, x##_h_w->pointer()->data(), x##_h_grad_w->pointer()->data());
 
 // Bidirectional Recurrent Cell impl by Long Short Term Memory
 template <typename ActivationFunction>
@@ -139,7 +139,7 @@ class LSTM_Unit : public Layer_Base {
         }
         // apply ouput gate after do tanh
         cache = c_state[cur_seqid]->copy(cache);
-        inner_activeFun.forward(cache->pointer());
+        inner_activeFun.forward(cache->pointer()->data(), cache->size());
         c_state_act[cur_seqid] = cache->copy(c_state_act[cur_seqid]);
         h_output[cur_seqid] = cache->copy(h_output[cur_seqid])->dotProduct(oup_gate[cur_seqid]);
 
@@ -182,7 +182,10 @@ class LSTM_Unit : public Layer_Base {
 
             { // output gate weight
                 oup_gate_delta = h_output_delta->copy(oup_gate_delta)->dotProduct(c_state_act[seqid]);
-                sigmoid.backward(oup_gate_delta->pointer(), oup_gate[seqid]->pointer(), oup_gate_delta->pointer());
+                sigmoid.backward(oup_gate_delta->pointer()->data(),
+                                 oup_gate[seqid]->pointer()->data(),
+                                 oup_gate_delta->pointer()->data(),
+                                 oup_gate_delta->size());
 
                 accumGrad(oup_grad_w, oup_gate_delta, input[seqid]);
                 if (seqid > 0) {
@@ -200,11 +203,17 @@ class LSTM_Unit : public Layer_Base {
                 if (seqid < (int)cur_seqid - 1) { // accumulate the last time c_state's delta and h_output's delta
                     assert(c_state_delta[seqid]);
                     cache = h_output_delta->copy(cache)->dotProduct(oup_gate[seqid]);
-                    inner_activeFun.backward(cache->pointer(), c_state_act[seqid]->pointer(), cache->pointer());
+                    inner_activeFun.backward(cache->pointer()->data(),
+                                             c_state_act[seqid]->pointer()->data(),
+                                             cache->pointer()->data(),
+                                             cache->size());
                     c_state_delta[seqid]->add(cache);
                 } else { // for the first time of bp, clear memory
                     c_state_delta[seqid] = h_output_delta->copy(c_state_delta[seqid])->dotProduct(oup_gate[seqid]);
-                    inner_activeFun.backward(c_state_delta[seqid]->pointer(), c_state_act[seqid]->pointer(), c_state_delta[seqid]->pointer());
+                    inner_activeFun.backward(c_state_delta[seqid]->pointer()->data(),
+                                             c_state_act[seqid]->pointer()->data(),
+                                             c_state_delta[seqid]->pointer()->data(),
+                                             c_state_delta[seqid]->size());
                 }
 
                 { // delta of c_state in t-1, forget gate weight and delta of extra_info
@@ -214,7 +223,10 @@ class LSTM_Unit : public Layer_Base {
                         // clear prev-time memory
                         c_state_delta[seqid - 1] = c_state_delta[seqid]->copy(c_state_delta[seqid - 1])->dotProduct(fg_gate[seqid]);
                         fg_gate_delta = c_state_delta[seqid]->copy(fg_gate_delta)->dotProduct(c_state[seqid - 1]);
-                        sigmoid.backward(fg_gate_delta->pointer(), fg_gate[seqid]->pointer(), fg_gate_delta->pointer());
+                        sigmoid.backward(fg_gate_delta->pointer()->data(),
+                                         fg_gate[seqid]->pointer()->data(),
+                                         fg_gate_delta->pointer()->data(),
+                                         fg_gate_delta->size());
 
                         accumGrad(fg_grad_w, fg_gate_delta, input[seqid]);
                         accumGrad(fg_h_grad_w, fg_gate_delta, h_output[seqid - 1]);
@@ -227,7 +239,10 @@ class LSTM_Unit : public Layer_Base {
 
                         // input gate weight
                         inp_gate_delta = c_state_delta[seqid]->copy(inp_gate_delta)->dotProduct(info[seqid]);
-                        sigmoid.backward(inp_gate_delta->pointer(), inp_gate[seqid]->pointer(), inp_gate_delta->pointer());
+                        sigmoid.backward(inp_gate_delta->pointer()->data(),
+                                         inp_gate[seqid]->pointer()->data(),
+                                         inp_gate_delta->pointer()->data(),
+                                         inp_gate_delta->size());
 
                         accumGrad(inp_grad_w, inp_gate_delta, input[seqid]);
                         if (seqid > 0) {
@@ -241,7 +256,10 @@ class LSTM_Unit : public Layer_Base {
 
                         // delta of input_act transform
                         input_act_delta = c_state_delta[seqid]->copy(input_act_delta)->dotProduct(inp_gate[seqid]);
-                        inner_activeFun.backward(input_act_delta->pointer(), info[seqid]->pointer(), input_act_delta->pointer());
+                        inner_activeFun.backward(input_act_delta->pointer()->data(),
+                                                 info[seqid]->pointer()->data(),
+                                                 input_act_delta->pointer()->data(),
+                                                 input_act_delta->size());
                         // input gate weight
                         accumGrad(info_grad_w, input_act_delta, input[seqid]);
                         if (seqid > 0) {
@@ -262,9 +280,9 @@ class LSTM_Unit : public Layer_Base {
         wrapper[0] = h_output[cur_seqid - 1];
         return wrapper;
     }
-    vector<Matrix*>* seq_output() { // get rnn encoder output sequence for attention decoder
+    const vector<Matrix*>& seq_output() { // get rnn encoder output sequence for attention decoder
         assert(cur_seqid == batch_size);
-        return &h_output;
+        return h_output;
     }
 
     void applyBatchGradient() {
@@ -311,7 +329,8 @@ class LSTM_Unit : public Layer_Base {
             target->at(cur_seqid)->add(cache);
         }
         target->at(cur_seqid)->add(bias);
-        actFun->forward(target->at(cur_seqid)->pointer());
+        actFun->forward(target->at(cur_seqid)->pointer()->data(),
+                        target->at(cur_seqid)->size());
     }
 
     size_t cur_seqid;