add flen

Add FLEN: Leveraging Field for Scalable CTR Prediction

Author: 浅梦

README.md

@@ -44,6 +44,8 @@ Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Star
 |                 FGCNN                  | [WWW 2019][Feature Generation by Convolutional Neural Network for Click-Through Rate Prediction ](https://arxiv.org/pdf/1904.04447)                             |
 |     Deep Session Interest Network      | [IJCAI 2019][Deep Session Interest Network for Click-Through Rate Prediction ](https://arxiv.org/abs/1905.06482)                                                |
 |                FiBiNET                 | [RecSys 2019][FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf)   |
+|                FLEN                    | [arxiv 2019][FLEN: Leveraging Field for Scalable CTR Prediction](https://arxiv.org/pdf/1911.04690.pdf)   |
+
 
 
 ## DisscussionGroup  

deepctr/__init__.py

@@ -1,4 +1,4 @@
 from .utils import check_version
 
-__version__ = '0.7.3'
+__version__ = '0.7.4'
 check_version(__version__)

deepctr/inputs.py

@@ -177,16 +177,17 @@ def create_embedding_matrix(feature_columns, l2_reg, init_std, seed, prefix="",
 
 def get_linear_logit(features, feature_columns, units=1, use_bias=False, init_std=0.0001, seed=1024, prefix='linear',
                      l2_reg=0):
-    for i in range(len(feature_columns)):
-        if isinstance(feature_columns[i], SparseFeat):
-            feature_columns[i] = feature_columns[i]._replace(embedding_dim=1)
-        if isinstance(feature_columns[i], VarLenSparseFeat):
-            feature_columns[i] = feature_columns[i]._replace(
-                sparsefeat=feature_columns[i].sparsefeat._replace(embedding_dim=1))
-
-    linear_emb_list = [input_from_feature_columns(features, feature_columns, l2_reg, init_std, seed,
+    linear_feature_columns = feature_columns.copy()
+    for i in range(len(linear_feature_columns)):
+        if isinstance(linear_feature_columns[i], SparseFeat):
+            linear_feature_columns[i] = linear_feature_columns[i]._replace(embedding_dim=1)
+        if isinstance(linear_feature_columns[i], VarLenSparseFeat):
+            linear_feature_columns[i] = linear_feature_columns[i]._replace(
+                sparsefeat=linear_feature_columns[i].sparsefeat._replace(embedding_dim=1))
+
+    linear_emb_list = [input_from_feature_columns(features, linear_feature_columns, l2_reg, init_std, seed,
                                                   prefix=prefix + str(i))[0] for i in range(units)]
-    _, dense_input_list = input_from_feature_columns(features, feature_columns, l2_reg, init_std, seed, prefix=prefix)
+    _, dense_input_list = input_from_feature_columns(features, linear_feature_columns, l2_reg, init_std, seed, prefix=prefix)
 
     linear_logit_list = []
     for i in range(units):

deepctr/layers/interaction.py

@@ -17,7 +17,7 @@
 from tensorflow.python.layers import utils
 
 from .activation import activation_layer
-from .utils import concat_func,reduce_sum,softmax,reduce_mean
+from .utils import concat_func, reduce_sum, softmax, reduce_mean
 
 
 class AFMLayer(Layer):
@@ -235,7 +235,7 @@ def build(self, input_shape):
                                                 shape=[1, self.field_nums[-1]
                                                        * self.field_nums[0], size],
                                                 dtype=tf.float32, initializer=glorot_uniform(
-                                                    seed=self.seed + i),
+                    seed=self.seed + i),
                                                 regularizer=l2(self.l2_reg)))
 
             self.bias.append(self.add_weight(name='bias' + str(i), shape=[size], dtype=tf.float32,
@@ -906,11 +906,10 @@ class SENETLayer(Layer):
         - **seed** : A Python integer to use as random seed.
 
       References
-        - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction
-Tongwen](https://arxiv.org/pdf/1905.09433.pdf)
+        - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf)
     """
 
-    def __init__(self, reduction_ratio=3,  seed=1024, **kwargs):
+    def __init__(self, reduction_ratio=3, seed=1024, **kwargs):
         self.reduction_ratio = reduction_ratio
 
         self.seed = seed
@@ -924,7 +923,7 @@ def build(self, input_shape):
 
         self.filed_size = len(input_shape)
         self.embedding_size = input_shape[0][-1]
-        reduction_size = max(1, self.filed_size//self.reduction_ratio)
+        reduction_size = max(1, self.filed_size // self.reduction_ratio)
 
         self.W_1 = self.add_weight(shape=(
             self.filed_size, reduction_size), initializer=glorot_normal(seed=self.seed), name="W_1")
@@ -944,7 +943,7 @@ def call(self, inputs, training=None, **kwargs):
                 "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))
 
         inputs = concat_func(inputs, axis=1)
-        Z = reduce_mean(inputs, axis=-1,)
+        Z = reduce_mean(inputs, axis=-1, )
 
         A_1 = tf.nn.relu(self.tensordot([Z, self.W_1]))
         A_2 = tf.nn.relu(self.tensordot([A_1, self.W_2]))
@@ -957,7 +956,7 @@ def compute_output_shape(self, input_shape):
         return input_shape
 
     def compute_mask(self, inputs, mask=None):
-        return [None]*self.filed_size
+        return [None] * self.filed_size
 
     def get_config(self, ):
         config = {'reduction_ratio': self.reduction_ratio, 'seed': self.seed}
@@ -980,8 +979,7 @@ class BilinearInteraction(Layer):
         - **seed** : A Python integer to use as random seed.
 
       References
-        - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction
-Tongwen](https://arxiv.org/pdf/1905.09433.pdf)
+        - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf)
 
     """
 
@@ -1003,10 +1001,11 @@ def build(self, input_shape):
                 seed=self.seed), name="bilinear_weight")
         elif self.bilinear_type == "each":
             self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(
-                seed=self.seed), name="bilinear_weight"+str(i)) for i in range(len(input_shape)-1)]
+                seed=self.seed), name="bilinear_weight" + str(i)) for i in range(len(input_shape) - 1)]
         elif self.bilinear_type == "interaction":
             self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(
-                seed=self.seed), name="bilinear_weight"+str(i)+'_'+str(j)) for i, j in itertools.combinations(range(len(input_shape)), 2)]
+                seed=self.seed), name="bilinear_weight" + str(i) + '_' + str(j)) for i, j in
+                           itertools.combinations(range(len(input_shape)), 2)]
         else:
             raise NotImplementedError
 
@@ -1036,7 +1035,7 @@ def compute_output_shape(self, input_shape):
         filed_size = len(input_shape)
         embedding_size = input_shape[0][-1]
 
-        return (None, 1, filed_size*(filed_size-1)//2 * embedding_size)
+        return (None, 1, filed_size * (filed_size - 1) // 2 * embedding_size)
 
     def get_config(self, ):
         config = {'bilinear_type': self.bilinear_type, 'seed': self.seed}
@@ -1056,15 +1055,15 @@ class FieldWiseBiInteraction(Layer):
      
       Arguments
         - **use_bias** : Boolean, if use bias.
-        - **l2_reg** : Float, l2 regularization coefficient.
         - **seed** : A Python integer to use as random seed.
-     
+
       References
-        [1] hen W, Zhan L, Ci Y, Lin C https://arxiv.org/pdf/1911.04690
+        - [FLEN: Leveraging Field for Scalable CTR Prediction](https://arxiv.org/pdf/1911.04690)
+
     """
-    def __init__(self, l2_reg=1e-5, seed=1024, **kwargs):
 
-        self.l2_reg = l2_reg
+    def __init__(self,use_bias=True, seed=1024, **kwargs):
+        self.use_bias = use_bias
         self.seed = seed
 
         super(FieldWiseBiInteraction, self).__init__(**kwargs)
@@ -1079,26 +1078,26 @@ def build(self, input_shape):
         self.num_fields = len(input_shape)
         embedding_size = input_shape[0][-1]
 
-        self.kernel_inter = self.add_weight(
-            name='kernel_inter',
+        self.kernel_mf = self.add_weight(
+            name='kernel_mf',
             shape=(int(self.num_fields * (self.num_fields - 1) / 2), 1),
-            initializer=glorot_normal(seed=self.seed),
-            regularizer=l2(self.l2_reg),
+            initializer=tf.keras.initializers.Ones(),
+            regularizer=None,
             trainable=True)
-        self.bias_inter = self.add_weight(name='bias_inter',
-                                          shape=(embedding_size),
-                                          initializer=Zeros(),
-                                          trainable=True)
-        self.kernel_intra = self.add_weight(
-            name='kernel_intra',
+
+        self.kernel_fm = self.add_weight(
+            name='kernel_fm',
             shape=(self.num_fields, 1),
-            initializer=glorot_normal(seed=self.seed),
-            regularizer=l2(self.l2_reg),
+            initializer=tf.keras.initializers.Constant(value=0.5),
+            regularizer=None,
             trainable=True)
-        self.bias_intra = self.add_weight(name='bias_intra',
-                                          shape=(embedding_size),
-                                          initializer=Zeros(),
-                                          trainable=True)
+        if self.use_bias:
+            self.bias_mf = self.add_weight(name='bias_mf',
+                                           shape=(embedding_size),
+                                           initializer=Zeros())
+            self.bias_fm = self.add_weight(name='bias_fm',
+                                           shape=(embedding_size),
+                                           initializer=Zeros())
 
         super(FieldWiseBiInteraction,
               self).build(input_shape)  # Be sure to call this somewhere!
@@ -1120,10 +1119,10 @@ def call(self, inputs, **kwargs):
 
         left = []
         right = []
-        for i in range(self.num_fields):
-            for j in range(i + 1, self.num_fields):
-                left.append(i)
-                right.append(j)
+
+        for i, j in itertools.combinations(list(range(self.num_fields)), 2):
+            left.append(i)
+            right.append(j)
 
         embeddings_left = tf.gather(params=field_wise_vectors,
                                     indices=left,
@@ -1133,9 +1132,10 @@ def call(self, inputs, **kwargs):
                                      axis=1)
 
         embeddings_prod = embeddings_left * embeddings_right
-        field_weighted_embedding = embeddings_prod * self.kernel_inter
+        field_weighted_embedding = embeddings_prod * self.kernel_mf
         h_mf = reduce_sum(field_weighted_embedding, axis=1)
-        h_mf = tf.nn.bias_add(h_mf, self.bias_inter)
+        if self.use_bias:
+            h_mf = tf.nn.bias_add(h_mf, self.bias_mf)
 
         # FM module
         square_of_sum_list = [
@@ -1154,11 +1154,16 @@ def call(self, inputs, **kwargs):
             zip(square_of_sum_list, sum_of_square_list)
         ], 1)
 
-        h_fm = reduce_sum(field_fm * self.kernel_intra, axis=1)
-
-        h_fm = tf.nn.bias_add(h_fm, self.bias_intra)
+        h_fm = reduce_sum(field_fm * self.kernel_fm, axis=1)
+        if self.use_bias:
+            h_fm = tf.nn.bias_add(h_fm, self.bias_fm)
 
         return h_mf + h_fm
 
     def compute_output_shape(self, input_shape):
-        return (None, input_shape[0][-1])
+        return (None, input_shape[0][-1])
+
+    def get_config(self, ):
+        config = {'use_bias': self.use_bias, 'seed': self.seed}
+        base_config = super(FieldWiseBiInteraction, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

deepctr/models/flen.py

@@ -4,41 +4,40 @@
     Tingyi Tan,[email protected]
 
 Reference:
-    [1] hen W, Zhan L, Ci Y, Lin C https://arxiv.org/pdf/1911.04690
+    [1] Chen W, Zhan L, Ci Y, Lin C. FLEN: Leveraging Field for Scalable CTR Prediction . arXiv preprint arXiv:1911.04690, 2019.(https://arxiv.org/pdf/1911.04690)
 
 """
 
-
 from itertools import chain
+
 import tensorflow as tf
-from tensorflow.python.keras.layers import Flatten
 
 from ..inputs import input_from_feature_columns, get_linear_logit, build_input_features, combined_dnn_input
 from ..layers.core import PredictionLayer, DNN
-from ..layers.utils import concat_func, add_func
 from ..layers.interaction import FieldWiseBiInteraction
+from ..layers.utils import concat_func, add_func
 
 
 def FLEN(linear_feature_columns,
-          dnn_feature_columns,
-          l2_reg_linear=0.00001,
-          l2_reg_embedding=0.00001,
-          l2_reg_dnn=0.00001,
-          l2_reg_fw=0.00001,
-          init_std=0.0001,
-          seed=1024,
-          dnn_dropout=0.2,
-          dnn_activation='relu',
-          dnn_use_bn=True,
-          task='binary'):
-    """Instantiates the DeepFM Network architecture.
+         dnn_feature_columns,
+         dnn_hidden_units=(128, 128),
+         l2_reg_linear=0.00001,
+         l2_reg_embedding=0.00001,
+         l2_reg_dnn=0,
+         init_std=0.0001,
+         seed=1024,
+         dnn_dropout=0.0,
+         dnn_activation='relu',
+         dnn_use_bn=False,
+         task='binary'):
+    """Instantiates the FLEN Network architecture.
 
     :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
     :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
+    :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net
     :param l2_reg_linear: float. L2 regularizer strength applied to linear part
     :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
     :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
-    :param l2_reg_fw: float. L2 regularizer strength applied to fwfm
     :param init_std: float,to use as the initialize std of embedding vector
     :param seed: integer ,to use as random seed.
     :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
@@ -61,35 +60,26 @@ def FLEN(linear_feature_columns,
         seed,
         support_group=True)
 
-    # S
     linear_logit = get_linear_logit(features,
                                     linear_feature_columns,
                                     init_std=init_std,
                                     seed=seed,
                                     prefix='linear',
                                     l2_reg=l2_reg_linear)
-    linear_logit = Flatten()(linear_logit)
 
-    # FM + MF
-    fm_mf_out = FieldWiseBiInteraction(l2_reg=l2_reg_fw, seed=seed)(
+    fm_mf_out = FieldWiseBiInteraction(seed=seed)(
         [concat_func(v, axis=1) for k, v in group_embedding_dict.items()])
-    fm_mf_out = DNN((32,), dnn_activation, l2_reg_dnn, dnn_dropout,
-                    dnn_use_bn, seed)(fm_mf_out)
 
-    # MLP
-    mlp_input = combined_dnn_input(
+    dnn_input = combined_dnn_input(
         list(chain.from_iterable(group_embedding_dict.values())),
         dense_value_list)
-    mlp_output = DNN((64,), dnn_activation, l2_reg_dnn, dnn_dropout,
-                     dnn_use_bn, seed)(mlp_input)
-    mlp_output = DNN((32,), dnn_activation, l2_reg_dnn, dnn_dropout,
-                     dnn_use_bn, seed)(mlp_output)
+    dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,
+                     dnn_use_bn, seed)(dnn_input)
+
+    dnn_logit = tf.keras.layers.Dense(1, use_bias=False, activation=None)(concat_func([fm_mf_out, dnn_output]))
 
-    # DNN
-    dnn_input = combined_dnn_input([fm_mf_out, mlp_output, linear_logit], dense_value_list)
-    dnn_output = dnn_input
-    dnn_logit = tf.keras.layers.Dense(1, use_bias=False, activation=None)(dnn_output)
-    output = PredictionLayer(task)(dnn_logit)
+    final_logit = add_func([linear_logit, dnn_logit])
+    output = PredictionLayer(task)(final_logit)
 
     model = tf.keras.models.Model(inputs=inputs_list, outputs=output)
-    return model
+    return model

docs/pics/FLEN.jpg

@@ -264,6 +264,17 @@ Feature Importance and Bilinear feature Interaction NETwork is proposed to dynam
 
 [Huang T, Zhang Z, Zhang J. FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction[J]. arXiv preprint arXiv:1905.09433, 2019.](https://arxiv.org/pdf/1905.09433.pdf)
 
+### FLEN(Field-Leveraged Embedding Network)
+
+A large-scale CTR prediction model with efficient usage of field information to alleviate gradient coupling problem.
+
+[**FLEN Model API**](./deepctr.models.flen.html)  
+
+[FLEN example](https://github.com/shenweichen/DeepCTR/tree/master/examples/run_flen.py)
+
+![FLEN](../pics/FLEN.jpg)
+
+[Chen W, Zhan L, Ci Y, Lin C. FLEN: Leveraging Field for Scalable CTR Prediction[J]. arXiv preprint arXiv:1911.04690, 2019.](https://arxiv.org/pdf/1911.04690.pdf)
 
 ## Layers
 

docs/source/Features.md

@@ -1,4 +1,5 @@
 # History
+- 03/15/2020 : [v0.7.4](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.4) released.Add [FLEN](./Features.html#flen-field-leveraged-embedding-network) and `FieldWiseBiInteraction`.
 - 03/04/2020 : [v0.7.3](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.3) released.Fix the inconsistency of prediction results when the model is loaded with trained weights.
 - 02/08/2020 : [v0.7.2](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.2) released.Fix some bugs.
 - 01/28/2020 : [v0.7.1](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.1) released.Simplify [VarLenSparseFeat](./Features.html#varlensparsefeat),support setting weight_normalization.Fix problem of embedding size of `SparseFeat` in `linear_feature_columns`.

docs/source/History.md

@@ -20,4 +20,5 @@ DeepCTR Models API
    ONN<deepctr.models.onn>
    FGCNN<deepctr.models.fgcnn>
    FiBiNET<deepctr.models.fibinet>
+   FLEN<deepctr.models.flen>
 

docs/source/Models.rst

@@ -26,7 +26,7 @@
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = '0.7.3'
+release = '0.7.4'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/conf.py

@@ -0,0 +1,7 @@
+deepctr.models.flen module
+=============================
+
+.. automodule:: deepctr.models.flen
+    :members:
+    :no-undoc-members:
+    :no-show-inheritance: