v 0.7.5

- Fix numerical instability in `LayerNormalization`
- update `PNN` api

Author: shenweichen

.github/workflows/ci.yml

@@ -17,8 +17,8 @@ jobs:
     timeout-minutes: 120
     strategy:
       matrix:
-        python-version: [3.5,3.6,3.7]
-        tf-version: [1.4.0,1.15.0,2.0.0]
+        python-version: [3.6,3.7]
+        tf-version: [1.4.0,1.15.0,2.1.0,2.2.0]
 
         exclude:
           - python-version: 3.7

deepctr/__init__.py

@@ -1,4 +1,4 @@
 from .utils import check_version
 
-__version__ = '0.7.4'
+__version__ = '0.7.5'
 check_version(__version__)

deepctr/layers/normalization.py

@@ -12,9 +12,12 @@
 
 
 class LayerNormalization(Layer):
-    def __init__(self, axis=-1, eps=1e-9, **kwargs):
+    def __init__(self, axis=-1, eps=1e-9, center=True,
+                 scale=True, **kwargs):
         self.axis = axis
         self.eps = eps
+        self.center = center
+        self.scale = scale
         super(LayerNormalization, self).__init__(**kwargs)
 
     def build(self, input_shape):
@@ -24,15 +27,21 @@ def build(self, input_shape):
                                     initializer=Zeros(), trainable=True)
         super(LayerNormalization, self).build(input_shape)
 
-    def call(self, x):
-        mean = K.mean(x, axis=self.axis, keepdims=True)
-        std = K.std(x, axis=self.axis, keepdims=True)
-        return self.gamma * (x - mean) / (std + self.eps) + self.beta
+    def call(self, inputs):
+        mean = K.mean(inputs, axis=self.axis, keepdims=True)
+        variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)
+        std = K.sqrt(variance + self.eps)
+        outputs = (inputs - mean) / std
+        if self.scale:
+            outputs *= self.gamma
+        if self.center:
+            outputs += self.beta
+        return outputs
 
     def compute_output_shape(self, input_shape):
         return input_shape
 
     def get_config(self, ):
-        config = {'axis': self.axis, 'eps': self.eps}
+        config = {'axis': self.axis, 'eps': self.eps, 'center': self.center, 'scale': self.scale}
         base_config = super(LayerNormalization, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))

deepctr/models/pnn.py

@@ -15,13 +15,12 @@
 from ..layers.utils import concat_func
 
 
-def PNN(dnn_feature_columns, embedding_size=8, dnn_hidden_units=(128, 128), l2_reg_embedding=1e-5, l2_reg_dnn=0,
-        init_std=0.0001, seed=1024, dnn_dropout=0, dnn_activation='relu', use_inner=True, use_outter=False,
-        kernel_type='mat', task='binary'):
+def PNN(dnn_feature_columns, dnn_hidden_units=(128, 128), l2_reg_embedding=1e-5, l2_reg_dnn=0, init_std=0.0001,
+        seed=1024, dnn_dropout=0, dnn_activation='relu', use_inner=True, use_outter=False, kernel_type='mat',
+        task='binary'):
     """Instantiates the Product-based Neural Network architecture.
 
     :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
-    :param embedding_size: positive integer,sparse feature embedding_size
     :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net
     :param l2_reg_embedding: float . L2 regularizer strength applied to embedding vector
     :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
@@ -51,7 +50,7 @@ def PNN(dnn_feature_columns, embedding_size=8, dnn_hidden_units=(128, 128), l2_r
 
     # ipnn deep input
     linear_signal = tf.keras.layers.Reshape(
-        [len(sparse_embedding_list) * embedding_size])(concat_func(sparse_embedding_list))
+        [sum(map(lambda x:int(x.shape[-1]) ,sparse_embedding_list))])(concat_func(sparse_embedding_list))
 
     if use_inner and use_outter:
         deep_input = tf.keras.layers.Concatenate()(

docs/source/History.md

@@ -1,4 +1,6 @@
 # History
+
+- 05/17/2020 : [v0.7.5](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.5) released.Fix numerical instability in `LayerNormalization`.
 - 03/15/2020 : [v0.7.4](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.4) released.Add [FLEN](./Features.html#flen-field-leveraged-embedding-network) and `FieldWiseBiInteraction`.
 - 03/04/2020 : [v0.7.3](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.3) released.Fix the inconsistency of prediction results when the model is loaded with trained weights.
 - 02/08/2020 : [v0.7.2](https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.2) released.Fix some bugs.

docs/source/Quick-Start.md

@@ -92,8 +92,8 @@ fixlen_feature_columns = [SparseFeat(feat, vocabulary_size=1e6,embedding_dim=4,
 ```
 - generate feature columns
 ```python
-dnn_feature_columns = sparse_feature_columns + dense_feature_columns
-linear_feature_columns = sparse_feature_columns + dense_feature_columns
+dnn_feature_columns = fixlen_feature_columns
+linear_feature_columns = fixlen_feature_columns
 
 feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)
 

docs/source/conf.py

@@ -26,7 +26,7 @@
 # The short X.Y version
 version = ''
 # The full version, including alpha/beta/rc tags
-release = '0.7.4'
+release = '0.7.5'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/index.rst

@@ -35,12 +35,12 @@ You can read the latest code at https://github.com/shenweichen/DeepCTR
 News
 -----
 
+05/17/2020 : Fix numerical instability in ``LayerNormalization``. `Changelog <https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.5>`_
+
 03/15/2020 : Add `FLEN <./Features.html#flen-field-leveraged-embedding-network>`_ (`中文介绍 <https://zhuanlan.zhihu.com/p/92787577>`_) and ``FieldWiseBiInteraction``. `Changelog <https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.4>`_
 
 03/04/2020 : Fix the inconsistency of prediction results when the model is loaded with trained weights. `Changelog <https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.3>`_
 
-02/08/2020 : Fix some bugs. `Changelog <https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.2>`_
-
 DisscussionGroup
 -----------------------
 

setup.py

@@ -9,7 +9,7 @@
 
 setuptools.setup(
     name="deepctr",
-    version="0.7.4",
+    version="0.7.5",
     author="Weichen Shen",
     author_email="[email protected]",
     description="Easy-to-use,Modular and Extendible package of deep learning based CTR(Click Through Rate) prediction models with tensorflow 1.x and 2.x .",

tests/layers/activations_test.py

@@ -1,5 +1,8 @@
 from deepctr.layers import activation
-from tensorflow.python.keras.utils import CustomObjectScope
+try:
+    from tensorflow.python.keras.utils import CustomObjectScope
+except:
+    from tensorflow.keras.utils import CustomObjectScope
 from tests.utils import layer_test
 
 

tests/layers/core_test.py

@@ -1,8 +1,10 @@
 import pytest
 import tensorflow as tf
 from tensorflow.python.keras.layers import PReLU
-from tensorflow.python.keras.utils import CustomObjectScope
-
+try:
+    from tensorflow.python.keras.utils import CustomObjectScope
+except:
+    from tensorflow.keras.utils import CustomObjectScope
 from deepctr import layers
 from deepctr.layers import Dice
 from tests.layers.interaction_test import BATCH_SIZE, EMBEDDING_SIZE, SEQ_LENGTH

tests/layers/interaction_test.py

@@ -1,7 +1,9 @@
 import pytest
-#from tensorflow.python.keras.layers import PReLU
-from tensorflow.python.keras.utils import CustomObjectScope
 
+try:
+    from tensorflow.python.keras.utils import CustomObjectScope
+except:
+    from tensorflow.keras.utils import CustomObjectScope
 from deepctr import layers
 
 from tests.utils import layer_test
@@ -16,13 +18,13 @@
 
     'layer_num',
 
-    [0,1]
+    [0, 1]
 
 )
-def test_CrossNet(layer_num,):
+def test_CrossNet(layer_num, ):
     with CustomObjectScope({'CrossNet': layers.CrossNet}):
         layer_test(layers.CrossNet, kwargs={
-                   'layer_num': layer_num, }, input_shape=(2, 3))
+            'layer_num': layer_num, }, input_shape=(2, 3))
 
 
 # def test_CrossNet_invalid():
@@ -41,7 +43,7 @@ def test_CrossNet(layer_num,):
 def test_InnerProductLayer(reduce_sum):
     with CustomObjectScope({'InnerProductLayer': layers.InnerProductLayer}):
         layer_test(layers.InnerProductLayer, kwargs={
-            'reduce_sum': reduce_sum}, input_shape=[(BATCH_SIZE, 1, EMBEDDING_SIZE)]*FIELD_SIZE)
+            'reduce_sum': reduce_sum}, input_shape=[(BATCH_SIZE, 1, EMBEDDING_SIZE)] * FIELD_SIZE)
 
 
 @pytest.mark.parametrize(
@@ -53,7 +55,7 @@ def test_InnerProductLayer(reduce_sum):
 def test_OutterProductLayer(kernel_type):
     with CustomObjectScope({'OutterProductLayer': layers.OutterProductLayer}):
         layer_test(layers.OutterProductLayer, kwargs={
-            'kernel_type': kernel_type}, input_shape=[(BATCH_SIZE, 1, EMBEDDING_SIZE)]*FIELD_SIZE)
+            'kernel_type': kernel_type}, input_shape=[(BATCH_SIZE, 1, EMBEDDING_SIZE)] * FIELD_SIZE)
 
 
 def test_BiInteractionPooling():
@@ -70,13 +72,13 @@ def test_FM():
 
 def test_AFMLayer():
     with CustomObjectScope({'AFMLayer': layers.AFMLayer}):
-        layer_test(layers.AFMLayer, kwargs={'dropout_rate':0.5}, input_shape=[(
-            BATCH_SIZE, 1, EMBEDDING_SIZE)]*FIELD_SIZE)
+        layer_test(layers.AFMLayer, kwargs={'dropout_rate': 0.5}, input_shape=[(
+            BATCH_SIZE, 1, EMBEDDING_SIZE)] * FIELD_SIZE)
 
 
 @pytest.mark.parametrize(
     'layer_size,split_half',
-    [((10,),False),((10,8),True)
+    [((10,), False), ((10, 8), True)
      ]
 )
 def test_CIN(layer_size, split_half):
@@ -99,17 +101,18 @@ def test_CIN(layer_size, split_half):
 
 @pytest.mark.parametrize(
     'head_num,use_res',
-    [(1,True),(2,False,)]
+    [(1, True), (2, False,)]
 )
-def test_InteractingLayer(head_num, use_res,):
+def test_InteractingLayer(head_num, use_res, ):
     with CustomObjectScope({'InteractingLayer': layers.InteractingLayer}):
         layer_test(layers.InteractingLayer, kwargs={"head_num": head_num, "use_res":
-                                                    use_res, }, input_shape=(
+            use_res, }, input_shape=(
             BATCH_SIZE, FIELD_SIZE, EMBEDDING_SIZE))
 
+
 def test_FGCNNLayer():
     with CustomObjectScope({'FGCNNLayer': layers.FGCNNLayer}):
-        layer_test(layers.FGCNNLayer, kwargs={'filters':(4, 6,),'kernel_width':(7, 7,)}, input_shape=(
+        layer_test(layers.FGCNNLayer, kwargs={'filters': (4, 6,), 'kernel_width': (7, 7,)}, input_shape=(
             BATCH_SIZE, FIELD_SIZE, EMBEDDING_SIZE))
 
 
@@ -121,12 +124,10 @@ def test_FGCNNLayer():
 
 @pytest.mark.parametrize(
     'bilinear_type',
-    ['all','each','interaction'
+    ['all', 'each', 'interaction'
      ]
 )
 def test_BilinearInteraction(bilinear_type):
     with CustomObjectScope({'BilinearInteraction': layers.BilinearInteraction}):
-        layer_test(layers.BilinearInteraction, kwargs={'bilinear_type':bilinear_type}, input_shape=[(
-            BATCH_SIZE, 1, EMBEDDING_SIZE)]*FIELD_SIZE)
-
-
+        layer_test(layers.BilinearInteraction, kwargs={'bilinear_type': bilinear_type}, input_shape=[(
+            BATCH_SIZE, 1, EMBEDDING_SIZE)] * FIELD_SIZE)

tests/layers/normalization_test.py

@@ -1,6 +1,8 @@
 import pytest
-from tensorflow.python.keras.utils import CustomObjectScope
-
+try:
+    from tensorflow.python.keras.utils import CustomObjectScope
+except:
+    from tensorflow.keras.utils import CustomObjectScope
 from deepctr import layers
 from tests.layers.interaction_test import BATCH_SIZE, FIELD_SIZE, EMBEDDING_SIZE
 from tests.utils import layer_test

tests/layers/sequence_test.py

@@ -1,6 +1,9 @@
 import pytest
 from packaging import version
-from tensorflow.python.keras.utils import CustomObjectScope
+try:
+    from tensorflow.python.keras.utils import CustomObjectScope
+except:
+    from tensorflow.keras.utils import CustomObjectScope
 import tensorflow as tf
 from deepctr.layers import sequence
 

tests/models/PNN_test.py

@@ -14,8 +14,7 @@ def test_PNN(use_inner, use_outter, sparse_feature_num):
     sample_size = SAMPLE_SIZE
     x, y, feature_columns = get_test_data(sample_size, sparse_feature_num=sparse_feature_num,
                                           dense_feature_num=sparse_feature_num)
-    model = PNN(feature_columns, embedding_size=4, dnn_hidden_units=[4, 4], dnn_dropout=0.5, use_inner=use_inner,
-                use_outter=use_outter)
+    model = PNN(feature_columns, dnn_hidden_units=[4, 4], dnn_dropout=0.5, use_inner=use_inner, use_outter=use_outter)
     check_model(model, model_name, x, y)