Implement the BST model. (shenweichen#327)

* Implement the BST model.

Author: codewithzichao

README.md

@@ -54,6 +54,7 @@ Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Star
 |     Deep Session Interest Network      | [IJCAI 2019][Deep Session Interest Network for Click-Through Rate Prediction ](https://arxiv.org/abs/1905.06482)                                                |
 |                FiBiNET                 | [RecSys 2019][FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf)   |
 |                FLEN                    | [arxiv 2019][FLEN: Leveraging Field for Scalable CTR Prediction](https://arxiv.org/pdf/1911.04690.pdf)   |
+|                 BST                   | [DLP-KDD 2019][Behavior sequence transformer for e-commerce recommendation in Alibaba](https://arxiv.org/pdf/1905.06874.pdf)                           | 
 |                DCN V2                    | [arxiv 2020][DCN V2: Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/abs/2008.13535)   |
 
 ## Citation

deepctr/__init__.py

@@ -1,4 +1,4 @@
-from .utils import check_version
-
-__version__ = '0.8.3'
-check_version(__version__)
+from .utils import check_version
+
+__version__ = '0.8.3'
+check_version(__version__)

deepctr/layers/sequence.py

@@ -79,7 +79,7 @@ def call(self, seq_value_len_list, mask=None, **kwargs):
         mask = tf.tile(mask, [1, 1, embedding_size])
 
         if self.mode == "max":
-            hist = uiseq_embed_list - (1-mask) * 1e9
+            hist = uiseq_embed_list - (1 - mask) * 1e9
             return reduce_max(hist, 1, keep_dims=True)
 
         hist = reduce_sum(uiseq_embed_list * mask, 1, keep_dims=False)
@@ -436,14 +436,16 @@ class Transformer(Layer):
             - **blinding**: bool. Whether or not use blinding.
             - **seed**: A Python integer to use as random seed.
             - **supports_masking**:bool. Whether or not support masking.
+            - **attention_type**: str, Type of attention, the value must be one of ["scaled_dot_product","additive"].
+            - **output_type**: str or None. Whether or not use average/sum pooling for output.
 
       References
             - [Vaswani, Ashish, et al. "Attention is all you need." Advances in Neural Information Processing Systems. 2017.](https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf)
     """
 
     def __init__(self, att_embedding_size=1, head_num=8, dropout_rate=0.0, use_positional_encoding=True, use_res=True,
                  use_feed_forward=True, use_layer_norm=False, blinding=True, seed=1024, supports_masking=False,
-                 **kwargs):
+                 attention_type="scaled_dot_product", output_type="mean", **kwargs):
         if head_num <= 0:
             raise ValueError('head_num must be a int > 0')
         self.att_embedding_size = att_embedding_size
@@ -456,6 +458,8 @@ def __init__(self, att_embedding_size=1, head_num=8, dropout_rate=0.0, use_posit
         self.dropout_rate = dropout_rate
         self.use_layer_norm = use_layer_norm
         self.blinding = blinding
+        self.attention_type = attention_type
+        self.output_type = output_type
         super(Transformer, self).__init__(**kwargs)
         self.supports_masking = supports_masking
 
@@ -464,7 +468,7 @@ def build(self, input_shape):
         if self.num_units != embedding_size:
             raise ValueError(
                 "att_embedding_size * head_num must equal the last dimension size of inputs,got %d * %d != %d" % (
-                self.att_embedding_size, self.head_num, embedding_size))
+                    self.att_embedding_size, self.head_num, embedding_size))
         self.seq_len_max = int(input_shape[0][-2])
         self.W_Query = self.add_weight(name='query', shape=[embedding_size, self.att_embedding_size * self.head_num],
                                        dtype=tf.float32,
@@ -475,6 +479,11 @@ def build(self, input_shape):
         self.W_Value = self.add_weight(name='value', shape=[embedding_size, self.att_embedding_size * self.head_num],
                                        dtype=tf.float32,
                                        initializer=tf.keras.initializers.TruncatedNormal(seed=self.seed + 2))
+        if self.attention_type == "additive":
+            self.b = self.add_weight('b', shape=[self.att_embedding_size], dtype=tf.float32,
+                                     initializer=tf.keras.initializers.glorot_uniform(seed=self.seed))
+            self.v = self.add_weight('v', shape=[self.att_embedding_size], dtype=tf.float32,
+                                     initializer=tf.keras.initializers.glorot_uniform(seed=self.seed))
         # if self.use_res:
         #     self.W_Res = self.add_weight(name='res', shape=[embedding_size, self.att_embedding_size * self.head_num], dtype=tf.float32,
         #                                  initializer=tf.keras.initializers.TruncatedNormal(seed=self.seed))
@@ -525,10 +534,18 @@ def call(self, inputs, mask=None, training=None, **kwargs):
         keys = tf.concat(tf.split(keys, self.head_num, axis=2), axis=0)
         values = tf.concat(tf.split(values, self.head_num, axis=2), axis=0)
 
-        # head_num*None T_q T_k
-        outputs = tf.matmul(querys, keys, transpose_b=True)
+        if self.attention_type == "scaled_dot_product":
+            # head_num*None T_q T_k
+            outputs = tf.matmul(querys, keys, transpose_b=True)
 
-        outputs = outputs / (keys.get_shape().as_list()[-1] ** 0.5)
+            outputs = outputs / (keys.get_shape().as_list()[-1] ** 0.5)
+        elif self.attention_type == "additive":
+            querys_reshaped = tf.expand_dims(querys, axis=-2)
+            keys_reshaped = tf.expand_dims(keys, axis=-3)
+            outputs = tf.tanh(tf.nn.bias_add(querys_reshaped + keys_reshaped, self.b))
+            outputs = tf.squeeze(tf.tensordot(outputs, tf.expand_dims(self.v, axis=-1), axes=[-1, 0]), axis=-1)
+        else:
+            NotImplementedError
 
         key_masks = tf.tile(key_masks, [self.head_num, 1])
 
@@ -579,7 +596,12 @@ def call(self, inputs, mask=None, training=None, **kwargs):
             if self.use_layer_norm:
                 result = self.ln(result)
 
-        return reduce_mean(result, axis=1, keep_dims=True)
+        if self.output_type == "mean":
+            return reduce_mean(result, axis=1, keep_dims=True)
+        elif self.output_type == "sum":
+            return reduce_sum(result, axis=1, keep_dims=True)
+        else:
+            return result
 
     def compute_output_shape(self, input_shape):
 
@@ -593,7 +615,7 @@ def get_config(self, ):
                   'dropout_rate': self.dropout_rate, 'use_res': self.use_res,
                   'use_positional_encoding': self.use_positional_encoding, 'use_feed_forward': self.use_feed_forward,
                   'use_layer_norm': self.use_layer_norm, 'seed': self.seed, 'supports_masking': self.supports_masking,
-                  'blinding': self.blinding}
+                  'blinding': self.blinding, 'attention_type': self.attention_type, 'output_type': self.output_type}
         base_config = super(Transformer, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 

deepctr/models/__init__.py

@@ -18,6 +18,7 @@
 from .fibinet import FiBiNET
 from .flen import FLEN
 from .fwfm import FwFM
+from .bst import BST
 
 __all__ = ["AFM", "CCPM", "DCN", "DCNMix", "MLR",  "DeepFM", "MLR", "NFM", "DIN", "DIEN", "FNN", "PNN",
-           "WDL", "xDeepFM", "AutoInt", "ONN", "FGCNN", "DSIN", "FiBiNET", 'FLEN', "FwFM"]
+           "WDL", "xDeepFM", "AutoInt", "ONN", "FGCNN", "DSIN", "FiBiNET", 'FLEN', "FwFM", "BST"]

deepctr/models/bst.py

@@ -0,0 +1,106 @@
+# -*- coding:utf-8 -*-
+"""
+Author:
+    Zichao Li, [email protected]
+
+Reference:
+    Qiwei Chen, Huan Zhao, Wei Li, Pipei Huang, and Wenwu Ou. 2019. Behavior sequence transformer for e-commerce recommendation in Alibaba. In Proceedings of the 1st International Workshop on Deep Learning Practice for High-Dimensional Sparse Data (DLP-KDD '19). Association for Computing Machinery, New York, NY, USA, Article 12, 1–4. DOI:https://doi.org/10.1145/3326937.3341261
+"""
+
+import tensorflow as tf
+from tensorflow.python.keras.layers import (Dense, LeakyReLU, Flatten)
+from ..feature_column import SparseFeat, VarLenSparseFeat, DenseFeat, build_input_features
+from ..inputs import get_varlen_pooling_list, create_embedding_matrix, embedding_lookup, varlen_embedding_lookup, \
+    get_dense_input
+from ..layers.core import DNN, PredictionLayer
+from ..layers.sequence import Transformer, AttentionSequencePoolingLayer
+from ..layers.utils import concat_func, combined_dnn_input
+
+
+def BST(dnn_feature_columns, history_feature_list, transformer_num=1, att_head_num=8,
+        use_bn=False, dnn_hidden_units=(1024, 512, 256), dnn_activation='relu', l2_reg_dnn=0,
+        l2_reg_embedding=1e-6, dnn_dropout=0.0, seed=1024, task='binary'):
+    """Instantiates the BST architecture.
+
+     :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
+     :param history_feature_list: list, to indicate sequence sparse field.
+     :param transformer_num: int, the number of transformer layer.
+     :param att_head_num: int, the number of heads in multi-head self attention.
+     :param use_bn: bool. Whether use BatchNormalization before activation or not in deep net
+     :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN
+     :param dnn_activation: Activation function to use in DNN
+     :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
+     :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
+     :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
+     :param seed: integer ,to use as random seed.
+     :param task: str, ``"binary"`` for  binary logloss or ``"regression"`` for regression loss
+     :return: A Keras model instance.
+
+     """
+
+    features = build_input_features(dnn_feature_columns)
+    inputs_list = list(features.values())
+
+    user_behavior_length = features["seq_length"]
+
+    sparse_feature_columns = list(
+        filter(lambda x: isinstance(x, SparseFeat), dnn_feature_columns)) if dnn_feature_columns else []
+    dense_feature_columns = list(
+        filter(lambda x: isinstance(x, DenseFeat), dnn_feature_columns)) if dnn_feature_columns else []
+    varlen_sparse_feature_columns = list(
+        filter(lambda x: isinstance(x, VarLenSparseFeat), dnn_feature_columns)) if dnn_feature_columns else []
+
+    history_feature_columns = []
+    sparse_varlen_feature_columns = []
+    history_fc_names = list(map(lambda x: "hist_" + x, history_feature_list))
+
+    for fc in varlen_sparse_feature_columns:
+        feature_name = fc.name
+        if feature_name in history_fc_names:
+            history_feature_columns.append(fc)
+        else:
+            sparse_varlen_feature_columns.append(fc)
+
+    embedding_dict = create_embedding_matrix(dnn_feature_columns, l2_reg_embedding, seed, prefix="",
+                                             seq_mask_zero=True)
+
+    query_emb_list = embedding_lookup(embedding_dict, features, sparse_feature_columns,
+                                      return_feat_list=history_feature_list, to_list=True)
+    hist_emb_list = embedding_lookup(embedding_dict, features, history_feature_columns,
+                                     return_feat_list=history_fc_names, to_list=True)
+    dnn_input_emb_list = embedding_lookup(embedding_dict, features, sparse_feature_columns,
+                                          mask_feat_list=history_feature_list, to_list=True)
+    dense_value_list = get_dense_input(features, dense_feature_columns)
+    sequence_embed_dict = varlen_embedding_lookup(embedding_dict, features, sparse_varlen_feature_columns)
+    sequence_embed_list = get_varlen_pooling_list(sequence_embed_dict, features, sparse_varlen_feature_columns,
+                                                  to_list=True)
+
+    dnn_input_emb_list += sequence_embed_list
+    query_emb = concat_func(query_emb_list)
+    deep_input_emb = concat_func(dnn_input_emb_list)
+    hist_emb = concat_func(hist_emb_list)
+
+    transformer_output = hist_emb
+    for i in range(transformer_num):
+        att_embedding_size = transformer_output.get_shape().as_list()[-1] // att_head_num
+        transformer_layer = Transformer(att_embedding_size=att_embedding_size, head_num=att_head_num,
+                                        dropout_rate=dnn_dropout, use_positional_encoding=True,use_res=True,
+                                        use_feed_forward=True, use_layer_norm=True,blinding=False, seed=seed,
+                                        supports_masking=False,output_type=None)
+        transformer_output = transformer_layer([transformer_output, transformer_output,
+                                                user_behavior_length, user_behavior_length])
+
+    attn_output = AttentionSequencePoolingLayer(att_hidden_units=(64, 16), weight_normalization=True,
+                                                supports_masking=False)([query_emb, transformer_output,
+                                                                         user_behavior_length])
+    deep_input_emb = concat_func([deep_input_emb, attn_output], axis=-1)
+    deep_input_emb = Flatten()(deep_input_emb)
+
+    dnn_input = combined_dnn_input([deep_input_emb], dense_value_list)
+    output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, use_bn, seed=seed)(dnn_input)
+    final_logit = Dense(1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(output)
+    output = PredictionLayer(task)(final_logit)
+
+    model = tf.keras.models.Model(inputs=inputs_list, outputs=output)
+
+    return model

tests/models/BST_test.py

@@ -0,0 +1,75 @@
+import numpy as np
+import pytest
+import tensorflow as tf
+from packaging import version
+
+from deepctr.feature_column import SparseFeat, VarLenSparseFeat, DenseFeat, get_feature_names
+from deepctr.models import BST
+from ..utils import check_model
+
+
+def get_xy_fd(use_neg=False, hash_flag=False):
+    feature_columns = [SparseFeat('user', 3, embedding_dim=12, use_hash=hash_flag),
+                       SparseFeat('gender', 2, embedding_dim=4, use_hash=hash_flag),
+                       SparseFeat('item_id', 3 + 1, embedding_dim=8, use_hash=hash_flag),
+                       SparseFeat('cate_id', 2 + 1, embedding_dim=4, use_hash=hash_flag),
+                       DenseFeat('pay_score', 1)]
+
+    feature_columns += [
+        VarLenSparseFeat(SparseFeat('hist_item_id', vocabulary_size=3 + 1, embedding_dim=8, embedding_name='item_id'),
+                         maxlen=4, length_name="seq_length"),
+        VarLenSparseFeat(SparseFeat('hist_cate_id', 2 + 1, embedding_dim=4, embedding_name='cate_id'), maxlen=4,
+                         length_name="seq_length")]
+
+    behavior_feature_list = ["item_id", "cate_id"]
+    uid = np.array([0, 1, 2])
+    ugender = np.array([0, 1, 0])
+    iid = np.array([1, 2, 3])  # 0 is mask value
+    cate_id = np.array([1, 2, 2])  # 0 is mask value
+    score = np.array([0.1, 0.2, 0.3])
+
+    hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
+    hist_cate_id = np.array([[1, 2, 2, 0], [1, 2, 2, 0], [1, 2, 0, 0]])
+
+    behavior_length = np.array([3, 3, 2])
+
+    feature_dict = {'user': uid, 'gender': ugender, 'item_id': iid, 'cate_id': cate_id,
+                    'hist_item_id': hist_iid, 'hist_cate_id': hist_cate_id,
+                    'pay_score': score, "seq_length": behavior_length}
+
+    if use_neg:
+        feature_dict['neg_hist_item_id'] = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
+        feature_dict['neg_hist_cate_id'] = np.array([[1, 2, 2, 0], [1, 2, 2, 0], [1, 2, 0, 0]])
+        feature_columns += [
+            VarLenSparseFeat(
+                SparseFeat('neg_hist_item_id', vocabulary_size=3 + 1, embedding_dim=8, embedding_name='item_id'),
+                maxlen=4, length_name="seq_length"),
+            VarLenSparseFeat(SparseFeat('neg_hist_cate_id', 2 + 1, embedding_dim=4, embedding_name='cate_id'),
+                             maxlen=4, length_name="seq_length")]
+
+    x = {name: feature_dict[name] for name in get_feature_names(feature_columns)}
+    y = np.array([1, 0, 1])
+    x["position_hist"] = np.array([[0, 1, 2, 3], [0, 1, 2, 3], [0, 1, 2, 3]])
+    return x, y, feature_columns, behavior_feature_list
+
+
+# @pytest.mark.xfail(reason="There is a bug when save model use Dice")
+# @pytest.mark.skip(reason="misunderstood the API")
+
+def test_BST():
+    if version.parse(tf.__version__) >= version.parse('2.0.0'):
+        tf.compat.v1.disable_eager_execution()
+    model_name = "BST"
+
+    x, y, feature_columns, behavior_feature_list = get_xy_fd(hash_flag=True)
+
+    model = BST(dnn_feature_columns=feature_columns,
+                history_feature_list=behavior_feature_list,
+                att_head_num=4)
+
+    check_model(model, model_name, x, y,
+                check_model_io=True)
+
+
+if __name__ == "__main__":
+    pass