FEFM/DeepFEFM (#364)

pandeconscious · Harshit Pande · web-flow · commit ef3eff6855a2 · 2021-06-12T21:36:10.000+08:00
add FEFM and DeepFEFM 
Co-authored-by: Harshit Pande &lt;harshitpande@Harshits-MacBook-Pro.local&gt;
diff --git a/README.md b/README.md
@@ -58,6 +58,7 @@ Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Star
 |                IFM                 | [IJCAI 2019][An Input-aware Factorization Machine for Sparse Prediction](https://www.ijcai.org/Proceedings/2019/0203.pdf)   |
 |                DCN V2                    | [arxiv 2020][DCN V2: Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/abs/2008.13535)   |
 |                DIFM                 | [IJCAI 2020][A Dual Input-aware Factorization Machine for CTR Prediction](https://www.ijcai.org/Proceedings/2020/0434.pdf)   |
+|   FEFM and DeepFEFM                    | [arxiv 2020][Field-Embedded Factorization Machines for Click-through rate prediction](https://arxiv.org/abs/2009.09931)                                         |
 
 ## Citation
 
diff --git a/deepctr/estimator/models/__init__.py b/deepctr/estimator/models/__init__.py
@@ -10,3 +10,4 @@
 from .pnn import PNNEstimator
 from .wdl import WDLEstimator
 from .xdeepfm import xDeepFMEstimator
+from .deepfefm import DeepFEFMEstimator
diff --git a/deepctr/estimator/models/deepfefm.py b/deepctr/estimator/models/deepfefm.py
@@ -0,0 +1,92 @@
+# -*- coding:utf-8 -*-
+"""
+Author:
+    Harshit Pande
+
+Reference:
+    [1] Field-Embedded Factorization Machines for Click-through Rate Prediction]
+    (https://arxiv.org/abs/2009.09931)
+
+"""
+
+import tensorflow as tf
+
+from ..feature_column import get_linear_logit, input_from_feature_columns
+from ..utils import DNN_SCOPE_NAME, deepctr_model_fn, variable_scope
+from ...layers.core import DNN
+from ...layers.interaction import FEFMLayer
+from ...layers.utils import concat_func, add_func, combined_dnn_input, reduce_sum
+
+
+def DeepFEFMEstimator(linear_feature_columns, dnn_feature_columns, embedding_size=48,
+                      dnn_hidden_units=(1024, 1024, 1024), l2_reg_linear=0.000001, l2_reg_embedding_feat=0.00001,
+                      l2_reg_embedding_field=0.0000001, l2_reg_dnn=0, seed=1024, dnn_dropout=0.2,
+                      dnn_activation='relu', dnn_use_bn=False, task='binary', model_dir=None,
+                      config=None, linear_optimizer='Ftrl', dnn_optimizer='Adagrad', training_chief_hooks=None):
+    """Instantiates the DeepFEFM Network architecture or the shallow FEFM architecture (Ablation support not provided
+    as estimator is meant for production, Ablation support provided in DeepFEFM implementation in models
+
+    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
+    :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
+    :param embedding_size: positive integer,sparse feature embedding_size
+    :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN
+    :param l2_reg_linear: float. L2 regularizer strength applied to linear part
+    :param l2_reg_embedding_feat: float. L2 regularizer strength applied to embedding vector of features
+    :param l2_reg_embedding_field: float, L2 regularizer to field embeddings
+    :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
+    :param seed: integer ,to use as random seed.
+    :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
+    :param dnn_activation: Activation function to use in DNN
+    :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN
+    :param task: str, ``"binary"`` for  binary logloss or  ``"regression"`` for regression loss
+    :param model_dir: Directory to save model parameters, graph and etc. This can
+        also be used to load checkpoints from the directory into a estimator
+        to continue training a previously saved model.
+    :param config: tf.RunConfig object to configure the runtime settings.
+    :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
+        the linear part of the model. Defaults to FTRL optimizer.
+    :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to
+        the deep part of the model. Defaults to Adagrad optimizer.
+    :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to
+        run on the chief worker during training.
+    :return: A Tensorflow Estimator  instance.
+    """
+
+    def _model_fn(features, labels, mode, config):
+        train_flag = (mode == tf.estimator.ModeKeys.TRAIN)
+
+        linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear)
+        final_logit_components = [linear_logits]
+
+        with variable_scope(DNN_SCOPE_NAME):
+            sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns,
+                                                                                 l2_reg_embedding=l2_reg_embedding_feat)
+
+            fefm_interaction_embedding = FEFMLayer(num_fields=len(sparse_embedding_list), embedding_size=embedding_size,
+                                   regularizer=l2_reg_embedding_field)(concat_func(sparse_embedding_list, axis=1))
+
+            fefm_logit = tf.keras.layers.Lambda(lambda x: reduce_sum(x, axis=1, keep_dims=True))(fefm_interaction_embedding)
+
+            final_logit_components.append(fefm_logit)
+
+            if dnn_hidden_units:
+                dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list)
+                dnn_input = concat_func([dnn_input, fefm_interaction_embedding], axis=1)
+
+                dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(
+                    dnn_input, training=train_flag)
+
+                dnn_logit = tf.keras.layers.Dense(
+                    1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_output)
+
+                final_logit_components.append(dnn_logit)
+
+        logits = add_func(final_logit_components)
+
+        return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer,
+                                training_chief_hooks=training_chief_hooks)
+
+    return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=config)
+
+
+
diff --git a/deepctr/feature_column.py b/deepctr/feature_column.py
@@ -181,7 +181,7 @@ def get_linear_logit(features, feature_columns, units=1, use_bias=False, seed=10
             dense_input = concat_func(dense_input_list)
             linear_logit = Linear(l2_reg, mode=1, use_bias=use_bias, seed=seed)(dense_input)
         else:   #empty feature_columns
-            return Lambda(lambda x: tf.constant([[0.0]]))(features.values()[0])
+            return Lambda(lambda x: tf.constant([[0.0]]))(list(features.values())[0])
         linear_logit_list.append(linear_logit)
 
     return concat_func(linear_logit_list)
diff --git a/deepctr/layers/__init__.py b/deepctr/layers/__init__.py
@@ -5,12 +5,13 @@
 from .interaction import (CIN, FM, AFMLayer, BiInteractionPooling, CrossNet, CrossNetMix,
                           InnerProductLayer, InteractingLayer,
                           OutterProductLayer, FGCNNLayer, SENETLayer, BilinearInteraction,
-                          FieldWiseBiInteraction, FwFMLayer)
+                          FieldWiseBiInteraction, FwFMLayer, FEFMLayer)
 from .normalization import LayerNormalization
 from .sequence import (AttentionSequencePoolingLayer, BiasEncoding, BiLSTM,
                        KMaxPooling, SequencePoolingLayer, WeightedSequenceLayer,
                        Transformer, DynamicGRU)
-from .utils import NoMask, Hash, Linear, Add, combined_dnn_input, softmax
+
+from .utils import NoMask, Hash, Linear, Add, combined_dnn_input, softmax, reduce_sum
 
 custom_objects = {'tf': tf,
                   'InnerProductLayer': InnerProductLayer,
@@ -45,4 +46,6 @@
                   'FieldWiseBiInteraction': FieldWiseBiInteraction,
                   'FwFMLayer': FwFMLayer,
                   'softmax': softmax,
+                  'FEFMLayer': FEFMLayer,
+                  'reduce_sum': reduce_sum
                   }
diff --git a/deepctr/layers/interaction.py b/deepctr/layers/interaction.py
@@ -1409,3 +1409,88 @@ def get_config(self):
             'regularizer': self.regularizer
         })
         return config
+
+
+class FEFMLayer(Layer):
+    """Field-Embedded Factorization Machines
+
+      Input shape
+        - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``.
+
+      Output shape
+        - 2D tensor with shape:
+            ``(batch_size, (num_fields * (num_fields-1))/2)`` # concatenated FEFM interaction embeddings
+
+      Arguments
+        - **num_fields** : integer for number of fields
+        - **embedding_size** : integer for embedding dimension
+        - **regularizer** : L2 regularizer weight for the field pair matrix embeddings parameters of FEFM
+
+      References
+        - [Field-Embedded Factorization Machines for Click-through Rate Prediction]
+         https://arxiv.org/pdf/2009.09931.pdf
+    """
+
+    def __init__(self, num_fields, embedding_size, regularizer, **kwargs):
+        self.num_fields = num_fields
+        self.embedding_size = embedding_size
+        self.regularizer = regularizer
+        super(FEFMLayer, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        if len(input_shape) != 3:
+            raise ValueError("Unexpected inputs dimensions % d,\
+                                expect to be 3 dimensions" % (len(input_shape)))
+
+        if input_shape[1] != self.num_fields:
+            raise ValueError("Mismatch in number of fields {} and \
+                    concatenated embeddings dims {}".format(self.num_fields, input_shape[2]))
+
+        self.field_embeddings = {}
+
+        for fi, fj in itertools.combinations(range(self.num_fields), 2):
+            field_pair_id = str(fi) + "-" + str(fj)
+            self.field_embeddings[field_pair_id] = self.add_weight(name='field_embeddings' + field_pair_id,
+                                                                   shape=(self.embedding_size, self.embedding_size),
+                                                                   initializer=TruncatedNormal(),
+                                                                   regularizer=l2(self.regularizer),
+                                                                   trainable=True)
+
+        super(FEFMLayer, self).build(input_shape)  # Be sure to call this somewhere!
+
+    def call(self, inputs, **kwargs):
+        if K.ndim(inputs) != 3:
+            raise ValueError(
+                "Unexpected inputs dimensions %d, expect to be 3 dimensions"
+                % (K.ndim(inputs)))
+
+        if inputs.shape[1] != self.num_fields:
+            raise ValueError("Mismatch in number of fields {} and \
+                    concatenated embeddings dims {}".format(self.num_fields, inputs.shape[1]))
+
+        pairwise_inner_prods = []
+        for fi, fj in itertools.combinations(range(self.num_fields), 2):
+            field_pair_id = str(fi) + "-" + str(fj)
+            feat_embed_i = tf.squeeze(inputs[0:, fi:fi + 1, 0:], axis=1)
+            feat_embed_j = tf.squeeze(inputs[0:, fj:fj + 1, 0:], axis=1)
+            field_pair_embed_ij = self.field_embeddings[field_pair_id]
+
+            feat_embed_i_tr = tf.matmul(feat_embed_i, field_pair_embed_ij + tf.transpose(field_pair_embed_ij))
+
+            f = batch_dot(feat_embed_i_tr, feat_embed_j, axes=1)
+            pairwise_inner_prods.append(f)
+
+        concat_vec = tf.concat(pairwise_inner_prods, axis=1)
+        return concat_vec
+
+    def compute_output_shape(self, input_shape):
+        return (None, (self.num_fields * (self.num_fields-1))/2)
+
+    def get_config(self):
+        config = super(FEFMLayer, self).get_config().copy()
+        config.update({
+            'num_fields': self.num_fields,
+            'regularizer': self.regularizer,
+            'embedding_size': self.embedding_size
+        })
+        return config
diff --git a/deepctr/models/__init__.py b/deepctr/models/__init__.py
@@ -21,6 +21,7 @@
 from .flen import FLEN
 from .fwfm import FwFM
 from .bst import BST
+from .deepfefm import DeepFEFM
 
 __all__ = ["AFM", "CCPM", "DCN", "IFM", "DIFM", "DCNMix", "MLR",  "DeepFM", "MLR", "NFM", "DIN", "DIEN", "FNN", "PNN",
-           "WDL", "xDeepFM", "AutoInt", "ONN", "FGCNN", "DSIN", "FiBiNET", 'FLEN', "FwFM", "BST"]
+           "WDL", "xDeepFM", "AutoInt", "ONN", "FGCNN", "DSIN", "FiBiNET", 'FLEN', "FwFM", "BST", "DeepFEFM"]
diff --git a/deepctr/models/deepfefm.py b/deepctr/models/deepfefm.py
@@ -0,0 +1,102 @@
+# -*- coding:utf-8 -*-
+"""
+Author:
+    Harshit Pande
+
+Reference:
+    [1] Field-Embedded Factorization Machines for Click-through Rate Prediction]
+    (https://arxiv.org/pdf/2009.09931.pdf)
+
+    this file also supports all the possible Ablation studies for reproducibility
+
+"""
+
+from itertools import chain
+
+import tensorflow as tf
+
+from ..feature_column import input_from_feature_columns, get_linear_logit, build_input_features, DEFAULT_GROUP_NAME
+from ..layers.core import PredictionLayer, DNN
+from ..layers.interaction import FEFMLayer
+from ..layers.utils import concat_func, combined_dnn_input, reduce_sum
+
+
+def DeepFEFM(linear_feature_columns, dnn_feature_columns, embedding_size=48, use_fefm=True,
+             dnn_hidden_units=(1024, 1024, 1024), l2_reg_linear=0.000001, l2_reg_embedding_feat=0.00001,
+             l2_reg_embedding_field=0.0000001, l2_reg_dnn=0, seed=1024, dnn_dropout=0.2, exclude_feature_embed_in_dnn=False,
+             use_linear=True, use_fefm_embed_in_dnn=True, dnn_activation='relu', dnn_use_bn=False, task='binary'):
+    """Instantiates the DeepFEFM Network architecture or the shallow FEFM architecture (Ablation studies supported)
+
+    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
+    :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
+    :param fm_group: list, group_name of features that will be used to do feature interactions.
+    :param embedding_size: positive integer,sparse feature embedding_size
+    :param use_fefm: bool,use FEFM logit or not (doesn't effect FEFM embeddings in DNN, controls only the use of final FEFM logit)
+    :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN
+    :param l2_reg_linear: float. L2 regularizer strength applied to linear part
+    :param l2_reg_embedding_feat: float. L2 regularizer strength applied to embedding vector of features
+    :param l2_reg_embedding_field: float, L2 regularizer to field embeddings
+    :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
+    :param seed: integer ,to use as random seed.
+    :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
+    :param exclude_feature_embed_in_dnn: bool, used in ablation studies for removing feature embeddings in DNN
+    :param use_linear: bool, used in ablation studies
+    :param use_fefm_embed_in_dnn: bool, True if FEFM interaction embeddings are to be used in FEFM (set False for Ablation)
+    :param dnn_activation: Activation function to use in DNN
+    :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN
+    :param task: str, ``"binary"`` for  binary logloss or  ``"regression"`` for regression loss
+    :return: A Keras model instance.
+    """
+
+    features = build_input_features(linear_feature_columns + dnn_feature_columns)
+
+    inputs_list = list(features.values())
+
+    linear_logit = get_linear_logit(features, linear_feature_columns, l2_reg=l2_reg_linear, seed=seed, prefix='linear')
+
+    group_embedding_dict, dense_value_list = input_from_feature_columns(features, dnn_feature_columns,
+                                                                        l2_reg_embedding_feat,
+                                                                        seed, support_group=True)
+
+    fefm_interaction_embedding = concat_func([FEFMLayer(num_fields=len(v), embedding_size=embedding_size,
+                                               regularizer=l2_reg_embedding_field)(concat_func(v, axis=1))
+                                     for k, v in group_embedding_dict.items() if k in [DEFAULT_GROUP_NAME]], axis=1)
+
+    dnn_input = combined_dnn_input(list(chain.from_iterable(group_embedding_dict.values())), dense_value_list)
+
+    # if use_fefm_embed_in_dnn is set to False it is Ablation4 (Use false only for Ablation)
+    if use_fefm_embed_in_dnn:
+        if exclude_feature_embed_in_dnn:
+            # Ablation3: remove feature vector embeddings from the DNN input
+            dnn_input = fefm_interaction_embedding
+        else:
+            # No ablation
+            dnn_input = concat_func([dnn_input, fefm_interaction_embedding], axis=1)
+
+    dnn_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input)
+
+    dnn_logit = tf.keras.layers.Dense(
+        1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_out)
+
+    fefm_logit = tf.keras.layers.Lambda(lambda x: reduce_sum(x, axis=1, keep_dims=True))(fefm_interaction_embedding)
+
+    if len(dnn_hidden_units) == 0 and use_fefm is False and use_linear is True:  # only linear
+        final_logit = linear_logit
+    elif len(dnn_hidden_units) == 0 and use_fefm is True and use_linear is True:  # linear + FEFM
+        final_logit = tf.keras.layers.add([linear_logit, fefm_logit])
+    elif len(dnn_hidden_units) > 0 and use_fefm is False and use_linear is True:  # linear +　Deep # Ablation1
+        final_logit = tf.keras.layers.add([linear_logit, dnn_logit])
+    elif len(dnn_hidden_units) > 0 and use_fefm is True and use_linear is True:  # linear + FEFM + Deep
+        final_logit = tf.keras.layers.add([linear_logit, fefm_logit, dnn_logit])
+    elif len(dnn_hidden_units) == 0 and use_fefm is True and use_linear is False:  # only FEFM (shallow)
+        final_logit = fefm_logit
+    elif len(dnn_hidden_units) > 0 and use_fefm is False and use_linear is False:  # only Deep
+        final_logit = dnn_logit
+    elif len(dnn_hidden_units) > 0 and use_fefm is True and use_linear is False:  # FEFM + Deep # Ablation2
+        final_logit = tf.keras.layers.add([fefm_logit, dnn_logit])
+    else:
+        raise NotImplementedError
+
+    output = PredictionLayer(task)(final_logit)
+    model = tf.keras.models.Model(inputs=inputs_list, outputs=output)
+    return model
diff --git a/examples/run_deepfefm.py b/examples/run_deepfefm.py
diff --git a/tests/layers/interaction_test.py b/tests/layers/interaction_test.py
diff --git a/tests/models/DCN_test.py b/tests/models/DCN_test.py
diff --git a/tests/models/DeepFEFM_test.py b/tests/models/DeepFEFM_test.py
