Skip to content

Commit 95cdbfe

Browse files
will-amwill-am
authored
Merge pull request #4859 from will-am/factorization_machine_layer
Add Factorization Machine Layer
2 parents ef3420e + 8a283db commit 95cdbfe

File tree

12 files changed

+445
-5
lines changed

12 files changed

+445
-5
lines changed

‎doc/api/v2/config/layer.rst

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ img_conv
5454

5555
.. _api_v2.layer_context_projection:
5656

57-
context_projection
57+
context_projection
5858
------------------
5959
.. autoclass:: paddle.v2.layer.context_projection
6060
:noindex:
@@ -70,7 +70,7 @@ Image Pooling Layer
7070
img_pool
7171
--------
7272
.. autoclass:: paddle.v2.layer.img_pool
73-
:noindex:
73+
:noindex:
7474

7575
spp
7676
---
@@ -104,7 +104,7 @@ sum_to_one_norm
104104
---------------
105105
.. autoclass:: paddle.v2.layer.sum_to_one_norm
106106
:noindex:
107-
107+
108108
cross_channel_norm
109109
------------------
110110
.. autoclass:: paddle.v2.layer.cross_channel_norm
@@ -114,7 +114,7 @@ row_l2_norm
114114
-----------
115115
.. autoclass:: paddle.v2.layer.row_l2_norm
116116
:noindex:
117-
117+
118118
Recurrent Layers
119119
================
120120

@@ -415,6 +415,13 @@ multiplex
415415
.. autoclass:: paddle.v2.layer.multiplex
416416
:noindex:
417417

418+
Factorization Machine Layer
419+
============================
420+
421+
factorization_machine
422+
---------------------
423+
.. autoclass:: paddle.v2.layer.factorization_machine
424+
:noindex:
418425

419426
Slicing and Joining Layers
420427
==========================

‎paddle/gserver/layers/FactorizationMachineLayer.cpp

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#include "FactorizationMachineLayer.h"
16+
#include <algorithm>
17+
#include <vector>
18+
#include "paddle/math/SparseMatrix.h"
19+
#include "paddle/utils/Logging.h"
20+
#include "paddle/utils/Stat.h"
21+
22+
namespace paddle {
23+
24+
REGISTER_LAYER(factorization_machine, FactorizationMachineLayer);
25+
26+
bool FactorizationMachineLayer::init(const LayerMap& layerMap,
27+
const ParameterMap& parameterMap) {
28+
/* Initialize the basic parent class */
29+
Layer::init(layerMap, parameterMap);
30+
31+
factorSize_ = config_.factor_size();
32+
33+
/* initialize the latentVectors_ */
34+
CHECK_EQ(inputLayers_.size(), 1UL);
35+
size_t inputSize = inputLayers_[0]->getSize();
36+
CHECK_EQ(parameters_[0]->getSize(), inputSize * factorSize_);
37+
latentVectors_ = std::unique_ptr<Weight>(
38+
new Weight(inputSize, factorSize_, parameters_[0]));
39+
40+
return true;
41+
}
42+
43+
void FactorizationMachineLayer::forward(PassType passType) {
44+
Layer::forward(passType);
45+
46+
const MatrixPtr& inputV = getInputValue(0);
47+
48+
size_t batchSize = inputV->getHeight();
49+
size_t outputSize = getSize();
50+
size_t inputSize = inputLayers_[0]->getSize();
51+
reserveOutput(batchSize, outputSize);
52+
53+
MatrixPtr outV = getOutputValue();
54+
55+
Matrix::resizeOrCreate(
56+
latentVectorsSquare_, inputSize, factorSize_, false, useGpu_);
57+
Matrix::resizeOrCreate(
58+
inputMulFactor_, batchSize, factorSize_, false, useGpu_);
59+
Matrix::resizeOrCreate(tmpOut_, batchSize, factorSize_, false, useGpu_);
60+
61+
REGISTER_TIMER_INFO("FmInputMulFactorTimer", getName().c_str());
62+
inputMulFactor_->mul(*inputV, *latentVectors_->getW());
63+
inputMulFactor_->square2(*tmpOut_);
64+
outV->sumRows(*tmpOut_, 0.5, 0);
65+
66+
if (dynamic_cast<CpuSparseMatrix*>(inputV.get())) {
67+
Matrix::resizeOrCreateSparseMatrix(inputSquare_,
68+
inputV->getHeight(),
69+
inputV->getWidth(),
70+
inputV->getElementCnt(),
71+
inputV->getValueType());
72+
inputSquare_->copyFrom(*inputV);
73+
(dynamic_cast<CpuSparseMatrix*>(inputSquare_.get()))->square2();
74+
} else {
75+
Matrix::resizeOrCreate(
76+
inputSquare_, inputV->getHeight(), inputV->getWidth(), false, useGpu_);
77+
inputV->square2(*inputSquare_);
78+
}
79+
latentVectors_->getW()->square2(*latentVectorsSquare_);
80+
tmpOut_->mul(*inputSquare_, *latentVectorsSquare_);
81+
outV->sumRows(*tmpOut_, -0.5, 1.0);
82+
83+
/* activation */ {
84+
REGISTER_TIMER_INFO("FmFwAtvTimer", getName().c_str());
85+
forwardActivation();
86+
}
87+
}
88+
89+
void FactorizationMachineLayer::backward(const UpdateCallback& callback) {
90+
/* Do derivation */ { backwardActivation(); }
91+
92+
const MatrixPtr& inputV = getInputValue(0);
93+
const MatrixPtr& oGrad = getOutputGrad();
94+
95+
Matrix::resizeOrCreate(
96+
tmpSum_, 1, latentVectors_->getW()->getHeight(), false, useGpu_);
97+
MatrixPtr tmpSumTrans = Matrix::create(tmpSum_->getRowBuf(0),
98+
latentVectors_->getW()->getHeight(),
99+
1,
100+
false,
101+
useGpu_);
102+
103+
/* Calculate the gradients of the latentVectors_ matrix */
104+
if (latentVectors_->getWGrad()) {
105+
if (dynamic_cast<CpuSparseMatrix*>(inputV.get())) {
106+
Matrix::resizeOrCreateSparseMatrix(tmpInput_,
107+
inputV->getHeight(),
108+
inputV->getWidth(),
109+
inputV->getElementCnt());
110+
111+
CpuSparseMatrix* sparseInputV =
112+
dynamic_cast<CpuSparseMatrix*>(inputV.get());
113+
CpuSparseMatrix* sparseInputSquare =
114+
dynamic_cast<CpuSparseMatrix*>(inputSquare_.get());
115+
CpuSparseMatrix* sparseTmpInput =
116+
dynamic_cast<CpuSparseMatrix*>(tmpInput_.get());
117+
sparseTmpInput->copyFrom(*sparseInputV);
118+
119+
sparseTmpInput->rowScale(0, *sparseInputV, *oGrad);
120+
latentVectors_->getWGrad()->mul(
121+
*sparseTmpInput->getTranspose(), *inputMulFactor_, 1, 1);
122+
sparseTmpInput->rowScale(0, *sparseInputSquare, *oGrad);
123+
124+
Matrix::resizeOrCreate(negOnes_, 1, inputV->getHeight(), false, useGpu_);
125+
negOnes_->zeroMem();
126+
negOnes_->add(-1);
127+
tmpSum_->mul(*negOnes_, *sparseTmpInput, 1, 0);
128+
} else {
129+
Matrix::resizeOrCreate(
130+
tmpInput_, inputV->getHeight(), inputV->getWidth(), false, useGpu_);
131+
132+
tmpInput_->rowScale(0, *inputV, *oGrad);
133+
latentVectors_->getWGrad()->mul(
134+
*tmpInput_->getTranspose(), *inputMulFactor_, 1, 1);
135+
tmpInput_->rowScale(0, *inputSquare_, *oGrad);
136+
137+
tmpSum_->sumCols(*tmpInput_, -1, 0);
138+
}
139+
140+
latentVectors_->getWGrad()->addRowScale(
141+
0, *latentVectors_->getW(), *tmpSumTrans);
142+
143+
/* Increasing the number of gradient */
144+
latentVectors_->getParameterPtr()->incUpdate(callback);
145+
}
146+
147+
/* Calculate the input layers gradient */
148+
MatrixPtr inGrad = getInputGrad(0);
149+
if (inGrad != NULL) {
150+
inGrad->mul(
151+
*inputMulFactor_, *latentVectors_->getW()->getTranspose(), 1, 1);
152+
tmpSumTrans->sumRows(*latentVectorsSquare_, -1, 0);
153+
inGrad->addColScale(0, *inputV, *tmpSum_);
154+
inGrad->rowScale(0, *inGrad, *oGrad);
155+
}
156+
}
157+
158+
} // namespace paddle

‎paddle/gserver/layers/FactorizationMachineLayer.h

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#pragma once
16+
17+
#include "Layer.h"
18+
#include "paddle/math/Matrix.h"
19+
#include "paddle/utils/ThreadLocal.h"
20+
21+
namespace paddle {
22+
/**
23+
* @brief The Factorization Machine models pairwise (order-2) feature
24+
* interactions as inner product of the learned latent vectors corresponding
25+
* to each input feature.
26+
*
27+
* The Factorization Machine can effectively capture feature interactions
28+
* especially when the input is sparse. While in principle FM can model higher
29+
* order feature interaction, in practice usually only order-2 feature
30+
* interactions are considered. The Factorization Machine Layer here only
31+
* computes the order-2 interations with the formula:
32+
*
33+
* \f[
34+
* y = \sum_{i=1}^{n-1}\sum_{j=i+1}^n\langle v_i, v_j \rangle x_i x_j
35+
* \f]
36+
*
37+
* The detailed calculation for forward and backward can be found at this paper:
38+
*
39+
* Factorization machines.
40+
*
41+
* The config file api is factorization_machine.
42+
*/
43+
44+
class FactorizationMachineLayer : public Layer {
45+
protected:
46+
// The latent vectors, shape: (size, factorSize_)
47+
// Each row of the latentVectors_ matrix is the latent vector
48+
// corresponding to one input feature dimension
49+
std::unique_ptr<Weight> latentVectors_;
50+
// The hyperparameter that defines the dimensionality of the factorization
51+
size_t factorSize_;
52+
53+
private:
54+
// Store the square values of the letent vectors matrix
55+
MatrixPtr latentVectorsSquare_;
56+
// Store the square values of input matrix
57+
MatrixPtr inputSquare_;
58+
// The result of input matrix * latent vector matrix that will be used in
59+
// both forward and backward step
60+
MatrixPtr inputMulFactor_;
61+
// Store temporary calculation result
62+
MatrixPtr tmpOut_;
63+
MatrixPtr tmpSum_;
64+
MatrixPtr tmpInput_;
65+
// Negative identity matrix
66+
MatrixPtr negOnes_;
67+
68+
public:
69+
explicit FactorizationMachineLayer(const LayerConfig& config)
70+
: Layer(config) {}
71+
~FactorizationMachineLayer() {}
72+
73+
bool init(const LayerMap& layerMap,
74+
const ParameterMap& parameterMap) override;
75+
76+
void forward(PassType passType) override;
77+
void backward(const UpdateCallback& callback = nullptr) override;
78+
};
79+
80+
} // namespace paddle

‎paddle/gserver/tests/test_LayerGrad.cpp

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2464,6 +2464,25 @@ TEST(Layer, L2DistanceLayer) {
24642464
}
24652465
}
24662466

2467+
void testFactorizationMachineLayer(InputType type, bool useGpu) {
2468+
const int FACTOR_SIZE = 10;
2469+
TestConfig config;
2470+
config.layerConfig.set_type("factorization_machine");
2471+
config.layerConfig.set_factor_size(FACTOR_SIZE);
2472+
config.layerConfig.set_size(1);
2473+
config.biasSize = 0;
2474+
config.inputDefs.push_back({type, "layer_0", 128, 1280});
2475+
config.layerConfig.add_inputs();
2476+
testLayerGrad(config, "factorization_machine", 16, false, useGpu, false);
2477+
}
2478+
2479+
TEST(Layer, FactorizationMachineLayer) {
2480+
for (auto useGpu : {false, true}) {
2481+
testFactorizationMachineLayer(INPUT_DATA, useGpu);
2482+
}
2483+
testFactorizationMachineLayer(INPUT_SPARSE_FLOAT_VALUE_DATA, false);
2484+
}
2485+
24672486
int main(int argc, char** argv) {
24682487
testing::InitGoogleTest(&argc, argv);
24692488
initMain(argc, argv);

‎paddle/math/CpuSparseMatrix.cpp

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,35 @@ void CpuSparseMatrix::printOneRow(std::ostream& os, size_t idx) const {
260260
os << ";";
261261
}
262262

263+
void CpuSparseMatrix::rowScale(size_t cCol, CpuSparseMatrix& b, Matrix& c) {
264+
CHECK(getFormat() != SPARSE_CSC) << "Not supported";
265+
CHECK_EQ(height_, b.getHeight());
266+
CHECK_EQ(width_, b.getWidth());
267+
real* A = getValue();
268+
real* B = b.getValue();
269+
if (b.getValueType() == FLOAT_VALUE) {
270+
for (size_t i = 0; i < height_; i++) {
271+
size_t start = getRowStartIdx(i);
272+
size_t end = getRowStartIdx(i + 1);
273+
CHECK_EQ(start, b.getRowStartIdx(i));
274+
CHECK_EQ(end, b.getRowStartIdx(i + 1));
275+
for (size_t j = start; j < end; j++) {
276+
A[j] = B[j] * c.getElement(i, cCol);
277+
}
278+
}
279+
} else if (b.getValueType() == NO_VALUE) {
280+
for (size_t i = 0; i < height_; i++) {
281+
size_t start = getRowStartIdx(i);
282+
size_t end = getRowStartIdx(i + 1);
283+
CHECK_EQ(start, b.getRowStartIdx(i));
284+
CHECK_EQ(end, b.getRowStartIdx(i + 1));
285+
for (size_t j = start; j < end; j++) {
286+
A[j] = c.getElement(i, cCol);
287+
}
288+
}
289+
}
290+
}
291+
263292
void CpuSparseMatrix::randomizeUniform() {
264293
CHECK_LE(elementCnt_, height_ * width_);
265294
if (valueType_ == FLOAT_VALUE) {

‎paddle/math/CpuSparseMatrix.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,15 @@ class CpuSparseMatrix : public Matrix {
239239
const unsigned int* cols,
240240
const real* values);
241241

242+
/**
243+
* @brief this_row = b_row * c_row[cCol]
244+
*
245+
* @param[in] cCol the column of matrix c used to scale each row of b
246+
* @param[in] b CpuSparseMatrix
247+
* @param[in] c Matrix
248+
*/
249+
void rowScale(size_t cCol, CpuSparseMatrix& b, Matrix& c);
250+
242251
void randomizeUniform();
243252

244253
void copyFrom(const GpuSparseMatrix& src, hl_stream_t stream);

‎proto/ModelConfig.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -544,6 +544,9 @@ message LayerConfig {
544544
// for batch normalization layer
545545
// The small constant added to the variance to improve numeric stability.
546546
optional double epsilon = 60 [ default = 0.00001 ];
547+
548+
// for factorization machine layer
549+
optional uint32 factor_size = 61;
547550
}
548551

549552
message EvaluatorConfig {

‎python/paddle/trainer/config_parser.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3870,6 +3870,21 @@ def __init__(self, name, inputs, value, **xargs):
38703870
image_conf.channels)
38713871

38723872

3873+
@config_layer('factorization_machine')
3874+
class FactorizationMachineLayer(LayerBase):
3875+
def __init__(self, name, inputs, factor_size, **xargs):
3876+
super(FactorizationMachineLayer, self).__init__(
3877+
name, 'factorization_machine', size=1, inputs=inputs, **xargs)
3878+
config_assert(
3879+
len(self.inputs) == 1,
3880+
'factorization machine layer must have one and only one input.')
3881+
self.config.factor_size = factor_size
3882+
input_layer = self.get_input_layer(0)
3883+
psize = input_layer.size * factor_size
3884+
dims = [input_layer.size, factor_size]
3885+
self.create_input_parameter(0, psize, dims)
3886+
3887+
38733888
# Deprecated, use a new layer specific class instead
38743889
@config_func
38753890
def Layer(name, type, **xargs):

0 commit comments

Comments
 (0)