Merge pull request #846 from reyoung/feature/remove_m4

Remove m4 when generate protobuf

Author: reyoung

paddle/gserver/layers/MultinomialSampler.h

@@ -14,8 +14,8 @@ limitations under the License. */
 
 #pragma once
 
+#include <memory>
 #include <random>
-
 #include "paddle/utils/TypeDefs.h"
 
 namespace paddle {
@@ -32,6 +32,17 @@ class MultinomialSampler {
 public:
   MultinomialSampler(const real* prob, int size);
 
+  //! protobuf always using double.
+  static MultinomialSampler* create(const double* prob, int size) {
+#ifdef PADDLE_TYPE_DOUBLE
+    return new MultinomialSampler(prob, size);
+#else
+    std::unique_ptr<real[]> tmp(new real[size]);
+    std::copy(prob, prob + size, tmp.get());
+    return new MultinomialSampler(tmp.get(), size);
+#endif
+  }
+
   /**
    * @brief Generate a random sample.
    * @param g is a random number engine. See <random>.

paddle/gserver/layers/NCELayer.cpp

@@ -99,8 +99,8 @@ class NCELayer : public Layer {
 
     if (config_.neg_sampling_dist_size()) {
       CHECK_EQ(numClasses_, config_.neg_sampling_dist_size());
-      sampler_.reset(new MultinomialSampler(config_.neg_sampling_dist().data(),
-                                            numClasses_));
+      sampler_.reset(MultinomialSampler::create(
+          config_.neg_sampling_dist().data(), numClasses_));
     }
 
     return true;

paddle/gserver/layers/WarpCTCLayer.cpp

@@ -31,7 +31,6 @@ bool WarpCTCLayer::init(const LayerMap& layerMap,
   CHECK_EQ(numClasses_, inputLayers_[0]->getSize());
 
   blank_ = config_.blank();
-  CHECK_GE(blank_, 0UL);
   CHECK_LT(blank_, numClasses_);
 
   normByTimes_ = config_.norm_by_times();

paddle/pserver/ParameterClient2.cpp

@@ -25,24 +25,17 @@ P_DEFINE_int32(parallel_thread_num, 1, "Thread number for parameter send");
 
 namespace paddle {
 
-template <class T>
-void copyToRepeatedField(google::protobuf::RepeatedField<T>* dest,
-                         const T* src,
+template <typename T1, typename T2>
+void copyToRepeatedField(google::protobuf::RepeatedField<T1>* dest,
+                         const T2* src,
                          size_t size) {
   dest->Clear();
   dest->Reserve(size);
-
   for (size_t i = 0; i < size; ++i) {
     dest->AddAlreadyReserved(src[i]);
   }
 }
 
-template <class T>
-void copyToRepeatedField(const std::vector<T>& src,
-                         google::protobuf::RepeatedField<T>* dest) {
-  copyToRepeatedField(dest, &src[0], src.size());
-}
-
 ParameterClient2::ParameterClient2(bool separate, int port, int numPorts)
     : BaseClient(separate, numPorts), port_(port) {
 #ifndef PADDLE_DISABLE_TIMER
@@ -618,6 +611,8 @@ void PreparedOperations::addOperationHelper(Operation* op, CpuMatrixPtr mat) {
       pmat.mutable_values(), mat->getData(), pmat.num_cols() * pmat.num_rows());
 }
 
+static inline real addTwo(real a, double b) { return a + b; }
+
 void ParameterClient2::doOperation(PreparedOperations& ops,
                                    bool waitForGradient,
                                    bool sendBackGradient,
@@ -682,8 +677,11 @@ void ParameterClient2::doOperation(PreparedOperations& ops,
         CpuVectorPtr rvec = resultVectors[i];
         if (!rvec) continue;
         CHECK_EQ(rvec->getSize(), (size_t)vec.dim());
-        CpuVector avec(rvec->getSize(), const_cast<real*>(vec.values().data()));
-        rvec->add(avec);
+        std::transform(rvec->getData(),
+                       rvec->getData() + rvec->getSize(),
+                       vec.values().data(),
+                       rvec->getData(),
+                       addTwo);
       }
 
       CHECK_EQ(resultMatrices.size(), (size_t)result.matrices_size());
@@ -693,11 +691,12 @@ void ParameterClient2::doOperation(PreparedOperations& ops,
         if (!rmat) continue;
         CHECK_EQ(rmat->getHeight(), (size_t)mat.num_rows());
         CHECK_EQ(rmat->getWidth(), (size_t)mat.num_cols());
-        CpuMatrixPtr amat =
-            std::make_shared<CpuMatrix>(const_cast<real*>(mat.values().data()),
-                                        rmat->getHeight(),
-                                        rmat->getWidth());
-        rmat->add(*amat);
+
+        std::transform(rmat->getData(),
+                       rmat->getData() + rmat->getElementCnt(),
+                       mat.values().data(),
+                       rmat->getData(),
+                       addTwo);
       }
     }
   }

proto/CMakeLists.txt

@@ -6,25 +6,6 @@ set(proto_filenames
     ParameterService.proto
     TrainerConfig.proto)
 
-set(real_proto_files)
-
-# TODO(yuyang18): Some internal proto will also be depended on.
-#                 Find a way to automatically calculate all depends.
-foreach(filename ${proto_filenames})
-    set(PROTOBUF_3_FLAGS "")
-    if (PROTOBUF_3)
-        set(PROTOBUF_3_FLAGS "-Dproto3")
-    endif()
-    add_custom_command(OUTPUT ${filename}
-	COMMAND ${M4_EXECUTABLE} -Dreal=${ACCURACY} ${PROTOBUF_3_FLAGS} -I '${INTERNAL_PROTO_PATH}'
-              ${PROJ_ROOT}/proto/${filename}.m4 > ${filename}
-        DEPENDS ${PROJ_ROOT}/proto/${filename}.m4
-        COMMENT "Generate ${filename}")
-endforeach()
-
-add_custom_target(proto_accuracy ALL
-                    DEPENDS ${proto_filenames})
-
 set(PROTO_GEN)
 set(PROTO_GEN_PY)
 
@@ -39,9 +20,8 @@ foreach(filename ${proto_filenames})
     add_custom_command(OUTPUT ${CUR_PROTO_GEN}
         COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} 
                   --cpp_out ${CMAKE_CURRENT_BINARY_DIR}
-                  --proto_path ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/${filename}
-        DEPENDS proto_accuracy
-                ${PROJ_ROOT}/proto/${filename}.m4)
+		  --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename}
+        DEPENDS ${filename})
 
     set(CUR_PROTO_GEN_PY
         ${PROJ_ROOT}/paddle/python/paddle/proto/${base_filename}_pb2.py)
@@ -50,9 +30,8 @@ foreach(filename ${proto_filenames})
         ${PROTO_GEN_PY})
     add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY}
         COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto
-                  --proto_path ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/${filename}
-        DEPENDS proto_accuracy
-                ${PROJ_ROOT}/proto/${filename}.m4)
+	--proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename}
+        DEPENDS ${filename})
 endforeach()
 
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/proto)
@@ -61,5 +40,4 @@ add_custom_target(gen_proto_cpp ALL DEPENDS ${PROTO_GEN})
 add_custom_target(gen_proto_py ALL DEPENDS ${PROTO_GEN_PY})
 add_library(paddle_proto STATIC
     ${PROTO_GEN})
-add_dependencies(paddle_proto proto_accuracy)
 target_include_directories(paddle_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

proto/DataConfig.proto.m4 renamed to proto/DataConfig.proto

@@ -11,11 +11,11 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
-ifdef(`proto3', `syntax = "proto2";')
+syntax = "proto2";
 
 package paddle;
 
-sinclude(`DataConfigExt.proto.m4')
+
 message FileGroupConf {
   optional uint32 queue_capacity = 1 [default = 1];
   // how many files to load for a load file thread
@@ -26,7 +26,7 @@ message FileGroupConf {
 };
 
 message DataConfig {
-sinclude(`DataConfigInter.proto.m4')
+
   required string type = 1;
 
   // name of a text file which contains a list of file names at each line
@@ -51,11 +51,11 @@ sinclude(`DataConfigInter.proto.m4')
 
   /// Note the field number 17, 18 and 19 have been deprecated.
 
-  // a list of values which will be used to create additional one dimensional real
+  // a list of values which will be used to create additional one dimensional float
   // values slots. These one dimensional slots can be used as the weight input
   // for cost layers.
   // Currently this is only supported by ProtoDataProvider.
-  repeated real constant_slots = 20;
+  repeated double constant_slots = 20;
 
   // for PyDataProvider.
   // Specify the load data script module name, object name and user args
@@ -80,6 +80,6 @@ sinclude(`DataConfigInter.proto.m4')
   optional bool is_main_data = 26 [default = true];
 
   // the usage ratio of instances. Setting to 1.0 means the use of all instances.
-  optional real usage_ratio = 27 [default = 1.0];
+  optional double usage_ratio = 27 [default = 1.0];
 };
 

proto/DataFormat.proto.m4 renamed to proto/DataFormat.proto

@@ -11,7 +11,7 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
-ifdef(`proto3', `syntax = "proto2";')
+syntax = "proto2";
 
 package paddle;
 

proto/ModelConfig.proto.m4 renamed to proto/ModelConfig.proto

@@ -11,7 +11,7 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
-ifdef(`proto3', `syntax = "proto2";')
+syntax = "proto2";
 
 import "ParameterConfig.proto";
 
@@ -20,7 +20,7 @@ package paddle;
 /**
  * Various structs for the configuration of a neural network
  */
-sinclude(`ModelConfigExt.proto.m4')
+
 
 message ExternalConfig {
   repeated string layer_names = 1;
@@ -146,8 +146,8 @@ message NormConfig {
 
   // the parameters for normalization
   // u = u / (1+scale*sum(u^2 in window))^pow
-  required real scale = 4;
-  required real pow = 5;
+  required double scale = 4;
+  required double pow = 5;
 
   // The size of output feature map.
   required uint32 output_x = 6;
@@ -223,7 +223,7 @@ message OperatorConfig {
   required uint64 output_size = 4;
 
   // For DotMulOperator
-  optional real dotmul_scale = 5 [default = 1.0];
+  optional double dotmul_scale = 5 [default = 1.0];
 
   // For ConvOperator
   optional ConvConfig conv_conf = 6;
@@ -266,7 +266,7 @@ message LayerInputConfig {
 }
 
 message LayerConfig {
-sinclude(`ModelConfigLayer.proto.m4')
+
   required string name = 1;
   required string type = 2;
   optional uint64 size = 3;
@@ -293,7 +293,7 @@ sinclude(`ModelConfigLayer.proto.m4')
   optional uint32 partial_sum = 9;
 
   // for dropout
-  optional real drop_rate = 10;
+  optional double drop_rate = 10;
 
   // for HierarchicalSoftmaxLayer and NCELayer
   // the number of classes
@@ -317,17 +317,17 @@ sinclude(`ModelConfigLayer.proto.m4')
   // For NCELayer
   // The distribution for generating the random negative labels.
   // A uniform distribution will be used if not provided
-  repeated real neg_sampling_dist = 17 [packed = true];
+  repeated double neg_sampling_dist = 17 [packed = true];
 
   // For MaxLayer
   // default: output VALUE of MaxLayer. set this flag to true for output INDEX
-  // INDEX will be put in Argument::value as real values.
+  // INDEX will be put in Argument::value as double values.
   optional bool output_max_index = 19 [default = false];
 
   /// The filed number 20 have been deprecated.
 
   // For self-normalized estimation
-  optional real softmax_selfnorm_alpha = 21 [default = 0.1];
+  optional double softmax_selfnorm_alpha = 21 [default = 0.1];
 
   /// The filed numbers 22 and 23 have been deprecated.
 
@@ -338,14 +338,14 @@ sinclude(`ModelConfigLayer.proto.m4')
   optional bool norm_by_times = 25;
 
   // for CostLayers
-  optional real coeff = 26 [default = 1.0];
+  optional double coeff = 26 [default = 1.0];
 
   // for AverageLayer
   // can be set to: 'average', 'sum' or 'squarerootn'
   optional string average_strategy = 27;
 
   // for error clipping
-  optional real error_clipping_threshold = 28 [default = 0.0];
+  optional double error_clipping_threshold = 28 [default = 0.0];
 
   // for operators used by mixed layer
   repeated OperatorConfig operator_confs = 29;
@@ -355,11 +355,11 @@ sinclude(`ModelConfigLayer.proto.m4')
   optional int32 max_sort_size = 31;
 
   // for SlopeInterceptLayer
-  optional real slope = 32;
-  optional real intercept = 33;
+  optional double slope = 32;
+  optional double intercept = 33;
 
   // for CosSimVecMatLayer and CosSimLayer
-  optional real cos_scale = 34;
+  optional double cos_scale = 34;
 
   // for DataNormLayer
   // can be set to: 'z-score', 'min-max' or 'decimal-scaling'
@@ -394,7 +394,7 @@ sinclude(`ModelConfigLayer.proto.m4')
   // if number of the selected columns is less than
   // sample number * selective_fc output size * selective_fc_mull_mull_ratio
   // sparse multiplication is used, otherwise, using full multiplication.
-  optional real selective_fc_full_mul_ratio = 44 [default = 0.02];
+  optional double selective_fc_full_mul_ratio = 44 [default = 0.02];
 
   // to indicate how many threads selective_fc use to to accelate
   // the plain_mul period
@@ -406,7 +406,7 @@ sinclude(`ModelConfigLayer.proto.m4')
   optional bool use_global_stats = 46;
 
   // use to compute moving mean and variance.
-  optional real moving_average_fraction = 47 [default = 0.9];
+  optional double moving_average_fraction = 47 [default = 0.9];
 
   // bias size
   optional uint32 bias_size = 48 [default = 0];
@@ -438,7 +438,7 @@ message EvaluatorConfig {
 
   // Used by PrecisionRecallEvaluator and ClassificationErrorEvaluator
   // For multi binary labels: true if output > classification_threshold
-  optional real classification_threshold = 6 [default = 0.5];
+  optional double classification_threshold = 6 [default = 0.5];
   // The positive label. -1 means average precision and recall
   optional int32 positive_label = 7 [default = -1];