Merge branch 'develop' into ner_ut2

Author: luotao1

paddle/fluid/API.spec

@@ -36,6 +36,7 @@ paddle.fluid.default_startup_program ArgSpec(args=[], varargs=None, keywords=Non
 paddle.fluid.default_main_program ArgSpec(args=[], varargs=None, keywords=None, defaults=None)
 paddle.fluid.program_guard ArgSpec(args=[], varargs='args', keywords='kwds', defaults=None)
 paddle.fluid.get_var ArgSpec(args=['name', 'program'], varargs=None, keywords=None, defaults=(None,))
+paddle.fluid.name_scope ArgSpec(args=[], varargs='args', keywords='kwds', defaults=None)
 paddle.fluid.Executor.__init__ ArgSpec(args=['self', 'place'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.Executor.close ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.Executor.run ArgSpec(args=['self', 'program', 'feed', 'fetch_list', 'feed_var_name', 'fetch_var_name', 'scope', 'return_numpy', 'use_program_cache'], varargs=None, keywords=None, defaults=(None, None, None, 'feed', 'fetch', None, True, False))

paddle/fluid/framework/details/multi_devices_graph_pass.cc

@@ -625,19 +625,11 @@ int MultiDevSSAGraphBuilder::GetVarDeviceID(const ir::Graph &graph,
 void MultiDevSSAGraphBuilder::CreateScaleLossGradOp(
     ir::Graph *result, const std::string &loss_grad_name) const {
   for (size_t i = 0; i < places_.size(); ++i) {
-// Insert ScaleCost OpHandle
-#ifdef PADDLE_WITH_CUDA
-    auto *communication_dev_ctx =
-        nccl_ctxs_ ? nccl_ctxs_->DevCtx(places_[i])
-                   : platform::DeviceContextPool::Instance().Get(places_[i]);
-#else
-    auto *communication_dev_ctx =
-        platform::DeviceContextPool::Instance().Get(platform::CPUPlace());
-#endif
+    // Insert ScaleCost OpHandle
+    auto *dev_ctx = platform::DeviceContextPool::Instance().Get(places_[i]);
     auto *op_handle = new ScaleLossGradOpHandle(
         result->CreateEmptyNode("scale_loss_grad", ir::Node::Type::kOperation),
-        local_scopes_.size(), local_scopes_[i], places_[i],
-        communication_dev_ctx);
+        local_scopes_.size(), local_scopes_[i], places_[i], dev_ctx);
     result->Get<GraphOps>(kGraphOps).emplace_back(op_handle);
 
     // FIXME: Currently ScaleLossGradOp only use device_count as scale
@@ -744,7 +736,7 @@ void MultiDevSSAGraphBuilder::CreateDistTrainOp(ir::Graph *result,
           .emplace(varname, op_dev_id);
     }
   } else {
-    PADDLE_ENFORCE(
+    PADDLE_THROW(
         "the distribute training related op should be in [split_byref, "
         "concat].");
   }

paddle/fluid/framework/ir/attention_lstm_fuse_pass.cc

@@ -59,7 +59,7 @@ void FindWhileOp(Graph* graph) {
 
   auto handle = [&](const GraphPatternDetector::subgraph_t& subgraph,
                     Graph* g) {
-    auto* while_pat_node = gpd.pattern().RetriveNode("while");
+    auto* while_pat_node = gpd.pattern().RetrieveNode("while");
     auto* while_node = subgraph.at(while_pat_node);
     marked_nodes.insert(while_node);
   };

paddle/fluid/framework/ir/fc_fuse_pass.cc

@@ -31,77 +31,34 @@ bool VarOutLinksToOp(Node* node, const std::string& op_type) {
 }
 
 void BuildFCPattern(PDPattern* pattern) {
-  // make sure the selected MUL op has one input argument is a parameter.
-  auto* mul_parameter_var = pattern->NewNode(
-      [](Node* node) {
-        return node->IsVar() && node->outputs.size() == 1UL &&
-               node->outputs.front()->Op()->Type() == "mul" && node->Var() &&
-               node->Var()->Persistable();  // check is a parameter
-      },
-      "mul_weight" /*name*/);
-
-  auto* mul_tmp_input_var = pattern->NewNode(
-      [](Node* node) {
-        bool result =
-            node->IsVar() && node->outputs.size() >= 1UL && node->Var() &&
-            !node->Var()->Persistable();  // this input is not an parameter.
-        if (!result) return false;
-        // check whether one output is MUL op.
-        for (auto* op : node->outputs) {
-          if (op->IsOp() && op->Op()->Type() == "mul") return true;
-        }
-        return false;
-      },
-      "mul_tmp_var" /*name*/);
-
-  // select a MUL op
-  auto* mul_op = pattern->NewNode(
-      [](Node* node) {
-        return node->IsOp() &&               // start from an Op
-               node->Op()->Type() == "mul";  // type is mul
-        // the output should be consumed only by one element_add, that check
-        // leaves in a Var PDNode.
-      },
-      "mul" /*name*/);
-
-  // make sure the MUL op's output has only one consumer and links to an
-  // ELEMENTWISE_ADD op.
-  auto* mul_out_var = pattern->NewNode(
-      [](Node* node) {
-        return node->IsVar() &&                  // starts from a Var
-               node->outputs.size() == 1UL &&    // only has one consumer
-               node->outputs.front()->IsOp() &&  // check basic logic
-               node->Var() &&                    // not a ControlDepVar
-               node->outputs.front()->Op()->Type() ==
-                   "elementwise_add";  // a very strong validation
-      },
-      "mul_out");
-  // this check is not essential, just to make the corresponding variable Node
-  // retrival easier.
-  auto* elementwise_add_tmp_var = pattern->NewNode(
-      [](Node* node) {
-        return node->IsVar() && node->outputs.size() >= 1UL && node->Var() &&
-               VarOutLinksToOp(node, "elementwise_add");
-      },
-      "elementwise_add_tmpvar");
-
-  // select an ELEMENTWISE_ADD op
-  auto* elementwise_add_op = pattern->NewNode(
-      [](Node* node) {
-        return node->IsOp() && node->Op()->Type() == "elementwise_add";
-      },
-      "elementwise_add" /*name*/);
-
-  // get the ELEMENTWISE_ADD op's output
-  auto* elementwise_add_out_var = pattern->NewNode(
-      [](Node* node) {
-        return node->IsVar() && node->inputs.size() == 1UL && node->Var() &&
-               node->inputs.front()->Op()->Type() == "elementwise_add";
-      },
-      "elementwise_add_out");
-
-  mul_op->LinksFrom({mul_parameter_var, mul_tmp_input_var})
-      .LinksTo({mul_out_var});
+  // Create Operators
+  auto* mul_op = pattern->NewNode("mul")->assert_is_op("mul");
+  auto* elementwise_add_op =
+      pattern->NewNode("elementwise_add")->assert_is_op("elementwise_add");
+  // Create variables
+  // w
+  auto* mul_weight_var = pattern->NewNode("mul_weight")
+                             ->AsInput()
+                             ->assert_is_op_nth_input("mul", "Y", 0);
+  // x
+  auto* mul_tmp_var = pattern->NewNode("mul_tmp_var")
+                          ->AsInput()
+                          ->assert_is_op_nth_input("mul", "X", 0);
+  // intermediate variable, will be removed in the IR after fuse.
+  auto* mul_out_var = pattern->NewNode("mul_out")
+                          ->AsIntermediate()
+                          ->assert_is_only_output_of_op("mul")
+                          ->assert_is_op_input("elementwise_add");
+  // bias
+  auto* elementwise_add_tmp_var = pattern->NewNode("elementwise_add_tmpvar")
+                                      ->assert_is_op_input("elementwise_add")
+                                      ->AsInput();
+  // output
+  auto* elementwise_add_out_var = pattern->NewNode("elementwise_add_out")
+                                      ->AsOutput()
+                                      ->assert_is_op_output("elementwise_add");
+
+  mul_op->LinksFrom({mul_weight_var, mul_tmp_var}).LinksTo({mul_out_var});
   elementwise_add_op->LinksFrom({mul_out_var, elementwise_add_tmp_var})
       .LinksTo({elementwise_add_out_var});
 }
@@ -120,18 +77,20 @@ bool LinksReplace(std::vector<Node*>* links, Node* from, Node* to) {
 std::unique_ptr<ir::Graph> FCFusePass::ApplyImpl(
     std::unique_ptr<ir::Graph> graph) const {
   PADDLE_ENFORCE(graph.get());
+  FusePassBase::Init("fc", graph.get());
 
   std::unordered_set<Node*> nodes2delete;
 
   GraphPatternDetector gpd;
   BuildFCPattern(gpd.mutable_pattern());
 
-#define GET_NODE(id)                                             \
-  PADDLE_ENFORCE(subgraph.count(gpd.pattern().RetriveNode(#id)), \
-                 "pattern has no Node called %s", #id);          \
-  auto* id = subgraph.at(gpd.pattern().RetriveNode(#id));        \
+#define GET_NODE(id)                                              \
+  PADDLE_ENFORCE(subgraph.count(gpd.pattern().RetrieveNode(#id)), \
+                 "pattern has no Node called %s", #id);           \
+  auto* id = subgraph.at(gpd.pattern().RetrieveNode(#id));        \
   PADDLE_ENFORCE_NOT_NULL(id, "subgraph has no node %s", #id);
 
+  int found_fc_count = 0;
   auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
                      Graph* g) {
     VLOG(4) << "handle FC fuse";
@@ -176,10 +135,13 @@ std::unique_ptr<ir::Graph> FCFusePass::ApplyImpl(
     graph->RemoveNode(mul);
     graph->RemoveNode(elementwise_add);
     graph->RemoveNode(mul_out);  // tmp variable
+
+    found_fc_count++;
   };
 
   gpd(graph.get(), handler);
 
+  AddStatis(found_fc_count);
   return graph;
 }
 

paddle/fluid/framework/ir/fc_fuse_pass.h

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "paddle/fluid/framework/ir/fuse_pass_base.h"
 #include "paddle/fluid/framework/ir/graph.h"
 #include "paddle/fluid/framework/ir/graph_pattern_detector.h"
 #include "paddle/fluid/framework/ir/pass.h"
@@ -23,7 +24,7 @@ namespace ir {
 /*
  * Fuse the MUL and ELEMENTWISE_ADD to a FCOp.
  */
-class FCFusePass : public Pass {
+class FCFusePass : public FusePassBase {
  public:
   virtual ~FCFusePass() {}
 

paddle/fluid/framework/ir/fc_fuse_pass_tester.cc

@@ -25,8 +25,13 @@ void SetOp(ProgramDesc* prog, const std::string& type,
            const std::vector<std::string>& outputs) {
   auto* op = prog->MutableBlock(0)->AppendOp();
   op->SetType(type);
-  op->SetInput("Xs", inputs);
-  op->SetOutput("Ys", outputs);
+  if (type == "mul") {
+    op->SetInput("X", {inputs[0]});
+    op->SetInput("Y", {inputs[1]});
+  } else if (type == "elementwise_add") {
+    op->SetInput("X", inputs);
+  }
+  op->SetOutput("Out", outputs);
 }
 
 // a->OP0->b

paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc

@@ -36,7 +36,7 @@ std::unique_ptr<ir::Graph> FCLstmFusePass::ApplyImpl(
   auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph,
                      Graph* g) {
 
-    auto* id = subgraph.at(gpd.pattern().RetriveNode("any_node"));
+    auto* id = subgraph.at(gpd.pattern().RetrieveNode("any_node"));
     marked_nodes.insert(id);
   };
   gpd(graph.get(), handler);
@@ -64,9 +64,9 @@ std::unique_ptr<ir::Graph> FCLstmFusePass::ApplyImpl(
 #undef GET_NODE
 #undef SET_IN
 
-    LOG(INFO) << "hidden_n: " << hidden_n->Name();
-    LOG(INFO) << "cell: " << cell_n->Name();
-    LOG(INFO) << "xx: " << xx_n->Name();
+    VLOG(4) << "hidden_n: " << hidden_n->Name();
+    VLOG(4) << "cell: " << cell_n->Name();
+    VLOG(4) << "xx: " << xx_n->Name();
 
     op_desc.SetInput("H0", {});
     op_desc.SetInput("C0", {});

paddle/fluid/framework/ir/fuse_pass_base.h

@@ -22,21 +22,37 @@ namespace paddle {
 namespace framework {
 namespace ir {
 
-static const char kParamScopeAttr[] = "param_scope";
+static const char kParamScopeAttr[] = "__param_scope__";
+static const char kFuseStatisAttr[] = "__fuse_statis__";
 
 class FusePassBase : public Pass {
  public:
-  void Init(Graph* graph) const { graph_ = graph; }
+  void Init(const std::string& repr, Graph* graph) const {
+    repr_ = repr;
+    graph_ = graph;
+  }
 
   Scope* param_scope() const {
     PADDLE_ENFORCE(graph_->Has(kParamScopeAttr));
     return graph_->Get<framework::Scope*>(kParamScopeAttr);
   }
 
+  void AddStatis(int count_of_fused) const {
+    PADDLE_ENFORCE(graph_);
+    PADDLE_ENFORCE(!repr_.empty());
+    if (!graph_->Has(kFuseStatisAttr)) {
+      graph_->Set(kFuseStatisAttr, new std::unordered_map<std::string, int>);
+    }
+    auto& info =
+        graph_->Get<std::unordered_map<std::string, int>>(kFuseStatisAttr);
+    info[repr_] = count_of_fused;
+  }
+
   virtual ~FusePassBase() {}
 
  protected:
   mutable Graph* graph_;
+  mutable std::string repr_;
 };
 
 }  // namespace ir