loglevel adjustment for distributed training (#31236)

seiriosPlus · web-flow · commit 188bcbb767b8 · 2021-02-26T14:36:36.000+08:00
Change-Id: I6210ce9c60bed48f3323c47b16500302b66cedf2
diff --git a/paddle/fluid/distributed/fleet.cc b/paddle/fluid/distributed/fleet.cc
@@ -508,7 +508,7 @@ void FleetWrapper::ShrinkDenseTable(int table_id, Scope* scope,
     if (name.find("batch_sum") != std::string::npos) {
       Variable* var = scope->FindVar(name);
       CHECK(var != nullptr) << "var[" << name << "] not found";
-      VLOG(0) << "prepare shrink dense batch_sum";
+      VLOG(3) << "prepare shrink dense batch_sum";
       LoDTensor* tensor = var->GetMutable<LoDTensor>();
       float* g = tensor->data<float>();
 
diff --git a/paddle/fluid/distributed/service/brpc_ps_server.cc b/paddle/fluid/distributed/service/brpc_ps_server.cc
@@ -76,16 +76,13 @@ uint64_t BrpcPsServer::start(const std::string &ip, uint32_t port) {
     }
   }
 
-  VLOG(0) << "BrpcPsServer::start registe_ps_server";
   _environment->registe_ps_server(ip, port, _rank);
-  VLOG(0) << "BrpcPsServer::start wait";
   cv_.wait(lock, [&] { return stoped_; });
 
   PSHost host;
   host.ip = ip;
   host.port = port;
   host.rank = _rank;
-  VLOG(0) << "BrpcPsServer::start return host.rank";
   return host.rank;
 }
 
@@ -461,7 +458,7 @@ int32_t BrpcPsService::save_one_table(Table *table,
 
   int32_t feasign_size = 0;
 
-  VLOG(0) << "save one table " << request.params(0) << " " << request.params(1);
+  VLOG(3) << "save table " << request.params(0) << " " << request.params(1);
   feasign_size = table->save(request.params(0), request.params(1));
   if (feasign_size < 0) {
     set_response_code(response, -1, "table save failed");
@@ -504,7 +501,7 @@ int32_t BrpcPsService::shrink_table(Table *table,
     set_response_code(response, -1, "table shrink failed");
     return -1;
   }
-  VLOG(0) << "Pserver Shrink Finished";
+  VLOG(3) << "Pserver Shrink Finished";
   return 0;
 }
 
diff --git a/paddle/fluid/distributed/service/communicator.cc b/paddle/fluid/distributed/service/communicator.cc
@@ -52,7 +52,7 @@ inline double GetCurrentUS() {
 Communicator::Communicator() {}
 
 void Communicator::init_gflag(const std::string &gflags) {
-  VLOG(0) << "Init With Gflags:" << gflags;
+  VLOG(3) << "Init With Gflags:" << gflags;
   std::vector<std::string> flags = paddle::string::split_string(gflags);
   if (flags.size() < 1) {
     flags.push_back("-max_body_size=314217728");
diff --git a/paddle/fluid/distributed/service/communicator.h b/paddle/fluid/distributed/service/communicator.h
@@ -194,10 +194,10 @@ class Communicator {
   Communicator();
 
   explicit Communicator(const std::map<std::string, std::string> &envs_) {
-    VLOG(0) << "Communicator Init Envs";
+    VLOG(3) << "Communicator Init Envs";
     for (auto &iter : envs_) {
       envs[iter.first] = iter.second;
-      VLOG(0) << iter.first << ": " << iter.second;
+      VLOG(3) << iter.first << ": " << iter.second;
     }
     barrier_table_id_ = std::stoi(envs.at("barrier_table_id"));
     trainer_id_ = std::stoi(envs.at("trainer_id"));
@@ -431,7 +431,7 @@ class HalfAsyncCommunicator : public AsyncCommunicator {
     need_global_step_ =
         static_cast<bool>(std::stoi(envs.at("need_global_step")));
 
-    VLOG(0) << "HalfAsyncCommunicator Initialized";
+    VLOG(1) << "HalfAsyncCommunicator Initialized";
   }
 
   void MainThread() override;
@@ -476,7 +476,7 @@ class SyncCommunicator : public HalfAsyncCommunicator {
     need_global_step_ =
         static_cast<bool>(std::stoi(envs.at("need_global_step")));
 
-    VLOG(0) << "SyncCommunicator Initialized";
+    VLOG(1) << "SyncCommunicator Initialized";
   }
 
   void BarrierSend();
@@ -520,7 +520,7 @@ class GeoCommunicator : public AsyncCommunicator {
     // id_queue's size
     max_merge_var_num_ = std::stoi(envs.at("communicator_max_merge_var_num"));
     send_queue_size_ = max_merge_var_num_;
-    VLOG(0) << "GeoCommunicator Initialized";
+    VLOG(1) << "GeoCommunicator Initialized";
   }
 
   void Send(const std::vector<std::string> &var_names,
diff --git a/paddle/fluid/distributed/service/heter_client.cc b/paddle/fluid/distributed/service/heter_client.cc
@@ -42,28 +42,28 @@ void HeterClient::MainThread() {
 void HeterClient::Stop() {
   running_ = false;
   if (!is_initialized_) {
-    VLOG(0) << "HeterClient is not inited, do nothing";
+    VLOG(3) << "HeterClient is not inited, do nothing";
   } else {
     if (main_thread_) {
       auto status = StopHeterWorker();
       status.wait();
       main_thread_->join();
       main_thread_.reset(nullptr);
     }
-    VLOG(1) << "HeterClient Stop Done";
+    VLOG(3) << "HeterClient Stop Done";
   }
 }
 
 void HeterClient::FinalizeWorker() {
   running_ = false;
   if (!is_initialized_) {
-    VLOG(0) << "HeterClient is not inited, do nothing";
+    VLOG(3) << "HeterClient is not inited, do nothing";
   } else {
     if (main_thread_) {
       main_thread_->join();
       main_thread_.reset(nullptr);
     }
-    VLOG(1) << "HeterClient Stop Done";
+    VLOG(3) << "HeterClient Stop Done";
   }
 }
 
diff --git a/paddle/fluid/distributed/service/heter_server.cc b/paddle/fluid/distributed/service/heter_server.cc
@@ -95,7 +95,7 @@ int32_t HeterService::stop_heter_worker(const PsRequestMessage& request,
   stop_cpu_worker_set_.insert(client_id);
   if (stop_cpu_worker_set_.size() == fan_in_) {
     is_exit_ = true;
-    VLOG(0) << "Stop heter Service done.";
+    VLOG(3) << "Stop heter Service done.";
   }
   return 0;
 }
diff --git a/paddle/fluid/distributed/service/heter_server.h b/paddle/fluid/distributed/service/heter_server.h
@@ -136,7 +136,7 @@ class HeterServer {
   virtual ~HeterServer() {}
 
   void Stop() {
-    VLOG(0) << "HeterServer Stop()";
+    VLOG(3) << "HeterServer Stop()";
     std::unique_lock<std::mutex> lock(mutex_);
     stoped_ = true;
     cv_.notify_all();
diff --git a/paddle/fluid/distributed/table/common_dense_table.cc b/paddle/fluid/distributed/table/common_dense_table.cc
@@ -91,7 +91,7 @@ int32_t CommonDenseTable::initialize_optimizer() {
   } else {
     VLOG(0) << "init optimizer failed";
   }
-  VLOG(0) << "init optimizer " << name << " done";
+  VLOG(3) << "init optimizer " << name << " done";
   return 0;
 }
 
diff --git a/paddle/fluid/distributed/table/common_dense_table.h b/paddle/fluid/distributed/table/common_dense_table.h
@@ -45,15 +45,12 @@ class CommonDenseTable : public DenseTable {
   virtual int32_t set_global_lr(float* lr) override;
 
   int32_t load(const std::string& path, const std::string& param) override {
-    VLOG(0) << "Dense table may load by "
-               "paddle.distributed.fleet.init_server";
+    VLOG(0) << "WARNING: dense variables will load on No.0 trainer";
     return 0;
   }
 
   int32_t save(const std::string& path, const std::string& param) override {
-    VLOG(0)
-        << "Dense table may be saved by "
-           "paddle.distributed.fleet.save_persistables/save_inference_model";
+    VLOG(0) << "WARNING: dense variables will save on No.0 trainer";
     return 0;
   }
 
diff --git a/paddle/fluid/distributed/table/common_sparse_table.cc b/paddle/fluid/distributed/table/common_sparse_table.cc
@@ -166,7 +166,7 @@ int64_t LoadFromText(const std::string& valuepath, const std::string& metapath,
     auto id = std::stoull(values[0]);
 
     if (id % pserver_num != pserver_id) {
-      VLOG(0) << "will not load " << values[0] << " from " << valuepath
+      VLOG(3) << "will not load " << values[0] << " from " << valuepath
               << ", please check id distribution";
       continue;
     }
@@ -259,7 +259,7 @@ int32_t CommonSparseTable::initialize_value() {
     }
   }
 
-  VLOG(0) << "has " << feasigns.size() << " ids need to be pre inited";
+  VLOG(3) << "has " << feasigns.size() << " ids need to be pre inited";
 
   auto buckets = bucket(feasigns.size(), 10);
   for (int x = 0; x < 10; ++x) {
@@ -291,10 +291,10 @@ int32_t CommonSparseTable::initialize_optimizer() {
     optimizer_ = std::make_shared<SSUM>(value_names_, value_dims_,
                                         value_offsets_, value_idx_);
   } else {
-    VLOG(0) << "init optimizer failed";
+    VLOG(3) << "init optimizer failed";
   }
 
-  VLOG(0) << "init optimizer " << name << " done";
+  VLOG(3) << "init optimizer " << name << " done";
   return 0;
 }
 
@@ -307,7 +307,7 @@ int32_t CommonSparseTable::set_global_lr(float* lr) {
 int32_t CommonSparseTable::load(const std::string& path,
                                 const std::string& param) {
   rwlock_->WRLock();
-  VLOG(0) << "sparse table load with " << path << " with meta " << param;
+  VLOG(3) << "sparse table load with " << path << " with meta " << param;
   LoadFromText(path, param, _shard_idx, _shard_num, task_pool_size_,
                &shard_values_);
   rwlock_->UNLock();
@@ -318,7 +318,7 @@ int32_t CommonSparseTable::save(const std::string& dirname,
                                 const std::string& param) {
   rwlock_->WRLock();
   int mode = std::stoi(param);
-  VLOG(0) << "sparse table save: " << dirname << " mode: " << mode;
+  VLOG(3) << "sparse table save: " << dirname << " mode: " << mode;
 
   auto varname = _config.common().table_name();
   std::string var_store =
@@ -534,11 +534,11 @@ int32_t CommonSparseTable::flush() { return 0; }
 int32_t CommonSparseTable::shrink(const std::string& param) {
   rwlock_->WRLock();
   int threshold = std::stoi(param);
-  VLOG(0) << "sparse table shrink: " << threshold;
+  VLOG(3) << "sparse table shrink: " << threshold;
 
   for (int shard_id = 0; shard_id < task_pool_size_; ++shard_id) {
     // shrink
-    VLOG(0) << shard_id << " " << task_pool_size_ << " begin shrink";
+    VLOG(4) << shard_id << " " << task_pool_size_ << " begin shrink";
     shard_values_[shard_id]->Shrink(threshold);
   }
   rwlock_->UNLock();

Original file line number	Diff line number	Diff line change
`@@ -76,16 +76,13 @@ uint64_t BrpcPsServer::start(const std::string &ip, uint32_t port) {`
`76`	`76`	`}`
`77`	`77`	`}`
`78`	`78`
`79`		`- VLOG(0) << "BrpcPsServer::start registe_ps_server";`
`80`	`79`	`_environment->registe_ps_server(ip, port, _rank);`
`81`		`- VLOG(0) << "BrpcPsServer::start wait";`
`82`	`80`	`cv_.wait(lock, [&] { return stoped_; });`
`83`	`81`
`84`	`82`	`PSHost host;`
`85`	`83`	`host.ip = ip;`
`86`	`84`	`host.port = port;`
`87`	`85`	`host.rank = _rank;`
`88`		`- VLOG(0) << "BrpcPsServer::start return host.rank";`
`89`	`86`	`return host.rank;`
`90`	`87`	`}`
`91`	`88`
`@@ -461,7 +458,7 @@ int32_t BrpcPsService::save_one_table(Table *table,`
`461`	`458`
`462`	`459`	`int32_t feasign_size = 0;`
`463`	`460`
`464`		`- VLOG(0) << "save one table " << request.params(0) << " " << request.params(1);`
	`461`	`+ VLOG(3) << "save table " << request.params(0) << " " << request.params(1);`
`465`	`462`	`feasign_size = table->save(request.params(0), request.params(1));`
`466`	`463`	`if (feasign_size < 0) {`
`467`	`464`	`set_response_code(response, -1, "table save failed");`
`@@ -504,7 +501,7 @@ int32_t BrpcPsService::shrink_table(Table *table,`
`504`	`501`	`set_response_code(response, -1, "table shrink failed");`
`505`	`502`	`return -1;`
`506`	`503`	`}`
`507`		`- VLOG(0) << "Pserver Shrink Finished";`
	`504`	`+ VLOG(3) << "Pserver Shrink Finished";`
`508`	`505`	`return 0;`
`509`	`506`	`}`
`510`	`507`
Original file line number	Diff line number	Diff line change
`@@ -42,28 +42,28 @@ void HeterClient::MainThread() {`
`42`	`42`	`void HeterClient::Stop() {`
`43`	`43`	`running_ = false;`
`44`	`44`	`if (!is_initialized_) {`
`45`		`- VLOG(0) << "HeterClient is not inited, do nothing";`
	`45`	`+ VLOG(3) << "HeterClient is not inited, do nothing";`
`46`	`46`	`} else {`
`47`	`47`	`if (main_thread_) {`
`48`	`48`	`auto status = StopHeterWorker();`
`49`	`49`	`status.wait();`
`50`	`50`	`main_thread_->join();`
`51`	`51`	`main_thread_.reset(nullptr);`
`52`	`52`	`}`
`53`		`- VLOG(1) << "HeterClient Stop Done";`
	`53`	`+ VLOG(3) << "HeterClient Stop Done";`
`54`	`54`	`}`
`55`	`55`	`}`
`56`	`56`
`57`	`57`	`void HeterClient::FinalizeWorker() {`
`58`	`58`	`running_ = false;`
`59`	`59`	`if (!is_initialized_) {`
`60`		`- VLOG(0) << "HeterClient is not inited, do nothing";`
	`60`	`+ VLOG(3) << "HeterClient is not inited, do nothing";`
`61`	`61`	`} else {`
`62`	`62`	`if (main_thread_) {`
`63`	`63`	`main_thread_->join();`
`64`	`64`	`main_thread_.reset(nullptr);`
`65`	`65`	`}`
`66`		`- VLOG(1) << "HeterClient Stop Done";`
	`66`	`+ VLOG(3) << "HeterClient Stop Done";`
`67`	`67`	`}`
`68`	`68`	`}`
`69`	`69`
Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,7 @@ int32_t HeterService::stop_heter_worker(const PsRequestMessage& request,`
`95`	`95`	`stop_cpu_worker_set_.insert(client_id);`
`96`	`96`	`if (stop_cpu_worker_set_.size() == fan_in_) {`
`97`	`97`	`is_exit_ = true;`
`98`		`- VLOG(0) << "Stop heter Service done.";`
	`98`	`+ VLOG(3) << "Stop heter Service done.";`
`99`	`99`	`}`
`100`	`100`	`return 0;`
`101`	`101`	`}`
Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,7 @@ int32_t CommonDenseTable::initialize_optimizer() {`
`91`	`91`	`} else {`
`92`	`92`	`VLOG(0) << "init optimizer failed";`
`93`	`93`	`}`
`94`		`- VLOG(0) << "init optimizer " << name << " done";`
	`94`	`+ VLOG(3) << "init optimizer " << name << " done";`
`95`	`95`	`return 0;`
`96`	`96`	`}`
`97`	`97`