code clean

Author: jacquesqiao

paddle/fluid/operators/reader/ctr_reader.cc

@@ -73,18 +73,6 @@ static inline void parse_line(
   }
 }
 
-static void print_map(
-    std::unordered_map<std::string, std::vector<int64_t>>* map) {
-  for (auto it = map->begin(); it != map->end(); ++it) {
-    std::cout << it->first << " -> ";
-    std::cout << "[";
-    for (auto& i : it->second) {
-      std::cout << i << " ";
-    }
-    std::cout << "]\n";
-  }
-}
-
 class Reader {
  public:
   virtual ~Reader() {}
@@ -162,10 +150,6 @@ void ReadThread(const std::vector<std::string>& file_list,
 
   VLOG(3) << "reader inited";
 
-  uint64_t t0 = GetTimeInSec();
-
-  int i = 0;
-
   while (reader.HasNext()) {
     batch_data.clear();
     batch_data.reserve(batch_size);
@@ -186,7 +170,6 @@ void ReadThread(const std::vector<std::string>& file_list,
         break;
       }
     }
-    //    print_map(&batch_data[0]);
 
     std::vector<framework::LoDTensor> lod_datas;
 
@@ -224,19 +207,10 @@ void ReadThread(const std::vector<std::string>& file_list,
 
     queue->Push(lod_datas);
     VLOG(4) << "push one data, queue_size=" << queue->Size();
-
-    if (i != 0 && i % 100 == 0) {
-      uint64_t t1 = GetTimeInSec();
-      float line_per_s = 100 * batch_size * 1000000 / (t1 - t0);
-      VLOG(3) << "[" << thread_id << "]"
-              << " line_per_second = " << line_per_s;
-      t0 = t1;
-    }
-    i++;
   }
 
   (*thread_status)[thread_id] = Stopped;
-  VLOG(3) << "thread " << thread_id << " exited";
+  VLOG(3) << "set status to stopped, thread " << thread_id << " exited";
 }
 
 }  // namespace reader

paddle/fluid/operators/reader/ctr_reader.h

@@ -39,15 +39,6 @@ void ReadThread(const std::vector<std::string>& file_list,
                 int thread_id, std::vector<ReaderThreadStatus>* thread_status,
                 std::shared_ptr<LoDTensorBlockingQueue> queue);
 
-inline uint64_t GetTimeInSec() {
-  using clock = std::conditional<std::chrono::high_resolution_clock::is_steady,
-                                 std::chrono::high_resolution_clock,
-                                 std::chrono::steady_clock>::type;
-  return std::chrono::duration_cast<std::chrono::microseconds>(
-             clock::now().time_since_epoch())
-      .count();
-}
-
 class CTRReader : public framework::FileReader {
  public:
   explicit CTRReader(const std::shared_ptr<LoDTensorBlockingQueue>& queue,

paddle/fluid/operators/reader/ctr_reader_test.cc

@@ -33,6 +33,7 @@ using paddle::operators::reader::LoDTensorBlockingQueueHolder;
 using paddle::operators::reader::CTRReader;
 using paddle::framework::LoDTensor;
 using paddle::framework::LoD;
+using paddle::framework::DDim;
 using paddle::platform::CPUPlace;
 
 static void generatedata(const std::vector<std::string>& data,
@@ -73,25 +74,25 @@ TEST(CTR_READER, read_data) {
   std::vector<int64_t> label_value = {0, 0, 1, 0, 1, 1, 0, 1, 1, 1};
 
   std::vector<std::tuple<LoD, std::vector<int64_t>>> data_slot_6002{
-      {{{0, 1, 2}}, {0, 0}},
-      {{{0, 5, 6}}, {10, 11, 12, 13, 14, 0}},
-      {{{0, 1, 2}}, {0, 0}},
-      {{{0, 1, 2}}, {30, 0}},
-      {{{0, 1, 2}}, {40, 0}}};
+      {{{0, 1, 2, 7}}, {0, 0, 10, 11, 12, 13, 14}},
+      {{{0, 1, 2, 3}}, {0, 0, 0}},
+      {{{0, 1, 2, 3}}, {30, 0, 40}},
+      {{{0, 1}}, {0}}};
   std::vector<std::tuple<LoD, std::vector<int64_t>>> data_slot_6003{
-      {{{0, 1, 4}}, {1, 5, 6, 7}},
-      {{{0, 1, 5}}, {0, 15, 16, 17, 18}},
-      {{{0, 1, 2}}, {0, 0}},
-      {{{0, 1, 3}}, {31, 35, 36}},
-      {{{0, 1, 4}}, {41, 47, 48, 49}}};
+      {{{0, 1, 4, 5}}, {1, 5, 6, 7, 0}},
+      {{{0, 4, 5, 6}}, {15, 16, 17, 18, 0, 0}},
+      {{{0, 1, 3, 4}}, {31, 35, 36, 41}},
+      {{{0, 3}}, {47, 48, 49}}};
+
+  std::vector<DDim> label_dims = {{1, 3}, {1, 3}, {1, 3}, {1, 1}};
 
   LoDTensorBlockingQueueHolder queue_holder;
   int capacity = 64;
   queue_holder.InitOnce(capacity, {}, false);
 
   std::shared_ptr<LoDTensorBlockingQueue> queue = queue_holder.GetQueue();
 
-  int batch_size = 2;
+  int batch_size = 3;
   int thread_num = 1;
   std::vector<std::string> slots = {"6002", "6003"};
   std::vector<std::string> file_list;
@@ -103,15 +104,15 @@ TEST(CTR_READER, read_data) {
 
   reader.Start();
 
-  size_t batch_num = std::ceil(ctr_data.size() / batch_size) * thread_num;
+  size_t batch_num =
+      std::ceil(static_cast<float>(ctr_data.size()) / batch_size) * thread_num;
 
   for (size_t i = 0; i < batch_num; ++i) {
     std::vector<LoDTensor> out;
     reader.ReadNext(&out);
     ASSERT_EQ(out.size(), slots.size() + 1);
     auto& label_tensor = out.back();
-    ASSERT_EQ(label_tensor.dims(),
-              paddle::framework::make_ddim({1, batch_size}));
+    ASSERT_EQ(label_tensor.dims(), label_dims[i]);
     for (size_t j = 0; j < batch_size && i * batch_num + j < ctr_data.size();
          ++j) {
       auto& label = label_tensor.data<int64_t>()[j];