Skip to content

Commit dd2dfeb

Browse files
jacquesqiaojacquesqiao
committed
add debug information
1 parent 803e2ed commit dd2dfeb

File tree

2 files changed

+116
-16
lines changed

2 files changed

+116
-16
lines changed

‎paddle/fluid/operators/reader/ctr_reader.cc

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,12 @@ void ReadThread(const std::vector<std::string>& file_list,
141141
const std::vector<std::string>& slots, int batch_size,
142142
int thread_id, std::vector<ReaderThreadStatus>* thread_status,
143143
std::shared_ptr<LoDTensorBlockingQueue> queue) {
144-
VLOG(3) << "reader thread start! thread_id = " << thread_id;
144+
VLOG(3) << "[" << thread_id << "]"
145+
<< " reader thread start! thread_id = " << thread_id;
146+
for (auto& file : file_list) {
147+
VLOG(3) << "[" << thread_id << "]"
148+
<< " file " << file;
149+
}
145150
(*thread_status)[thread_id] = Running;
146151
VLOG(3) << "set status to running";
147152

@@ -159,6 +164,10 @@ void ReadThread(const std::vector<std::string>& file_list,
159164

160165
VLOG(3) << "reader inited";
161166

167+
clock_t t0 = clock();
168+
169+
int i = 0;
170+
162171
while (reader.HasNext()) {
163172
batch_data.clear();
164173
batch_label.clear();
@@ -176,9 +185,7 @@ void ReadThread(const std::vector<std::string>& file_list,
176185
break;
177186
}
178187
}
179-
180-
VLOG(3) << "read one batch, batch_size = " << batch_data.size();
181-
print_map(&batch_data[0]);
188+
// print_map(&batch_data[0]);
182189

183190
std::vector<framework::LoDTensor> lod_datas;
184191

@@ -204,8 +211,6 @@ void ReadThread(const std::vector<std::string>& file_list,
204211
lod_datas.push_back(lod_tensor);
205212
}
206213

207-
VLOG(3) << "convert data to tensor";
208-
209214
// insert label tensor
210215
framework::LoDTensor label_tensor;
211216
int64_t* label_tensor_data = label_tensor.mutable_data<int64_t>(
@@ -214,8 +219,18 @@ void ReadThread(const std::vector<std::string>& file_list,
214219
memcpy(label_tensor_data, batch_label.data(), batch_label.size());
215220
lod_datas.push_back(label_tensor);
216221

217-
VLOG(3) << "push one data";
218-
queue->Push(lod_datas);
222+
// queue->Push(lod_datas);
223+
VLOG(4) << "push one data, queue_size=" << queue->Size();
224+
225+
if (i != 0 && i % 100 == 0) {
226+
clock_t t1 = clock();
227+
float line_per_s = 100 * batch_size * static_cast<int64>(CLOCKS_PER_SEC) /
228+
static_cast<int>(t1 - t0);
229+
VLOG(3) << "[" << thread_id << "]"
230+
<< " line_per_second = " << line_per_s;
231+
t0 = t1;
232+
}
233+
i++;
219234
}
220235

221236
(*thread_status)[thread_id] = Stopped;

‎paddle/fluid/operators/reader/ctr_reader_test.cc

Lines changed: 93 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,34 +12,119 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
#include "paddle/fluid/operators/reader/ctr_reader.h"
16+
17+
#include <time.h>
18+
1519
#include "gtest/gtest.h"
1620

1721
#include "paddle/fluid/framework/lod_tensor.h"
1822
#include "paddle/fluid/operators/reader/blocking_queue.h"
19-
#include "paddle/fluid/operators/reader/ctr_reader.h"
2023

2124
using paddle::operators::reader::LoDTensorBlockingQueue;
2225
using paddle::operators::reader::LoDTensorBlockingQueueHolder;
2326
using paddle::operators::reader::CTRReader;
27+
using paddle::framework::LoDTensor;
2428

2529
TEST(CTR_READER, read_data) {
2630
LoDTensorBlockingQueueHolder queue_holder;
2731
int capacity = 64;
28-
queue_holder.InitOnce(capacity, {}, false);
32+
queue_holder.InitOnce(capacity, {}, true);
2933

3034
std::shared_ptr<LoDTensorBlockingQueue> queue = queue_holder.GetQueue();
3135

3236
int batch_size = 10;
33-
int thread_num = 1;
34-
std::vector<std::string> slots = {"6003", "6004"};
37+
int thread_num = 2;
38+
std::vector<std::string> slots = {
39+
"6002", "6003", "6004", "6005", "6006", "6007", "6008", "6009", "6010",
40+
"6011", "6012", "6013", "6014", "6015", "6016", "6017", "6018", "6019",
41+
"6020", "6021", "6023", "6024", "6025", "6026", "6027", "6028", "6029",
42+
"6030", "6031", "6032", "6033", "6034", "6035", "6036", "6037", "6038",
43+
"6039", "6040", "6041", "6042", "6043", "6044", "6045", "6046", "6047",
44+
"6048", "6050", "6051", "6052", "6054", "6055", "6056", "6057", "6058",
45+
"6059", "6060", "6061", "6062", "6063", "6064", "6065", "6066", "6067",
46+
"6068", "6069", "6070", "6071", "6072", "6073", "6074", "6075", "6076",
47+
"6077", "6078", "6079", "6080", "6081", "6082", "6083", "6084", "6085",
48+
"6086", "6087", "6088", "6089", "6090", "6091", "6092", "6093", "6094",
49+
"6095", "6096", "6097", "6098", "6099", "6100", "6101", "6102", "6103",
50+
"6104", "6105", "6106", "6107", "6108", "6109", "6110", "6111", "6112",
51+
"6113", "6114", "6115", "6116", "6117", "6118", "6119", "6120", "6121",
52+
"6122", "6123", "6124", "6125", "6126", "6127", "6128", "6129", "6130",
53+
"6131", "6132", "6133", "6134", "6135", "6136", "6137", "6138", "6139",
54+
"6140", "6141", "6142", "6143", "6144", "6145", "6146", "6147", "6148",
55+
"6149", "6150", "6151", "6152", "6153", "6155", "6156", "6157", "6158",
56+
"6160", "6161", "6162", "6163", "6164", "6165", "6166", "6167", "6168",
57+
"6169", "6170", "6171", "6172", "6173", "6174", "6175", "6176", "6177",
58+
"6178", "6181", "6182", "6183", "6184", "6185", "6186", "6188", "6189",
59+
"6190", "6191", "6192", "6194", "6195", "6196", "6197", "6198", "6199",
60+
"6200", "6201", "6202", "6203", "6204", "6205", "6206", "6207", "6208",
61+
"6209", "6210", "6211", "6212", "6213", "6214", "6215", "6216", "6217",
62+
"6218", "6220", "6222", "6223", "6224", "6225", "6226", "6227", "6228",
63+
"6229", "6230", "6231", "6232", "6233", "6234", "6235", "6236", "6237",
64+
"6238", "6239", "6240", "6241", "6242", "6243", "6244", "6245", "6247",
65+
"6248", "6250", "6251", "6253", "6254", "6255", "6256", "6257", "6258",
66+
"6259", "6260", "6261", "6262", "6263", "6264", "6265", "6350", "6351",
67+
"6352", "6353", "6354", "6355", "6356", "6738", "6739", "6740", "6741",
68+
"6751", "6753", "6754", "6755", "6756", "6757", "6759", "6760", "6763",
69+
"6764", "6765", "6766", "6767", "6768", "6769", "6770", "6806", "6807",
70+
"6808", "6809", "6810", "6811", "6812", "6813", "6814", "6815", "6816",
71+
"6817", "6818", "6819", "6820", "6821", "6822", "6823", "6824", "6825",
72+
"6826", "6827", "6828", "6829", "6830", "6831", "6832", "6833", "6834",
73+
"6835", "6836", "6837", "6838", "6839", "6840", "6841", "6842", "6843",
74+
"6844", "6845", "6846", "6847", "6848", "6849", "6850", "6851", "6852",
75+
"6853", "6854", "6855", "6856", "6857", "6858", "6859", "6860", "6861",
76+
"6862", "6863", "6864", "6865", "6866", "6867", "6868", "6869", "6870",
77+
"6871", "6872", "6873", "6874", "6875", "6876", "6877", "6878", "6879",
78+
"6880", "6881", "6882", "6883", "6884", "6885", "6886", "6887", "6888",
79+
"6889", "6890", "6891", "6892", "6893", "6894", "6895", "6896", "6897",
80+
"6898", "6899", "6900", "6901", "6902", "6903", "6904", "6905", "6906",
81+
"6907", "6908", "6909", "6910", "6911", "6912", "6913", "6914", "6915",
82+
"6916", "6917", "6918", "6919", "6920", "6921", "6922", "6923", "6924",
83+
"6925", "6926", "6927", "6928", "6929", "6930", "6931", "6932", "6933",
84+
"6934", "6935", "6936", "6937", "6938", "6939", "6940", "6941", "6942",
85+
"6943", "6944", "6945", "6946", "6947", "6948", "6949", "6950", "6951",
86+
"6952", "6953", "6954", "6955", "6956", "6957", "6958", "6959", "6960",
87+
"6961", "6962", "6963", "7001", "7002", "7003", "7004", "7005", "7006",
88+
"7007", "7008", "7009", "7010", "7011", "7012", "7013", "7014", "7015",
89+
"7016", "7017", "7018", "7019", "7020", "7021", "7022", "7023", "7024",
90+
"7025", "7026", "7027", "7028", "7029", "7030", "7031", "7032", "7033",
91+
"7034", "7035", "7036", "7037", "7038", "7039", "7040", "7041", "7042",
92+
"7043", "7044", "7045", "7046", "7047", "7048", "7049", "7050", "7051",
93+
"7052", "7053", "7054", "7055", "7056", "7057", "7058", "7060", "7062",
94+
"7063", "7064", "7065", "7066", "7067", "7068", "7069", "7070", "7071",
95+
"7072", "7073", "7074", "7075", "7076", "7077", "7078", "7079", "7080",
96+
"7081", "7082", "7083", "7084", "7085", "7086", "7087", "7088", "7089",
97+
"7090", "7091", "7092", "7093", "7094", "7095", "7096", "7097", "7098",
98+
"7099", "7100", "7101", "7102", "7103", "7104", "7105", "7106", "7107",
99+
"7108", "7109", "7110", "7120", "7122", "7123", "7124", "7125", "7126",
100+
"7127", "7128", "7129", "7131", "7133", "7134", "7135", "7136", "7137",
101+
"7138", "7139", "7140", "7141", "7142", "7143", "7144", "7145", "7146",
102+
"7147", "7148", "7149", "7150", "7151", "7152", "7153", "7154", "7155",
103+
"7156", "7157", "7158", "7159", "7160", "7161", "7162", "7163", "7164",
104+
"7165", "7166", "7167", "7168", "7169", "7170", "7171", "7172", "7173",
105+
"7174", "7175", "7176", "7177", "7178", "7179", "7180", "7181", "7182",
106+
"7183", "7184", "7185", "7186", "7187", "7500", "7501", "7502", "7503",
107+
"7504", "7505", "7506", "7507", "7508", "7509", "7510", "7511", "7512",
108+
"7513", "7514", "7515", "7516", "7517", "7750"};
35109
std::vector<std::string> file_list = {
36110
"/Users/qiaolongfei/project/gzip_test/part-00000-A.gz",
37-
"/Users/qiaolongfei/project/gzip_test/part-00000-A.gz"};
111+
"/Users/qiaolongfei/project/gzip_test/part-00001-A.gz",
112+
"/Users/qiaolongfei/project/gzip_test/part-00002-A.gz"};
38113

39114
CTRReader reader(queue, batch_size, thread_num, slots, file_list);
40115

41116
reader.Start();
42-
//
43-
// std::vector<LoDTensor> out;
44-
// reader.ReadNext(&out);
117+
118+
std::cout << "start to reader data" << std::endl;
119+
std::vector<LoDTensor> out;
120+
int read_batch = 1000;
121+
clock_t t0 = clock();
122+
for (int i = 0; i < read_batch; ++i) {
123+
reader.ReadNext(&out);
124+
}
125+
clock_t t1 = clock();
126+
float line_per_s = read_batch * batch_size *
127+
static_cast<int64>(CLOCKS_PER_SEC) /
128+
static_cast<int>(t1 - t0);
129+
VLOG(3) << "line_per_second = " << line_per_s;
45130
}

0 commit comments

Comments
 (0)