@@ -101,23 +101,22 @@ void SplitData(
101
101
}
102
102
103
103
void ThreadRunInfer (
104
- const int tid, paddle::framework::Executor* executor,
105
- paddle::framework::Scope* scope,
106
- const std::unique_ptr<paddle::framework::ProgramDesc>& inference_program,
104
+ const int tid, paddle::framework::Scope* scope,
107
105
const std::vector<std::vector<const paddle::framework::LoDTensor*>>& jobs) {
108
- auto copy_program = std::unique_ptr<paddle::framework::ProgramDesc>(
109
- new paddle::framework::ProgramDesc (*inference_program));
106
+ // maybe framework:ProgramDesc is not thread-safe
110
107
auto & sub_scope = scope->NewScope ();
108
+ auto place = paddle::platform::CPUPlace ();
109
+ auto executor = paddle::framework::Executor (place);
110
+ auto inference_program =
111
+ paddle::inference::Load (&executor, scope, FLAGS_model_path);
111
112
112
- std::string feed_holder_name = " feed_" + paddle::string::to_string (tid);
113
- std::string fetch_holder_name = " fetch_" + paddle::string::to_string (tid);
114
- copy_program->SetFeedHolderName (feed_holder_name);
115
- copy_program->SetFetchHolderName (fetch_holder_name);
113
+ auto ctx = executor.Prepare (*inference_program, /* block_id*/ 0 );
114
+ executor.CreateVariables (*inference_program, &sub_scope, /* block_id*/ 0 );
116
115
117
116
const std::vector<std::string>& feed_target_names =
118
- copy_program ->GetFeedTargetNames ();
117
+ inference_program ->GetFeedTargetNames ();
119
118
const std::vector<std::string>& fetch_target_names =
120
- copy_program ->GetFetchTargetNames ();
119
+ inference_program ->GetFetchTargetNames ();
121
120
122
121
PADDLE_ENFORCE_EQ (fetch_target_names.size (), 1UL );
123
122
std::map<std::string, paddle::framework::LoDTensor*> fetch_targets;
@@ -131,9 +130,8 @@ void ThreadRunInfer(
131
130
auto start_ms = GetCurrentMs ();
132
131
for (size_t i = 0 ; i < inputs.size (); ++i) {
133
132
feed_targets[feed_target_names[0 ]] = inputs[i];
134
- executor->Run (*copy_program, &sub_scope, &feed_targets, &fetch_targets,
135
- true /* create_local_scope*/ , true /* create_vars*/ ,
136
- feed_holder_name, fetch_holder_name);
133
+ executor.RunPreparedContext (ctx.get (), &sub_scope, &feed_targets,
134
+ &fetch_targets, false /* create_local_scope*/ );
137
135
}
138
136
auto stop_ms = GetCurrentMs ();
139
137
scope->DeleteScope (&sub_scope);
@@ -158,22 +156,10 @@ TEST(inference, nlp) {
158
156
LOG (INFO) << " Number of samples (seq_len<1024): " << datasets.size ();
159
157
LOG (INFO) << " Total number of words: " << num_total_words;
160
158
161
- const bool model_combined = false ;
162
159
// 0. Call `paddle::framework::InitDevices()` initialize all the devices
163
- // 1. Define place, executor, scope
164
- auto place = paddle::platform::CPUPlace ();
165
- auto executor = paddle::framework::Executor (place);
166
160
std::unique_ptr<paddle::framework::Scope> scope (
167
161
new paddle::framework::Scope ());
168
162
169
- // 2. Initialize the inference_program and load parameters
170
- std::unique_ptr<paddle::framework::ProgramDesc> inference_program;
171
- inference_program =
172
- InitProgram (&executor, scope.get (), FLAGS_model_path, model_combined);
173
- if (FLAGS_use_mkldnn) {
174
- EnableMKLDNN (inference_program);
175
- }
176
-
177
163
#ifdef PADDLE_WITH_MKLML
178
164
// only use 1 thread number per std::thread
179
165
omp_set_dynamic (0 );
@@ -189,21 +175,30 @@ TEST(inference, nlp) {
189
175
start_ms = GetCurrentMs ();
190
176
for (int i = 0 ; i < FLAGS_num_threads; ++i) {
191
177
threads.emplace_back (
192
- new std::thread (ThreadRunInfer, i, &executor, scope.get (),
193
- std::ref (inference_program), std::ref (jobs)));
178
+ new std::thread (ThreadRunInfer, i, scope.get (), std::ref (jobs)));
194
179
}
195
180
for (int i = 0 ; i < FLAGS_num_threads; ++i) {
196
181
threads[i]->join ();
197
182
}
198
183
stop_ms = GetCurrentMs ();
199
184
} else {
200
- if (FLAGS_prepare_vars) {
201
- executor.CreateVariables (*inference_program, scope.get (), 0 );
185
+ // 1. Define place, executor, scope
186
+ auto place = paddle::platform::CPUPlace ();
187
+ auto executor = paddle::framework::Executor (place);
188
+
189
+ // 2. Initialize the inference_program and load parameters
190
+ std::unique_ptr<paddle::framework::ProgramDesc> inference_program;
191
+ inference_program = InitProgram (&executor, scope.get (), FLAGS_model_path,
192
+ /* model combined*/ false );
193
+ if (FLAGS_use_mkldnn) {
194
+ EnableMKLDNN (inference_program);
202
195
}
203
196
// always prepare context
204
197
std::unique_ptr<paddle::framework::ExecutorPrepareContext> ctx;
205
198
ctx = executor.Prepare (*inference_program, 0 );
206
-
199
+ if (FLAGS_prepare_vars) {
200
+ executor.CreateVariables (*inference_program, scope.get (), 0 );
201
+ }
207
202
// preapre fetch
208
203
const std::vector<std::string>& fetch_target_names =
209
204
inference_program->GetFetchTargetNames ();
0 commit comments