Skip to content

Commit 1cfd3cb

Browse files
velconiatyphoonzero
velconia
authored and
typhoonzero
committed
Add some dist-training robust cases into fluid benchmark test (#11207)
* 1. add weight decay feature into fluid benchmark test 2. add learning rate decay feature into fluid benchmark test 3. add L1&L2 regularization feature into fluid benchmark test 4. add error clipping feature into fluid benchmark test 5. add gradient clipping feature into fluid benchmark test * Add some document to README.md under benchmark/fluid/ repo * Add model_base.py * Fix bugs in test_listen_and_serv_op * 1. remove args out of fluid_benchmark.py 2. remove lr_decay, regularization, clipping out of fluid_benchmark.py * add async_mode description to doc and remove the clipping description out * for restart build * to restart build * remove optimization args from args.py * 1. remove optimization from models 2. fix bug in test_listen_and_serv_op * change the name retry_times to left_time * change retry_times to the pserver start left time
1 parent 3fd3e50 commit 1cfd3cb

File tree

4 files changed

+143
-110
lines changed

4 files changed

+143
-110
lines changed

‎benchmark/fluid/README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@ Currently supported `--model` argument include:
2424

2525
* Run the following command to start a benchmark job locally:
2626
```bash
27-
python fluid_benchmark.py --model mnist --device GPU
27+
python fluid_benchmark.py --model mnist --device GPU
2828
```
2929
You can choose to use GPU/CPU training. With GPU training, you can specify
3030
`--gpus <gpu_num>` to run multi GPU training.
31+
You can set async mode parameter server. With async mode, you can specify
32+
`--async_mode` to train model asynchronous.
3133
* Run distributed training with parameter servers:
3234
* see [run_fluid_benchmark.sh](https://github.com/PaddlePaddle/Paddle/blob/develop/benchmark/fluid/run_fluid_benchmark.sh) as an example.
3335
* start parameter servers:

‎benchmark/fluid/args.py

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import argparse
16+
17+
__all__ = ['parse_args', ]
18+
19+
BENCHMARK_MODELS = [
20+
"machine_translation", "resnet", "vgg", "mnist", "stacked_dynamic_lstm"
21+
]
22+
23+
24+
def parse_args():
25+
parser = argparse.ArgumentParser('Fluid model benchmarks.')
26+
parser.add_argument(
27+
'--model',
28+
type=str,
29+
choices=BENCHMARK_MODELS,
30+
default='resnet',
31+
help='The model to run benchmark with.')
32+
parser.add_argument(
33+
'--batch_size', type=int, default=32, help='The minibatch size.')
34+
# args related to learning rate
35+
parser.add_argument(
36+
'--learning_rate', type=float, default=0.001, help='The learning rate.')
37+
# TODO(wuyi): add "--use_fake_data" option back.
38+
parser.add_argument(
39+
'--skip_batch_num',
40+
type=int,
41+
default=5,
42+
help='The first num of minibatch num to skip, for better performance test'
43+
)
44+
parser.add_argument(
45+
'--iterations', type=int, default=80, help='The number of minibatches.')
46+
parser.add_argument(
47+
'--pass_num', type=int, default=100, help='The number of passes.')
48+
parser.add_argument(
49+
'--data_format',
50+
type=str,
51+
default='NCHW',
52+
choices=['NCHW', 'NHWC'],
53+
help='The data data_format, now only support NCHW.')
54+
parser.add_argument(
55+
'--device',
56+
type=str,
57+
default='GPU',
58+
choices=['CPU', 'GPU'],
59+
help='The device type.')
60+
parser.add_argument(
61+
'--gpus',
62+
type=int,
63+
default=1,
64+
help='If gpus > 1, will use ParallelExecutor to run, else use Executor.')
65+
# this option is available only for vgg and resnet.
66+
parser.add_argument(
67+
'--cpus',
68+
type=int,
69+
default=1,
70+
help='If cpus > 1, will use ParallelDo to run, else use Executor.')
71+
parser.add_argument(
72+
'--data_set',
73+
type=str,
74+
default='flowers',
75+
choices=['cifar10', 'flowers'],
76+
help='Optional dataset for benchmark.')
77+
parser.add_argument(
78+
'--infer_only', action='store_true', help='If set, run forward only.')
79+
parser.add_argument(
80+
'--use_cprof', action='store_true', help='If set, use cProfile.')
81+
parser.add_argument(
82+
'--use_nvprof',
83+
action='store_true',
84+
help='If set, use nvprof for CUDA.')
85+
parser.add_argument(
86+
'--no_test',
87+
action='store_true',
88+
help='If set, do not test the testset during training.')
89+
parser.add_argument(
90+
'--memory_optimize',
91+
action='store_true',
92+
help='If set, optimize runtime memory before start.')
93+
parser.add_argument(
94+
'--use_fake_data',
95+
action='store_true',
96+
help='If set ommit the actual read data operators.')
97+
parser.add_argument(
98+
'--profile', action='store_true', help='If set, profile a few steps.')
99+
parser.add_argument(
100+
'--update_method',
101+
type=str,
102+
default='local',
103+
choices=['local', 'pserver', 'nccl2'],
104+
help='Choose parameter update method, can be local, pserver, nccl2.')
105+
parser.add_argument(
106+
'--no_split_var',
107+
action='store_true',
108+
default=False,
109+
help='Whether split variables into blocks when update_method is pserver')
110+
parser.add_argument(
111+
'--async_mode',
112+
action='store_true',
113+
default=False,
114+
help='Whether start pserver in async mode to support ASGD')
115+
parser.add_argument(
116+
'--use_reader_op',
117+
action='store_true',
118+
help='Whether to use reader op, and must specify the data path if set this to true.'
119+
)
120+
parser.add_argument(
121+
'--data_path',
122+
type=str,
123+
default="",
124+
help='Directory that contains all the training recordio files.')
125+
args = parser.parse_args()
126+
return args

‎benchmark/fluid/fluid_benchmark.py

Lines changed: 9 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -24,108 +24,7 @@
2424
import paddle.fluid.profiler as profiler
2525
import paddle.fluid.transpiler.distribute_transpiler as distribute_transpiler
2626

27-
BENCHMARK_MODELS = [
28-
"machine_translation", "resnet", "vgg", "mnist", "stacked_dynamic_lstm"
29-
]
30-
31-
32-
def parse_args():
33-
parser = argparse.ArgumentParser('Fluid model benchmarks.')
34-
parser.add_argument(
35-
'--model',
36-
type=str,
37-
choices=BENCHMARK_MODELS,
38-
default='resnet',
39-
help='The model to run benchmark with.')
40-
parser.add_argument(
41-
'--batch_size',
42-
type=int,
43-
default=32,
44-
help='The batch size on each gpu.')
45-
parser.add_argument(
46-
'--learning_rate', type=float, default=0.001, help='The learning rate.')
47-
parser.add_argument(
48-
'--skip_batch_num',
49-
type=int,
50-
default=5,
51-
help='The first num of minibatch num to skip, for better performance test'
52-
)
53-
parser.add_argument(
54-
'--iterations',
55-
type=int,
56-
default=80,
57-
help='The number of minibatches, set to -1 to run all batches.')
58-
parser.add_argument(
59-
'--pass_num', type=int, default=100, help='The number of passes.')
60-
parser.add_argument(
61-
'--data_format',
62-
type=str,
63-
default='NCHW',
64-
choices=['NCHW', 'NHWC'],
65-
help='The data data_format, now only support NCHW.')
66-
parser.add_argument(
67-
'--device',
68-
type=str,
69-
default='GPU',
70-
choices=['CPU', 'GPU'],
71-
help='The device type.')
72-
parser.add_argument(
73-
'--gpus',
74-
type=int,
75-
default=1,
76-
help='If gpus > 1, will use ParallelExecutor to run, else use Executor.')
77-
# this option is available only for vgg and resnet.
78-
parser.add_argument(
79-
'--cpus',
80-
type=int,
81-
default=1,
82-
help='If cpus > 1, will use ParallelDo to run, else use Executor.')
83-
parser.add_argument(
84-
'--data_set',
85-
type=str,
86-
default='flowers',
87-
choices=['cifar10', 'flowers', 'imagenet'],
88-
help='Optional dataset for benchmark.')
89-
parser.add_argument(
90-
'--infer_only', action='store_true', help='If set, run forward only.')
91-
parser.add_argument(
92-
'--use_cprof', action='store_true', help='If set, use cProfile.')
93-
parser.add_argument(
94-
'--use_nvprof',
95-
action='store_true',
96-
help='If set, use nvprof for CUDA.')
97-
parser.add_argument(
98-
'--no_test',
99-
action='store_true',
100-
help='If set, do not test the testset during training.')
101-
parser.add_argument(
102-
'--memory_optimize',
103-
action='store_true',
104-
help='If set, optimize runtime memory before start.')
105-
parser.add_argument(
106-
'--use_fake_data',
107-
action='store_true',
108-
help='If set ommit the actual read data operators.')
109-
parser.add_argument(
110-
'--profile', action='store_true', help='If set, profile a few steps.')
111-
parser.add_argument(
112-
'--update_method',
113-
type=str,
114-
default='local',
115-
choices=['local', 'pserver', 'nccl2'],
116-
help='Choose parameter update method, can be local, pserver, nccl2.')
117-
parser.add_argument(
118-
'--use_reader_op',
119-
action='store_true',
120-
help='Whether to use reader op, and must specify the data path if set this to true.'
121-
)
122-
parser.add_argument(
123-
'--data_path',
124-
type=str,
125-
default="",
126-
help='Directory that contains all the training recordio files.')
127-
args = parser.parse_args()
128-
return args
27+
from args import *
12928

13029

13130
def append_nccl2_prepare(trainer_id):
@@ -160,7 +59,7 @@ def append_nccl2_prepare(trainer_id):
16059
"nccl-based dist train.")
16160

16261

163-
def dist_transpile(trainer_id):
62+
def dist_transpile(trainer_id, args):
16463
if trainer_id < 0:
16564
return None, None
16665

@@ -182,7 +81,12 @@ def dist_transpile(trainer_id):
18281
training_role = os.getenv("PADDLE_TRAINING_ROLE")
18382

18483
t = distribute_transpiler.DistributeTranspiler()
185-
t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
84+
t.transpile(
85+
trainer_id,
86+
pservers=pserver_endpoints,
87+
trainers=trainers,
88+
sync_mode=not args.async_mode,
89+
slice_var_up=not args.no_split_var)
18690
if training_role == "PSERVER":
18791
pserver_program = t.get_pserver_program(current_endpoint)
18892
pserver_startup_program = t.get_startup_program(current_endpoint,
@@ -417,7 +321,7 @@ def main():
417321
fluid.memory_optimize(fluid.default_main_program())
418322

419323
if args.update_method == "pserver":
420-
train_prog, startup_prog = dist_transpile(trainer_id)
324+
train_prog, startup_prog = dist_transpile(trainer_id, args)
421325
if not train_prog:
422326
raise Exception(
423327
"Must configure correct environments to run dist train.")

‎python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,17 +70,18 @@ def _start_pserver(self, use_cuda, sync_mode):
7070
return p.pid
7171

7272
def _wait_ps_ready(self, pid):
73-
retry_times = self.ps_timeout
73+
start_left_time = self.ps_timeout
74+
sleep_time = 0.5
7475
while True:
75-
assert retry_times >= 0, "wait ps ready failed"
76-
time.sleep(0.5)
76+
assert start_left_time >= 0, "wait ps ready failed"
77+
time.sleep(sleep_time)
7778
try:
7879
# the listen_and_serv_op would touch a file which contains the listen port
7980
# on the /tmp directory until it was ready to process all the RPC call.
8081
os.stat("/tmp/paddle.%d.port" % pid)
8182
return
8283
except os.error:
83-
retry_times -= 1
84+
start_left_time -= sleep_time
8485

8586
def test_rpc_interfaces(self):
8687
# TODO(Yancey1989): need to make sure the rpc interface correctly.

0 commit comments

Comments
 (0)