Skip to content

Commit 77200a7

Browse files
JiayiFengJiayiFeng
committed
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into dev_double_buffer_for_cpp_reader
2 parents af64f39 + 42e65a2 commit 77200a7

File tree

99 files changed

+2474
-1338
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

99 files changed

+2474
-1338
lines changed

‎.travis.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ script:
5656
export DEPLOY_DOCS_SH=https://raw.githubusercontent.com/PaddlePaddle/PaddlePaddle.org/master/scripts/deploy/deploy_docs.sh
5757
export DOCS_DIR=`pwd`
5858
cd ..
59-
curl $DEPLOY_DOCS_SH | bash -s $CONTENT_DEC_PASSWD $TRAVIS_BRANCH $DOCS_DIR $DOCS_DIR/build/doc
59+
curl $DEPLOY_DOCS_SH | bash -s $CONTENT_DEC_PASSWD $TRAVIS_BRANCH $DOCS_DIR $DOCS_DIR/build/doc/v2
6060
notifications:
6161
email:
6262
on_success: change

‎CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,8 @@ include(external/eigen) # download eigen3
144144
include(external/pybind11) # download pybind11
145145
include(external/cares)
146146
include(external/grpc)
147+
include(external/snappy) # download snappy
148+
include(external/snappystream)
147149

148150
include(cudnn) # set cudnn libraries, must before configure
149151
include(cupti)

‎benchmark/cluster/vgg16/vgg16_fluid.py

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2-
#
2+
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
55
# You may obtain a copy of the License at
6-
#
6+
#
77
# http://www.apache.org/licenses/LICENSE-2.0
8-
#
8+
#
99
# Unless required by applicable law or agreed to in writing, software
1010
# distributed under the License is distributed on an "AS IS" BASIS,
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -138,13 +138,14 @@ def main():
138138
avg_cost = fluid.layers.mean(x=cost)
139139

140140
# Evaluator
141-
accuracy = fluid.evaluator.Accuracy(input=predict, label=label)
141+
batch_size = fluid.layers.create_tensor(dtype='int64')
142+
batch_acc = fluid.layers.accuracy(
143+
input=predict, label=label, total=batch_size)
142144

143145
# inference program
144146
inference_program = fluid.default_main_program().clone()
145147
with fluid.program_guard(inference_program):
146-
test_target = accuracy.metrics + accuracy.states
147-
inference_program = fluid.io.get_inference_program(test_target)
148+
inference_program = fluid.io.get_inference_program(batch_acc)
148149

149150
# Optimization
150151
optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate)
@@ -157,27 +158,30 @@ def main():
157158

158159
# test
159160
def test(exe):
160-
accuracy.reset(exe)
161+
test_pass_acc = fluid.average.WeightedAverage()
161162
for batch_id, data in enumerate(test_reader()):
162163
img_data = np.array(map(lambda x: x[0].reshape(data_shape),
163164
data)).astype("float32")
164165
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
165166
y_data = y_data.reshape([-1, 1])
166167

167-
exe.run(inference_program,
168-
feed={"pixel": img_data,
169-
"label": y_data})
168+
outs = exe.run(inference_program,
169+
feed={"pixel": img_data,
170+
"label": y_data},
171+
fetch_list=[batch_acc, batch_size])
172+
test_pass_acc.add(value=np.array(outs[0]), weight=np.array(outs[1]))
170173

171-
return accuracy.eval(exe)
174+
return test_pass_acc.eval()
172175

173176
def train_loop(exe, trainer_prog):
174177
iters = 0
175178
ts = time.time()
179+
train_pass_acc = fluid.average.WeightedAverage()
176180
for pass_id in range(args.num_passes):
177181
# train
178182
start_time = time.time()
179183
num_samples = 0
180-
accuracy.reset(exe)
184+
train_pass_acc.reset()
181185
with profiler.profiler("CPU", 'total') as prof:
182186
for batch_id, data in enumerate(train_reader()):
183187
ts = time.time()
@@ -187,21 +191,22 @@ def train_loop(exe, trainer_prog):
187191
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
188192
y_data = y_data.reshape([-1, 1])
189193

190-
loss, acc = exe.run(
194+
loss, acc, b_size = exe.run(
191195
trainer_prog,
192196
feed={"pixel": img_data,
193197
"label": y_data},
194-
fetch_list=[avg_cost] + accuracy.metrics)
198+
fetch_list=[avg_cost, batch_acc, batch_size])
195199
iters += 1
196200
num_samples += len(data)
201+
train_pass_acc.add(value=acc, weight=b_size)
197202
print(
198203
"Pass = %d, Iters = %d, Loss = %f, Accuracy = %f, Speed = %.2f img/s"
199204
% (pass_id, iters, loss, acc,
200205
len(data) / (time.time() - ts))
201206
) # The accuracy is the accumulation of batches, but not the current batch.
202207

203208
pass_elapsed = time.time() - start_time
204-
pass_train_acc = accuracy.eval(exe)
209+
pass_train_acc = train_pass_acc.eval()
205210
pass_test_acc = test(exe)
206211
print(
207212
"Pass = %d, Training performance = %f imgs/s, Train accuracy = %f, Test accuracy = %f\n"

‎cmake/external/snappy.cmake

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
16+
IF(MOBILE_INFERENCE)
17+
return()
18+
ENDIF()
19+
20+
include (ExternalProject)
21+
22+
# NOTE: snappy is needed when linking with recordio
23+
24+
SET(SNAPPY_SOURCES_DIR ${THIRD_PARTY_PATH}/snappy)
25+
SET(SNAPPY_INSTALL_DIR ${THIRD_PARTY_PATH}/install/snappy)
26+
SET(SNAPPY_INCLUDE_DIR "${SNAPPY_INSTALL_DIR}/include/" CACHE PATH "snappy include directory." FORCE)
27+
28+
ExternalProject_Add(
29+
extern_snappy
30+
GIT_REPOSITORY "https://github.com/google/snappy"
31+
GIT_TAG "1.1.7"
32+
PREFIX ${SNAPPY_SOURCES_DIR}
33+
UPDATE_COMMAND ""
34+
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
35+
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
36+
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
37+
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
38+
-DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR}
39+
-DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib
40+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
41+
-DBUILD_TESTING=OFF
42+
-DSNAPPY_BUILD_TESTS:BOOL=OFF
43+
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
44+
${EXTERNAL_OPTIONAL_ARGS}
45+
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_INSTALL_DIR}
46+
-DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib
47+
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
48+
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
49+
BUILD_COMMAND make -j8
50+
INSTALL_COMMAND make install
51+
)
52+
53+
add_library(snappy STATIC IMPORTED GLOBAL)
54+
set_property(TARGET snappy PROPERTY IMPORTED_LOCATION
55+
"${SNAPPY_INSTALL_DIR}/lib/libsnappy.a")
56+
57+
include_directories(${SNAPPY_INCLUDE_DIR})
58+
add_dependencies(snappy extern_snappy)

‎cmake/external/snappystream.cmake

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
16+
IF(MOBILE_INFERENCE)
17+
return()
18+
ENDIF()
19+
20+
include (ExternalProject)
21+
22+
# NOTE: snappy is needed when linking with recordio
23+
24+
SET(SNAPPYSTREAM_SOURCES_DIR ${THIRD_PARTY_PATH}/snappy_stream)
25+
SET(SNAPPYSTREAM_INSTALL_DIR ${THIRD_PARTY_PATH}/install/snappy_stream)
26+
SET(SNAPPYSTREAM_INCLUDE_DIR "${SNAPPYSTREAM_INSTALL_DIR}/include/" CACHE PATH "snappy stream include directory." FORCE)
27+
28+
ExternalProject_Add(
29+
extern_snappystream
30+
GIT_REPOSITORY "https://github.com/hoxnox/snappystream.git"
31+
GIT_TAG "0.2.8"
32+
PREFIX ${SNAPPYSTREAM_SOURCES_DIR}
33+
UPDATE_COMMAND ""
34+
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
35+
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
36+
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
37+
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
38+
-DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR}
39+
-DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib
40+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
41+
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
42+
-DSNAPPY_ROOT=${SNAPPY_INSTALL_DIR}
43+
${EXTERNAL_OPTIONAL_ARGS}
44+
CMAKE_CACHE_ARGS
45+
-DCMAKE_INSTALL_PREFIX:PATH=${SNAPPYSTREAM_INSTALL_DIR}
46+
-DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPYSTREAM_INSTALL_DIR}/lib
47+
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
48+
BUILD_COMMAND make -j8
49+
INSTALL_COMMAND make install
50+
DEPENDS snappy
51+
)
52+
53+
add_library(snappystream STATIC IMPORTED GLOBAL)
54+
set_property(TARGET snappystream PROPERTY IMPORTED_LOCATION
55+
"${SNAPPYSTREAM_INSTALL_DIR}/lib/libsnappystream.a")
56+
57+
include_directories(${SNAPPYSTREAM_INCLUDE_DIR})
58+
add_dependencies(snappystream extern_snappystream)

‎doc/CMakeLists.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
1-
add_subdirectory(api)
21
add_subdirectory(v2)

‎doc/v2/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,5 @@ sphinx_add_target(paddle_docs_cn
4747
${SPHINX_CACHE_DIR_CN}
4848
${CMAKE_CURRENT_SOURCE_DIR}
4949
${SPHINX_HTML_DIR_CN})
50+
51+
add_subdirectory(api)

‎doc/api/CMakeLists.txt renamed to ‎doc/v2/api/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ set(SPHINX_CACHE_DIR_EN "${CMAKE_CURRENT_BINARY_DIR}/en/_doctrees")
88
set(SPHINX_HTML_DIR_EN "${CMAKE_CURRENT_BINARY_DIR}/en/html")
99

1010
configure_file(
11-
"${CMAKE_CURRENT_SOURCE_DIR}/../templates/conf.py.en.in"
11+
"${CMAKE_CURRENT_SOURCE_DIR}/../../templates/conf.py.en.in"
1212
"${BINARY_BUILD_DIR_EN}/conf.py"
1313
@ONLY)
1414

‎doc/api/v2/config/activation.rst renamed to ‎doc/v2/api/config/activation.rst

File renamed without changes.

‎doc/api/v2/config/attr.rst renamed to ‎doc/v2/api/config/attr.rst

File renamed without changes.

0 commit comments

Comments
 (0)