From 705318456e829d1ef0c0125c66bcf20a851eca95 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 11 Jan 2017 19:07:54 +0800 Subject: [PATCH 1/6] add api_train for image_classification --- demo/image_classification/api_train.py | 177 ++++++++++++++++++++++++ demo/image_classification/cifar_util.py | 138 ++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 demo/image_classification/api_train.py create mode 100644 demo/image_classification/cifar_util.py diff --git a/demo/image_classification/api_train.py b/demo/image_classification/api_train.py new file mode 100644 index 00000000000000..607efece038eb1 --- /dev/null +++ b/demo/image_classification/api_train.py @@ -0,0 +1,177 @@ +""" +A very basic example for how to use current Raw SWIG API to train mnist network. + +Current implementation uses Raw SWIG, which means the API call is directly \ +passed to C++ side of Paddle. + +The user api could be simpler and carefully designed. +""" +import numpy as np +import py_paddle.swig_paddle as api +from paddle.trainer_config_helpers import * + +import cifar_util + +data_size = 3 * 32 * 32 +label_size = 10 + + +def optimizer_config(): + """Function to config optimizer.""" + settings( + batch_size=128, + learning_rate=0.1 / 128.0, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * 128)) + + +def network_config(): + """Function to config neural network.""" + img = data_layer(name='image', size=data_size) + lbl = data_layer(name='label', size=label_size) + hidden1 = fc_layer(input=img, size=200) + hidden2 = fc_layer(input=hidden1, size=200) + inference = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) + cost = classification_cost(input=inference, label=lbl) + outputs(cost) + + +def init_parameter(gradient_machine): + """Function to init parameter inside gradient machine""" + assert isinstance(gradient_machine, api.GradientMachine) + for each_param in gradient_machine.getParameters(): + assert isinstance(each_param, api.Parameter) + array_size = len(each_param) + array = np.random.uniform(-1.0, 1.0, array_size).astype('float32') + each_param.getBuf(api.PARAMETER_VALUE).copyFromNumpyArray(array) + + +def main(): + api.initPaddle("-use_gpu=false", "-trainer_count=4") # use 4 cpu cores + + # prepare cifar-10 data. + cifar_data = cifar_util.Cifar10Data( + img_size=32, + mean_img_size=32, + num_classes=10, + train_file_list='data/cifar-out/batches/train.txt', + test_file_list='data/cifar-out/batches/test.txt', + meta='data/cifar-out/batches/batches.meta') + + # get enable_types for each optimizer. + # enable_types = [value, gradient, momentum, etc] + # For each optimizer(SGD, Adam), GradientMachine should enable different + # buffers. + opt_config_proto = parse_optimizer_config(optimizer_config) + opt_config = api.OptimizationConfig.createFromProto(opt_config_proto) + _temp_optimizer_ = api.ParameterOptimizer.create(opt_config) + enable_types = _temp_optimizer_.getParameterTypes() + + # Create Simple Gradient Machine. + model_config = parse_network_config(network_config) + gradient_machine = api.GradientMachine.createFromConfigProto( + model_config, api.CREATE_MODE_NORMAL, enable_types) + + # This type check is not useful. Only enable type hint in IDE. + # Such as PyCharm + assert isinstance(gradient_machine, api.GradientMachine) + + # Initialize Parameter by numpy. + init_parameter(gradient_machine=gradient_machine) + + # Create Local Updater. Local means not run in cluster. + # For a cluster training, here we can change to createRemoteUpdater + # in future. + updater = api.ParameterUpdater.createLocalUpdater(opt_config) + assert isinstance(updater, api.ParameterUpdater) + + # Initialize ParameterUpdater. + updater.init(gradient_machine) + + # start gradient machine. + # the gradient machine must be started before invoke forward/backward. + # not just for training, but also for inference. + gradient_machine.start() + + # evaluator can print error rate, etc. It is a C++ class. + batch_evaluator = gradient_machine.makeEvaluator() + test_evaluator = gradient_machine.makeEvaluator() + + # output_arguments is Neural Network forward result. Here is not useful, just passed + # to gradient_machine.forward + output_arguments = api.Arguments.createArguments(0) + + for pass_id in xrange(3): # we train 2 passes. + updater.startPass() + + for batch_id, data_batch in enumerate(cifar_data.train_data()()): + # data_batch is input images. + # here, for online learning, we could get data_batch from network. + + # Start update one batch. + pass_type = updater.startBatch(len(data_batch)) + + # Start BatchEvaluator. + # batch_evaluator can be used between start/finish. + batch_evaluator.start() + + # forwardBackward is a shortcut for forward and backward. + # It is sometimes faster than invoke forward/backward separately, + # because in GradientMachine, it may be async. + gradient_machine.forwardBackward( + cifar_data.data_converter.convert(data_batch), output_arguments, + pass_type) + + for each_param in gradient_machine.getParameters(): + updater.update(each_param) + + # Get cost. We use numpy to calculate total cost for this batch. + cost_vec = output_arguments.getSlotValue(0) + cost_vec = cost_vec.copyToNumpyMat() + cost = cost_vec.sum() / len(data_batch) + + # Make evaluator works. + gradient_machine.eval(batch_evaluator) + + # Print logs. + print 'Pass id', pass_id, 'Batch id', batch_id, 'with cost=', \ + cost, batch_evaluator + + batch_evaluator.finish() + # Finish batch. + # * will clear gradient. + # * ensure all values should be updated. + updater.finishBatch(cost) + + # testing stage. use test data set to test current network. + updater.apply() + test_evaluator.start() + for data_batch in cifar_data.test_data(): + # in testing stage, only forward is needed. + gradient_machine.forward( + cifar_data.data_converter.convert(data_batch), output_arguments, + api.PASS_TEST) + gradient_machine.eval(test_evaluator) + + # print error rate for test data set + print 'Pass', pass_id, ' test evaluator: ', test_evaluator + test_evaluator.finish() + updater.restore() + + updater.catchUpWith() + params = gradient_machine.getParameters() + for each_param in params: + assert isinstance(each_param, api.Parameter) + value = each_param.getBuf(api.PARAMETER_VALUE) + value = value.copyToNumpyArray() + + # Here, we could save parameter to every where you want + print each_param.getName(), value + + updater.finishPass() + + gradient_machine.finish() + + +if __name__ == '__main__': + main() diff --git a/demo/image_classification/cifar_util.py b/demo/image_classification/cifar_util.py new file mode 100644 index 00000000000000..92fd92b5f8005b --- /dev/null +++ b/demo/image_classification/cifar_util.py @@ -0,0 +1,138 @@ +import io +import random + +import paddle.trainer.PyDataProvider2 as dataprovider +import paddle.utils.image_util as image_util +from paddle.trainer.PyDataProvider2 import * +from py_paddle import DataProviderConverter + + +class BatchPool(object): + """Proxy of C++ ParameterUpdater class.""" + + def __init__(self, generator, batch_size): + self.data = list(generator) + self.batch_size = batch_size + + def __call__(self): + random.shuffle(self.data) + for offset in xrange(0, len(self.data), self.batch_size): + limit = min(offset + self.batch_size, len(self.data)) + yield self.data[offset:limit] + + +class Cifar10Data(object): + """ + Class to prepare cifar-10 data, have all the message to read and convert + image data to the data the can send to paddle to train or test. + """ + + def __init__(self, + meta, + train_file_list, + test_file_list, + img_size=32, + mean_img_size=32, + num_classes=10, + is_color=True, + use_jpeg=True): + self.mean_img_size = mean_img_size + self.img_size = img_size + self.num_classes = num_classes + self.train_file_list = train_file_list + self.test_file_list = test_file_list + self.is_color = is_color + if self.is_color: + self.img_raw_size = self.img_size * self.img_size * 3 + else: + self.img_raw_size = self.img_size * self.img_size + self.meta_path = meta + self.use_jpeg = use_jpeg + self.batch_size = 128 + self.img_mean = image_util.load_meta(self.meta_path, self.mean_img_size, + self.img_size, self.is_color) + + # DataProvider Converter is a utility convert Python Object to Paddle C++ + # Input. The input format is as same as Paddle's DataProvider. + # input_types = { + # 'image': dp.dense_vector(data_size), + # 'label': dp.integer_value(label_size) + # } + input_types = [ + dataprovider.dense_vector(self.img_raw_size), + dataprovider.integer_value(self.num_classes) + ] + self.data_converter = DataProviderConverter(input_types) + + @staticmethod + def _input_order_converter(generator): + for item in generator: + yield item['image'], item['label'] + + @staticmethod + def generator_to_batch(generator, batch_size): + ret_val = list() + for each_item in generator: + ret_val.append(each_item) + if len(ret_val) == batch_size: + yield ret_val + ret_val = list() + if len(ret_val) != 0: + yield ret_val + + def _read_data(self, is_train): + """ + The main function for loading data. + Load the batch, iterate all the images and labels in this batch. + file_list: the batch file list. + """ + if is_train: + file_list = self.train_file_list + else: + file_list = self.test_file_list + + with open(file_list, 'r') as fdata: + lines = [line.strip() for line in fdata] + random.shuffle(lines) + for file_name in lines: + with io.open(file_name.strip(), 'rb') as file: + data = cPickle.load(file) + indexes = list(range(len(data['images']))) + if is_train: + random.shuffle(indexes) + for i in indexes: + if self.use_jpeg == 1: + img = image_util.decode_jpeg(data['images'][i]) + else: + img = data['images'][i] + img_feat = image_util.preprocess_img( + img, self.img_mean, self.img_size, is_train, + self.is_color) + label = data['labels'][i] + yield { + 'image': img_feat.astype('float32'), + 'label': int(label) + } + + def train_data(self): + """ + Get Train Data. + TrainData will stored in a data pool. Currently implementation is not care + about memory, speed. Just a very naive implementation. + """ + train_data_generator = self._input_order_converter( + self._read_data(True)) + train_data = BatchPool(train_data_generator, self.batch_size) + return train_data + + def test_data(self): + """ + Get Test Data. + TrainData will stored in a data pool. Currently implementation is not care + about memory, speed. Just a very naive implementation. + """ + test_data_generator = self._input_order_converter( + self._read_data(False)) + test_data = self.generator_to_batch(test_data_generator, + self.batch_size) + return test_data From 28e2bef9a2eef68f497b5a5aef81a062269d82fc Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 11 Jan 2017 19:18:02 +0800 Subject: [PATCH 2/6] add some comment --- demo/image_classification/api_train.py | 17 +++++++++++++++-- demo/image_classification/cifar_util.py | 18 +++++++++++++++++- demo/mnist/api_train.py | 13 +++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/demo/image_classification/api_train.py b/demo/image_classification/api_train.py index 607efece038eb1..343540f45aa004 100644 --- a/demo/image_classification/api_train.py +++ b/demo/image_classification/api_train.py @@ -1,10 +1,23 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ -A very basic example for how to use current Raw SWIG API to train mnist network. +An example to show how to use current Raw SWIG API to train cifar-10 network. Current implementation uses Raw SWIG, which means the API call is directly \ passed to C++ side of Paddle. -The user api could be simpler and carefully designed. +The user api should be simpler and carefully designed. """ import numpy as np import py_paddle.swig_paddle as api diff --git a/demo/image_classification/cifar_util.py b/demo/image_classification/cifar_util.py index 92fd92b5f8005b..52cef4b8abfa31 100644 --- a/demo/image_classification/cifar_util.py +++ b/demo/image_classification/cifar_util.py @@ -1,3 +1,19 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Some class for prepare cifar-10 image data. +""" import io import random @@ -8,7 +24,7 @@ class BatchPool(object): - """Proxy of C++ ParameterUpdater class.""" + """A class to get all data in memory and do data shuffle.""" def __init__(self, generator, batch_size): self.data = list(generator) diff --git a/demo/mnist/api_train.py b/demo/mnist/api_train.py index f301da382ff8a5..bb47a4bc94e663 100644 --- a/demo/mnist/api_train.py +++ b/demo/mnist/api_train.py @@ -1,3 +1,16 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ A very basic example for how to use current Raw SWIG API to train mnist network. From 37ae99e0895dbebededdd6b6f47795b5040f1f8c Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 11 Jan 2017 19:23:01 +0800 Subject: [PATCH 3/6] add gloabl parameter --- demo/image_classification/api_train.py | 11 +++++++---- demo/image_classification/cifar_util.py | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/demo/image_classification/api_train.py b/demo/image_classification/api_train.py index 343540f45aa004..642e6d894cb8e2 100644 --- a/demo/image_classification/api_train.py +++ b/demo/image_classification/api_train.py @@ -25,7 +25,9 @@ import cifar_util -data_size = 3 * 32 * 32 +# some global parameter. +image_size = 32 +data_size = 3 * image_size * image_size label_size = 10 @@ -64,9 +66,10 @@ def main(): # prepare cifar-10 data. cifar_data = cifar_util.Cifar10Data( - img_size=32, - mean_img_size=32, - num_classes=10, + img_size=image_size, + mean_img_size=image_size, + num_classes=label_size, + batch_size=128, train_file_list='data/cifar-out/batches/train.txt', test_file_list='data/cifar-out/batches/test.txt', meta='data/cifar-out/batches/batches.meta') diff --git a/demo/image_classification/cifar_util.py b/demo/image_classification/cifar_util.py index 52cef4b8abfa31..b41b016c257b70 100644 --- a/demo/image_classification/cifar_util.py +++ b/demo/image_classification/cifar_util.py @@ -47,6 +47,7 @@ def __init__(self, meta, train_file_list, test_file_list, + batch_size, img_size=32, mean_img_size=32, num_classes=10, @@ -64,7 +65,7 @@ def __init__(self, self.img_raw_size = self.img_size * self.img_size self.meta_path = meta self.use_jpeg = use_jpeg - self.batch_size = 128 + self.batch_size = batch_size self.img_mean = image_util.load_meta(self.meta_path, self.mean_img_size, self.img_size, self.is_color) From 68b8b11e644e4df03de4fdac2c182a321763e25e Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 11 Jan 2017 19:37:56 +0800 Subject: [PATCH 4/6] use complete python lib path --- demo/image_classification/api_train.py | 63 ++++++++++++++------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/demo/image_classification/api_train.py b/demo/image_classification/api_train.py index 642e6d894cb8e2..435d82372dc9bf 100644 --- a/demo/image_classification/api_train.py +++ b/demo/image_classification/api_train.py @@ -20,8 +20,8 @@ The user api should be simpler and carefully designed. """ import numpy as np -import py_paddle.swig_paddle as api -from paddle.trainer_config_helpers import * +import paddle.trainer_config_helpers as config_helpers +import py_paddle.swig_paddle as swig_api import cifar_util @@ -33,36 +33,39 @@ def optimizer_config(): """Function to config optimizer.""" - settings( + config_helpers.optimizers.settings( batch_size=128, learning_rate=0.1 / 128.0, - learning_method=MomentumOptimizer(0.9), - regularization=L2Regularization(0.0005 * 128)) + learning_method=config_helpers.optimizers.MomentumOptimizer(0.9), + regularization=config_helpers.optimizers.L2Regularization(0.0005 * 128)) def network_config(): """Function to config neural network.""" - img = data_layer(name='image', size=data_size) - lbl = data_layer(name='label', size=label_size) - hidden1 = fc_layer(input=img, size=200) - hidden2 = fc_layer(input=hidden1, size=200) - inference = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) - cost = classification_cost(input=inference, label=lbl) - outputs(cost) + img = config_helpers.layers.data_layer(name='image', size=data_size) + lbl = config_helpers.layers.data_layer(name='label', size=label_size) + hidden1 = config_helpers.layers.fc_layer(input=img, size=200) + hidden2 = config_helpers.layers.fc_layer(input=hidden1, size=200) + inference = config_helpers.layers.fc_layer( + input=hidden2, + size=10, + act=config_helpers.activations.SoftmaxActivation()) + cost = config_helpers.layers.classification_cost(input=inference, label=lbl) + config_helpers.networks.outputs(cost) def init_parameter(gradient_machine): """Function to init parameter inside gradient machine""" - assert isinstance(gradient_machine, api.GradientMachine) + assert isinstance(gradient_machine, swig_api.GradientMachine) for each_param in gradient_machine.getParameters(): - assert isinstance(each_param, api.Parameter) + assert isinstance(each_param, swig_api.Parameter) array_size = len(each_param) array = np.random.uniform(-1.0, 1.0, array_size).astype('float32') - each_param.getBuf(api.PARAMETER_VALUE).copyFromNumpyArray(array) + each_param.getBuf(swig_api.PARAMETER_VALUE).copyFromNumpyArray(array) def main(): - api.initPaddle("-use_gpu=false", "-trainer_count=4") # use 4 cpu cores + swig_api.initPaddle("-use_gpu=false", "-trainer_count=4") # use 4 cpu cores # prepare cifar-10 data. cifar_data = cifar_util.Cifar10Data( @@ -78,19 +81,21 @@ def main(): # enable_types = [value, gradient, momentum, etc] # For each optimizer(SGD, Adam), GradientMachine should enable different # buffers. - opt_config_proto = parse_optimizer_config(optimizer_config) - opt_config = api.OptimizationConfig.createFromProto(opt_config_proto) - _temp_optimizer_ = api.ParameterOptimizer.create(opt_config) + opt_config_proto = config_helpers.config_parser_utils.parse_optimizer_config( + optimizer_config) + opt_config = swig_api.OptimizationConfig.createFromProto(opt_config_proto) + _temp_optimizer_ = swig_api.ParameterOptimizer.create(opt_config) enable_types = _temp_optimizer_.getParameterTypes() # Create Simple Gradient Machine. - model_config = parse_network_config(network_config) - gradient_machine = api.GradientMachine.createFromConfigProto( - model_config, api.CREATE_MODE_NORMAL, enable_types) + model_config = config_helpers.config_parser_utils.parse_network_config( + network_config) + gradient_machine = swig_api.GradientMachine.createFromConfigProto( + model_config, swig_api.CREATE_MODE_NORMAL, enable_types) # This type check is not useful. Only enable type hint in IDE. # Such as PyCharm - assert isinstance(gradient_machine, api.GradientMachine) + assert isinstance(gradient_machine, swig_api.GradientMachine) # Initialize Parameter by numpy. init_parameter(gradient_machine=gradient_machine) @@ -98,8 +103,8 @@ def main(): # Create Local Updater. Local means not run in cluster. # For a cluster training, here we can change to createRemoteUpdater # in future. - updater = api.ParameterUpdater.createLocalUpdater(opt_config) - assert isinstance(updater, api.ParameterUpdater) + updater = swig_api.ParameterUpdater.createLocalUpdater(opt_config) + assert isinstance(updater, swig_api.ParameterUpdater) # Initialize ParameterUpdater. updater.init(gradient_machine) @@ -115,7 +120,7 @@ def main(): # output_arguments is Neural Network forward result. Here is not useful, just passed # to gradient_machine.forward - output_arguments = api.Arguments.createArguments(0) + output_arguments = swig_api.Arguments.createArguments(0) for pass_id in xrange(3): # we train 2 passes. updater.startPass() @@ -166,7 +171,7 @@ def main(): # in testing stage, only forward is needed. gradient_machine.forward( cifar_data.data_converter.convert(data_batch), output_arguments, - api.PASS_TEST) + swig_api.PASS_TEST) gradient_machine.eval(test_evaluator) # print error rate for test data set @@ -177,8 +182,8 @@ def main(): updater.catchUpWith() params = gradient_machine.getParameters() for each_param in params: - assert isinstance(each_param, api.Parameter) - value = each_param.getBuf(api.PARAMETER_VALUE) + assert isinstance(each_param, swig_api.Parameter) + value = each_param.getBuf(swig_api.PARAMETER_VALUE) value = value.copyToNumpyArray() # Here, we could save parameter to every where you want From 89d8564d06ed265a4c10a5c2a600703f24a8fbf3 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 11 Jan 2017 19:43:05 +0800 Subject: [PATCH 5/6] rm unused image_util.py --- demo/image_classification/image_util.py | 221 ------------------------ 1 file changed, 221 deletions(-) delete mode 100644 demo/image_classification/image_util.py diff --git a/demo/image_classification/image_util.py b/demo/image_classification/image_util.py deleted file mode 100644 index f09605394a19e0..00000000000000 --- a/demo/image_classification/image_util.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np -from PIL import Image -from cStringIO import StringIO - - -def resize_image(img, target_size): - """ - Resize an image so that the shorter edge has length target_size. - img: the input image to be resized. - target_size: the target resized image size. - """ - percent = (target_size / float(min(img.size[0], img.size[1]))) - resized_size = int(round(img.size[0] * percent)), int( - round(img.size[1] * percent)) - img = img.resize(resized_size, Image.ANTIALIAS) - return img - - -def flip(im): - """ - Return the flipped image. - Flip an image along the horizontal direction. - im: input image, (H x W x K) ndarrays - """ - if len(im.shape) == 3: - return im[:, :, ::-1] - else: - return im[:, ::-1] - - -def crop_img(im, inner_size, color=True, test=True): - """ - Return cropped image. - The size of the cropped image is inner_size * inner_size. - im: (K x H x W) ndarrays - inner_size: the cropped image size. - color: whether it is color image. - test: whether in test mode. - If False, does random cropping and flipping. - If True, crop the center of images. - """ - if color: - height, width = max(inner_size, im.shape[1]), max(inner_size, - im.shape[2]) - padded_im = np.zeros((3, height, width)) - startY = (height - im.shape[1]) / 2 - startX = (width - im.shape[2]) / 2 - endY, endX = startY + im.shape[1], startX + im.shape[2] - padded_im[:, startY:endY, startX:endX] = im - else: - im = im.astype('float32') - height, width = max(inner_size, im.shape[0]), max(inner_size, - im.shape[1]) - padded_im = np.zeros((height, width)) - startY = (height - im.shape[0]) / 2 - startX = (width - im.shape[1]) / 2 - endY, endX = startY + im.shape[0], startX + im.shape[1] - padded_im[startY:endY, startX:endX] = im - if test: - startY = (height - inner_size) / 2 - startX = (width - inner_size) / 2 - else: - startY = np.random.randint(0, height - inner_size + 1) - startX = np.random.randint(0, width - inner_size + 1) - endY, endX = startY + inner_size, startX + inner_size - if color: - pic = padded_im[:, startY:endY, startX:endX] - else: - pic = padded_im[startY:endY, startX:endX] - if (not test) and (np.random.randint(2) == 0): - pic = flip(pic) - return pic - - -def decode_jpeg(jpeg_string): - np_array = np.array(Image.open(StringIO(jpeg_string))) - if len(np_array.shape) == 3: - np_array = np.transpose(np_array, (2, 0, 1)) - return np_array - - -def preprocess_img(im, img_mean, crop_size, is_train, color=True): - """ - Does data augmentation for images. - If is_train is false, cropping the center region from the image. - If is_train is true, randomly crop a region from the image, - and randomy does flipping. - im: (K x H x W) ndarrays - """ - im = im.astype('float32') - test = not is_train - pic = crop_img(im, crop_size, color, test) - pic -= img_mean - return pic.flatten() - - -def load_meta(meta_path, mean_img_size, crop_size, color=True): - """ - Return the loaded meta file. - Load the meta image, which is the mean of the images in the dataset. - The mean image is subtracted from every input image so that the expected mean - of each input image is zero. - """ - mean = np.load(meta_path)['data_mean'] - border = (mean_img_size - crop_size) / 2 - if color: - assert (mean_img_size * mean_img_size * 3 == mean.shape[0]) - mean = mean.reshape(3, mean_img_size, mean_img_size) - mean = mean[:, border:border + crop_size, border:border + - crop_size].astype('float32') - else: - assert (mean_img_size * mean_img_size == mean.shape[0]) - mean = mean.reshape(mean_img_size, mean_img_size) - mean = mean[border:border + crop_size, border:border + - crop_size].astype('float32') - return mean - - -def load_image(img_path, is_color=True): - """ - Load image and return. - img_path: image path. - is_color: is color image or not. - """ - img = Image.open(img_path) - img.load() - return img - - -def oversample(img, crop_dims): - """ - image : iterable of (H x W x K) ndarrays - crop_dims: (height, width) tuple for the crops. - Returned data contains ten crops of input image, namely, - four corner patches and the center patch as well as their - horizontal reflections. - """ - # Dimensions and center. - im_shape = np.array(img[0].shape) - crop_dims = np.array(crop_dims) - im_center = im_shape[:2] / 2.0 - - # Make crop coordinates - h_indices = (0, im_shape[0] - crop_dims[0]) - w_indices = (0, im_shape[1] - crop_dims[1]) - crops_ix = np.empty((5, 4), dtype=int) - curr = 0 - for i in h_indices: - for j in w_indices: - crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) - curr += 1 - crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate( - [-crop_dims / 2.0, crop_dims / 2.0]) - crops_ix = np.tile(crops_ix, (2, 1)) - - # Extract crops - crops = np.empty( - (10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), - dtype=np.float32) - ix = 0 - for im in img: - for crop in crops_ix: - crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] - ix += 1 - crops[ix - 5:ix] = crops[ix - 5:ix, :, ::-1, :] # flip for mirrors - return crops - - -class ImageTransformer: - def __init__(self, - transpose=None, - channel_swap=None, - mean=None, - is_color=True): - self.transpose = transpose - self.channel_swap = None - self.mean = None - self.is_color = is_color - - def set_transpose(self, order): - if self.is_color: - assert 3 == len(order) - self.transpose = order - - def set_channel_swap(self, order): - if self.is_color: - assert 3 == len(order) - self.channel_swap = order - - def set_mean(self, mean): - # mean value, may be one value per channel - if mean.ndim == 1: - mean = mean[:, np.newaxis, np.newaxis] - else: - # elementwise mean - if self.is_color: - assert len(mean.shape) == 3 - self.mean = mean - - def transformer(self, data): - if self.transpose is not None: - data = data.transpose(self.transpose) - if self.channel_swap is not None: - data = data[self.channel_swap, :, :] - if self.mean is not None: - data -= self.mean - return data From b95e5b1d1e3f9b90dcf7d2abf6bcb9c30b6eac48 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 12 Jan 2017 09:59:31 +0800 Subject: [PATCH 6/6] change global var to upper case --- demo/image_classification/api_train.py | 16 ++++++++-------- demo/image_classification/cifar_util.py | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/demo/image_classification/api_train.py b/demo/image_classification/api_train.py index 435d82372dc9bf..a28cb5a0e3b440 100644 --- a/demo/image_classification/api_train.py +++ b/demo/image_classification/api_train.py @@ -26,9 +26,9 @@ import cifar_util # some global parameter. -image_size = 32 -data_size = 3 * image_size * image_size -label_size = 10 +IMAGE_SIZE = 32 +DATA_SIZE = 3 * IMAGE_SIZE * IMAGE_SIZE +LABEL_SIZE = 10 def optimizer_config(): @@ -42,8 +42,8 @@ def optimizer_config(): def network_config(): """Function to config neural network.""" - img = config_helpers.layers.data_layer(name='image', size=data_size) - lbl = config_helpers.layers.data_layer(name='label', size=label_size) + img = config_helpers.layers.data_layer(name='image', size=DATA_SIZE) + lbl = config_helpers.layers.data_layer(name='label', size=LABEL_SIZE) hidden1 = config_helpers.layers.fc_layer(input=img, size=200) hidden2 = config_helpers.layers.fc_layer(input=hidden1, size=200) inference = config_helpers.layers.fc_layer( @@ -69,9 +69,9 @@ def main(): # prepare cifar-10 data. cifar_data = cifar_util.Cifar10Data( - img_size=image_size, - mean_img_size=image_size, - num_classes=label_size, + img_size=IMAGE_SIZE, + mean_img_size=IMAGE_SIZE, + num_classes=LABEL_SIZE, batch_size=128, train_file_list='data/cifar-out/batches/train.txt', test_file_list='data/cifar-out/batches/test.txt', diff --git a/demo/image_classification/cifar_util.py b/demo/image_classification/cifar_util.py index b41b016c257b70..c3d2e37edf0941 100644 --- a/demo/image_classification/cifar_util.py +++ b/demo/image_classification/cifar_util.py @@ -17,7 +17,7 @@ import io import random -import paddle.trainer.PyDataProvider2 as dataprovider +import paddle.trainer.PyDataProvider2 as data_provider import paddle.utils.image_util as image_util from paddle.trainer.PyDataProvider2 import * from py_paddle import DataProviderConverter @@ -76,8 +76,8 @@ def __init__(self, # 'label': dp.integer_value(label_size) # } input_types = [ - dataprovider.dense_vector(self.img_raw_size), - dataprovider.integer_value(self.num_classes) + data_provider.dense_vector(self.img_raw_size), + data_provider.integer_value(self.num_classes) ] self.data_converter = DataProviderConverter(input_types)