Remove MonoChainItem

Author: reyoung

demo/mnist/api_train.py

@@ -7,11 +7,10 @@
 The user api could be simpler and carefully designed.
 """
 
-import mnist_provider
 import paddle.trainer.PyDataProvider2 as dp
-import py_paddle.swig_paddle as api
-from mnist_util import read_from_mnist
 from paddle.trainer_config_helpers import *
+
+import mnist_provider
 from trainer import *
 
 
@@ -33,41 +32,6 @@ def mnist_network(pixel, label):
     return cost
 
 
-def generator_to_batch(generator, batch_size):
-    ret_val = list()
-    for each_item in generator:
-        ret_val.append(each_item)
-        if len(ret_val) == batch_size:
-            yield ret_val
-            ret_val = list()
-    if len(ret_val) != 0:
-        yield ret_val
-
-
-def input_order_converter(generator):
-    for each_item in generator:
-        yield each_item['pixel'], each_item['label']
-
-
-class MonolithicChainItem(RunnerChainItem):
-    def finalize(self, context, next_callback):
-        context.gradient_machine.finish()
-
-    def on_pass_end(self, context, next_callback):
-        context.updater.catchUpWith()
-        params = context.gradient_machine.getParameters()
-        for each_param in params:
-            assert isinstance(each_param, api.Parameter)
-            value = each_param.getBuf(api.PARAMETER_VALUE)
-            value = value.copyToNumpyArray()
-
-            # Here, we could save parameter to every where you want
-            print each_param.getName(), value
-
-    def __init__(self):
-        RunnerChainItem.__init__(self)
-
-
 def main():
     mnist = mnist_network()
 
@@ -91,7 +55,7 @@ def main():
             method=mnist_provider.process,
             file_list=['./data/raw_data/t10k'],
             batch_size=256))
-    runner.add_chain_item(MonolithicChainItem())
+    runner.add_chain_item(SaveParamsOnPassEnd())
     with runner.use():
         for _ in xrange(2):
             runner.run_one_pass()

demo/mnist/mnist_provider.py

@@ -1,5 +1,5 @@
 from paddle.trainer.PyDataProvider2 import *
-from mnist_util import read_from_mnist
+import numpy
 
 
 # Define a py data provider
@@ -8,5 +8,27 @@
                  'label': integer_value(10)},
     cache=CacheType.CACHE_PASS_IN_MEM)
 def process(settings, filename):  # settings is not used currently.
-    for each in read_from_mnist(filename):
-        yield each
+    imgf = filename + "-images-idx3-ubyte"
+    labelf = filename + "-labels-idx1-ubyte"
+    f = open(imgf, "rb")
+    l = open(labelf, "rb")
+
+    f.read(16)
+    l.read(8)
+
+    # Define number of samples for train/test
+    if "train" in filename:
+        n = 60000
+    else:
+        n = 10000
+
+    images = numpy.fromfile(
+        f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)).astype('float32')
+    images = images / 255.0 * 2.0 - 1.0
+    labels = numpy.fromfile(l, 'ubyte', count=n).astype("int")
+
+    for i in xrange(n):
+        yield {"pixel": images[i, :], 'label': labels[i]}
+
+    f.close()
+    l.close()

demo/mnist/mnist_util.py

@@ -10,7 +10,7 @@
     'RandomInitializeParams', 'BasicLocalParameterUpdater', 'network',
     'BasicTrainerDataProvider', 'BasicDataProviderOps',
     'BasicGradientMachineTrainOps', 'Counter', 'BatchEvaluate',
-    'BasicTestDataProvider', 'TestOnPassEnd'
+    'BasicTestDataProvider', 'TestOnPassEnd', 'SaveParamsOnPassEnd'
 ]
 
 
@@ -580,3 +580,17 @@ def initialize(self, context, next_callback):
 
 BasicTrainerDataProvider = data_provider_creator(True)
 BasicTestDataProvider = data_provider_creator(False)
+
+
+class SaveParamsOnPassEnd(RunnerChainItem):
+    def __init__(self):
+        RunnerChainItem.__init__(self)
+
+    def on_pass_end(self, context, next_callback):
+
+        context.updater.catchUpWith()
+        params = context.gradient_machine.getParameters()
+        for param in params:
+            param.save(param.getName())
+
+        next_callback(context)