Merge pull request #1464 from reyoung/feature/clean_mnist_v2

Combine Reader/Feeder together in trainer.train

Author: reyoung

demo/mnist/api_train_v2.py

@@ -1,14 +1,5 @@
 import paddle.v2 as paddle
 
-import mnist_util
-
-
-def train_reader():
-    train_file = './data/raw_data/train'
-    generator = mnist_util.read_from_mnist(train_file)
-    for item in generator:
-        yield item
-
 
 def main():
     paddle.init(use_gpu=False, trainer_count=1)
@@ -40,11 +31,13 @@ def event_handler(event):
     trainer = paddle.trainer.SGD(update_equation=adam_optimizer)
 
     trainer.train(
-        train_data_reader=train_reader,
+        reader=paddle.reader.batched(
+            paddle.reader.shuffle(
+                paddle.dataset.mnist.train(), buf_size=8192),
+            batch_size=32),
         cost=cost,
         parameters=parameters,
         event_handler=event_handler,
-        batch_size=32,  # batch size should be refactor in Data reader
         reader_dict={images.name: 0,
                      label.name: 1})
 

python/CMakeLists.txt

@@ -4,7 +4,7 @@ set(OUTPUT_DIR
 file(GLOB TRAINER_PY_FILES . ./paddle/trainer/*.py)
 file(GLOB HELPERS_PY_FILES . ./paddle/trainer_config_helpers/*.py)
 file(GLOB UTILS_PY_FILES . ./paddle/utils/*.py)
-file(GLOB V2_PY_FILES . ./paddle/v2/*.py)
+file(GLOB_RECURSE V2_PY_FILES ./paddle/v2/ *.py)
 
 set(PY_FILES paddle/__init__.py
              ${TRAINER_PY_FILES}
@@ -24,7 +24,7 @@ add_custom_target(paddle_python ALL DEPENDS
     ${OUTPUT_DIR}/.timestamp)
 
 add_subdirectory(paddle/trainer_config_helpers/tests)
-add_subdirectory(paddle/reader/tests)
+add_subdirectory(paddle/v2/reader/tests)
 add_subdirectory(paddle/v2/tests)
 
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/

python/paddle/reader/tests/CMakeLists.txt

@@ -20,13 +20,16 @@
 import data_type
 import topology
 import data_feeder
+from . import dataset
+from . import reader
 import attr
 import pooling
 import py_paddle.swig_paddle as api
 
 __all__ = [
     'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer',
-    'event', 'data_type', 'attr', 'pooling', 'data_feeder', 'topology'
+    'event', 'data_type', 'attr', 'pooling', 'data_feeder', 'dataset', 'reader',
+    'topology'
 ]
 
 

python/paddle/v2/__init__.py

@@ -0,0 +1,3 @@
+import mnist
+
+__all__ = ['mnist']

python/paddle/v2/dataset/__init__.py

@@ -1,10 +1,10 @@
 """
 MNIST dataset.
 """
-import numpy
 import paddle.v2.dataset.common
 import subprocess
-
+import numpy
+import platform
 __all__ = ['train', 'test']
 
 URL_PREFIX = 'http://yann.lecun.com/exdb/mnist/'
@@ -20,12 +20,19 @@
 
 def reader_creator(image_filename, label_filename, buffer_size):
     def reader():
+        if platform.system() == 'Darwin':
+            zcat_cmd = 'gzcat'
+        elif platform.system() == 'Linux':
+            zcat_cmd = 'zcat'
+        else:
+            raise NotImplementedError()
+
         # According to http://stackoverflow.com/a/38061619/724872, we
         # cannot use standard package gzip here.
-        m = subprocess.Popen(["zcat", image_filename], stdout=subprocess.PIPE)
+        m = subprocess.Popen([zcat_cmd, image_filename], stdout=subprocess.PIPE)
         m.stdout.read(16)  # skip some magic bytes
 
-        l = subprocess.Popen(["zcat", label_filename], stdout=subprocess.PIPE)
+        l = subprocess.Popen([zcat_cmd, label_filename], stdout=subprocess.PIPE)
         l.stdout.read(8)  # skip some magic bytes
 
         while True:

python/paddle/v2/dataset/mnist.py

@@ -14,7 +14,7 @@
 
 __all__ = [
     'map_readers', 'buffered', 'compose', 'chain', 'shuffle',
-    'ComposeNotAligned'
+    'ComposeNotAligned', 'batched'
 ]
 
 from Queue import Queue
@@ -191,3 +191,25 @@ def data_reader():
             e = q.get()
 
     return data_reader
+
+
+def batched(reader, batch_size):
+    """
+    Create a batched reader.
+    :param reader: the data reader to read from.
+    :param batch_size: batch_size
+    :return: the batched reader.
+    """
+
+    def batched_reader():
+        r = reader()
+        batch = []
+        for instance in r:
+            batch.append(instance)
+            if len(batch) == batch_size:
+                yield batch
+                batch = []
+        if batch:
+            yield batch
+
+    return batched_reader

python/paddle/reader/__init__.py renamed to python/paddle/v2/reader/__init__.py

@@ -0,0 +1,3 @@
+add_test(NAME reader_tests
+  COMMAND bash ${PROJ_ROOT}/python/paddle/v2/reader/tests/run_tests.sh
+  ${PYTHON_EXECUTABLE})