fix gpubox trainning scripts and fix cuda error

Author: danleifeng

tools/run_gpubox.sh

@@ -1,4 +1,44 @@
-#!/bin/bash
+# !/bin/bash
+
+if [ ! -d "./log" ]; then
+  mkdir ./log
+  echo "Create log floder for store running log"
+fi
 
 export FLAGS_LAUNCH_BARRIER=0
-fleetrun --worker_num=1 --server_num=1 tools/static_gpubox_trainer.py -m models/rank/dnn/config_gpubox.yaml
+export PADDLE_TRAINER_ID=0
+export PADDLE_PSERVER_NUMS=1
+export PADDLE_TRAINERS=1
+export PADDLE_TRAINERS_NUM=${PADDLE_TRAINERS}
+
+
+# set free port if 29011 is occupied
+export PADDLE_PSERVERS_IP_PORT_LIST="127.0.0.1:29011"
+export PADDLE_PSERVER_PORT_ARRAY=(29011)
+
+# set gpu numbers according to your device
+export FLAGS_selected_gpus="0,1,2,3,4,5,6,7"
+
+# set your model yaml
+SC="tools/static_gpubox_trainer.py -m models/rank/dnn/config_gpubox.yaml"
+
+# run pserver
+export TRAINING_ROLE=PSERVER
+for((i=0;i<$PADDLE_PSERVER_NUMS;i++))
+do
+    cur_port=${PADDLE_PSERVER_PORT_ARRAY[$i]}
+    echo "PADDLE WILL START PSERVER "$cur_port
+    export PADDLE_PORT=${cur_port}
+    python -u $SC &> ./log/pserver.$i.log &
+done
+
+# run trainer
+export TRAINING_ROLE=TRAINER
+for((i=0;i<$PADDLE_TRAINERS;i++))
+do
+    echo "PADDLE WILL START Trainer "$i
+    PADDLE_TRAINER_ID=$i
+    python -u $SC &> ./log/worker.$i.log
+done
+
+echo "Training log stored in ./log/"

tools/static_gpubox_trainer.py

@@ -74,6 +74,7 @@ def run(self):
             fleet.stop_worker()
             self.record_result()
         logger.info("Run Success, Exit.")
+        logger.info("-" * 100)
 
     def network(self):
         self.model = get_model(self.config)
@@ -150,6 +151,9 @@ def run_worker(self):
                     self.exe, model_dir,
                     [feed.name for feed in self.input_data],
                     self.inference_target_var)
+            self.reader.release_memory()
+            self.PSGPU.end_pass()
+            logger.info("finish {} epoch training....".format(epoch))
 
     def init_reader(self):
         if fleet.is_server():