python transport sdk support initLog

Author: cyjseagull

cpp/wedpr-transport/sdk-wrapper/python/bindings/wedpr_python_gateway_sdk/transport/impl/transport_loader.py

@@ -47,6 +47,7 @@ def __repr__(self):
 
 class TransportLoader:
     transport_builder = TransportBuilder()
+    transport_builder.initLog("conf/wedpr_sdk_log_config.ini")
 
     @staticmethod
     def load(transport_config: TransportConfig) -> Transport:

python/ppc_model/conf/logging.conf

@@ -3,7 +3,7 @@ keys=root,wsgi
 
 [logger_root]
 level=INFO
-handlers=consoleHandler,fileHandler
+handlers=fileHandler
 
 [logger_wsgi]
 level = INFO
@@ -12,7 +12,7 @@ qualname = wsgi
 propagate = 0
 
 [handlers]
-keys=fileHandler,consoleHandler,accessHandler
+keys=fileHandler,accessHandler
 
 [handler_accessHandler]
 class=handlers.TimedRotatingFileHandler

python/ppc_model/conf/wedpr_sdk_log_config.ini

@@ -0,0 +1,17 @@
+[log]
+enable = true
+log_path = ./log
+; info debug trace
+level = info
+; MB
+max_log_file_size = 200
+; LineID, TimeStamp, ProcessID, ThreadName, ThreadID and Message
+;format = %Severity%|ppcs-gateway|system-id|%TimeStamp%|%ThreadName%-%ThreadID%|%Message%
+format = %Severity%|%TimeStamp%|%Message%
+enable_rotate_by_hour = true
+; if archive_path is empty, the archive function will be disabled
+; archive_path = ./log/
+;compress_archive_file = true
+; ; 0 = no limit, in MB
+; max_archive_size = 0
+; min_free_space = 0

python/ppc_model/ppc_model_app.py

@@ -68,7 +68,7 @@ def register_task_handler():
                         TransLogger(app, setup_console_handler=False), numthreads=2)
 
     protocol = 'http'
-    message = f"Starting ppc model server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']} successfully"
+    message = f"* Starting ppc model server at {protocol}://{app.config['HOST']}:{app.config['HTTP_PORT']} successfully"
     print(message)
     components.logger().info(message)
     server.start()

python/ppc_model/task/task_manager.py

@@ -244,8 +244,8 @@ def _terminate_timeout_tasks(self):
         with self._rw_lock.gen_rlock():
             for task_id, value in self._tasks.items():
                 alive_time = (datetime.datetime.now() -
-                              value[1]).total_seconds()
-                if alive_time >= self._task_timeout_s and value[0] == TaskStatus.RUNNING.value:
+                              value.start_time).total_seconds()
+                if alive_time >= self._task_timeout_s and value.task_status == TaskStatus.RUNNING.value:
                     tasks_to_kill.append(task_id)
 
         for task_id in tasks_to_kill:
@@ -257,9 +257,9 @@ def _cleanup_finished_tasks(self):
         with self._rw_lock.gen_rlock():
             for task_id, value in self._tasks.items():
                 alive_time = (datetime.datetime.now() -
-                              value[1]).total_seconds()
+                              value.start_time).total_seconds()
                 if alive_time >= self._task_timeout_s + 3600:
-                    tasks_to_cleanup.append((task_id, value[3]))
+                    tasks_to_cleanup.append((task_id, value.job_id))
         with self._rw_lock.gen_wlock():
             for task_id, job_id in tasks_to_cleanup:
                 if task_id in self._tasks:

python/ppc_model/tools/start.sh

@@ -11,14 +11,19 @@ LOG_INFO() {
 }
 binary_path=${SHELL_FOLDER}/ppc_model_app.py
 cd ${SHELL_FOLDER}
+log_dir=logs
+if [ ! -f ${log_dir} ];then
+   mkdir -p logs
+fi
 node=$(basename ${SHELL_FOLDER})
 node_pid=$(ps aux|grep ${binary_path}|grep -v grep|awk '{print $2}')
 
 if [ ! -z ${node_pid} ];then
     echo " ${node} is running, pid is $node_pid."
     exit 0
 else
-    nohup python ${binary_path} > start.out 2>&1 &
+    # -u means Force the stdout and stderr streams to be unbuffered
+    nohup python -u ${binary_path} > start.out 2>&1 &
     sleep 1.5
 fi
 try_times=4
@@ -34,4 +39,4 @@ do
     sleep 0.5
     ((i=i+1))
 done
-echo -e "\033[31m  Exceed waiting time. Please try again to start ${node} \033[0m"
+echo -e "\033[31m  Exceed waiting time. Please try again to start ${node} \033[0m"