+ add server for training

* unify client

Author: HYLcool

trinity/data/client.py trinity/cli/client.pytrinity/data/client.py renamed to trinity/cli/client.py

@@ -1,6 +1,7 @@
 import requests
 
-LOCAL_SERVER_URL = "http://127.0.0.1:5000/data_workflow"
+LOCAL_DATA_WORKFLOW_SERVER_URL = "http://127.0.0.1:5005/data_workflow"
+LOCAL_TRINITY_TRAINING_SERVER_URL = "http://127.0.0.1:5006/trinity_training"
 
 
 def send_get_request(url: str, params: dict) -> None:
@@ -33,7 +34,9 @@ def request(url, **kwargs):
 
 if __name__ == "__main__":
     res = request(
-        url=LOCAL_SERVER_URL,
+        url=LOCAL_DATA_WORKFLOW_SERVER_URL,
         configPath="scripts/config/gsm8k.yaml",
     )
-    print(res)
+    if res:
+        print(res)
+        print(res['message'])

trinity/cli/launcher.py

@@ -100,17 +100,33 @@ def both(config: Config) -> None:
 
 def activate_data_module(config_path: str):
     """Check whether to activate data module and preprocess datasets."""
-    from trinity.data.client import LOCAL_SERVER_URL, request
+    from trinity.cli.client import LOCAL_DATA_WORKFLOW_SERVER_URL, request
 
     logger.info("Activating data module...")
     res = request(
-        url=LOCAL_SERVER_URL,
+        url=LOCAL_DATA_WORKFLOW_SERVER_URL,
         configPath=config_path,
     )
     if res["return_code"] != 0:
         logger.error(f"Failed to activate data module: {res['return_msg']}.")
         return
 
+def run(config_path: str):
+    # TODO: support parse all args from command line
+    config = load_config(config_path)
+    config.check_and_update()
+    # try to activate data module
+    data_config = config.data
+    if data_config.dj_config_path or data_config.dj_process_desc:
+        activate_data_module(config_path)
+    ray.init()
+    if config.mode == "explore":
+        explore(config)
+    elif config.mode == "train":
+        train(config)
+    elif config.mode == "both":
+        both(config)
+
 
 def main() -> None:
     """The main entrypoint."""
@@ -126,19 +142,7 @@ def main() -> None:
     args = parser.parse_args()
     if args.command == "run":
         # TODO: support parse all args from command line
-        config = load_config(args.config)
-        config.check_and_update()
-        # try to activate data module
-        data_config = config.data
-        if data_config.dj_config_path or data_config.dj_process_desc:
-            activate_data_module(args.config)
-        ray.init()
-        if config.mode == "explore":
-            explore(config)
-        elif config.mode == "train":
-            train(config)
-        elif config.mode == "both":
-            both(config)
+        run(args.config)
 
 
 if __name__ == "__main__":

trinity/cli/server.py

@@ -0,0 +1,26 @@
+import traceback
+from flask import Flask, jsonify, request
+
+app = Flask(__name__)
+
+APP_NAME = 'trinity_training'
+PORT = 5006
+
+@app.route(f"/{APP_NAME}", methods=["GET"])
+def trinity_training():
+    config_path = request.args.get("configPath")
+    try:
+        from trinity.cli.launcher import run
+
+        run(config_path)
+        ret = 0
+        msg = "Training Success."
+    except:
+        traceback.print_exc()
+        msg = traceback.format_exc()
+        ret = 1
+    return jsonify({"return_code": ret, "message": msg})
+
+
+if __name__ == "__main__":
+    app.run(port=PORT, debug=True)

trinity/data/readme.md

@@ -92,10 +92,10 @@ synth_data = synthesizer.process(clean_data)
   - Request using our simple client:
 
   ```python
-  from trinity.data.client import request
+  from trinity.cli.client import request
 
   res = request(
-    url="http://127.0.0.1:5000/data_workflow",
+    url="http://127.0.0.1:5005/data_workflow",
     configPath="tests/test_configs/active_iterator_test_cfg.yaml"
   )
 

trinity/data/server.py

@@ -1,13 +1,15 @@
 from flask import Flask, jsonify, request
 
-from trinity.common.config import load_config
-from trinity.data.controllers.active_iterator import DataActiveIterator
-
 app = Flask(__name__)
 
+APP_NAME = 'data_workflow'
+PORT = 5005
 
-@app.route("/data_workflow", methods=["GET"])
+@app.route(f"/{APP_NAME}", methods=["GET"])
 def data_workflow():
+    from trinity.common.config import load_config
+    from trinity.data.controllers.active_iterator import DataActiveIterator
+
     config_path = request.args.get("configPath")
     config = load_config(config_path)
 
@@ -17,4 +19,4 @@ def data_workflow():
 
 
 if __name__ == "__main__":
-    app.run(debug=True)
+    app.run(port=PORT, debug=True)