Merge pull request #28 from datakind/input_config_manager

Refactor input and output data.

Author: Zebreu

.gitignore

@@ -3,6 +3,9 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+gdrive_creds.json
+dkroutingtool/src/creds
+
 # C extensions
 *.so
 

dkroutingtool/README.md

@@ -14,7 +14,7 @@ Within this folder (dkroutingtool) run:
 `sh s_build_docker_dev.sh`
 
 
-### Running Model
+### Running Application
 You can run the image interactively with the following:
 
 `sh local_start.sh`
@@ -55,16 +55,42 @@ data otherwise), you can force an osm update by running the following script (be
 <br>
 
 -----------------------------------------------------------
+### File input and output structure
 
-## Other Options
+Running in cloud mode:
+When running using the cloud provider the tool will download all input data from the cloud
+into the local data directory in WORKING_DATA_DIR/input_data/<scenario>-datetime/ folder and
+then read from that directory. All outputs are put into WORKING_DATA_DIR/output_data/<scenario>-datetime/.
+This is then zipped and uploaded to the output/ directory in the root cloud folder.
 
-### Running the Manual Update (Dev)
-Download `manual_routes_edit.xlsx` file from the docker container, edit as needed and copy back into the docker container.
-Then get into the docker container and run:
+Running in local mode:
+When running locally the tool will read from the data/ directory and write to the same
+WORKING_DATA_DIR/output_data/<scenario>-datetime/ directory. for the output.
 
-`PYTHONPATH=./py-lib:src/py /opt/conda/bin/python src/py/main_application.py --local --manual`
+------------------------------------------------------------------------------
+### Running the model with google drive output (for DEVELOPERS)
+You can run the model with google drive by putting your credentials in src/creds/gdrive_creds.json
 
-<br>
-<br>
+Then create a version of scripts/run_app.sh with the folder ids filled in (you can access these) by right
+clicking your files in google drive -> copy link -> and extracting the id in the URI.
+
+Then you can run it in the docker environment as follows:
+
+export CLOUDCONTEXT=gdrive
+export GDRIVECUSTOMERFILEID=...
+export GDRIVEEXTRAFILEID=...
+export GDRIVEROOTFOLDERID=...
+export GDRIVECREDSFILE=src/creds/gdrive_creds.json
+
+# Local mode
+/opt/conda/bin/python src/py/main_application.py --input test_scenario --local
+
+# Local Manual Mode
+/opt/conda/bin/python src/py/main_application.py --input test_scenario --manual_input_path WORKING_DATA_DIR/output_data/test_scenario_2022_09_16_19_38/manual_edits --local
+
+# Cloud mode
+/opt/conda/bin/python src/py/main_application.py --input test_scenario
+
+# Cloud manual mode
+/opt/conda/bin/python src/py/main_application.py --input test_scenario --manual
 
-------------------------------------------------------------------------------

dkroutingtool/local_start.sh

@@ -1,5 +1,7 @@
 docker run -it \
   --mount src=`pwd`/src,target=/src,type=bind \
   --mount src=`pwd`/scripts,target=/scripts,type=bind \
-  -p 8080:8080 dkroutingtool:dev bash
+  -p 8080:8080 \
+  -p 5001:5001 \
+  dkroutingtool:dev bash
 

dkroutingtool/requirements.txt

@@ -10,6 +10,7 @@ cryptography==3.2
 decorator==4.4.0
 docutils==0.14
 et-xmlfile==1.0.1
+flask==2.1.3
 folium==0.9.1
 geojson==2.4.1
 idna==2.8
@@ -55,5 +56,6 @@ google-api-python-client==1.12.5
 google-auth==1.23.0
 google-auth-httplib2==0.0.4
 google-auth-oauthlib==0.4.2
+oauth2client==3.0.0
 googleapis-common-protos==1.52.0
 ruamel.yaml==0.16.12

dkroutingtool/scripts/run_app_gdrive.sh

@@ -0,0 +1,18 @@
+export CLOUDCONTEXT=gdrive
+export GDRIVECUSTOMERFILEID=
+export GDRIVEEXTRAFILEID=
+export GDRIVEROOTFOLDERID=
+export GDRIVECREDSFILE=src/creds/gdrive_creds.json
+
+# Local mode
+#/opt/conda/bin/python src/py/main_application.py --input test_scenario --local
+
+# Local Manual Mode
+#/opt/conda/bin/python src/py/main_application.py --input test_scenario --manual_input_path WORKING_DATA_DIR/output_data/test_scenario_2022_09_16_19_38/manual_edits --local
+
+# Cloud mode
+/opt/conda/bin/python src/py/main_application.py --input test_scenario
+
+# Cloud manual mode
+#/opt/conda/bin/python src/py/main_application.py --input test_scenario --manual
+

dkroutingtool/scripts/run_manual_mode.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+export PYTHONPATH=py-lib:src/py
+
+/opt/conda/bin/python src/py/main_application.py --local --manual

dkroutingtool/scripts/start_server.sh

@@ -0,0 +1,3 @@
+export PYTHONPATH=./py-lib:./src/py
+
+/opt/conda/bin/python src/py/api/main.py

dkroutingtool/src/py/api/main.py

@@ -0,0 +1,42 @@
+"""Simple Flask API stub.
+
+ /route
+ Currently runs the route optimization and returns the solution text
+
+"""
+
+import time
+from flask import Flask
+from run_routing import run_routing_from_config
+from config.config_manager import ConfigManager, GPSInputPaths, ConfigFileLocations
+from output.route_solution_data import SolutionOutput
+
+
+app = Flask(__name__)
+PORT = 5001
+
+@app.route('/route', methods=['POST'])
+def route():
+    root_dir = './'
+    config_manager = ConfigManager.load(
+        ConfigFileLocations(
+            routing_config_file=f"{root_dir}/data/config.json",
+            build_parameters_file=f"{root_dir}build_parameters.yml",
+            gps_input_files=GPSInputPaths(
+                gps_file=f"{root_dir}data/customer_data.xlsx",
+                custom_header_file=f"{root_dir}data/custom_header.yaml",
+                gps_extra_input_file=f"{root_dir}data/extra_points.csv"
+            ),
+            manual_edits_input_files=None
+        )
+    )
+    solution_data, vis_data = run_routing_from_config(config_manager)
+    # TODO: come up with better way to serialize this data.
+    solution_output = SolutionOutput(solution_data)
+    solution_txt = solution_output.serialize_solution()
+
+    # Return the solution text.
+    return {"solution": solution_txt}
+
+if __name__ == '__main__':
+    app.run(host="0.0.0.0", port=PORT)