Initial commit

Author: eshaan7

.flake8

@@ -0,0 +1,9 @@
+[flake8]
+max-line-length = 88
+ignore =
+    W503, # line break before binary operator
+    E231, # missing whitespace after ','
+exclude =
+    Dockerfile,
+    venv,
+    virtualenv

.github/workflows/python-publish.yml

@@ -0,0 +1,31 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.gitignore

@@ -0,0 +1,8 @@
+.vscode
+venv/
+virtualenv/
+__pycache__/
+app.py
+build/
+dist/
+*.egg-info

README.md

@@ -0,0 +1,100 @@
+# Flask-Shell2HTTP
+
+A minimalist REST API wrapper for python's subprocess API.<br/>
+Execute shell commands asynchronously and safely from flask's endpoints.
+
+Inspired by the work of awesome folks over at [msoap/shell2http](https://github.com/msoap/shell2http).
+
+## You can use this for
+
+- Set a script that runs on a succesful POST request to an endpoint of your choice. See [Example code](examples/run_script.py)
+- Map a base command to an endpoint and passing dynamic arguments to it. See [Example code](examples/basic.py)
+- Can also process uploaded files. See [Example code](examples/multiple_files.py)
+- Choose to run a command asynchronously or not. (upcoming feature)
+
+## Quick Start
+
+#### Dependencies
+
+- Python: `>=v3.6`
+- [Flask](https://pypi.org/project/Flask/)
+- [Flask-Executor](https://pypi.org/project/Flask-Executor)
+
+#### Install
+
+```bash
+$ pip install flask_shell2http flask_executor
+```
+
+#### Example
+
+```python
+from flask import Flask
+from flask_executor import Executor
+from flask_shell2http import Shell2HTTP
+
+# Flask application instance
+app = Flask(__name__)
+
+executor = Executor(app)
+shell2http = Shell2HTTP(app=app, executor=executor, base_url_prefix="/commands/")
+
+shell2http.register_command(endpoint="saythis", command_name="echo")
+```
+
+Run the application server with, `$ flask run -p 4000`.
+
+#### Make HTTP calls
+
+```bash
+$ curl -X POST -d '{"args": ["Hello", "World!"]}' http://localhost:4000/commands/saythis
+```
+
+<details><summary>or using python's requests module,</summary>
+
+```python
+data = {"args": ["Hello", "World!"]}
+resp = requests.post("http://localhost:4000/commands/saythis", json=data)
+print("Result:", resp.json())
+```
+
+</details>
+
+returns JSON,
+
+```json
+{
+   "key": "ddbe0a94847c65f9b8198424ffd07c50",
+   "status": "running"
+}
+```
+
+Then using this `key` you can query for the result,
+
+```bash
+$ curl http://localhost:4000/commands/saythis?key=ddbe0a94847c65f9b8198424ffd07c50
+```
+
+Returns result in JSON,
+
+```json
+{
+  "end_time": 1593019807.782958, 
+  "error": "", 
+  "md5": "ddbe0a94847c65f9b8198424ffd07c50", 
+  "process_time": 0.00748753547668457, 
+  "report": {
+    "result": "Hello World!\n"
+  }, 
+  "start_time": 1593019807.7754705, 
+  "status": "success"
+}
+```
+
+## Why?
+
+This was made to integrate various command-line tools easily with [IntelOwl](https://github.com/intelowlproject/IntelOwl).
+
+## Various examples
+
+You can find various examples under [examples](examples/)

examples/basic.py

@@ -0,0 +1,44 @@
+# system imports
+import requests
+
+# web imports
+from flask import Flask
+from flask_executor import Executor
+from flask_shell2http import Shell2HTTP
+
+# Flask application instance
+app = Flask(__name__)
+
+# application factory
+executor = Executor(app)
+shell2http = Shell2HTTP(app, executor, base_url_prefix="/cmd/")
+
+ENDPOINT = "echo"
+
+shell2http.register_command(endpoint=ENDPOINT, command_name="echo")
+
+
+@app.route("/")
+def test():
+    """
+    The final executed command becomes:
+    ```bash
+    $ echo hello world
+    ```
+    """
+    url = f"http://localhost:4000/cmd/{ENDPOINT}"
+    data = {"args": ["hello", "world"]}
+    resp = requests.post(url, json=data)
+    resp_data = resp.json()
+    print(resp_data)
+    key = resp_data["key"]
+    if key:
+        resp2 = requests.get(f"{url}?key={key}")
+        return resp2.json()
+    else:
+        return resp_data
+
+
+# Application Runner
+if __name__ == "__main__":
+    app.run(port=4000)

examples/multiple_files.py

@@ -0,0 +1,54 @@
+# system imports
+import requests
+import tempfile
+
+# web imports
+from flask import Flask
+from flask_executor import Executor
+from flask_shell2http import Shell2HTTP
+
+# Flask application instance
+app = Flask(__name__)
+
+# application factory
+executor = Executor()
+executor.init_app(app)
+shell2http = Shell2HTTP(base_url_prefix="/cmd/")
+shell2http.init_app(app, executor)
+
+ENDPOINT = "catthisformeplease"
+
+shell2http.register_command(endpoint=ENDPOINT, command_name="strings")
+
+
+@app.route("/")
+def test():
+    """
+    Prefix each filename with @ in arguments.\n
+    Files are stored in temporary directories which are flushed on command completion.\n
+    The final executed command becomes:
+    ```bash
+    $ strings /tmp/inputfile /tmp/someotherfile
+    ```
+    """
+    url = f"http://localhost:4000/cmd/{ENDPOINT}"
+    data = {"args": ["@inputfile", "@someotherfile"]}
+    with tempfile.TemporaryFile() as fp:
+        fp.write(b"Hello world!")
+        fp.seek(0)
+        f = fp.read()
+        files = {"inputfile": f, "someotherfile": f}
+    resp = requests.post(url=url, files=files, data=data)
+    resp_data = resp.json()
+    print(resp_data)
+    key = resp_data["key"]
+    if key:
+        resp2 = requests.get(f"{url}?key={key}")
+        return resp2.json()
+    else:
+        return resp_data
+
+
+# Application Runner
+if __name__ == "__main__":
+    app.run(port=4000)

examples/run_script.py

@@ -0,0 +1,41 @@
+# system imports
+import requests
+
+# web imports
+from flask import Flask
+from flask_executor import Executor
+from flask_shell2http import Shell2HTTP
+
+# Flask application instance
+app = Flask(__name__)
+
+# application factory
+executor = Executor(app)
+shell2http = Shell2HTTP(app, executor, base_url_prefix="/scripts/")
+
+shell2http.register_command(endpoint="hacktheplanet", command_name="./fuxsocy.py")
+
+
+@app.route("/")
+def test():
+    """
+    The final executed command becomes:
+    ```bash
+    $ ./fuxsocy.py
+    ```
+    """
+    url = "http://localhost:4000/scripts/hacktheplanet"
+    resp = requests.post(url)
+    resp_data = resp.json()
+    print(resp_data)
+    key = resp_data["key"]
+    if key:
+        resp2 = requests.get(f"{url}?key={key}")
+        return resp2.json()
+    else:
+        return resp_data
+
+
+# Application Runner
+if __name__ == "__main__":
+    app.run(port=4000)

flask_shell2http/__init__.py

@@ -0,0 +1,2 @@
+# flake8: noqa
+from .base_entrypoint import Shell2HTTP

flask_shell2http/api.py

@@ -0,0 +1,68 @@
+# system imports
+from http import HTTPStatus
+
+# web imports
+from flask import request, jsonify, make_response
+from flask.views import MethodView
+
+# lib imports
+from .classes import JobExecutor, ReportStore, RequestParser
+
+
+class shell2httpAPI(MethodView):
+    command_name: str
+    executor: JobExecutor
+    store: ReportStore
+    request_parser: RequestParser
+
+    def get(self):
+        try:
+            md5: str = request.args.get("key")
+            if not md5:
+                raise Exception("No key provided in arguments.")
+            # check if job has been finished
+            future = self.executor.get_job(md5)
+            if future:
+                if not future.done:
+                    return make_response(jsonify(status="running", md5=md5), 200)
+
+                # pop future object since it has been finished
+                self.executor.pop_job(md5)
+
+            # if yes, get result from store
+            report = self.store.get_one(md5)
+            if not report:
+                raise Exception(f"Report does not exist for key:{md5}")
+
+            return make_response(report.to_json(), HTTPStatus.OK)
+
+        except Exception as e:
+            return make_response(jsonify(error=str(e)), HTTPStatus.NOT_FOUND)
+
+    def post(self):
+        try:
+            # Check if command is correct and parse it
+            cmd, md5 = self.request_parser.parse_req(request)
+
+            # run executor job in background
+            job_key = JobExecutor.make_key(md5)
+            future = self.executor.new_job(
+                future_key=job_key, fn=self.executor.run_command, cmd=cmd, md5=md5
+            )
+            # callback that adds result to store
+            future.add_done_callback(self.store.save_result)
+            # callback that removes the temporary directory
+            future.add_done_callback(self.request_parser.cleanup_temp_dir)
+
+            return make_response(
+                jsonify(status="running", key=md5), HTTPStatus.ACCEPTED,
+            )
+
+        except Exception as e:
+            return make_response(jsonify(error=str(e)), HTTPStatus.BAD_REQUEST)
+
+    def __init__(self, command_name, executor):
+        self.command_name = command_name
+        self.executor = JobExecutor(executor)
+        self.store = ReportStore()
+        self.request_parser = RequestParser(command_name)