force_unique_key feature with docs, example | code improve | logging docs refactor

Author: eshaan7

docs/source/Logging.md docs/source/Configuration.mddocs/source/Logging.md renamed to docs/source/Configuration.md

@@ -1,4 +1,14 @@
-## Logging Configuration
+## Configuration
+
+### POST Request Options
+
+One can read [post-request-schema.json](https://github.com/Eshaan7/Flask-Shell2HTTP/blob/master/post-request-schema.json)
+to see and understand the various _optional_ tweaks which can be done when making requests to the API. 
+
+There are many [example programs](Examples.md) with client requests given which demonstrate these different behaviours.
+
+
+### Logging Configuration
 
 This extension logs messages of different severity `INFO`, `DEBUG`, `ERROR` 
 using the python's inbuilt [logging](https://docs.python.org/3/library/logging.html) module.

docs/source/index.rst

@@ -36,7 +36,7 @@ more detailed :doc:`Examples` that shows different use-cases for this package.
 
    Quickstart
    Examples
-   Logging
+   Configuration
 
 API Reference
 -------------------------------

examples/basic.py

@@ -41,7 +41,8 @@
     # make new request for a command with arguments
     uri = f"/cmd/{ENDPOINT_AND_CMD}"
     # timeout in seconds, default value is 3600
-    data = {"args": ["hello", "world"], "timeout": 60}
+    # force_unique_key disables rate-limiting
+    data = {"args": ["hello", "world"], "timeout": 60, "force_unique_key": True}
     resp1 = c.post(uri, json=data).get_json()
     print(resp1)
     # fetch result

flask_shell2http/api.py

@@ -18,15 +18,15 @@
 from flask_executor.futures import Future
 
 # lib imports
-from .classes import RequestParser, run_command
+from .classes import RunnerParser
 from .helpers import get_logger
 
 
 logger = get_logger()
-request_parser = RequestParser()
+runner_parser = RunnerParser()
 
 
-class shell2httpAPI(MethodView):
+class Shell2HttpAPI(MethodView):
     """
     Flask.MethodView that registers GET and POST methods for a given endpoint.
     This is invoked on `Shell2HTTP.register_command`.
@@ -75,16 +75,20 @@ def post(self):
                 f"Requester: '{request.remote_addr}'."
             )
             # Check if request data is correct and parse it
-            cmd, timeout, callback_context, key = request_parser.parse_req(
+            cmd, timeout, callback_context, key = runner_parser.parse_req(
                 request, self.command_name
             )
 
             # run executor job in background
             future = self.executor.submit_stored(
-                future_key=key, fn=run_command, cmd=cmd, timeout=timeout, key=key,
+                future_key=key,
+                fn=runner_parser.run_command,
+                cmd=cmd,
+                timeout=timeout,
+                key=key,
             )
             # callback that removes the temporary directory
-            future.add_done_callback(request_parser.cleanup_temp_dir)
+            future.add_done_callback(runner_parser.cleanup_temp_dir)
             if self.user_callback_fn:
                 # user defined callback fn with callback_context if any
                 future.add_done_callback(

flask_shell2http/base_entrypoint.py

@@ -7,7 +7,7 @@
 from flask_executor.futures import Future
 
 # lib imports
-from .api import shell2httpAPI
+from .api import Shell2HttpAPI
 from .helpers import get_logger
 
 
@@ -126,7 +126,7 @@ def my_callback_fn(context: dict, future: Future) -> None:
             return None
 
         # else, add new URL rule
-        view_func = shell2httpAPI.as_view(
+        view_func = Shell2HttpAPI.as_view(
             endpoint,
             command_name=command_name,
             user_callback_fn=callback_fn,

flask_shell2http/classes.py

@@ -12,67 +12,15 @@
 from flask_executor.futures import Future
 
 # lib imports
-from .helpers import list_replace, calc_hash, get_logger, DEFAULT_TIMEOUT
+from .helpers import list_replace, gen_key, get_logger, DEFAULT_TIMEOUT
 
 logger = get_logger()
 
 
-def run_command(cmd: List[str], timeout: int, key: str) -> Dict:
+class RunnerParser:
     """
-    This function is called by the executor to run given command
-    using a subprocess asynchronously.
-
-    :param cmd: List[str]
-        command to run split as a list
-    :param key: str
-        future_key of particular Future instance
-    :param timeout: int
-        maximum timeout in seconds (default = 3600)
-
-    :rtype: Dict
-
-    :returns:
-        A Concurrent.Future object where future.result() is the report
-    """
-    start_time: float = time.time()
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,)
-    try:
-        outs, errs = proc.communicate(timeout=int(timeout))
-        stdout = outs.decode("utf-8")
-        stderr = errs.decode("utf-8")
-        returncode = proc.returncode
-        logger.info(f"Job: '{key}' --> finished with returncode: '{returncode}'.")
-
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        stdout, _ = [s.decode("utf-8") for s in proc.communicate()]
-        stderr = f"command timedout after {timeout} seconds."
-        returncode = proc.returncode
-        logger.error(f"Job: '{key}' --> failed. Reason: \"{stderr}\".")
-
-    except Exception as e:
-        proc.kill()
-        returncode = -1
-        stdout = None
-        stderr = str(e)
-        logger.error(f"Job: '{key}' --> failed. Reason: \"{stderr}\".")
-
-    end_time: float = time.time()
-    process_time = end_time - start_time
-    return dict(
-        key=key,
-        report=stdout,
-        error=stderr,
-        returncode=returncode,
-        start_time=start_time,
-        end_time=end_time,
-        process_time=process_time,
-    )
-
-
-class RequestParser:
-    """
-    Utility class to parse incoming POST request data into meaningful arguments.
+    Utility class to parse incoming POST request
+    data into meaningful arguments, generate key and run command.
     Internal use Only.
     """
 
@@ -112,28 +60,30 @@ def __parse_multipart_req(args: List[str], files) -> (List[str], str):
     def parse_req(self, request, base_command: str) -> (str, int, Dict, str):
         # default values if request is w/o any data
         # i.e. just run-script
+        tmpdir = None
+        # default values
         args: List[str] = []
         timeout: int = DEFAULT_TIMEOUT
-        tmpdir = None
         callback_context = {}
+        randomize_key = False
         if request.is_json:
             # request does not contain a file
             args = request.json.get("args", [])
             timeout: int = request.json.get("timeout", DEFAULT_TIMEOUT)
             callback_context = request.json.get("callback_context", {})
+            randomize_key = request.json.get("force_unique_key", False)
         elif request.files:
             # request contains file and form_data
             data = json.loads(request.form.get("request_json", "{}"))
             received_args = data.get("args", [])
             timeout: int = data.get("timeout", DEFAULT_TIMEOUT)
             callback_context = data.get("callback_context", {})
-            args, tmpdir = RequestParser.__parse_multipart_req(
-                received_args, request.files
-            )
+            randomize_key = data.get("force_unique_key", False)
+            args, tmpdir = self.__parse_multipart_req(received_args, request.files)
 
         cmd: List[str] = base_command.split(" ")
         cmd.extend(args)
-        key: str = calc_hash(cmd)
+        key: str = gen_key(cmd, randomize=randomize_key)
         if tmpdir:
             self.__tmpdirs.update({key: tmpdir})
 
@@ -158,3 +108,56 @@ def cleanup_temp_dir(self, future: Future) -> None:
             logger.debug(
                 f"Job: '{key}' --> Failed to clear Temporary directory: '{tmpdir}'."
             )
+
+    @staticmethod
+    def run_command(cmd: List[str], timeout: int, key: str) -> Dict:
+        """
+        This function is called by the executor to run given command
+        using a subprocess asynchronously.
+
+        :param cmd: List[str]
+            command to run split as a list
+        :param key: str
+            future_key of particular Future instance
+        :param timeout: int
+            maximum timeout in seconds (default = 3600)
+
+        :rtype: Dict
+
+        :returns:
+            A Concurrent.Future object where future.result() is the report
+        """
+        start_time: float = time.time()
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,)
+        try:
+            outs, errs = proc.communicate(timeout=int(timeout))
+            stdout = outs.decode("utf-8")
+            stderr = errs.decode("utf-8")
+            returncode = proc.returncode
+            logger.info(f"Job: '{key}' --> finished with returncode: '{returncode}'.")
+
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            stdout, _ = [s.decode("utf-8") for s in proc.communicate()]
+            stderr = f"command timedout after {timeout} seconds."
+            returncode = proc.returncode
+            logger.error(f"Job: '{key}' --> failed. Reason: \"{stderr}\".")
+
+        except Exception as e:
+            proc.kill()
+            returncode = -1
+            stdout = None
+            stderr = str(e)
+            logger.error(f"Job: '{key}' --> failed. Reason: \"{stderr}\".")
+
+        end_time: float = time.time()
+        process_time = end_time - start_time
+        return dict(
+            key=key,
+            report=stdout,
+            error=stderr,
+            returncode=returncode,
+            start_time=start_time,
+            end_time=end_time,
+            process_time=process_time,
+        )

flask_shell2http/helpers.py

@@ -1,6 +1,10 @@
 import logging
+import hashlib
+import uuid
 from typing import List
 
+from flask import current_app
+
 DEFAULT_TIMEOUT = 3600
 
 
@@ -17,15 +21,22 @@ def list_replace(lst: List, old, new) -> None:
         pass
 
 
+def gen_key(lst: List, randomize=False) -> str:
+    if randomize:
+        return str(uuid.uuid4())[:8]
+    return calc_hash(lst)[:8]
+
+
 def calc_hash(lst: List) -> str:
     """
     Internal use only.
     Calculates sha1sum of given command with it's byte-string.
     This is for non-cryptographic purpose,
     that's why a faster and insecure hashing algorithm is chosen.
     """
-    to_hash = " ".join(lst).encode("ascii")
-    return __import__("hashlib").sha1(to_hash).hexdigest()[:8]
+    current_app.config.get("Shell2HTTP_")
+    to_hash = " ".join(lst).encode("utf-8")
+    return hashlib.sha1(to_hash).hexdigest()
 
 
 def get_logger() -> logging.Logger:

post-request-schema.json

@@ -7,6 +7,7 @@
     {
       "args": ["hello", "world"],
       "timeout": 60,
+      "force_unique_key": true,
       "callback_context": {
         "read_result_from": "/path/to/saved/file",
         "force_success": true
@@ -30,6 +31,12 @@
       "description": "Maximum timeout after which subprocess fails if not already complete.",
       "default": 3600
     },
+    "force_unique_key": {
+      "type": "boolean",
+      "title": "Flag to enable/disable internal rate limiting mechanism",
+      "description": "By default, the key is the SHA1 sum of the command + args POSTed to the API. This is done as a rate limiting measure so as to prevent multiple jobs with same parameters, if one such job is already running. If force_unique_key is set to true, the API will bypass this default behaviour and a psuedorandom key will be returned instead",
+      "default": false
+    },
     "callback_context": {
       "type": "object",
       "title": "Additional context for user defined callback function",