feat(g-code-testing): Add functionality to run HTTP requests through G-Code Parser (RET-148, RET-149, RET-151) (#8316)

* Filter out __init__.py files

* feat(g-code-testing): Add robot HTTP support to G-Code Project

* Convert ProtocolRunner into an ABC called GCodeEngine
    * In ABC define an abstract `run` method
    * Rename all references
* Create ProtocolGCodeEngine which runs protocols
* Create HTTPGCodeEngine which runs HTTP requests

* Push name change

* Create first configuration

* Rename references in README

* Fix import paths in README

* Formatting

* Change folder name

* Turn into package

* Add new test data

* Add configuration file to define different test data

* Update cli.py to work with configuration file

* Fix JSON file

* Update Makefile

* Formatting

* Use hardcoded versions

* Fix tests

* Fix variable name

* Refactor from_config_file to add dependency injection for testing

* Also renamed classes to not start with `Test` because it was messing up pytest

* Add testing

* Formatting

* Linting

* Add Github Action

* Split up GCodeTestData

Break out GCodeTestData into HTTPTestData and ProtocolTestData. 
Did this so parameters on each class can be more specific than just path.

* Update GCodeEngine to account for TestData breakout

Instead of subclassing and having a single method just have more specialized methods on GCodeEngine. 

Pass HTTPTestData or ProtocolTestData classes itself instead of parameter from class

* Change from JSON to Py. Update config files

Changed json file to py file so for the HTTP types I can specify paths to the main function

Changed all HTTP configs to inherit from HTTPBase and defined a main function

* Forgot to ignore imports from robot_server

* Update tests

* Update to account for TestData breakout

* Fix whitespace

Author: DerekMaggio

.github/workflows/g-code-testing-lint-test.yaml

@@ -0,0 +1,50 @@
+# This workflow runs test and lint on branch pushes that touch the
+# notify-server project or its dependencies
+
+name: 'G-Code Testing Lint & Test'
+
+on:
+  # Most of the time, we run on pull requests, which lets us handle external PRs
+  push:
+    paths:
+      - 'Makefile'
+      - 'g-code-testing/**'
+      - '.github/workflows/g-code-testing-lint-test.yaml'
+      - '.github/actions/python/**'
+    tags-ignore:
+      - '*'
+  pull_request:
+    paths:
+      - 'Makefile'
+      - 'g-code-testing/**'
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  lint-test:
+    name: 'g-code-testing package linting and tests'
+    runs-on: 'ubuntu-18.04'
+    steps:
+      - uses: 'actions/checkout@v2'
+      - uses: 'actions/setup-node@v1'
+        with:
+          node-version: '12'
+      - uses: 'actions/setup-python@v2'
+        with:
+          python-version: '3.7'
+
+      - uses: './.github/actions/python/setup'
+        with:
+          project: 'g-code-testing'
+      - name: Lint
+        run: make -C g-code-testing lint
+      - name: Test
+        run: make -C g-code-testing test
+      - name: 'Upload coverage report'
+        uses: 'codecov/codecov-action@v2'
+        with:
+          files: ./g-code-testing/coverage.xml
+          flags: g-code-testing

g-code-testing/Pipfile

@@ -10,10 +10,10 @@ robot-server = {editable = true, path = "./../robot-server"}
 opentrons-shared-data = {editable = true, path = "../shared-data/python"}
 g-code-testing = {editable = true, path = "."}
 anyio = "==3.3.0"
-
+pydantic = "==1.4"
 
 [dev-packages]
-diff-match-patch = "*"
+diff-match-patch = "==20200713"
 pytest = "~=6.1"
 pytest-aiohttp = "==0.3.0"
 pytest-cov = "==2.10.1"
@@ -27,6 +27,5 @@ flake8-noqa = "~=1.1.0"
 black = "==21.7b0"
 decoy = "~=1.6.7"
 
-
 [requires]
 python_version = "3.7"

g-code-testing/Pipfile.lock

@@ -17,8 +17,8 @@ Framework supporting the following:
 
 ### G-Code Parser
 
-Using the `ProtocolRunner` you can run a Python protocol file against emulation.
-The `ProtocolRunner` instance will return a `GCodeProgram` instance
+Using the `GCodeEngine` you can run a Python protocol file against emulation.
+The `GCodeEngine` instance will return a `GCodeProgram` instance
 
 **Example:**
 
@@ -29,16 +29,15 @@ against emulation.
 In "Concise" format store parsed G-Code to /where/I/want/to/store/my/output.txt
 """
 import os.path
-from protocol_runner import ProtocolRunner
-from g_code_program.supported_text_modes \
-    import SupportedTextModes
+from g_code_parsing.g_code_engine import HTTPGCodeEngine
+from g_code_parsing.g_code_program.supported_text_modes import SupportedTextModes
 from opentrons.hardware_control.emulation.settings import Settings
 
 PROTOCOL_PATH = os.path.join('my', 'absolute', 'path', 'to', 'my', 'protocol.py')
 OUTPUT_PATH = os.path.join('where', 'I', 'want', 'to', 'store', 'my', 'output.txt')
 
 settings = Settings()  # Using default settings defined in class
-with ProtocolRunner(settings).run_protocol(PROTOCOL_PATH) as program:
+with HTTPGCodeEngine(settings).run_protocol(PROTOCOL_PATH) as program:
     program.save_text_explanation_to_file(OUTPUT_PATH, SupportedTextModes.CONCISE)
 ```
 
@@ -54,7 +53,7 @@ them in HTML format.
 Compare 2 files and save the diff to a file
 """
 import os.path
-from g_code_differ import GCodeDiffer
+from g_code_parsing.g_code_differ import GCodeDiffer
 FILE_1_PATH = os.path.join('tmp', 'file_1.txt')
 FILE_2_PATH = os.path.join('tmp', 'file_2.txt')
 HTML_PATH = os.path.join('home', 'derek_maggio', 'Desktop', 'my_diff.html')

g-code-testing/README.md

@@ -52,7 +52,7 @@ g-code-run:
 		--text-mode '$(text_mode)' \
 		--left-pipette '$(left_pipette)' \
 		--right-pipette '$(right_pipette)' \
-		$(protocol_path) \
+		$(configuration_name) \
 		2> /dev/null
 
 master_file = /tmp/master_file.txt  # File pulled from S3
@@ -99,7 +99,7 @@ g-code-smoothie-protocol-run:
 	@$(MAKE) --no-print-directory g-code-run \
 		left_pipette='{"model": "p20_single_v2.0", "id": "P20SV202020070101"}' \
 		right_pipette='{"model": "p20_single_v2.0", "id": "P20SV202020070101"}' \
-		protocol_path='protocols/smoothie_protocol.py' \
+		configuration_name='protocol_smoothie' \
   		> $(file_under_test)
 
 # g-code-smoothie-protocol-run
@@ -135,7 +135,7 @@ g-code-2-modules-1s-1m-v2-protocol-run:
 	@$(MAKE) --no-print-directory g-code-run \
 		left_pipette='{"model": "p300_single_v2.1", "id": "P20SV202020070101"}' \
 		right_pipette='{"model": "p20_multi_v2.1", "id": "P20SV202020070101"}' \
-		protocol_path='protocols/2_modules_1s_1m_v2.py' \
+		configuration_name='protocol_2_modules' \
   		> $(file_under_test)
 
 # g-code-2-modules-1s-1m-v2-protocol-diff
@@ -170,7 +170,7 @@ g-code-swift-turbo-protocol-run:
 	@$(MAKE) --no-print-directory g-code-run \
 		left_pipette='{"model": "p20_single_v2.0", "id": "P20SV202020070101"}' \
 		right_pipette='{"model": "p300_multi_v2.1", "id": "P20SV202020070101"}' \
-		protocol_path='protocols/swift_turbo.py' \
+		configuration_name='protocol_swift_turbo' \
   		> $(file_under_test)
 
 # g-code-swift-turbo-protocol-diff
@@ -205,7 +205,7 @@ g-code-swift-smoke-protocol-run:
 	@$(MAKE) --no-print-directory g-code-run \
 		left_pipette='{"model": "p20_single_v2.0", "id": "P20SV202020070101"}' \
 		right_pipette='{"model": "p300_multi_v2.1", "id": "P20SV202020070101"}' \
-		protocol_path='protocols/swift_smoke.py' \
+		configuration_name='protocol_swift_smoke' \
   		> $(file_under_test)
 
 # g-code-swift-smoke-protocol-diff
@@ -240,7 +240,7 @@ g-code-2-single-channel-v2-protocol-run:
 	@$(MAKE) --no-print-directory g-code-run \
 		left_pipette='{"model": "p20_single_v2.0", "id": "P20SV202020070101"}' \
 		right_pipette='{"model": "p300_single_v2.1", "id": "P20SV202020070101"}' \
-		protocol_path='protocols/2_single_channel_v2.py' \
+		configuration_name='protocol_2_single_channel' \
   		> $(file_under_test)
 
 # g-code-2-single-channel-v2-protocol-diff

g-code-testing/g_code_parsing.mk

@@ -1,7 +1,6 @@
 import abc
 import argparse
 import json
-import os
 from typing import Union, Dict, Any, List
 
 import sys
@@ -18,14 +17,18 @@
 from g_code_parsing.g_code_program.supported_text_modes import (
     SupportedTextModes,
 )
-from g_code_parsing.protocol_runner import ProtocolRunner
-from g_code_parsing.utils import get_configuration_dir
+from g_code_parsing.g_code_engine import GCodeEngine
+from g_code_parsing.g_code_test_data import (
+    GCodeTestFile,
+    ProtocolTestData,
+    HTTPTestData,
+)
+from g_code_test_data.configurations import CONFIGURATIONS
 
 
 class CLICommand(abc.ABC):
     """ABC which all CLI command classes should inherit from"""
 
-    CONFIGURATION_DIR_LOCATION = get_configuration_dir()
     able_to_respond_with_error_code = False
     respond_with_error_code = False
     error_message = "Error message not defined"
@@ -40,7 +43,7 @@ def execute(self) -> str:
 class RunCommand(CLICommand):
     """The "run" command of the CLI."""
 
-    configuration_path: str
+    configuration_name: str
     text_mode: str
     left_pipette_config: PipetteSettings
     right_pipette_config: PipetteSettings
@@ -55,11 +58,22 @@ def execute(self) -> str:
             left=self.left_pipette_config, right=self.right_pipette_config
         )
         settings = Settings(smoothie=smoothie_settings, host=self.host)
-        file_path = os.path.join(
-            self.CONFIGURATION_DIR_LOCATION, self.configuration_path
-        )
-        with ProtocolRunner(settings).run_protocol(file_path) as program:
-            return program.get_text_explanation(self.text_mode)
+        # Ignoring because mypy is throwing an error saying that configs
+        # is of type List[BaseModel] when it for sure isn't
+        configuration = GCodeTestFile(
+            configs=CONFIGURATIONS  # type: ignore
+        ).get_by_name(self.configuration_name)
+
+        engine = GCodeEngine(settings)
+
+        if isinstance(configuration, ProtocolTestData):
+            with engine.run_protocol(configuration) as program:
+                text = program.get_text_explanation(self.text_mode)
+        elif isinstance(configuration, HTTPTestData):
+            with engine.run_http(configuration) as program:
+                text = program.get_text_explanation(self.text_mode)
+
+        return text
 
 
 @dataclass
@@ -99,16 +113,11 @@ def execute(self) -> str:
 
 class ConfigurationCommand(CLICommand):
     def execute(self) -> str:
-        paths = [
-            os.path.join(root, name).replace(self.CONFIGURATION_DIR_LOCATION + "/", "")
-            for root, dirs, files in os.walk(
-                self.CONFIGURATION_DIR_LOCATION, topdown=False
-            )
-            for name in files
-        ]
-
-        path_string = "\n".join(paths)
-
+        # Ignoring because mypy is throwing an error saying that configs
+        # is of type List[BaseModel] when it for sure isn't
+        path_string = "\n".join(
+            GCodeTestFile(configs=CONFIGURATIONS).names  # type: ignore
+        )
         return f"Runnable Configurations:\n{path_string}"
 
 
@@ -122,7 +131,7 @@ class GCodeCLI:
     COMMAND_KEY = "command"
 
     RUN_PROTOCOL_COMMAND = "run"
-    CONFIGURATION_PATH = "configuration_path"
+    CONFIGURATION_NAME = "configuration_name"
     TEXT_MODE_KEY_NAME = "text_mode"
     LEFT_PIPETTE_KEY_NAME = "left_pipette"
     RIGHT_PIPETTE_KEY_NAME = "right_pipette"
@@ -170,7 +179,7 @@ def to_command(cls, processed_args) -> CLICommand:
         command: Union[RunCommand, DiffCommand, ConfigurationCommand]
         if cls.RUN_PROTOCOL_COMMAND == passed_command_name:
             command = RunCommand(
-                configuration_path=processed_args[cls.CONFIGURATION_PATH],
+                configuration_name=processed_args[cls.CONFIGURATION_NAME],
                 text_mode=processed_args[cls.TEXT_MODE_KEY_NAME],
                 left_pipette_config=processed_args[cls.LEFT_PIPETTE_KEY_NAME],
                 right_pipette_config=processed_args[cls.RIGHT_PIPETTE_KEY_NAME],
@@ -236,7 +245,7 @@ def parser(cls) -> argparse.ArgumentParser:
             formatter_class=argparse.RawTextHelpFormatter,
         )
         run_parser.add_argument(
-            "configuration_path", type=str, help="Path to configuration you want to run"
+            "configuration_name", type=str, help="Name of configuration you want to run"
         )
         run_parser.add_argument(
             "--text-mode",

g-code-testing/g_code_parsing/cli.py

@@ -43,3 +43,10 @@ def __init__(self, invalid_g_code) -> None:
             f"a polling command"
         )
         self.invalid_g_code = invalid_g_code
+
+
+class ConfigurationNotFoundError(ValueError):
+    def __init__(self, configuration_name) -> None:
+
+        super().__init__(f'Configuration "{configuration_name}" not found')
+        self.configuration_name = configuration_name