- Consolidation getting closer.

Author: general-kroll-4-life

.github/workflows/build.yml

@@ -119,7 +119,7 @@ jobs:
     
     - name: Generate rewritten registry for simulations
       run: |
-        python3 test\python\registry_rewrite.py
+        python3 test\python\registry_rewrite.py --srcdir "$pwd\test\registry\src" --destdir "$pwd\test\registry-mocked\src"
 
     - name: Get dependencies
       run: |
@@ -256,7 +256,7 @@ jobs:
     
     - name: Generate rewritten registry for simulations
       run: |
-        python3 test/python/registry_rewrite.py
+        python3 test/python/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
       
     - name: Get dependencies
       run: |
@@ -523,7 +523,7 @@ jobs:
     - name: Generate rewritten registry for simulations
       if: ${{ matrix.registry  != 'test/registry' }}
       run: |
-        python3 test/python/registry_rewrite.py
+        python3 test/python/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
 
     - name: Create certificates for robot tests
       run: |
@@ -653,7 +653,7 @@ jobs:
     
     - name: Generate rewritten registry for simulations
       run: |
-        python3 test/python/registry_rewrite.py
+        python3 test/python/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
       
     - name: Get dependencies
       run: |
@@ -937,7 +937,7 @@ jobs:
 
     - shell: wsl-bash {0}
       name: Generate rewritten registry for simulations
-      run: python3 test/python/registry_rewrite.py
+      run: python3 test/python/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
 
     - shell: wsl-bash {0}
       name: Create certificates for robot tests
@@ -1037,7 +1037,7 @@ jobs:
     
     - name: Generate rewritten registry for simulations
       run: |
-        python3 test/python/registry_rewrite.py
+        python3 test/python/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
 
     - name: Get dependencies
       run: |
@@ -1557,7 +1557,7 @@ jobs:
     - name: Generate rewritten registry for simulations
       if: env.BUILD_IMAGE_REQUIRED == 'true'
       run: |
-        python3 test/python/registry_rewrite.py --replacement-host=host.docker.internal
+        python3 test/python/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src" --replacement-host=host.docker.internal
 
     
     - name: Create certificates for robot tests

.github/workflows/scenario.yaml

@@ -99,23 +99,23 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }}
         run: |
-          python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/readonly 2>&1 | tee cicd/log/markdown-readonly-testing-results.log
+          python3 test/python/markdown_testing/markdown_testing.py $(pwd) --test-root=docs/walkthroughs/readonly 2>&1 | tee cicd/log/markdown-readonly-testing-results.log
       
       - name: Run Read Write Walkthrough Scenarios
         if: ${{ env.runType == env.RUNTYPE_READ_WRITE || env.runType == env.RUNTYPE_ALL }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }}
         run: |
-          python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/readwrite 2>&1 | tee cicd/log/markdown-readwrite-testing-results.log
+          python3 test/python/markdown_testing/markdown_testing.py $(pwd) --test-root=docs/walkthroughs/readwrite 2>&1 | tee cicd/log/markdown-readwrite-testing-results.log
       
       - name: Run Deploy Walkthrough Scenarios
         if: ${{ env.runType == env.RUNTYPE_DEPLOY || env.runType == env.RUNTYPE_ALL }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }}
         run: |
-          python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/deploy 2>&1 | tee cicd/log/markdown-deploy-testing-results.log
+          python3 test/python/markdown_testing/markdown_testing.py $(pwd) --test-root=docs/walkthroughs/deploy 2>&1 | tee cicd/log/markdown-deploy-testing-results.log
 
       - name: Upload Test Results
         uses: actions/[email protected]

Dockerfile

@@ -101,7 +101,8 @@ ADD cicd ${TEST_ROOT_DIR}/cicd
 COPY --from=certificates /opt/test/stackql ${TEST_ROOT_DIR}/
 
 RUN pip3 install PyYaml \
-    && python3 ${TEST_ROOT_DIR}/test/python/registry_rewrite.py
+    && python3 ${TEST_ROOT_DIR}/test/python/registry_rewrite.py \
+      --srcdir ${TEST_ROOT_DIR}/test/registry/src --destdir ${TEST_ROOT_DIR}/test/registry-mocked/src
 
 FROM utility AS integration
 

docs/developer_guide.md

@@ -186,7 +186,7 @@ Local testing of the application:
 
 1. Run `go test --tags "sqlite_stackql" ./...` tests.
 2. Build the executable [as per the root README](/README.md#build)
-3. Perform registry rewrites as needed for mocking `python3 test/python/registry_rewrite.py`.
+3. Perform registry rewrites as needed for mocking `python3 test/python/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"`.
 3. Run robot tests:
     - Functional tests, mocked as needed `robot -d test/robot/functional test/robot/functional`.
     - Integration tests `robot -d test/robot/integration test/robot/integration`.  For these, you will need to set various envirnonment variables as per the github actions.

test/python/markdown_testing/markdown_testing.py

@@ -26,12 +26,11 @@ def _parse_args() -> argparse.Namespace:
     Parse the arguments.
     """
     parser = argparse.ArgumentParser(description='Create a token.')
-    parser.add_argument('--test-root', type=str, help='The test root.', default=os.path.join(_REPOSITORY_ROOT_PATH, 'docs', 'walkthroughs'))
+    parser.add_argument('cwd', type=str, help='The working dir to stasrt and finish in.')
+    parser.add_argument('--test-root', type=str, help='The test root.', required=True)
     return parser.parse_args()
 
 
-_REPOSITORY_ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..'))
-
 
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
@@ -236,23 +235,25 @@ class MdOrchestrator(object):
 
     def __init__(
         self,
+        cwd: str,
         parser: MdParser, 
         max_setup_blocks: int = 1,
         max_invocations_blocks: int = 1,
         max_teardown_blocks: int = 1,
         setup_contains_shell_invocation: bool = True
     ):
+        self._cwd = cwd
         self._parser = parser
         self._max_setup_blocks = max_setup_blocks
         self._max_invocations_blocks = max_invocations_blocks
         self._max_teardown_blocks = max_teardown_blocks
         self._setup_contains_shell_invocation = setup_contains_shell_invocation
 
     def _get_teardown_base(self) -> str:
-        return f'set -e;\ncd {_REPOSITORY_ROOT_PATH};\n'
+        return f'set -e;\ncd {self._cwd};\n'
 
     def _get_setup_base(self) -> str:
-        return f'cd {_REPOSITORY_ROOT_PATH};\n'
+        return f'cd {self._cwd};\n'
 
     def orchestrate(self, file_path: str) -> List[WorkloadDTO]:
         return self._orchestrate_block(file_path, 0, self._parser.parse_markdown_file(file_path).get_ordered())
@@ -425,9 +426,9 @@ def run(self) -> WalkthroughResult:
 
 class AllWalkthroughsRunner(object):
 
-    def __init__(self):
+    def __init__(self, cwd: str):
         md_parser = MdParser()
-        self._orchestrator: MdOrchestrator = MdOrchestrator(md_parser)
+        self._orchestrator: MdOrchestrator = MdOrchestrator(cwd, md_parser)
 
     def run_all(self, walkthrough_inodes: List[str], recursive=True, skip_readme=True) -> List[WalkthroughResult]:
         results: List[WalkthroughResult] = []
@@ -480,7 +481,7 @@ def _run_tests(root_dir: str) -> List[WalkthroughResult]:
 
     :return: The results.
     """
-    runner: AllWalkthroughsRunner = AllWalkthroughsRunner()
+    runner: AllWalkthroughsRunner = AllWalkthroughsRunner(root_dir)
     results: List[WalkthroughResult] = runner.run_all([root_dir])
     return results
 

test/python/registry_rewrite.py

@@ -9,11 +9,6 @@
 import shutil
 import yaml
 
-CURDIR :str = os.path.dirname(os.path.realpath(__file__))
-_TEST_ROOT_DIR :str = os.path.abspath(os.path.join(CURDIR, '..'))
-
-_DEFAULT_SRC_DIR = os.path.join(_TEST_ROOT_DIR, 'registry', 'src')
-_DEFAULT_DST_DIR = os.path.join(_TEST_ROOT_DIR, 'registry-mocked', 'src')
 DEFAULT_PORT = 1070
 GOOGLE_DEFAULT_PORT = 1080
 GOOGLEADMIN_DEFAULT_PORT = 1098
@@ -33,13 +28,13 @@
 parser.add_argument(
     '--srcdir', 
     type=str,
-    default=_DEFAULT_SRC_DIR,
+    required=True,
     help='directory containing executable'
 )
 parser.add_argument(
     '--destdir',
     type=str,
-    default=_DEFAULT_DST_DIR,
+    required=True,
     help='directory containing config and cache'
 )
 parser.add_argument(

test/python/stackql_context.py

@@ -7,7 +7,7 @@
 
 from typing import Optional
 
-from typed_python_responses import SELECT_AWS_CLOUD_CONTROL_EVENTS_MINIMAL_EXPECTED
+from .typed_python_responses import SELECT_AWS_CLOUD_CONTROL_EVENTS_MINIMAL_EXPECTED
 
 _exe_name = 'stackql'
 
@@ -96,8 +96,6 @@ def get_source_path_for_docker(self) -> str:
 
 DB_INTERNAL_CFG_LAX :str = "{ \"tableRegex\": \"(?i)^(?:public\\\\.)?(?:pg_.*|current_schema|information_schema)\" }"
 
-_TEST_APP_CACHE_ROOT = os.path.abspath(os.path.join(REPOSITORY_ROOT, "test", ".stackql"))
-
 ROBOT_TEST_ROOT = os.path.abspath(os.path.join(REPOSITORY_ROOT, 'test', 'robot'))
 
 ROBOT_PROD_REG_DIR = os.path.abspath(os.path.join(ROBOT_TEST_ROOT, 'registry', 'prod'))