scenario-testing-evolution

Summary:

- Supporting different testing patterns:
    - `readonly`; read ops only.
    - `readwrite`; read and mutate with `stackql`.
    - `deploy`; lifecycle management with `stackql-deploy`.
- CI improvments to elide needless builds.

Author: general-kroll-4-life

.github/workflows/build.yml

@@ -11,6 +11,8 @@ on:
       - "**/*.md"
       - "docs/**"
       - "licenses/**"
+      - "test/python/markdown_testing/**"
+      - ".github/workflows/scenario.yaml"
   pull_request:
     branches:
       - main
@@ -20,6 +22,8 @@ on:
       - "docs/**"
       - "examples/**"
       - "licenses/**"
+      - "test/python/markdown_testing/**"
+      - ".github/workflows/scenario.yaml"
 
 env:
   GOTESTCMD: "go test -timeout 1200s --tags \"sqlite_stackql\" -v ./..."

.github/workflows/scenario.yaml

@@ -7,6 +7,10 @@ on:
 
 env:
   TAG_NAME: ${{ github.ref_name }}
+  RUNTYPE_READ_ONLY: readonly
+  RUNTYPE_READ_WRITE: readwrite
+  RUNTYPE_DEPLOY: deploy
+  RUNTYPE_ALL: all
 
 jobs:
   scenario-testing:
@@ -15,8 +19,28 @@ jobs:
     steps:
       - name: Ref Parse
         run: |
+          _defaultRunType="${{ env.RUNTYPE_READ_ONLY }}"
+          runID=$(echo -n '${{ github.ref_name }}' | cut -d '-' -f 2)"
+          runType=$(echo -n '${{ github.ref_name }}' | cut -d '-' -f 3)"
+          repositoryShorthand=$(echo -n '${{ github.ref_name }}' | rev | cut -d '-' -f 1 | rev)
+          artifactRepositoryFullName='stackql/stackql'
+          if [ "$repositoryShorthand" = "devel" ]; then
+            artifactRepositoryFullName='stackql/stackql-devel'
+          fi
+          if [ "$runType" = "" ]; then
+            runType="$_defaultRunType"
+          elif [ "$runType" != "${{ env.RUNTYPE_READ_ONLY }}" ] \
+                 && [ "$runType" != "${{ env.RUNTYPE_READ_WRITE }}" ] \
+                 && [ "$runType" != "${{ env.RUNTYPE_DEPLOY }}" \
+                 && [ "$runType" != "${{ env.RUNTYPE_ALL }}" \
+            ]; then
+            echo "Invalid run type: $runType"
+            exit 1
+          fi
           {
             echo "runID=$(echo -n '${{ github.ref_name }}' | cut -d '-' -f 2)"
+            echo "artifactRepositoryFullName=$artifactRepositoryFullName"
+            echo "runType=$runType"
           } >> "${GITHUB_ENV}"
 
       - name: Check out code into the Go module directory
@@ -28,7 +52,7 @@ jobs:
           name: stackql_linux_amd64
           path: build
           github-token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }}
-          repository: stackql/stackql
+          repository: ${{ env.artifactRepositoryFullName }}
           run-id: ${{ env.runID }}
 
       - name: Stackql permissions
@@ -46,26 +70,45 @@ jobs:
       - name: Persist secrets
         run: |
           echo "$GCP_RO_SECRET" >> cicd/keys/testing/google-ro-credentials.json
+          echo "$GCP_RW_SECRET" >> cicd/keys/testing/google-rw-credentials.json
         shell: bash
         env:
           GCP_RO_SECRET: ${{ secrets.CI_SCENARIO_GCP_RO_SECRET }}
+          GCP_RW_SECRET: ${{ secrets.CI_SCENARIO_GCP_RW_SECRET }}
 
       - name: Install Python dependencies
         run: |
           pip3 install -r cicd/requirements.txt
       
-      - name: Run Walkthrough Scenarios
+      - name: Run Read Only Walkthrough Scenarios
+        if: ${{ env.runType == env.RUNTYPE_READ_ONLY || env.runType == env.RUNTYPE_ALL }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }}
         run: |
-          python3 test/python/markdown_testing/markdown_testing.py 2>&1 | tee cicd/log/markdown-testing-results.log 
+          python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/readonly 2>&1 | tee cicd/log/markdown-readonly-testing-results.log
       
+      - name: Run Read Write Walkthrough Scenarios
+        if: ${{ env.runType == env.RUNTYPE_READ_WRITE || env.runType == env.RUNTYPE_ALL }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/readwrite 2>&1 | tee cicd/log/markdown-readwrite-testing-results.log
+      
+      - name: Run Deploy Walkthrough Scenarios
+        if: ${{ env.runType == env.RUNTYPE_DEPLOY || env.runType == env.RUNTYPE_ALL }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/deploy 2>&1 | tee cicd/log/markdown-deploy-testing-results.log
+
       - name: Upload Test Results
         uses: actions/[email protected]
         with:
           name: scenario_test_results
-          path: cicd/log/markdown-testing-results.log
+          path: cicd/log/markdown-*.log
 
       - name: Cleanup
         if: always()

docs/walkthroughs/deploy/README.md

@@ -0,0 +1,4 @@
+
+## Infra lifecycle managment with `stackql`
+
+

docs/walkthroughs/list-aws-instances.md …kthroughs/readonly/list-aws-instances.mddocs/walkthroughs/list-aws-instances.md renamed to docs/walkthroughs/readonly/list-aws-instances.md docs/walkthroughs/list-aws-instances.md renamed to docs/walkthroughs/readonly/list-aws-instances.md

@@ -0,0 +1,4 @@
+
+## Lifecycle managment with `stackql-deploy`
+
+

…hroughs/list-google-accelerator-types.md …eadonly/list-google-accelerator-types.mddocs/walkthroughs/list-google-accelerator-types.md renamed to docs/walkthroughs/readonly/list-google-accelerator-types.md docs/walkthroughs/list-google-accelerator-types.md renamed to docs/walkthroughs/readonly/list-google-accelerator-types.md

@@ -10,7 +10,8 @@
 
 from tabulate import tabulate
 
-_REPOSITORY_ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..'))
+import argparse
+
 
 """
 Intentions:
@@ -19,6 +20,19 @@
   - Support sequential markdown code block execution, leveraging [info strings](https://spec.commonmark.org/0.30/#info-string).
 """
 
+
+def parse_args() -> argparse.Namespace:
+    """
+    Parse the arguments.
+    """
+    parser = argparse.ArgumentParser(description='Create a token.')
+    parser.add_argument('--test-root', type=str, help='The test root.', default=os.path.join(_REPOSITORY_ROOT_PATH, 'docs', 'walkthroughs'))
+    return parser.parse_args()
+
+
+_REPOSITORY_ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..'))
+
+
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
@@ -363,10 +377,6 @@ def run_all(self, walkthrough_inodes: List[str], recursive=True, skip_readme=Tru
                         e2e: SimpleRunner = SimpleRunner(workload)
                         result = e2e.run()
                         results.append(result)
-                    if recursive:
-                        for dir in dirs:
-                            dir_path = os.path.join(root, dir)
-                            results += self.run_all([dir_path], recursive)
                 continue
             is_file = os.path.isfile(inode_path)
             if is_file:
@@ -382,7 +392,7 @@ def run_all(self, walkthrough_inodes: List[str], recursive=True, skip_readme=Tru
             raise FileNotFoundError(f'Path not tractable: {inode_path}')
         return results
 
-def collate_results(results: List[WalkthroughResult]) -> bool:
+def _collate_results(results: List[WalkthroughResult]) -> bool:
     failed: int = 0
     for result in results:
         if result.rc != 0 or not result.passes_stdout_check or not result.passes_stderr_check:
@@ -391,14 +401,32 @@ def collate_results(results: List[WalkthroughResult]) -> bool:
     print(tabulate([[result.name, result.rc, result.passes_stdout_check, result.passes_stderr_check] for result in results], headers=['Test Name', 'Return Code', 'Passes Stdout Checks', 'Passes Stderr Checks']))
     return failed == 0
 
-def main():
+def run_tests(root_dir: str) -> List[WalkthroughResult]:
+    """
+    Run all tests.
+    A decent entry point for a test harness.
+
+    :param root_dir: The root directory.
+
+    :return: The results.
+    """
     runner: AllWalkthroughsRunner = AllWalkthroughsRunner()
-    results: List[WalkthroughResult] = runner.run_all([os.path.join(_REPOSITORY_ROOT_PATH, 'docs', 'walkthroughs')])
-    if collate_results(results):
+    results: List[WalkthroughResult] = runner.run_all([root_dir])
+    return results
+
+
+def _process_tests(root_dir: str) -> List[WalkthroughResult]:
+    results: List[WalkthroughResult] = run_tests(root_dir)
+    if _collate_results(results):
         print('All tests passed.')
         sys.exit(0)
     print('Some tests failed.')
     sys.exit(1)
 
+def _main() -> None:
+    args :argparse.Namespace = parse_args()
+    _process_tests(args.test_root)
+
+
 if __name__ == '__main__':
-    main()
+    _main()