advanced-regression-testing

.github/workflows/main.yml

@@ -57,7 +57,7 @@ jobs:
 
       - name: "[SETUP] get updated providers"
         run: |
-          pip install -q -r scripts/setup/requirements.txt
+          pip install -q -r requirements.txt
           python scripts/setup/get-updated-providers.py
 
       - name: "[SETUP] prepare dist and test dirs"

.github/workflows/regression.yml

@@ -1,7 +1,14 @@
 name: Integration Testing and Analysis
 
 on:
+  pull_request:
+    branches:
+      - main
+      - dev
   push:
+    branches:
+      - main
+      - dev
     tags:
       - robot*
       - regression*
@@ -15,8 +22,8 @@ env:
   STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }}
 
 jobs:
-  build-and-deploy:
-    name: build-and-deploy
+  regression-testing:
+    name: regression-testing
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -79,15 +86,36 @@ jobs:
           go get ./...
           python3 cicd/python/build.py --build
 
+      - name: Build any-sdk cli from source
+        working-directory: stackql-any-sdk
+        run: |
+
+          go get ./...
+
+          go build -x -v \
+            -o build/anysdk ./cmd/interrogate
+
       - name: Parse tag
         id: parse_tag
         run: |
-          tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)"
-          echo "tag_obj: $tag_obj"
-          {
-            echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')"
-            echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')"
-          } | tee -a "$GITHUB_ENV"
+          if [ "${{ github.ref_type }}" = "tag" ]; then
+            tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)"
+            echo "tag_obj: $tag_obj"
+            {
+              echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')"
+              echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')"
+            } | tee -a "$GITHUB_ENV"
+          else
+            {
+            echo "IS_BRANCH=true"
+            } >> $GITHUB_ENV
+          fi
+
+
+      - name: Generate rewritten registry for simulations
+        working-directory: stackql-core
+        run: |
+          python3 test/python/registry-rewrite.py
 
 
       - name: Prepare load balancing materials
@@ -118,8 +146,36 @@ jobs:
           openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
           openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
           openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 
+
+
+      - name: Start Core Test Mocks
+        working-directory: stackql-core
+        run: |
+          pgrep -f flask | xargs kill -9 || true
+          flask --app=./test/python/flask/gcp/app          run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 & 
+          flask --app=./test/python/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 & 
+
+      - name: Run any-sdk cli mocked testing
+        working-directory: stackql-core
+        run: |
+          export GCP_SERVICE_ACCOUNT_KEY="$(cat test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json)"
+          bucketsListIDs="$(${{ github.workspace }}/stackql-any-sdk/build/anysdk query \
+            --svc-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/services/storage-v1.yaml" \
+            --tls.allowInsecure \
+            --prov-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/provider.yaml" \
+            --resource buckets \
+            --method list \
+            --parameters '{ "project": "stackql-demo" }' \
+            | jq -r '.items[].id')" 
+          matchingBuckets="$(echo "${bucketsListIDs}" | grep "stackql-demo" )"
+          if [ "${matchingBuckets}" = "" ]; then
+            echo "Core Test Failed with no matching buckets"
+            exit 1
+          else
+            echo "Core Test passed with matching buckets: $matchingBuckets"
+          fi
 
-      - name: Run core robot functional tests
+      - name: Run core proxied robot functional tests against local registry
         if: success()
         working-directory: stackql-core
         run: |
@@ -130,12 +186,21 @@ jobs:
             --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
             --include registry \
             -d test/robot/reports \
-            test/robot/functional
+            test/robot/functional  || true
 
-      - name: Output from functional tests
+      - name: Output from core proxied functional tests
         if: always()
         run: |
           cat stackql-core/test/robot/reports/output.xml
+          python3 scripts/cicd/python/robot-parse.py  --robot-output-file stackql-core/test/robot/reports/output.xml > stackql-core/test/robot/reports/proxied_parsed_output.json
+
+      - name: Upload core traffic lights
+        uses: actions/[email protected]
+        if: success()
+        with:
+          name: proxied-core-traffic-lights
+          path: stackql-core/test/robot/reports/proxied_parsed_output.json
+
 
       - name: Post core test cleanup
         run: |
@@ -149,13 +214,21 @@ jobs:
           robot \
             --variable "${sundryCfg}" \
             --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
-            -d test/robot/reports \
-            test/robot/stackql/mocked
+            -d test/robot/reports/mocked \
+            test/robot/stackql/mocked || true
 
       - name: Output from local registry mocked functional tests
         if: always()
         run: |
-          cat test/robot/reports/output.xml
+          cat test/robot/reports/mocked/output.xml
+          python3 scripts/cicd/python/robot-parse.py  --robot-output-file test/robot/reports/mocked/output.xml > test/robot/reports/mocked/parsed_output.json
+
+      - name: Upload local registry mocked traffic lights
+        uses: actions/[email protected]
+        if: success()
+        with:
+          name: local-registry-mocked-traffic-lights
+          path: test/robot/reports/mocked/parsed_output.json
 
       - name: Post registry mocked test cleanup
         run: |
@@ -164,23 +237,74 @@ jobs:
           sudo cp /etc/hosts.bak /etc/hosts || true
           rm -f test/robot/reports/*.xml || true
 
-      - name: Run live robot functional tests
+      - name: Run live readonly robot functional tests
         if: success()
-        id: live_integration_tests
+        id: live_integration_tests_readonly
         env:
           GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RO_SECRET }}
           AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }}
         run: |
           providerRoot="$(realpath $(pwd)/providers)"
-          sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
+          sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}'
           robot \
             --variable "${sundryCfg}" \
             --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
-            -d test/robot/reports \
-            test/robot/stackql/live
+            -d test/robot/reports/readonly \
+            test/robot/stackql/live/readonly || true
 
-      - name: Output from live functional tests
+      - name: Output from live readonly functional tests
         if: always()
         run: |
-          cat test/robot/reports/output.xml
+          cat test/robot/reports/readonly/output.xml
+          python3 scripts/cicd/python/robot-parse.py  --robot-output-file test/robot/reports/readonly/output.xml > test/robot/reports/readonly/parsed_output.json
+
+      - name: Upload readonly traffic lights
+        uses: actions/[email protected]
+        if: success()
+        with:
+          name: local-registry-readonly-traffic-lights
+          path: test/robot/reports/readonly/parsed_output.json
+
+      - name: Run live readwrite robot functional tests
+        if: github.ref_type == 'tag'
+        id: live_integration_tests_readwrite
+        env:
+          GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RW_SECRET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }}
+        run: |
+          providerRoot="$(realpath $(pwd)/providers)"
+          sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}'
+          robot \
+            --variable "${sundryCfg}" \
+            --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
+            -d test/robot/reports/readwrite \
+            test/robot/stackql/live/readwrite || true
+
+      - name: Output from live readwrite functional tests
+        if: github.ref_type == 'tag'
+        run: |
+          cat test/robot/reports/readwrite/output.xml
+          python3 scripts/cicd/python/robot-parse.py  --robot-output-file test/robot/reports/readwrite/output.xml > test/robot/reports/readwrite/parsed_output.json
+
+      - name: Upload readonly traffic lights
+        uses: actions/[email protected]
+        if: success()
+        with:
+          name: local-registry-readwrite-traffic-lights
+          path: test/robot/reports/readwrite/parsed_output.json
+
+      - name: Display traffic lights
+        run: |
+          for i in $(ls test/robot/reports/*/parsed_output.json); do
+            echo "Traffic light for $i"
+            if [ -f "$i" ]; then
+              python3 scripts/cicd/python/display-parsed.py  --traffic-light-file $i
+            else
+              echo "File $i does not exist 🛑"
+            fi
+          done
+          echo "Traffic light for proxied"
+          python3 scripts/cicd/python/display-parsed.py  --traffic-light-file  stackql-core/test/robot/reports/proxied_parsed_output.json
+          echo "traffic lights completed"

.gitignore

@@ -8,3 +8,4 @@ stackql-zip
 .stackql/
 stackql-core/
 stackql-any-sdk/
+.venv/

requirements.txt

@@ -0,0 +1,4 @@
+boto3>=1.37.24
+PyYaml>=6.0.1
+requests==2.32.3
+robotframework==6.1.1

scripts/cicd/python/display-parsed.py

@@ -0,0 +1,58 @@
+import json
+
+from argparse import ArgumentParser, Namespace
+
+from typing import Iterable
+
+def parse_args() -> Namespace:
+    parser = ArgumentParser(description="Parse Robot Framework output XML file.")
+    parser.add_argument(
+        "--traffic-light-file",
+        help="Path to the traffic lights json file.",
+    )
+    return parser.parse_args()
+
+class _TrafficLightDisplayer(object):
+
+    _EMOJI_MAP = {
+        "red": "🛑",
+        "yellow": "🟡",
+        "orange": "🟠",
+        "grey": "⚪",
+        "blue": "🔵",
+        "green": "🟢",
+    }
+
+    def __init__(self, traffic_light_file: str):
+        self._traffic_light_file = traffic_light_file
+        with open(traffic_light_file, "r") as f:
+            data = json.load(f)
+        self._data = data
+
+    def _display(self, key: str, traffic_light :str) -> str:
+        return f'{key}: {self._EMOJI_MAP[traffic_light]}'
+
+    def _simple_assemble(self) -> Iterable[str]:
+        result = []
+        for key, value in self._data.get('tags', {}).items():
+            result.append(f'Tag: {self._display(key, value)}')
+        result.append(f'Total: {self._display("total", self._data.get("total"))}')
+        return result
+
+    def render(self) -> None:
+        result = self._simple_assemble()
+        print(f'Traffic Light Summary for file: {self._traffic_light_file}')
+        print("    ")
+        print("\n    ".join(result))
+        print("")
+
+
+def main() -> None:
+    args = parse_args()
+    displayer = _TrafficLightDisplayer(args.traffic_light_file)
+    displayer.render()
+
+if __name__ == "__main__":
+    main()
+
+