diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 11e36c48..908c2d4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,7 +57,7 @@ jobs: - name: "[SETUP] get updated providers" run: | - pip install -q -r scripts/setup/requirements.txt + pip install -q -r requirements.txt python scripts/setup/get-updated-providers.py - name: "[SETUP] prepare dist and test dirs" diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 6bfcb5f0..e6a72d62 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -1,7 +1,14 @@ name: Integration Testing and Analysis on: + pull_request: + branches: + - main + - dev push: + branches: + - main + - dev tags: - robot* - regression* @@ -15,8 +22,8 @@ env: STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }} jobs: - build-and-deploy: - name: build-and-deploy + regression-testing: + name: regression-testing runs-on: ubuntu-latest permissions: id-token: write @@ -79,15 +86,36 @@ jobs: go get ./... python3 cicd/python/build.py --build + - name: Build any-sdk cli from source + working-directory: stackql-any-sdk + run: | + + go get ./... + + go build -x -v \ + -o build/anysdk ./cmd/interrogate + - name: Parse tag id: parse_tag run: | - tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)" - echo "tag_obj: $tag_obj" - { - echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')" - echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')" - } | tee -a "$GITHUB_ENV" + if [ "${{ github.ref_type }}" = "tag" ]; then + tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)" + echo "tag_obj: $tag_obj" + { + echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')" + echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')" + } | tee -a "$GITHUB_ENV" + else + { + echo "IS_BRANCH=true" + } >> $GITHUB_ENV + fi + + + - name: Generate rewritten registry for simulations + working-directory: stackql-core + run: | + python3 test/python/registry-rewrite.py - name: Prepare load balancing materials @@ -118,8 +146,36 @@ jobs: openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365 openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365 openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 + + + - name: Start Core Test Mocks + working-directory: stackql-core + run: | + pgrep -f flask | xargs kill -9 || true + flask --app=./test/python/flask/gcp/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 & + flask --app=./test/python/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 & + + - name: Run any-sdk cli mocked testing + working-directory: stackql-core + run: | + export GCP_SERVICE_ACCOUNT_KEY="$(cat test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json)" + bucketsListIDs="$(${{ github.workspace }}/stackql-any-sdk/build/anysdk query \ + --svc-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/services/storage-v1.yaml" \ + --tls.allowInsecure \ + --prov-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/provider.yaml" \ + --resource buckets \ + --method list \ + --parameters '{ "project": "stackql-demo" }' \ + | jq -r '.items[].id')" + matchingBuckets="$(echo "${bucketsListIDs}" | grep "stackql-demo" )" + if [ "${matchingBuckets}" = "" ]; then + echo "Core Test Failed with no matching buckets" + exit 1 + else + echo "Core Test passed with matching buckets: $matchingBuckets" + fi - - name: Run core robot functional tests + - name: Run core proxied robot functional tests against local registry if: success() working-directory: stackql-core run: | @@ -130,12 +186,21 @@ jobs: --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \ --include registry \ -d test/robot/reports \ - test/robot/functional + test/robot/functional || true - - name: Output from functional tests + - name: Output from core proxied functional tests if: always() run: | cat stackql-core/test/robot/reports/output.xml + python3 scripts/cicd/python/robot-parse.py --robot-output-file stackql-core/test/robot/reports/output.xml > stackql-core/test/robot/reports/proxied_parsed_output.json + + - name: Upload core traffic lights + uses: actions/upload-artifact@v4.3.1 + if: success() + with: + name: proxied-core-traffic-lights + path: stackql-core/test/robot/reports/proxied_parsed_output.json + - name: Post core test cleanup run: | @@ -149,13 +214,21 @@ jobs: robot \ --variable "${sundryCfg}" \ --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \ - -d test/robot/reports \ - test/robot/stackql/mocked + -d test/robot/reports/mocked \ + test/robot/stackql/mocked || true - name: Output from local registry mocked functional tests if: always() run: | - cat test/robot/reports/output.xml + cat test/robot/reports/mocked/output.xml + python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/mocked/output.xml > test/robot/reports/mocked/parsed_output.json + + - name: Upload local registry mocked traffic lights + uses: actions/upload-artifact@v4.3.1 + if: success() + with: + name: local-registry-mocked-traffic-lights + path: test/robot/reports/mocked/parsed_output.json - name: Post registry mocked test cleanup run: | @@ -164,23 +237,74 @@ jobs: sudo cp /etc/hosts.bak /etc/hosts || true rm -f test/robot/reports/*.xml || true - - name: Run live robot functional tests + - name: Run live readonly robot functional tests if: success() - id: live_integration_tests + id: live_integration_tests_readonly env: GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RO_SECRET }} AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }} run: | providerRoot="$(realpath $(pwd)/providers)" - sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}' + sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}' robot \ --variable "${sundryCfg}" \ --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \ - -d test/robot/reports \ - test/robot/stackql/live + -d test/robot/reports/readonly \ + test/robot/stackql/live/readonly || true - - name: Output from live functional tests + - name: Output from live readonly functional tests if: always() run: | - cat test/robot/reports/output.xml + cat test/robot/reports/readonly/output.xml + python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/readonly/output.xml > test/robot/reports/readonly/parsed_output.json + + - name: Upload readonly traffic lights + uses: actions/upload-artifact@v4.3.1 + if: success() + with: + name: local-registry-readonly-traffic-lights + path: test/robot/reports/readonly/parsed_output.json + + - name: Run live readwrite robot functional tests + if: github.ref_type == 'tag' + id: live_integration_tests_readwrite + env: + GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RW_SECRET }} + AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }} + run: | + providerRoot="$(realpath $(pwd)/providers)" + sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}' + robot \ + --variable "${sundryCfg}" \ + --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \ + -d test/robot/reports/readwrite \ + test/robot/stackql/live/readwrite || true + + - name: Output from live readwrite functional tests + if: github.ref_type == 'tag' + run: | + cat test/robot/reports/readwrite/output.xml + python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/readwrite/output.xml > test/robot/reports/readwrite/parsed_output.json + + - name: Upload readonly traffic lights + uses: actions/upload-artifact@v4.3.1 + if: success() + with: + name: local-registry-readwrite-traffic-lights + path: test/robot/reports/readwrite/parsed_output.json + + - name: Display traffic lights + run: | + for i in $(ls test/robot/reports/*/parsed_output.json); do + echo "Traffic light for $i" + if [ -f "$i" ]; then + python3 scripts/cicd/python/display-parsed.py --traffic-light-file $i + else + echo "File $i does not exist 🛑" + fi + done + echo "Traffic light for proxied" + python3 scripts/cicd/python/display-parsed.py --traffic-light-file stackql-core/test/robot/reports/proxied_parsed_output.json + echo "traffic lights completed" diff --git a/.gitignore b/.gitignore index bf80845f..ba377109 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ stackql-zip .stackql/ stackql-core/ stackql-any-sdk/ +.venv/ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..07b66f1d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +boto3>=1.37.24 +PyYaml>=6.0.1 +requests==2.32.3 +robotframework==6.1.1 \ No newline at end of file diff --git a/scripts/cicd/python/display-parsed.py b/scripts/cicd/python/display-parsed.py new file mode 100644 index 00000000..d4df475d --- /dev/null +++ b/scripts/cicd/python/display-parsed.py @@ -0,0 +1,58 @@ +import json + +from argparse import ArgumentParser, Namespace + +from typing import Iterable + +def parse_args() -> Namespace: + parser = ArgumentParser(description="Parse Robot Framework output XML file.") + parser.add_argument( + "--traffic-light-file", + help="Path to the traffic lights json file.", + ) + return parser.parse_args() + +class _TrafficLightDisplayer(object): + + _EMOJI_MAP = { + "red": "🛑", + "yellow": "🟡", + "orange": "🟠", + "grey": "⚪", + "blue": "🔵", + "green": "🟢", + } + + def __init__(self, traffic_light_file: str): + self._traffic_light_file = traffic_light_file + with open(traffic_light_file, "r") as f: + data = json.load(f) + self._data = data + + def _display(self, key: str, traffic_light :str) -> str: + return f'{key}: {self._EMOJI_MAP[traffic_light]}' + + def _simple_assemble(self) -> Iterable[str]: + result = [] + for key, value in self._data.get('tags', {}).items(): + result.append(f'Tag: {self._display(key, value)}') + result.append(f'Total: {self._display("total", self._data.get("total"))}') + return result + + def render(self) -> None: + result = self._simple_assemble() + print(f'Traffic Light Summary for file: {self._traffic_light_file}') + print(" ") + print("\n ".join(result)) + print("") + + +def main() -> None: + args = parse_args() + displayer = _TrafficLightDisplayer(args.traffic_light_file) + displayer.render() + +if __name__ == "__main__": + main() + + diff --git a/scripts/cicd/python/robot-parse.py b/scripts/cicd/python/robot-parse.py new file mode 100644 index 00000000..475aeecf --- /dev/null +++ b/scripts/cicd/python/robot-parse.py @@ -0,0 +1,112 @@ + + +# per https://docs.robotframework.org/docs/parsing_results + +from robot.api import ExecutionResult +import sys +import json + +from argparse import ArgumentParser, Namespace + + +def parse_args() -> Namespace: + parser = ArgumentParser(description="Parse Robot Framework output XML file.") + parser.add_argument( + "--robot-output-file", + help="Path to the Robot Framework output XML file to parse.", + ) + return parser.parse_args() + + + + +class _CustomSummary(object): + + def __init__(self, result_file_path: str): + self._result_file_path: str = result_file_path + self._result = ExecutionResult(self._result_file_path) + + @property + def statistics(self): + return self._result.statistics + +class _CustomAnalyzer(object): + + def __init__(self, summary: _CustomSummary): + self._summary: _CustomSummary = summary + + + def get_statistics(self) -> dict: + stats = self._summary.statistics + tag_dict = {} + for k, v in stats.tags.tags.items(): + tag_dict[k] = { + "failed": v.failed, + "passed": v.passed, + "skipped": v.skipped, + "elapsed": v.elapsed, + } + summary_dict = { + "failed": stats.total.failed, + "passed": stats.total.passed, + "skipped": stats.total.skipped, + "total": stats.total.total, + "message": stats.total.message, + "tags_expanded": tag_dict, + # "suite": suite_dict, + } + return summary_dict + + +class _DefaultTrafficLightAnalytics(object): + + def __init__(self, analyzer: _CustomAnalyzer): + self._analyzer: _CustomAnalyzer = analyzer + + def _get_traffic_light(self, v: dict) -> str: + total = v["passed"] + v["failed"] + if total == 0: + return "grey" + fail_ratio = v["failed"] / total if total > 0 else 0 + if fail_ratio == 0: + return "green" + elif v["failed"] > 0 and v["passed"] == 0: + return "red" + elif fail_ratio < 0.3: + return "yellow" + elif fail_ratio < 0.7: + return "orange" + return "red" + + def get_result_traffic_lights(self) -> dict: + rv = {} + tags_dict = {} + stats = self._analyzer.get_statistics() + for k, v in stats.get("tags_expanded", {}).items(): + tags_dict[k] = self._get_traffic_light(v) + rv['tags'] = tags_dict + rv['total'] = self._get_traffic_light(stats) + return rv + + + + +def main(): + args = parse_args() + summary = _CustomSummary(args.robot_output_file) + analyzer = _CustomAnalyzer(summary) + traffic_lights = _DefaultTrafficLightAnalytics(analyzer) + traffic_lights = traffic_lights.get_result_traffic_lights() + print(f"{json.dumps(traffic_lights, sort_keys=True, indent=2)}") + # analysis_dict = analyzer.get_statistics() + # print(f"{json.dumps(analysis_dict, sort_keys=True, indent=2)}") + + + +if __name__ == "__main__": + main() + +# summary = _CustomSummary('output.xml') +# stats = summary.statistics +# print(f"Number of Failed Tests: {stats.total.failed}") +# print(f"Total number of Tests: {stats.total.passed}") \ No newline at end of file diff --git a/scripts/local/ci/01-gather.sh b/scripts/local/ci/01-gather.sh new file mode 100755 index 00000000..026bfb21 --- /dev/null +++ b/scripts/local/ci/01-gather.sh @@ -0,0 +1,14 @@ +#! /usr/bin/env bash + +>&2 echo "requires git version >= 2.45" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)" + +cd "${REPOSITORY_ROOT_DIR}" + +git clone --revision=refs/heads/main https://github.com/stackql/stackql.git stackql-core + +git clone --revision=refs/heads/main https://github.com/stackql/any-sdk.git stackql-any-sdk + diff --git a/scripts/local/ci/02-setup.sh b/scripts/local/ci/02-setup.sh new file mode 100755 index 00000000..bfa24113 --- /dev/null +++ b/scripts/local/ci/02-setup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +>&2 echo "requires all of requirements.txt" + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)" + +STACKQL_CORE_DIR="${REPOSITORY_ROOT_DIR}/stackql-core" + +if [ ! -d "${STACKQL_CORE_DIR}/.venv" ]; then + >&2 echo "No existing virtual environment, creating one..." + >&2 echo "Creating virtual environment in ${STACKQL_CORE_DIR}/.venv" + python -m venv "${STACKQL_CORE_DIR}/.venv" + >&2 echo "Virtual environment created." +fi + +source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate" + +pip install -r "${STACKQL_CORE_DIR}/cicd/requirements.txt" + +pip install -r "${REPOSITORY_ROOT_DIR}/requirements.txt" + +cd "${STACKQL_CORE_DIR}" + +python cicd/python/build.py --build + +python test/python/registry-rewrite.py + +openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365 +openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365 +openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365 + + + + diff --git a/scripts/local/ci/03-run-live-readwrite.sh b/scripts/local/ci/03-run-live-readwrite.sh new file mode 100755 index 00000000..3a911de3 --- /dev/null +++ b/scripts/local/ci/03-run-live-readwrite.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +>&2 echo "requires all of requirements.txt" + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)" + +_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-readwrite.sh" + +if [ -f "${_SEC_FILE}" ]; then + source "${_SEC_FILE}" +fi + +if [ "${GOOGLE_CREDENTIALS}" = "" ]; then + >&2 echo "Required env var GOOGLE_CREDENTIALS is not set" + exit 1 +fi + +cd "${REPOSITORY_ROOT_DIR}" + +source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate" + +robot -d test/robot/reports/readwrite test/robot/stackql/live/readwrite + diff --git a/scripts/local/ci/04-run-live-readonly.sh b/scripts/local/ci/04-run-live-readonly.sh new file mode 100755 index 00000000..36ea4398 --- /dev/null +++ b/scripts/local/ci/04-run-live-readonly.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +>&2 echo "requires all of requirements.txt" + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)" + +_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-readwrite.sh" + +if [ -f "${_SEC_FILE}" ]; then + source "${_SEC_FILE}" +fi + +if [ -f "scripts/sec/aws-ro-stackql.sh" ]; then + source "scripts/sec/aws-ro-stackql.sh" +fi + +if [ "${AWS_ACCESS_KEY_ID}" = "" ]; then + >&2 echo "Required env var AWS_ACCESS_KEY_ID is not set" + exit 1 +fi + +if [ "${AWS_SECRET_ACCESS_KEY}" = "" ]; then + >&2 echo "Required env var AWS_SECRET_ACCESS_KEY is not set" + exit 1 +fi + +if [ "${GOOGLE_CREDENTIALS}" = "" ]; then + >&2 echo "Required env var GOOGLE_CREDENTIALS is not set" + exit 1 +fi + +cd "${REPOSITORY_ROOT_DIR}" + +source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate" + +robot -d test/robot/reports/readonly test/robot/stackql/live/readonly + diff --git a/test/robot/reports/.gitignore b/scripts/sec/.gitignore similarity index 100% rename from test/robot/reports/.gitignore rename to scripts/sec/.gitignore diff --git a/scripts/setup/requirements.txt b/scripts/setup/requirements.txt deleted file mode 100644 index 1db657b6..00000000 --- a/scripts/setup/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -boto3 \ No newline at end of file diff --git a/test/robot/reports/mocked/.gitignore b/test/robot/reports/mocked/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/test/robot/reports/mocked/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/test/robot/reports/readonly/.gitignore b/test/robot/reports/readonly/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/test/robot/reports/readonly/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/test/robot/reports/readwrite/.gitignore b/test/robot/reports/readwrite/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/test/robot/reports/readwrite/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/test/robot/stackql/live/live.robot b/test/robot/stackql/live/live.robot deleted file mode 100644 index cadbf742..00000000 --- a/test/robot/stackql/live/live.robot +++ /dev/null @@ -1,18 +0,0 @@ -*** Settings *** -Resource ${CURDIR}/stackql.resource -Test Teardown Stackql Per Test Teardown - -*** Test Cases *** -Google Buckets List With Date Logic Contains Exemplifies Use of SQLite Math Functions - Pass Execution If "${SQL_BACKEND}" == "postgres_tcp" This is a valid case where the test is targetted at SQLite only - ${inputStr} = Catenate - ... SELECT name, timeCreated, floor(julianday('2025-01-27')-julianday(timeCreated)) as days_since_ceiling - ... FROM google.storage.buckets - ... WHERE project = 'stackql-demo' - ... order by name desc - ... ; - Stock Stackql Exec Inline Contains Both Streams - ... ${inputStr} - ... days_since_ceiling - ... ${EMPTY} - ... Google-Buckets-List-With-Date-Logic-Contains-Exemplifies-Use-of-SQLite-Math-Functions diff --git a/test/robot/stackql/live/__init__.robot b/test/robot/stackql/live/readonly/__init__.robot similarity index 100% rename from test/robot/stackql/live/__init__.robot rename to test/robot/stackql/live/readonly/__init__.robot diff --git a/test/robot/stackql/live/readonly/__pycache__/readonly_variables.cpython-313.pyc b/test/robot/stackql/live/readonly/__pycache__/readonly_variables.cpython-313.pyc new file mode 100644 index 00000000..35dbc01b Binary files /dev/null and b/test/robot/stackql/live/readonly/__pycache__/readonly_variables.cpython-313.pyc differ diff --git a/test/robot/stackql/live/readonly/live_readonly.robot b/test/robot/stackql/live/readonly/live_readonly.robot new file mode 100644 index 00000000..86a9fb65 --- /dev/null +++ b/test/robot/stackql/live/readonly/live_readonly.robot @@ -0,0 +1,46 @@ +*** Settings *** +Resource ${CURDIR}/stackql.resource +Test Teardown Stackql Per Test Teardown + +*** Test Cases *** +Simple Google Buckets List With Date Logic Contains Exemplifies Use of SQLite Math Functions + Pass Execution If "${SQL_BACKEND}" == "postgres_tcp" This is a valid case where the test is targetted at SQLite only + [Tags] google storage buckets gooogle.storage google.storage.buckets tier_1 + ${inputStr} = Catenate + ... SELECT name, timeCreated, floor(julianday('2025-01-27')-julianday(timeCreated)) as days_since_ceiling + ... FROM google.storage.buckets + ... WHERE project = 'stackql-demo' + ... order by name desc + ... ; + Stock Stackql Exec Inline Contains Both Streams + ... ${inputStr} + ... days_since_ceiling + ... ${EMPTY} + ... Google-Buckets-List-With-Date-Logic-Contains-Exemplifies-Use-of-SQLite-Math-Functions + +AWS Route53 List Record Sets Simple + [Documentation] It is fine for this to dump 404 infor to stderr. So long as the empty reusult is represented with a header row, all good. + [Tags] aws route53 resource_record_sets aws.route53 aws.route53.resource_record_sets tier_1 + ${inputStr} = Catenate + ... select Name, Type, ResourceRecords + ... from aws.route53.resource_record_sets + ... where Id = '${AWS_RECORD_SET_ID}' + ... and region = '${AWS_RECORD_SET_REGION}' + ... order by Name, Type + ... ; + Stock Stackql Exec Inline Contains Both Streams + ... ${inputStr} + ... ResourceRecords + ... ${EMPTY} + ... AWS-Route53-List-Record-Sets-Simple + +# AWS IAM Users Subquery Left Joined With Aliasing and Name Collision +# [Documentation] AWS IAM Users Complex Query. Acceptable to hardcoode region for global resource. +# [Tags] aws iam users aws.iam aws.iam.users tier_1 +# ${inputStr} = Catenate +# ... select u1.UserName, u.UserId, u.Arn, u1.region from ( select Arn, UserName, UserId from aws.iam.users where region = 'us-east-1' ) u inner join aws.iam.users u1 on u1.Arn = u.Arn where region = 'us-east-1' order by u1.UserName desc; +# Stock Stackql Exec Inline Contains Both Streams +# ... ${inputStr} +# ... UserName +# ... ${EMPTY} +# ... AWS-IAM-Users-Subquery-Left-Joined-With-Aliasing-and-Name-Collision diff --git a/test/robot/stackql/live/readonly/readonly_variables.py b/test/robot/stackql/live/readonly/readonly_variables.py new file mode 100644 index 00000000..7df5ce87 --- /dev/null +++ b/test/robot/stackql/live/readonly/readonly_variables.py @@ -0,0 +1,46 @@ + +import json +import os +from copy import deepcopy + + +_EMPTY_GCS_BUCKET_CHECK = """|------|----------------|----------------| +| name | softDeleteTime | hardDeleteTime | +|------|----------------|----------------|""" + +_REPOSITORY_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..")) +_CORE_REPOSITORY_ROOT = os.path.abspath(os.path.join(_REPOSITORY_ROOT, "stackql-core")) +_REGISTRY_PATH = os.path.abspath(os.path.join(_REPOSITORY_ROOT, "providers")) + +def _get_expected_gcs_bucket_check( + gcs_bucket_name: str, + gcp_project: str +) -> str: + """ + Expected GCS bucket check. + """ + return '' + \ + '|------------------------|----------------|----------------|\n' + \ + '| name | softDeleteTime | hardDeleteTime |\n' + \ + '|------------------------|----------------|----------------|\n' + \ + '| stackql-demo-bucket-01 | null | null |\n' + \ + '|------------------------|----------------|----------------|' + + +def get_variables( + sundry_config: str # a json string with arbitrary config +) -> dict: + """ + Robot variables. + """ + sundry_config_dict: dict = json.loads(sundry_config) + return { + "sundry_config_dict": sundry_config_dict, + "GCS_BUCKET_NAME": sundry_config_dict["GCS_BUCKET_NAME"], + "GCP_PROJECT": sundry_config_dict["GCP_PROJECT"], + "AWS_RECORD_SET_ID": sundry_config_dict["AWS_RECORD_SET_ID"], + "AWS_RECORD_SET_REGION": sundry_config_dict["AWS_RECORD_SET_REGION"], + "REPOSITORY_ROOT": _REPOSITORY_ROOT, + "registry_path": _REGISTRY_PATH, + "CORE_REPOSITORY_ROOT": _CORE_REPOSITORY_ROOT, + } \ No newline at end of file diff --git a/test/robot/stackql/live/stackql.resource b/test/robot/stackql/live/readonly/stackql.resource similarity index 85% rename from test/robot/stackql/live/stackql.resource rename to test/robot/stackql/live/readonly/stackql.resource index 89a7aed6..86e0b899 100644 --- a/test/robot/stackql/live/stackql.resource +++ b/test/robot/stackql/live/readonly/stackql.resource @@ -1,20 +1,28 @@ *** Variables *** -${REPOSITORY_ROOT} ${CURDIR}${/}..${/}..${/}..${/}.. +${REPOSITORY_ROOT} ${CURDIR}${/}..${/}..${/}..${/}..${/}.. ${CORE_REPOSITORY_ROOT} ${REPOSITORY_ROOT}${/}stackql-core ${CORE_LIB_HOME} ${CORE_REPOSITORY_ROOT}${/}test${/}robot${/}lib -${LOCAL_LIB_HOME} ${CURDIR}${/}..${/}..${/}lib ${EXECUTION_PLATFORM} native # to be overridden from command line, eg "docker" ${SQL_BACKEND} sqlite_embedded # to be overridden from command line, eg "postgres_tcp" ${IS_WSL} false # to be overridden from command line, with string "true" ${SHOULD_RUN_DOCKER_EXTERNAL_TESTS} false # to be overridden from command line, with string "true" ${CONCURRENCY_LIMIT} 1 # to be overridden from command line, with integer value, -1 for no limit ${USE_STACKQL_PREINSTALLED} false # to be overridden from command line, with string "true" -${SUNDRY_CONFIG} {} # to be overridden from command line, with string value ${CORE_PREFIX} stackql-core +${SUNDRY_CONFIG}= SEPARATOR= +... { +... "GCS_BUCKET_NAME": "stackql-demo-bucket-01", +... "GCP_PROJECT": "stackql-demo", +... "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", +... "AWS_RECORD_SET_REGION": "us-east-1", +... "registry_path": "${CURDIR}${/}..${/}..${/}..${/}..${/}..${/}providers" +... } *** Settings *** Library Process -Library OperatingSystem +Library OperatingSystem +# Variable first defined clobbers later defined therefore most specific variable file first. +Variables ${CURDIR}${/}readonly_variables.py ${SUNDRY_CONFIG} Variables ${CORE_LIB_HOME}/stackql_context.py ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${USE_STACKQL_PREINSTALLED} ... ${SUNDRY_CONFIG} Library Process diff --git a/test/robot/stackql/live/tmp/.gitignore b/test/robot/stackql/live/readonly/tmp/.gitignore similarity index 100% rename from test/robot/stackql/live/tmp/.gitignore rename to test/robot/stackql/live/readonly/tmp/.gitignore diff --git a/test/robot/stackql/live/readwrite/__init__.robot b/test/robot/stackql/live/readwrite/__init__.robot new file mode 100644 index 00000000..cb2865fe --- /dev/null +++ b/test/robot/stackql/live/readwrite/__init__.robot @@ -0,0 +1,5 @@ +*** Settings *** +Resource ${CURDIR}/stackql.resource +Suite Setup Prepare StackQL Environment +Suite Teardown Terminate All Processes kill=True + diff --git a/test/robot/stackql/live/readwrite/__pycache__/readwrite_variables.cpython-313.pyc b/test/robot/stackql/live/readwrite/__pycache__/readwrite_variables.cpython-313.pyc new file mode 100644 index 00000000..9e403ab0 Binary files /dev/null and b/test/robot/stackql/live/readwrite/__pycache__/readwrite_variables.cpython-313.pyc differ diff --git a/test/robot/stackql/live/readwrite/live_readwrite.robot b/test/robot/stackql/live/readwrite/live_readwrite.robot new file mode 100644 index 00000000..4886168f --- /dev/null +++ b/test/robot/stackql/live/readwrite/live_readwrite.robot @@ -0,0 +1,35 @@ +*** Settings *** +Resource ${CURDIR}/stackql.resource +Test Teardown Stackql Per Test Teardown + +*** Test Cases *** +Google Buckets Lifecycle + [Documentation] This test case inserts a bucket ("row") into the google.storage.buckets "table", checks the row was inserted, deletes the row, and checks the row was deleted. + [Tags] google storage buckets gooogle.storage google.storage.buckets + ${insertInputStr} = Catenate + ... insert into google.storage.buckets(data__name, project) + ... select '${GCS_BUCKET_NAME}', '${GCP_PROJECT}'; + ${checkInputStr} = Catenate + ... select name, "softDeleteTime", "hardDeleteTime" from google.storage.buckets where bucket = '${GCS_BUCKET_NAME}'; + ${deleteInputStr} = Catenate + ... delete from google.storage.buckets where bucket = '${GCS_BUCKET_NAME}'; + Stock Stackql Exec Inline Equals Both Streams + ... ${insertInputStr} + ... ${EMPTY} + ... The operation was despatched successfully + ... Google-Buckets-Lifecycle-Insert + Stock Stackql Exec Inline Equals Both Streams + ... ${checkInputStr} + ... ${EXPECTED_GCS_BUCKET_CHECK} + ... ${EMPTY} + ... Google-Buckets-Lifecycle-Post-Insert-Check + Stock Stackql Exec Inline Equals Both Streams + ... ${deleteInputStr} + ... ${EMPTY} + ... The operation was despatched successfully + ... Google-Buckets-Lifecycle-Delete + Sleep 5s + Stock Stackql Exec Inline Equals Stdout + ... ${checkInputStr} + ... ${EXPECTED_EMPTY_GCS_BUCKET_CHECK} + ... Google-Buckets-Lifecycle-Post-Delete-Check diff --git a/test/robot/stackql/live/readwrite/readwrite_variables.py b/test/robot/stackql/live/readwrite/readwrite_variables.py new file mode 100644 index 00000000..13f7241a --- /dev/null +++ b/test/robot/stackql/live/readwrite/readwrite_variables.py @@ -0,0 +1,56 @@ + +import json + +import os + +from copy import deepcopy + +_REPOSITORY_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..")) +_CORE_REPOSITORY_ROOT = os.path.abspath(os.path.join(_REPOSITORY_ROOT, "stackql-core")) +_REGISTRY_PATH = os.path.abspath(os.path.join(_REPOSITORY_ROOT, "providers")) +_REGISTRY_NO_VERIFY_CFG_STR = f'{{ "url": "file://{_REGISTRY_PATH}", "verifyConfig": {{ "nopVerify": true }} }}' + +_EMPTY_GCS_BUCKET_CHECK = """|------|----------------|----------------| +| name | softDeleteTime | hardDeleteTime | +|------|----------------|----------------|""" + +def _get_expected_gcs_bucket_check( + gcs_bucket_name: str, + gcp_project: str +) -> str: + """ + Expected GCS bucket check. + """ + return '' + \ + '|------------------------|----------------|----------------|\n' + \ + '| name | softDeleteTime | hardDeleteTime |\n' + \ + '|------------------------|----------------|----------------|\n' + \ + '| stackql-demo-bucket-02 | null | null |\n' + \ + '|------------------------|----------------|----------------|' + + +def get_variables( + sundry_config: str # a json string with arbitrary config +) -> dict: + """ + Robot variables. + """ + sundry_config_dict: dict = json.loads(sundry_config) + adornment = deepcopy(sundry_config_dict) + base_rv = { + "sundry_config_dict": adornment, + "GCS_BUCKET_NAME": sundry_config_dict["GCS_BUCKET_NAME"], + "REPOSITORY_ROOT": _REPOSITORY_ROOT, + "registry_path": _REGISTRY_PATH, + "CORE_REPOSITORY_ROOT": _CORE_REPOSITORY_ROOT, + "GCP_PROJECT": sundry_config_dict["GCP_PROJECT"], + "EXPECTED_GCS_BUCKET_CHECK": _get_expected_gcs_bucket_check( + sundry_config_dict["GCS_BUCKET_NAME"], + sundry_config_dict["GCP_PROJECT"] + ), + "EXPECTED_EMPTY_GCS_BUCKET_CHECK": _EMPTY_GCS_BUCKET_CHECK, + # "REGISTRY_NO_VERIFY_CFG_STR": _REGISTRY_NO_VERIFY_CFG_STR, + } + for k, v in base_rv.items(): + sundry_config_dict[k] = v + return sundry_config_dict \ No newline at end of file diff --git a/test/robot/stackql/live/readwrite/stackql.resource b/test/robot/stackql/live/readwrite/stackql.resource new file mode 100644 index 00000000..2c5209d5 --- /dev/null +++ b/test/robot/stackql/live/readwrite/stackql.resource @@ -0,0 +1,93 @@ +*** Variables *** +${CORE_LIB_HOME} ${CURDIR}${/}..${/}..${/}..${/}..${/}..${/}stackql-core${/}test${/}robot${/}lib +${EXECUTION_PLATFORM} native # to be overridden from command line, eg "docker" +${SQL_BACKEND} sqlite_embedded # to be overridden from command line, eg "postgres_tcp" +${IS_WSL} false # to be overridden from command line, with string "true" +${SHOULD_RUN_DOCKER_EXTERNAL_TESTS} false # to be overridden from command line, with string "true" +${CONCURRENCY_LIMIT} 1 # to be overridden from command line, with integer value, -1 for no limit +${USE_STACKQL_PREINSTALLED} false # to be overridden from command line, with string "true" +${CORE_PREFIX} stackql-core +${SUNDRY_CONFIG}= SEPARATOR= +... { +... "GCS_BUCKET_NAME": "stackql-demo-bucket-02", +... "GCP_PROJECT": "stackql-demo", +... "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", +... "AWS_RECORD_SET_REGION": "us-east-1", +... "registry_path": "${CURDIR}${/}..${/}..${/}..${/}..${/}..${/}providers" +... } + +*** Settings *** +Library Process +Library OperatingSystem +Variables ${CURDIR}${/}readwrite_variables.py ${SUNDRY_CONFIG} +Variables ${CORE_LIB_HOME}${/}stackql_context.py ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${USE_STACKQL_PREINSTALLED} +... ${SUNDRY_CONFIG} +Library Process +Library OperatingSystem +Library String +Library ${CORE_LIB_HOME}/StackQLInterfaces.py ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${CONCURRENCY_LIMIT} +Library ${CORE_LIB_HOME}/CloudIntegration.py + +*** Keywords *** + + +Prepare StackQL Environment + Sleep 10s + +Stock Stackql Exec Inline Equals Stdout + [Arguments] ${inputStr} ${outputStr} ${tmpFileTrunk} + Should Stackql Exec Inline Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... {} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... stdout=${CURDIR}${/}tmp${/}${tmpFileTrunk}.tmp + ... stderr=${CURDIR}${/}tmp${/}${tmpFileTrunk}-stderr.tmp + +Stock Stackql Exec Inline Equals Both Streams + [Arguments] ${inputStr} ${outputStr} ${outputStderrStr} ${tmpFileTrunk} + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... {} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${outputStderrStr} + ... stdout=${CURDIR}${/}tmp${/}${tmpFileTrunk}.tmp + ... stderr=${CURDIR}${/}tmp${/}${tmpFileTrunk}-stderr.tmp + +Stock Stackql Exec Inline Contains Both Streams + [Arguments] ${inputStr} ${outputStr} ${outputStderrStr} ${tmpFileTrunk} + Should Stackql Exec Inline Contain Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... {} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${outputStderrStr} + ... stdout=${CURDIR}${/}tmp${/}${tmpFileTrunk}.tmp + ... stderr=${CURDIR}${/}tmp${/}${tmpFileTrunk}-stderr.tmp + + +Stackql Per Test Teardown + IF "${EXECUTION_PLATFORM}" == "docker" and "${SQL_BACKEND}" == "postgres_tcp" + ${res} = Run Process bash \-c docker kill $(docker ps \-\-filter name\=execrun \-q) + Log Container killed + # Should Be Equal As Integers ${res.rc} 0 + ${restwo} = Run Process bash \-c docker rm $(docker ps \-\-filter status\=exited \-q) + Log Container removed + # Should Be Equal As Integers ${restwo.rc} 0 + END diff --git a/test/robot/stackql/live/readwrite/tmp/.gitignore b/test/robot/stackql/live/readwrite/tmp/.gitignore new file mode 100644 index 00000000..c96a04f0 --- /dev/null +++ b/test/robot/stackql/live/readwrite/tmp/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file