- Added robot test Select Paginated Star From Transformed JSON Response Body.

- Added robot test `Select Paginated Projection From Transformed JSON Response Body`.

Author: general-kroll-4-life

.vscode/launch.json

@@ -175,6 +175,7 @@
         "select BackupId, BackupState from aws.cloudhsm.backups where region = 'rubbish-region' order by BackupId;",
         "select  rings.projectsId as project,  rings.locationsId as locale,  split_part(rings.name, '/', -1) as key_ring_name,  split_part(keys.name, '/', -1) as key_name,  json_extract(keys.\"versionTemplate\", '$.algorithm') as key_algorithm,  json_extract(keys.\"versionTemplate\", '$.protectionLevel') as key_protection_level  from  google.cloudkms.key_rings rings  inner join google.cloudkms.crypto_keys keys  on  keys.keyRingsId = split_part(rings.name, '/', -1)  and keys.projectsId = rings.projectsId  and keys.locationsId = rings.locationsId  where  rings.projectsId in ('testing-project', 'testing-project-two', 'testing-project-three')  and rings.locationsId in ('global', 'australia-southeast1', 'australia-southeast2')  order by project, locale, key_name ;",
         "delete from aws.cloud_control.resources  where  region = 'ap-southeast-1'  and data__TypeName = 'AWS::Logs::LogGroup'  and data__Identifier = 'LogGroupResourceExampleThird' ;",
+        "select * from azure.network.virtual_networks where subscriptionId = 'subid';",
       ],
       "default": "show providers;"
     },
@@ -183,13 +184,13 @@
       "id": "registryString",
       "description": "Registry Configuration",
       "options": [
-        "{ \"url\": \"file://${workspaceFolder}/test/registry-sandbox\", \"localDocRoot\": \"${workspaceFolder}/test/registry-sandbox\", \"verifyConfig\": { \"nopVerify\": true } }",
-        "{ \"url\": \"file://${workspaceFolder}/test/registry\", \"localDocRoot\": \"${workspaceFolder}/test/registry\", \"verifyConfig\": { \"nopVerify\": true } }",
-        "{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }",
-        "{ \"url\": \"file://${workspaceFolder}/test/registry-mocked-native\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked-native\", \"verifyConfig\": { \"nopVerify\": true } }",
-        "{ \"url\": \"file://${workspaceFolder}/test/registry-advanced\", \"localDocRoot\": \"${workspaceFolder}/test/registry-advanced\", \"verifyConfig\": { \"nopVerify\": true } }",
-        "{ \"url\": \"file://${workspaceFolder}/build/.stackql\", \"localDocRoot\": \"${workspaceFolder}/build/.stackql\", \"verifyConfig\": { \"nopVerify\": true } }",
-        "{ \"url\": \"file://${workspaceFolder}/docs/examples/empty-registry\", \"localDocRoot\": \"${workspaceFolder}/docs/examples/empty-registry\" }",
+        "{ \"url\": \"file:///Users/admin/stackql/stackql-devel/test/registry-sandbox\", \"localDocRoot\": \"${workspaceFolder}/test/registry-sandbox\", \"verifyConfig\": { \"nopVerify\": true } }",
+        "{ \"url\": \"file:///Users/admin/stackql/stackql-devel/test/registry\", \"localDocRoot\": \"${workspaceFolder}/test/registry\", \"verifyConfig\": { \"nopVerify\": true } }",
+        "{ \"url\": \"file:///Users/admin/stackql/stackql-devel/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }",
+        "{ \"url\": \"file:///Users/admin/stackql/stackql-devel/test/registry-mocked-native\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked-native\", \"verifyConfig\": { \"nopVerify\": true } }",
+        "{ \"url\": \"file:///Users/admin/stackql/stackql-devel/test/registry-advanced\", \"localDocRoot\": \"${workspaceFolder}/test/registry-advanced\", \"verifyConfig\": { \"nopVerify\": true } }",
+        "{ \"url\": \"file:///Users/admin/stackql/stackql-devel/build/.stackql\", \"localDocRoot\": \"${workspaceFolder}/build/.stackql\", \"verifyConfig\": { \"nopVerify\": true } }",
+        "{ \"url\": \"file:///Users/admin/stackql/stackql-devel/docs/examples/empty-registry\", \"localDocRoot\": \"${workspaceFolder}/docs/examples/empty-registry\" }",
         "{ \"url\": \"https://cdn.statically.io/gh/stackql/stackql-provider-registry/main/providers\", \"localDocRoot\": \"${workspaceFolder}/test/registry\" }",
         "{ \"url\": \"https://cdn.statically.io/gh/stackql/stackql-provider-registry/dev/providers\" }",
         "{ \"url\": \"https://registry-dev.stackql.app/providers\" }",
@@ -203,7 +204,7 @@
       "description": "Auth Input arg String",
       "default": "{}",
       "options": [
-        "{ \"local_openssl\": { \"type\": \"null_auth\"}, \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" },  \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
+        "{ \"local_openssl\": { \"type\": \"null_auth\"}, \"azure\": { \"type\": \"null_auth\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" },  \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
         "{ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" },s \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/netlify/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/k8s/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/sumologic/sumologic-token.txt\", \"type\": \"basic\" } }",
         "{ \"pgi\": { \"type\": \"sql_data_source::postgres\", \"sqlDataSource\": { \"dsn\": \"postgres://stackql:[email protected]:8432\" } }, \"azure\": { \"type\": \"azure_default\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
         "{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" }, \"azure\": {\"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsenvvar\": \"AZ_ACCESS_TOKEN\"} }",

cicd/scripts/context.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+export REPOSITORY_ROOT="$(realpath ${CUR_DIR}/../..)"
+
+
+

cicd/scripts/testing-env.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${CUR_DIR}/context.sh"
+
+export REPOSITORY_ROOT="${REPOSITORY_ROOT}"
+export workspaceFolder="${REPOSITORY_ROOT}"
+
+export OKTA_SECRET_KEY='some-dummy-api-key'
+export GITHUB_SECRET_KEY='some-dummy-github-key'
+export K8S_SECRET_KEY='some-k8s-token'
+export AZ_ACCESS_TOKEN='dummy_azure_token'
+export SUMO_CREDS='somesumologictoken'
+export DIGITALOCEAN_TOKEN='somedigitaloceantoken'
+export DUMMY_DIGITALOCEAN_USERNAME='myusername'
+export DUMMY_DIGITALOCEAN_PASSWORD='mypassword'
+
+
+googleCredentialsFilePath="${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json"
+
+export stackqlMockedRegistryStr="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }"
+
+export stackqlAuthStr="{\"google\": {\"credentialsfilepath\": \"${googleCredentialsFilePath}\", \"type\": \"service_account\"}, \"okta\": {\"credentialsenvvar\": \"OKTA_SECRET_KEY\", \"type\": \"api_key\"}, \"aws\": {\"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"/Users/admin/stackql/stackql-devel/test/robot/functional/../../../test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"NON_SECRET\"}, \"github\": {\"type\": \"basic\", \"credentialsenvvar\": \"GITHUB_SECRET_KEY\"}, \"k8s\": {\"credentialsenvvar\": \"K8S_SECRET_KEY\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \"}, \"azure\": {\"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsenvvar\": \"AZ_ACCESS_TOKEN\"}, \"sumologic\": {\"type\": \"basic\", \"credentialsenvvar\": \"SUMO_CREDS\"}, \"digitalocean\": {\"type\": \"bearer\", \"username\": \"myusername\", \"password\": \"mypassword\"}}"
+
+
+
+

test/python/stackql_test_tooling/flask/README.md

@@ -81,17 +81,25 @@ flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask
 With embedded `sqlite` (default), from the root of this repository:
 
 ```bash
-export workspaceFolder="$(pwd)"
+source cicd/scripts/testing-env.sh
 
-stackql --registry="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }" --tls.allowInsecure shell
+stackql --registry="${stackqlMockedRegistryStr}" --auth="${stackqlAuthStr}" --tls.allowInsecure shell
 ```
 
 With `postgres`, from the root of this repository:
 
 ```bash
 docker compose -f docker-compose-externals.yml up postgres_stackql -d
 
-export workspaceFolder="$(pwd)"
+source cicd/scripts/testing-env.sh
 
-stackql --registry="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }" --tls.allowInsecure --sqlBackend="{ \"dbEngine\": \"postgres_tcp\", \"sqlDialect\": \"postgres\", \"dsn\": \"postgres://stackql:[email protected]:7432/stackql\" }" shell
+stackql --registry="${stackqlMockedRegistryStr}" --tls.allowInsecure --sqlBackend="{ \"dbEngine\": \"postgres_tcp\", \"sqlDialect\": \"postgres\", \"dsn\": \"postgres://stackql:[email protected]:7432/stackql\" }" shell
 ```
+
+## Sources of Mock Data
+
+There are some decent examples in vendor documentation, eg:
+
+- [Azure vendor documenation](https://learn.microsoft.com/en-us/rest/api/azure/).
+
+