Skip to content

Commit 7a4e3a0

Browse files
general-kroll-4-lifegeneral-kroll-4-life
committed
improvement-of-scenarios
Summary: - Change to walkthrough user input delimiters. - Added `aws` walkthrough. - Tabulation of results. - Return code nonzero if **any** test fails.
1 parent 912e485 commit 7a4e3a0

File tree

11 files changed

+173
-53
lines changed

11 files changed

+173
-53
lines changed

‎.github/workflows/scenario.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ jobs:
5555
pip3 install -r cicd/requirements.txt
5656
5757
- name: Run Walkthrough Scenarios
58+
env:
59+
AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }}
60+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }}
5861
run: |
5962
python3 test/python/markdown_testing/markdown_testing.py 2>&1 | tee cicd/log/markdown-testing-results.log
6063

‎.vscode/.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
*
22
!.gitignore
33
!launch.json
4-
!settings.json
4+
!settings.json
5+
!example.env

‎.vscode/example.env‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
DUMMY_DIGITALOCEAN_USERNAME=myusername
2+
DUMMY_DIGITALOCEAN_PASSWORD=mypassword
3+
DD_API_KEY=myusername
4+
DD_APPLICATION_KEY=mypassword

‎.vscode/launch.json‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,10 @@
155155
"select JSON_EXTRACT(saml.samlIdentity, '$.username') as saml_username from github.scim.saml_ids saml where saml.org = 'dummyorg';",
156156
"select kind, name, maximumCardsPerInstance from google.compute.acceleratorTypes where project = 'defective-response-content-project' and zone = 'australia-southeast1-a' order by name desc;",
157157
"registry pull google;",
158-
"create materialized view nv as select BackupId, BackupState from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;"
158+
"create materialized view nv as select BackupId, BackupState from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;",
159+
"SELECT instance_id FROM aws.ec2.instances WHERE region IN ('us-east-1', 'ap-southeast-2');",
160+
"SELECT instance_id FROM aws.ec2_solid_gold.instances WHERE region IN ('us-east-1', 'ap-southeast-2');",
161+
"SELECT region FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id;",
159162
],
160163
"default": "show providers;"
161164
},
@@ -182,12 +185,13 @@
182185
"type": "pickString",
183186
"id": "authString",
184187
"description": "Auth Input arg String",
185-
"default": "{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
188+
"default": "{}",
186189
"options": [
187190
"{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
188191
"{ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" },s \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/netlify/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/k8s/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/sumologic/sumologic-token.txt\", \"type\": \"basic\" } }",
189192
"{ \"pgi\": { \"type\": \"sql_data_source::postgres\", \"sqlDataSource\": { \"dsn\": \"postgres://stackql:[email protected]:8432\" } }, \"azure\": { \"type\": \"azure_default\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
190-
"{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }"
193+
"{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }",
194+
"{}"
191195
]
192196
},
193197
{
@@ -381,12 +385,7 @@
381385
{
382386
"name": "generic exec",
383387
"type": "go",
384-
"env": {
385-
"DUMMY_DIGITALOCEAN_USERNAME": "myusername",
386-
"DUMMY_DIGITALOCEAN_PASSWORD": "mypassword",
387-
"DD_API_KEY": "myusername",
388-
"DD_APPLICATION_KEY": "mypassword"
389-
},
388+
"envFile": "${workspaceFolder}/.vscode/.env",
390389
"request": "launch",
391390
"mode": "debug",
392391
"program": "${workspaceFolder}/stackql",

‎cicd/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
Flask==3.0.3
2+
Jinja2==3.1.4
23
mistune==3.0.2
34
psycopg2-binary>=2.9.9
45
psycopg[binary]>=3.1.16
56
PyYaml>=6.0.1
67
requests==2.32.3
78
robotframework==6.1.1
89
sqlalchemy==1.4.44
10+
tabulate==0.9.0

‎docs/developer_guide.md‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ The short of things is that for basic build and unit testing, these are needed:
1515

1616
- Install `golang` on your system **if you do not already have version >= 1.21**, per [the `golang` doco](https://go.dev/doc/install).
1717
- Install `python` on your system **if you do not already have version >= 3.11**, available from [the `python` website](https://www.python.org/downloads/) and numerous package managers.
18+
- Using a `venv` or otherwise, install the requisite python packages, eg: (system permitting) from the repository root: `pip install -r cicd/requirements.txt`.
1819

1920
Then, each of these should be run from the repository root:
2021

@@ -25,7 +26,6 @@ Then, each of these should be run from the repository root:
2526

2627
For serious development, simulated integration tests are essential. So, there are more dependencies:
2728

28-
- Install the python dependencies (including `robot` framework). Simplest way, system permitting, is `pip install -r cicd/requirements.txt`.
2929
- Install `psql`. On some systems, this can be done as client only and/or with various package managers; fallback is to just [install postgres manually](https://www.postgresql.org/download/).
3030

3131
Having installed all dependencies, the `robot` tests should be run from the repository root directory (this relies upon the executable in `./build/stackql`, built above):
@@ -88,9 +88,15 @@ robot --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true -d test/robot/functional
8888

8989
### Manually Testing
9090

91-
Please see [the mock testing doco](/test/mockserver/README.md#manually-testing-mocks).
91+
Please see [the mock testing doco](/test/python/flask/README.md).
9292

9393

94+
## Debuggers
95+
96+
The `vscode` tooling configuration is mostly ready to use, as seen in the `.vscode` directory. You will need to create a file at the `.gitignore`d location `.vscode/.env`. Simplest thing just copy the example to get going: `cp .vscode/example.env .vscode/.env`.
97+
98+
The debugger config is pretty messy, and probably with time we will slim it down. That said, it is far from useless as an example.
99+
94100
## Provider development
95101

96102
Keen to expose some new functionality though `stackql`? We are very keen on this!

‎docs/walkthroughs/README.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@ in order to setup, run, verify and tear down testing scenarios. The tests *can*
1616

1717
## Running from CI
1818

19-
The canonical, **ruleset-protected** tag form is `scenario-<run_number>-<anything>`. At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run.
19+
The canonical, **ruleset-protected** tag form is `scenario-<<run_number>>-<<anything>>`. At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run.
2020

2121

22+
## Plumbing
23+
24+
These walkthroughs are runnable using CI. This is built upon:
25+
26+
- `jinja2` templates, with `<<` and `>>` as delimiters.
27+
2228

‎docs/walkthroughs/list-aws-instances.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
2+
## Setup
3+
4+
First, for whichever AWS user you would like to use, grant read only privileges on EC2 (eg: using `arn:aws:iam::aws:policy/ReadOnlyAccess`). Then, create a set of AWS CLI credentials per [the AWS documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-authentication-user.html#cli-authentication-user-get), and store them in the appropriate environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
5+
6+
Then, do this in bash:
7+
8+
```bash setup stackql-shell app_root_path=./test/tmp/.list-aws-instances.stackql
9+
10+
stackql shell --approot=<<app_root_path>> --registry="{ \"url\": \"file://$(pwd)/test/registry\", \"localDocRoot\": \"$(pwd)/test/registry\", \"verifyConfig\": { \"nopVerify\": true } }"
11+
```
12+
13+
## Method
14+
15+
Do this in the `stackql` shell, replacing the tuple of regions with whichever AWS regions hold interest for you (these are not templated in the example):
16+
17+
```sql stackql-shell
18+
19+
20+
SELECT instance_id, region
21+
FROM aws.ec2_nextgen.instances
22+
WHERE region IN ('us-east-1', 'ap-southeast-2', 'eu-west-1');
23+
24+
```
25+
26+
## Result
27+
28+
29+
Assuming you have chosen regions wisely, you will see something like this included in the output:
30+
31+
```sql stackql stdout expectation stdout-table-contains-data
32+
|---------------------|----------------|
33+
| instance_id | region |
34+
|---------------------|----------------|
35+
| i-some-silly-id-011 | us-east-1 |
36+
|---------------------|----------------|
37+
| i-some-other-id-011 | ap-southeast-2 |
38+
|---------------------|----------------|
39+
```
40+
41+
## Cleanup
42+
43+
```bash teardown best-effort app_root_path=./test/tmp/.list-aws-instances.stackql
44+
45+
rm -rf <<app_root_path>>
46+
47+
```

docs/walkthroughs/get-google-accelerator-types.md renamed to docs/walkthroughs/list-google-accelerator-types.md

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11

22
## Setup
33

4-
First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
4+
First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
55

66
Then, do this in bash:
77

8-
```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-accel.stackql
8+
```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-accel.stackql
99

10-
export GOOGLE_CREDENTIALS="$(cat <credentials-path>)";
10+
export GOOGLE_CREDENTIALS="$(cat <<credentials_path>>)";
1111

12-
stackql shell --approot=<app-root-path>
12+
stackql shell --approot=<<app_root_path>>
1313
```
1414

1515
## Method
1616

17-
Do this in the `stackql` shell, replacing `<project>` with your GCP project name, and `<zone>` as desired, eg: `australia-southeast1-a`:
17+
Do this in the `stackql` shell, replacing `<<project>>` with your GCP project name, and `<<zone>>` as desired, eg: `australia-southeast1-a`:
1818

1919
```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a
2020

@@ -25,8 +25,8 @@ select
2525
kind
2626
FROM google.compute.accelerator_types
2727
WHERE
28-
project = '<project>'
29-
AND zone = '<zone>'
28+
project = '<<project>>'
29+
AND zone = '<<zone>>'
3030
ORDER BY
3131
name desc
3232
;
@@ -36,7 +36,7 @@ ORDER BY
3636
## Result
3737

3838

39-
You will see something very much like this included in the output, presuming you have one VM (if you have zero, only the headers should appper, more VMs means more rows):
39+
You will see exactly this included in the output:
4040

4141
```sql expectation stdout-contains-all
4242
|---------------------|-------------------------|
@@ -52,19 +52,10 @@ You will see something very much like this included in the output, presuming you
5252
|---------------------|-------------------------|
5353
```
5454

55-
<!--- EXPECTATION
56-
google\ provider,\ version\ 'v24.11.00274'\ successfully\ installed
57-
goodbye
58-
-->
59-
60-
<x-expectation style="display: none;">
61-
<stdout-contains-nonempty-table></stdout-contains-nonempty-table>
62-
</x-expectation>
63-
6455
## Cleanup
6556

66-
```bash teardown best-effort app-root-path=./test/tmp/.get-google-accel.stackql
57+
```bash teardown best-effort app_root_path=./test/tmp/.get-google-accel.stackql
6758

68-
rm -rf <app-root-path>
59+
rm -rf <<app_root_path>>
6960
7061
```

docs/walkthroughs/get-google-vms.md renamed to docs/walkthroughs/list-google-vms.md

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11

22
## Setup
33

4-
First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
4+
First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
55

66
Then, do this in bash:
77

8-
```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-vms.stackql
8+
```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-vms.stackql
99

10-
export GOOGLE_CREDENTIALS="$(cat <credentials-path>)";
10+
export GOOGLE_CREDENTIALS="$(cat <<credentials_path>>)";
1111

12-
stackql shell --approot=<app-root-path>
12+
stackql shell --approot=<<app_root_path>>
1313
```
1414

1515
## Method
1616

17-
Do this in the `stackql` shell, replacing `<project>` with your GCP project name, and `<zone>` as desired, eg: `australia-southeast1-a`:
17+
Do this in the `stackql` shell, replacing `<<project>>` with your GCP project name, and `<<zone>>` as desired, eg: `australia-southeast1-a`:
1818

1919
```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a
2020

@@ -25,8 +25,8 @@ select
2525
id
2626
FROM google.compute.instances
2727
WHERE
28-
project = '<project>'
29-
AND zone = '<zone>'
28+
project = '<<project>>'
29+
AND zone = '<<zone>>'
3030
;
3131

3232
```
@@ -55,8 +55,8 @@ goodbye
5555

5656
## Cleanup
5757

58-
```bash teardown best-effort app-root-path=./test/tmp/.get-google-vms.stackql
58+
```bash teardown best-effort app_root_path=./test/tmp/.get-google-vms.stackql
5959

60-
rm -rf <app-root-path>
60+
rm -rf <<app_root_path>>
6161
6262
```

0 commit comments

Comments
 (0)