Skip to content

Commit 8e69e58

Browse files
general-kroll-4-lifegeneral-kroll-4-life
authored
expanded-analytics (#597)
Summary: - `postgres` analytics dev and test pattern. - Harden analytics against regressions. - Added robot test `Materialized View of Filtered Multi Level Table Valued Function In Subquery Returns Expected Results`. - Upgraded `macos` runners to `15`.
1 parent 4cb9ea0 commit 8e69e58

File tree

13 files changed

+1942
-9
lines changed

13 files changed

+1942
-9
lines changed

‎.github/actionlint.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@ self-hosted-runner:
55
- ubuntu-22-04-m
66
- ubuntu-22-04-arm64-m
77
- arm-ubuntu-22-04-runner-one
8-
- macos-13 # should not be necessary
9-
- macos-14 # should not be necessary
8+
- macos-14 # should not be necessary
9+
- macos-15 # should not be necessary

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1207,7 +1207,7 @@ jobs:
12071207
12081208
macosbuild:
12091209
name: MacOS Build
1210-
runs-on: macos-13
1210+
runs-on: macos-15
12111211
timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
12121212
steps:
12131213
- name: Check out code into the Go module directory
@@ -1347,7 +1347,7 @@ jobs:
13471347

13481348
macosarmbuild:
13491349
name: MacOS ARM Build
1350-
runs-on: macos-13
1350+
runs-on: macos-15
13511351
timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
13521352
steps:
13531353

‎cicd/scripts/context.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env bash
22

3-
CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
3+
CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-${(%):-%x}}")" && pwd)"
44

55
export REPOSITORY_ROOT="$(realpath ${CUR_DIR}/../..)"
66

‎cicd/scripts/testing-env.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env bash
22

3-
CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
3+
CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-${(%):-%x}}")" && pwd)"
44

55
source "${CUR_DIR}/context.sh"
66

‎cicd/util/01-build-robot-lib.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ checkPoetry () {
88
fi
99
}
1010

11-
CURDIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
11+
CURDIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-${(%):-%x}}" )" &> /dev/null && pwd )"
1212

1313
REPOSITORY_ROOT="$(realpath ${CURDIR}/../..)"
1414

‎cicd/util/02-install-robot-lib.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
#! /usr/bin/env bash
22

3-
CURDIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
3+
CURDIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-${(%):-%x}}" )" &> /dev/null && pwd )"
44

55
REPOSITORY_ROOT="$(realpath ${CURDIR}/../..)"
66

77
PACKAGE_ROOT="${REPOSITORY_ROOT}/test"
88

99
venv_path="${REPOSITORY_ROOT}/.venv"
1010

11-
CURDIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
11+
CURDIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-${(%):-%x}}" )" &> /dev/null && pwd )"
1212

1313
REPOSITORY_ROOT="$(realpath ${CURDIR}/../..)"
1414

‎cicd/vol/stackql-data/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*
2+
!.gitignore

‎cicd/vol/vendor-secrets/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*
2+
!.gitignore

‎docker-compose-live.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
version: "3.9"
2+
3+
services:
4+
postgres_stackql:
5+
image: postgres:14.5-bullseye
6+
hostname: postgres_stackql
7+
volumes:
8+
- ./cicd/vol/postgres/setup:/docker-entrypoint-initdb.d:ro
9+
environment:
10+
- POSTGRES_PASSWORD=stackql
11+
ports:
12+
- "7632:5432/tcp"
13+
stackql:
14+
# Update image tag as needed
15+
image: stackql/stackql:latest
16+
hostname: stackql
17+
restart: no
18+
volumes:
19+
- ./cicd/vol/vendor-secrets:/opt/stackql/vendor-secrets:ro
20+
- ./cicd/vol/stackql-data:/opt/stackql/.stackql:rw
21+
ports:
22+
- "8632:8432/tcp"
23+
command:
24+
- bash
25+
- -c
26+
- |
27+
while ! nc -z postgres_stackql 5432; do sleep 1; done;
28+
source /opt/stackql/vendor-secrets/secrets.sh
29+
stackql srv \
30+
--http.log.enabled=true \
31+
--pgsrv.port=8432 \
32+
--pgsrv.address=0.0.0.0 \
33+
--export.alias=stackql_export \
34+
--auth='{ "github": { "credentialsenvvar": "STACKQL_GITHUB_TOKEN", "type": "api_key", "valuePrefix": "Bearer " }, "googleadmin": { "type": "service_account", "credentialsenvvar": "GOOGLEADMIN_CREDENTIALS" } }' \
35+
--sqlBackend='{ "dbEngine": "postgres_tcp", "sqlDialect": "postgres", "dsn": "postgres://stackql:stackql@postgres_stackql:5432" }'

‎docs/analytics.md‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
2+
3+
# Analytics with stackql
4+
5+
The canonical pattern is a postgres backend. To meaningfully develop analytics capability, **real** authenticated access to providers plus a postgres backend is needed. Therefore for local development:
6+
7+
- Ensure that all env var secrets are exported from the `.gitignore`d file `cicd/vol/vendor-secrets/secrets.sh`.
8+
- Run and kill development containers with `docker compose -f docker-compose-live.yml down --volumes` / `docker compose -f docker-compose-live.yml up --force-recreate`.
9+
- Connect and develop queries with `psql "postgresql://stackql:[email protected]:8632/stackql"`.
10+
11+
12+
## TODO
13+
14+
Robot tests for:
15+
16+
- Support for `current_date`.
17+
- Support for `current_timestamp`.
18+
- Support for multi-layered table valued functions in subqueries with outside filters, per Figure MLS-01.
19+
20+
---
21+
22+
```sql
23+
24+
-- sqlite version
25+
26+
CREATE OR REPLACE MATERIALIZED VIEW gcp_compute_public_ip_exposure AS
27+
select
28+
resource_type,
29+
resource_id,
30+
resource_name,
31+
cloud,
32+
region,
33+
protocol,
34+
from_port,
35+
to_port,
36+
cidr,
37+
direction,
38+
public_access_type,
39+
public_principal,
40+
access_mechanism
41+
from
42+
(
43+
SELECT
44+
'compute' AS resource_type,
45+
vms.id AS resource_id,
46+
vms.name AS resource_name,
47+
'google' AS cloud,
48+
split_part(vms.zone, '/', -1) AS region,
49+
NULL AS protocol,
50+
NULL AS from_port,
51+
NULL AS to_port,
52+
NULL AS cidr,
53+
NULL AS direction,
54+
NULL AS public_access_type,
55+
NULL AS public_principal,
56+
NULL AS access_mechanism,
57+
json_extract(ac.value, '$.natIP') as external_ip
58+
FROM google.compute.instances vms,
59+
json_each(vms.networkInterfaces) AS ni,
60+
json_each(json_extract(ni.value, '$.accessConfigs')) AS ac
61+
WHERE
62+
vms.project in (
63+
'testing-project'
64+
)
65+
) foo
66+
where external_ip != ''
67+
;
68+
69+
70+
-- postgres version
71+
72+
CREATE OR REPLACE MATERIALIZED VIEW gcp_compute_public_ip_exposure AS
73+
select
74+
resource_type,
75+
resource_id,
76+
resource_name,
77+
cloud,
78+
region,
79+
protocol,
80+
from_port,
81+
to_port,
82+
cidr,
83+
direction,
84+
public_access_type,
85+
public_principal,
86+
access_mechanism
87+
from
88+
(
89+
SELECT
90+
'compute' AS resource_type,
91+
vms.id AS resource_id,
92+
vms.name AS resource_name,
93+
'google' AS cloud,
94+
split_part(vms.zone, '/', -1) AS region,
95+
NULL AS protocol,
96+
NULL AS from_port,
97+
NULL AS to_port,
98+
NULL AS cidr,
99+
NULL AS direction,
100+
NULL AS public_access_type,
101+
NULL AS public_principal,
102+
NULL AS access_mechanism,
103+
json_extract_path_text(ac.value, 'natIP') as external_ip
104+
FROM google.compute.instances vms,
105+
json_array_elements_text(vms.networkInterfaces) AS ni,
106+
json_array_elements_text(json_extract_path_text(ni.value, 'accessConfigs')) AS ac
107+
WHERE
108+
vms.project in (
109+
'stackql-interesting'
110+
)
111+
) foo
112+
where external_ip != ''
113+
;
114+
115+
```
116+
117+
**Figure MLS-01**: Multi-layered table valued functions in subqueries with outside filters.
118+
119+
---
120+

0 commit comments

Comments
 (0)