Skip to content

Commit a9cc164

Browse files
consume-improved-response-transform-interface (#547)
Summary: - Consume new reponse transform interface. - Support for `json` response transforms. - Support for `text` response transforms. - Added robot test `Select Paginated Star From Transformed JSON Response Body`. - Added robot test `Select Paginated Projection From Transformed JSON Response Body`. - Added robot test `Select Join of Paginated Projection From Transformed JSON and XML Response Bodies`. - Added robot test `Select View of Join of Paginated Projection From Transformed JSON and XML Response Bodies`. - Added robot test `Select Materialized View of Join of Paginated Projection From Transformed JSON and XML Response Bodies`.
1 parent d81d930 commit a9cc164

File tree

12 files changed

+582
-33
lines changed

12 files changed

+582
-33
lines changed

cicd/scripts/context.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/env bash
2+
3+
CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4+
5+
export REPOSITORY_ROOT="$(realpath ${CUR_DIR}/../..)"
6+
7+
8+

cicd/scripts/testing-env.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4+
5+
source "${CUR_DIR}/context.sh"
6+
7+
export REPOSITORY_ROOT="${REPOSITORY_ROOT}"
8+
export workspaceFolder="${REPOSITORY_ROOT}"
9+
10+
export OKTA_SECRET_KEY='some-dummy-api-key'
11+
export GITHUB_SECRET_KEY='some-dummy-github-key'
12+
export K8S_SECRET_KEY='some-k8s-token'
13+
export AZ_ACCESS_TOKEN='dummy_azure_token'
14+
export SUMO_CREDS='somesumologictoken'
15+
export DIGITALOCEAN_TOKEN='somedigitaloceantoken'
16+
export DUMMY_DIGITALOCEAN_USERNAME='myusername'
17+
export DUMMY_DIGITALOCEAN_PASSWORD='mypassword'
18+
19+
20+
googleCredentialsFilePath="${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json"
21+
22+
export stackqlMockedRegistryStr="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }"
23+
24+
export stackqlAuthStr="{\"google\": {\"credentialsfilepath\": \"${googleCredentialsFilePath}\", \"type\": \"service_account\"}, \"okta\": {\"credentialsenvvar\": \"OKTA_SECRET_KEY\", \"type\": \"api_key\"}, \"aws\": {\"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"/Users/admin/stackql/stackql-devel/test/robot/functional/../../../test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"NON_SECRET\"}, \"github\": {\"type\": \"basic\", \"credentialsenvvar\": \"GITHUB_SECRET_KEY\"}, \"k8s\": {\"credentialsenvvar\": \"K8S_SECRET_KEY\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \"}, \"azure\": {\"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsenvvar\": \"AZ_ACCESS_TOKEN\"}, \"sumologic\": {\"type\": \"basic\", \"credentialsenvvar\": \"SUMO_CREDS\"}, \"digitalocean\": {\"type\": \"bearer\", \"username\": \"myusername\", \"password\": \"mypassword\"}}"
25+
26+
27+
28+

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ require (
1919
github.com/spf13/cobra v1.4.0
2020
github.com/spf13/pflag v1.0.5
2121
github.com/spf13/viper v1.10.1
22-
github.com/stackql/any-sdk v0.1.3-alpha13
22+
github.com/stackql/any-sdk v0.1.3-beta02
2323
github.com/stackql/go-suffix-map v0.0.1-alpha01
2424
github.com/stackql/psql-wire v0.1.1-beta23
2525
github.com/stackql/stackql-parser v0.0.14-alpha05

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -484,8 +484,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
484484
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
485485
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
486486
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
487-
github.com/stackql/any-sdk v0.1.3-alpha13 h1:IJLnZojR7JU6f3cixL0U8KdJva+qociPb7Uu3Vctq9w=
488-
github.com/stackql/any-sdk v0.1.3-alpha13/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
487+
github.com/stackql/any-sdk v0.1.3-beta02 h1:0jSwyYFddjAN++U+yNNLF7SMLPIIaRhv522UzeWDf2E=
488+
github.com/stackql/any-sdk v0.1.3-beta02/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
489489
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
490490
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
491491
github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pguvTFuUhhVQ=

internal/stackql/execution/mono_valent_execution.go

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package execution
22

33
import (
4-
"bytes"
54
"encoding/json"
65
"fmt"
76
"io"
@@ -1330,19 +1329,34 @@ func (mv *monoValentExecution) GetExecutor() (func(pc primitive.IPrimitiveCtx) i
13301329
expectedResponse, isExpectedResponse := m.GetResponse()
13311330
if isExpectedResponse {
13321331
responseTransform, responseTransformExists := expectedResponse.GetTransform()
1333-
if responseTransformExists && responseTransform.GetType() == "golang_template_v0.1.0" {
1332+
if responseTransformExists {
13341333
input := stdoutStr
1335-
tmpl := responseTransform.GetBody()
1336-
inStream := stream_transform.NewTextReader(bytes.NewBufferString(input))
1337-
outStream := bytes.NewBuffer(nil)
1338-
tfm, setupErr := stream_transform.NewTemplateStreamTransformer(tmpl, inStream, outStream)
1339-
if setupErr != nil {
1340-
return internaldto.NewErroneousExecutorOutput(fmt.Errorf("template stream transform error: %w", setupErr))
1334+
streamTransformerFactory := stream_transform.NewStreamTransformerFactory(
1335+
responseTransform.GetType(),
1336+
responseTransform.GetBody(),
1337+
)
1338+
if !streamTransformerFactory.IsTransformable() {
1339+
return internaldto.NewErroneousExecutorOutput(
1340+
fmt.Errorf("unsupported template type: %s", responseTransform.GetType()),
1341+
)
13411342
}
1342-
if tfErr := tfm.Transform(); tfErr != nil {
1343-
return internaldto.NewErroneousExecutorOutput(fmt.Errorf("failed to transform: %w", tfErr))
1343+
tfm, getTfmErr := streamTransformerFactory.GetTransformer(input)
1344+
if getTfmErr != nil {
1345+
return internaldto.NewErroneousExecutorOutput(
1346+
fmt.Errorf("failed to transform: %w", getTfmErr))
13441347
}
1345-
outputStr := outStream.String()
1348+
transformError := tfm.Transform()
1349+
if transformError != nil {
1350+
return internaldto.NewErroneousExecutorOutput(
1351+
fmt.Errorf("failed to transform: %w", transformError))
1352+
}
1353+
outStream := tfm.GetOutStream()
1354+
outputBytes, readErr := io.ReadAll(outStream)
1355+
if readErr != nil {
1356+
return internaldto.NewErroneousExecutorOutput(
1357+
fmt.Errorf("failed to read transformed stream: %w", readErr))
1358+
}
1359+
outputStr := string(outputBytes)
13461360
stdoutStr = outputStr
13471361
}
13481362
}

test/python/stackql_test_tooling/flask/README.md

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,17 +81,25 @@ flask --app=${HOME}/stackql/stackql-devel/test/python/stackql_test_tooling/flask
8181
With embedded `sqlite` (default), from the root of this repository:
8282

8383
```bash
84-
export workspaceFolder="$(pwd)"
84+
source cicd/scripts/testing-env.sh
8585

86-
stackql --registry="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }" --tls.allowInsecure shell
86+
stackql --registry="${stackqlMockedRegistryStr}" --auth="${stackqlAuthStr}" --tls.allowInsecure shell
8787
```
8888

8989
With `postgres`, from the root of this repository:
9090

9191
```bash
9292
docker compose -f docker-compose-externals.yml up postgres_stackql -d
9393

94-
export workspaceFolder="$(pwd)"
94+
source cicd/scripts/testing-env.sh
9595

96-
stackql --registry="{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }" --tls.allowInsecure --sqlBackend="{ \"dbEngine\": \"postgres_tcp\", \"sqlDialect\": \"postgres\", \"dsn\": \"postgres://stackql:[email protected]:7432/stackql\" }" shell
96+
stackql --registry="${stackqlMockedRegistryStr}" --tls.allowInsecure --sqlBackend="{ \"dbEngine\": \"postgres_tcp\", \"sqlDialect\": \"postgres\", \"dsn\": \"postgres://stackql:[email protected]:7432/stackql\" }" shell
9797
```
98+
99+
## Sources of Mock Data
100+
101+
There are some decent examples in vendor documentation, eg:
102+
103+
- [Azure vendor documenation](https://learn.microsoft.com/en-us/rest/api/azure/).
104+
105+

test/python/stackql_test_tooling/flask/azure/app.py

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,44 @@
1010
app.template_folder = os.path.join(os.path.dirname(__file__), "templates")
1111

1212
# Configure logging
13-
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
13+
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
1414
logger = logging.getLogger(__name__)
1515

1616
@app.before_request
1717
def log_request_info():
1818
logger.info(f"Request: {request.method} {request.path}\n - Query: {request.args}\n - Headers: {request.headers}\n - Body: {request.get_data()}")
1919

20+
21+
def _extract_req_adornments(req: Request) -> dict:
22+
"""
23+
Extracts the request adornments from the request object.
24+
"""
25+
req_adornments = {}
26+
if req.headers.get('Authorization'):
27+
auth_header = req.headers.get('Authorization')
28+
if auth_header.startswith('Basic '):
29+
auth_value = auth_header.split(' ')[1]
30+
decoded_value = base64.b64decode(auth_value).decode('utf-8')
31+
username, password = decoded_value.split(':', 1)
32+
req_adornments['username'] = username
33+
req_adornments['password'] = password
34+
elif auth_header.startswith('Bearer '):
35+
token = auth_header.split(' ')[1]
36+
req_adornments['token'] = token
37+
logger.debug(f"Host url: {request.host_url}")
38+
host_components = request.host_url.split(':')
39+
logger.debug(f"Host components: {host_components}")
40+
if len(host_components) == 3:
41+
req_adornments['scheme'] = host_components[0]
42+
req_adornments['host_name'] = host_components[1].lstrip('/')
43+
req_adornments['port'] = int(host_components[2].strip('/'))
44+
elif len(host_components) == 2:
45+
req_adornments['scheme'] = host_components[0]
46+
req_adornments['host_name'] = host_components[1].lstrip('/')
47+
48+
logger.debug(f"Request adornments:\n {json.dumps(req_adornments, indent=2)}\n\n")
49+
50+
return req_adornments
2051
class GetMatcherConfig:
2152

2253
_ROOT_PATH_CFG: List[dict] = {}
@@ -45,6 +76,7 @@ def _match_json_strict(self, lhs: dict, rhs: dict) -> bool:
4576

4677
def _match_json_by_key(self, lhs: dict, rhs: dict) -> bool:
4778
for key, value in rhs.items():
79+
logger.debug(f"Matching key: {key} from {json.dumps(lhs)} with value: {value}")
4880
if key not in lhs:
4981
return False
5082
if isinstance(value, dict):
@@ -53,6 +85,7 @@ def _match_json_by_key(self, lhs: dict, rhs: dict) -> bool:
5385
elif isinstance(value, list):
5486
for item in value:
5587
if not self._match_string(lhs[key], item):
88+
logger.debug(f"Matching item {item} in list {lhs[key]} failed")
5689
return False
5790
elif isinstance(value, str):
5891
if not self._match_string(lhs[key], value):
@@ -138,7 +171,7 @@ def match_route(self, req: Request) -> dict:
138171
for i in range(len(self._ROOT_PATH_CFG)):
139172
route_name: str = f"route_{i}"
140173
cfg: dict = self._ROOT_PATH_CFG[i]
141-
logger.debug(f"Evaluating route: {route_name}")
174+
logger.debug(f"Evaluating route: {route_name} with config: \n{json.dumps(cfg, indent=2, sort_keys=True)}\n\n")
142175

143176
is_method_match: bool = self._is_method_match(req, cfg)
144177
if not is_method_match:
@@ -200,87 +233,87 @@ def match_route(self, req: Request) -> dict:
200233
@app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/', methods=['GET'])
201234
def keys_list_01(resourceGroupName):
202235
template_name = "keys-list-01.json"
203-
response = make_response(render_template(template_name, request=request))
236+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
204237
response.headers.update({"Content-Type": "application/json"})
205238
response.status_code = 200
206239
return response
207240

208241
@app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-alt-keyvault/keys/', methods=['GET'])
209242
def keys_list_02(resourceGroupName):
210243
template_name = "keys-list-02.json"
211-
response = make_response(render_template(template_name, request=request))
244+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
212245
response.headers.update({"Content-Type": "application/json"})
213246
response.status_code = 200
214247
return response
215248

216249
@app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/dummy-key-01/', methods=['GET'])
217250
def key_detail_01(resourceGroupName):
218251
template_name = "key-detail-01.json"
219-
response = make_response(render_template(template_name, request=request))
252+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
220253
response.headers.update({"Content-Type": "application/json"})
221254
response.status_code = 200
222255
return response
223256

224257
@app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/dummy-key-02/', methods=['GET'])
225258
def key_detail_02(resourceGroupName):
226259
template_name = "key-detail-02.json"
227-
response = make_response(render_template(template_name, request=request))
260+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
228261
response.headers.update({"Content-Type": "application/json"})
229262
response.status_code = 200
230263
return response
231264

232265
@app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/alt-dummy-key-02/', methods=['GET'])
233266
def key_detail_03(resourceGroupName):
234267
template_name = "key-detail-03.json"
235-
response = make_response(render_template(template_name, request=request))
268+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
236269
response.headers.update({"Content-Type": "application/json"})
237270
response.status_code = 200
238271
return response
239272

240273
@app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-alt-keyvault/keys/alt-dummy-key-01/', methods=['GET'])
241274
def key_detail_04(resourceGroupName):
242275
template_name = "key-detail-04.json"
243-
response = make_response(render_template(template_name, request=request))
276+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
244277
response.headers.update({"Content-Type": "application/json"})
245278
response.status_code = 200
246279
return response
247280

248281
@app.route('/subscriptions/000000-0000-0000-0000-000000000022/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-alt-keyvault/keys/alt-dummy-key-02/', methods=['GET'])
249282
def key_detail_05(resourceGroupName):
250283
template_name = "key-detail-05.json"
251-
response = make_response(render_template(template_name, request=request))
284+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
252285
response.headers.update({"Content-Type": "application/json"})
253286
response.status_code = 200
254287
return response
255288

256289
@app.route('/subscriptions/000000-0000-0000-0000-000000000011/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/dummy-key-01/', methods=['GET'])
257290
def key_detail_06(resourceGroupName):
258291
template_name = "key-detail-06.json"
259-
response = make_response(render_template(template_name, request=request))
292+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
260293
response.headers.update({"Content-Type": "application/json"})
261294
response.status_code = 200
262295
return response
263296

264297
@app.route('/subscriptions/000000-0000-0000-0000-000000000011/resourceGroups/<resourceGroupName>/providers/Microsoft.KeyVault/vaults/stackql-testing-keyvault/keys/', methods=['GET'])
265298
def keys_list_03(resourceGroupName):
266299
template_name = "keys-list-03.json"
267-
response = make_response(render_template(template_name, request=request))
300+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
268301
response.headers.update({"Content-Type": "application/json"})
269302
response.status_code = 200
270303
return response
271304

272305
@app.route('/subscriptions/<subscriptionId>/providers/Microsoft.Compute/sshPublicKeys/', methods=['GET'])
273306
def ssh_public_keys_list(subscriptionId):
274307
template_name = "ssh-public-keys-list-01.json"
275-
response = make_response(render_template(template_name, request=request))
308+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
276309
response.headers.update({"Content-Type": "application/json"})
277310
response.status_code = 200
278311
return response
279312

280313
@app.route('/subscriptions/<subscriptionId>/resourceGroups/<resourceGroupId>/providers/Microsoft.Compute/virtualMachines/', methods=['GET'])
281314
def virtual_machines_list(subscriptionId, resourceGroupId):
282315
template_name = "virtual-machines-list-01.json"
283-
response = make_response(render_template(template_name, request=request))
316+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
284317
response.headers.update({"Content-Type": "application/json"})
285318
response.status_code = 200
286319
return response
@@ -295,11 +328,11 @@ def generic_handler(request: Request):
295328
route_cfg: dict = cfg_obj.match_route(request)
296329
template_name = route_cfg.get("httpResponse", {}).get("template", "")
297330
if not template_name:
298-
rv = make_response(render_template('nil-response.json', request=request))
331+
rv = make_response(render_template('nil-response.json', request=request, **_extract_req_adornments(request)))
299332
rv.status_code = 404
300333
return rv
301334
logger.info(f"routing to template: {template_name}")
302-
response = make_response(render_template(template_name, request=request))
335+
response = make_response(render_template(template_name, request=request, **_extract_req_adornments(request)))
303336
response.headers.update(route_cfg.get("httpResponse", {}).get("headers", {}))
304337
response.status_code = route_cfg.get("httpResponse", {}).get("status", 200)
305338
return response

test/python/stackql_test_tooling/flask/azure/expectations.json

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,41 @@
11
[
2+
{
3+
"httpRequest": {
4+
"headers": {
5+
"Accept": [ "application/json" ]
6+
},
7+
"method": "GET",
8+
"path": "/subscriptions/subid/providers/Microsoft.Network/virtualNetworks/",
9+
"queryStringParameters" : {
10+
"api-version" : [ "2023-11-01" ],
11+
"$skiptoken": [ "0011" ]
12+
}
13+
},
14+
"httpResponse": {
15+
"template": "virtual-networks-list-all-paginated-02.json",
16+
"headers": {
17+
"Content-Type": "application/json; charset=utf-8"
18+
}
19+
}
20+
},
21+
{
22+
"httpRequest": {
23+
"headers": {
24+
"Accept": [ "application/json" ]
25+
},
26+
"method": "GET",
27+
"path": "/subscriptions/subid/providers/Microsoft.Network/virtualNetworks/",
28+
"queryStringParameters" : {
29+
"api-version" : [ "2023-11-01" ]
30+
}
31+
},
32+
"httpResponse": {
33+
"template": "virtual-networks-list-all-paginated-01.json",
34+
"headers": {
35+
"Content-Type": "application/json; charset=utf-8"
36+
}
37+
}
38+
},
239
{
340
"httpRequest": {
441
"headers": {

0 commit comments

Comments
 (0)