Skip to content

Commit 3c27c1c

Browse files
xml-http-respomse-transform
Summary: - Support for: - Textual transformation of `xml` responses. - Presentation through schema overrrides. - Added robot test `Select Star From Transformed XML Response Body`. - Added robot test `Select Projection From Transformed XML Response Body`. - Added robot test `Describe Transformed XML Response Body`.
1 parent faab96b commit 3c27c1c

File tree

11 files changed

+386
-33
lines changed

11 files changed

+386
-33
lines changed

.vscode/launch.json

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,10 @@
171171
"insert into local_openssl.keys.rsa(config_file, key_out_file, cert_out_file, days) select '${workspaceFolder}/test/server/mtls/openssl.cnf', '${workspaceFolder}/test/tmp/manual_key.pem', '${workspaceFolder}/test/tmp/manuial_cert.pem', 90;",
172172
"describe local_openssl.keys.x509;",
173173
"select * from local_openssl.keys.x509 where cert_file = '/Users/admin/stackql/stackql-devel/test/tmp/manual_cert.pem';",
174+
"select * from aws.ec2.volumes_presented where region = 'ap-southeast-2';",
175+
"select BackupId, BackupState from aws.cloudhsm.backups where region = 'rubbish-region' order by BackupId;",
176+
"select rings.projectsId as project, rings.locationsId as locale, split_part(rings.name, '/', -1) as key_ring_name, split_part(keys.name, '/', -1) as key_name, json_extract(keys.\"versionTemplate\", '$.algorithm') as key_algorithm, json_extract(keys.\"versionTemplate\", '$.protectionLevel') as key_protection_level from google.cloudkms.key_rings rings inner join google.cloudkms.crypto_keys keys on keys.keyRingsId = split_part(rings.name, '/', -1) and keys.projectsId = rings.projectsId and keys.locationsId = rings.locationsId where rings.projectsId in ('testing-project', 'testing-project-two', 'testing-project-three') and rings.locationsId in ('global', 'australia-southeast1', 'australia-southeast2') order by project, locale, key_name ;",
177+
"delete from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::Logs::LogGroup' and data__Identifier = 'LogGroupResourceExampleThird' ;",
174178
],
175179
"default": "show providers;"
176180
},
@@ -180,8 +184,8 @@
180184
"description": "Registry Configuration",
181185
"options": [
182186
"{ \"url\": \"file://${workspaceFolder}/test/registry-sandbox\", \"localDocRoot\": \"${workspaceFolder}/test/registry-sandbox\", \"verifyConfig\": { \"nopVerify\": true } }",
183-
"{ \"url\": \"file://${workspaceFolder}/test/registry\", \"localDocRoot\": \"${workspaceFolder}/test/registry\", \"verifyConfig\": { \"nopVerify\": true } }",
184-
"{ \"url\": \"file://${workspaceFolder}/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }",
187+
"{ \"url\": \"file:///Users/admin/stackql/stackql-devel/test/registry\", \"localDocRoot\": \"/Users/admin/stackql/stackql-devel/test/registry\", \"verifyConfig\": { \"nopVerify\": true } }",
188+
"{ \"url\": \"file:///Users/admin/stackql/stackql-devel/test/registry-mocked\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked\", \"verifyConfig\": { \"nopVerify\": true } }",
185189
"{ \"url\": \"file://${workspaceFolder}/test/registry-mocked-native\", \"localDocRoot\": \"${workspaceFolder}/test/registry-mocked-native\", \"verifyConfig\": { \"nopVerify\": true } }",
186190
"{ \"url\": \"file://${workspaceFolder}/test/registry-advanced\", \"localDocRoot\": \"${workspaceFolder}/test/registry-advanced\", \"verifyConfig\": { \"nopVerify\": true } }",
187191
"{ \"url\": \"file://${workspaceFolder}/build/.stackql\", \"localDocRoot\": \"${workspaceFolder}/build/.stackql\", \"verifyConfig\": { \"nopVerify\": true } }",

docs/developer_guide.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ go test -timeout 1200s --tags "sqlite_stackql" ./...
7777
**Note**: this requires the local build (above) to have been completed successfully, which builds a binary in `./build/`.
7878

7979
```bash
80-
robot -d test/robot/functional test/robot/functional
80+
env PYTHONPATH="$PYTHONPATH:$(pwd)/test/python" robot -d test/robot/functional test/robot/functional
8181
```
8282

8383
Or better yet, if you have docker desktop and the `postgres` image cited in the docker compose files:

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ require (
1919
github.com/spf13/cobra v1.4.0
2020
github.com/spf13/pflag v1.0.5
2121
github.com/spf13/viper v1.10.1
22-
github.com/stackql/any-sdk v0.1.2-beta02
22+
github.com/stackql/any-sdk v0.1.3-alpha12
2323
github.com/stackql/go-suffix-map v0.0.1-alpha01
2424
github.com/stackql/psql-wire v0.1.1-beta23
2525
github.com/stackql/stackql-parser v0.0.14-alpha05

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -484,8 +484,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
484484
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
485485
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
486486
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
487-
github.com/stackql/any-sdk v0.1.2-beta02 h1:+XXSHnnKlGYWIDgBcVEiiC9HEFq2iTXGwKmHkFsBv8Q=
488-
github.com/stackql/any-sdk v0.1.2-beta02/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
487+
github.com/stackql/any-sdk v0.1.3-alpha12 h1:cvg0N3LLSd+upSxPyPdwq8JJrCQfOl7hWaPge8zlJD4=
488+
github.com/stackql/any-sdk v0.1.3-alpha12/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
489489
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
490490
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
491491
github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pguvTFuUhhVQ=

internal/stackql/dependencyplanner/dependencyplanner.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,9 @@ func (dp *standardDependencyPlanner) processAcquire(
531531
return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err
532532
}
533533
selectItemsKey := annotationCtx.GetTableMeta().GetSelectItemsKey()
534+
if selectItemsKey == "" {
535+
selectItemsKey = m.GetSelectItemsKey()
536+
}
534537
var defaultColName string
535538
if selectItemsKey != "" {
536539
defaultColName = util.TrimSelectItemsKey(selectItemsKey)

internal/stackql/execution/mono_valent_execution.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1155,6 +1155,10 @@ func (sp *standardProcessor) Process() ProcessorResponse {
11551155
polyHandler.LogHTTPResponseMap(res.GetProcessedBody())
11561156
logging.GetLogger().Infoln(fmt.Sprintf("monoValentExecution.Execute() response = %v", res))
11571157

1158+
if selectItemsKey == "" {
1159+
selectItemsKey = method.GetSelectItemsKey()
1160+
}
1161+
11581162
itemisationResult := itemise(res.GetProcessedBody(), resErr, selectItemsKey)
11591163

11601164
if itemisationResult.IsNilPayload() {
@@ -1189,12 +1193,12 @@ func (sp *standardProcessor) Process() ProcessorResponse {
11891193
)
11901194
housekeepingDone = insertPrepResult.IsHousekeepingDone()
11911195
insertPrepErr, hasInsertPrepErr := insertPrepResult.GetError()
1196+
if isSkipResponse && isMutation && httpResponse.StatusCode < 300 {
1197+
return newHTTPProcessorResponse(
1198+
nil, reversalStream, false, nil,
1199+
).WithSuccessMessages([]string{"The operation was despatched successfully"})
1200+
}
11921201
if hasInsertPrepErr {
1193-
if isSkipResponse && isMutation && httpResponse.StatusCode < 300 {
1194-
return newHTTPProcessorResponse(
1195-
nil, reversalStream, false, nil,
1196-
).WithSuccessMessages([]string{"The operation was despatched successfully"})
1197-
}
11981202
return newHTTPProcessorResponse(nil, reversalStream, false, insertPrepErr)
11991203
}
12001204

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
2+
import argparse
3+
import json
4+
import re
5+
6+
from typing import List, Optional
7+
8+
class _Tag(object):
9+
10+
def __init__(
11+
self,
12+
raw_tag: str,
13+
is_integration: bool = False,
14+
is_release: bool = False,
15+
is_hotfix: bool = False,
16+
is_prerelease: bool = False,
17+
is_build: bool = False,
18+
is_invalid: bool = False,
19+
is_regression: bool = False,
20+
is_robot: bool = False,
21+
repository_shorthand: Optional[str] = None,
22+
run_type: Optional[str] = None,
23+
run_id: Optional[str] = None,
24+
):
25+
self._raw_tag = raw_tag
26+
self._is_integration = is_integration
27+
self._is_release = is_release
28+
self._is_hotfix = is_hotfix
29+
self._is_prerelease = is_prerelease
30+
self._is_build = is_build
31+
self._is_invalid = is_invalid
32+
self._is_regression = is_regression
33+
self._is_robot = is_robot
34+
self._repository_shorthand = repository_shorthand if repository_shorthand else ''
35+
self._run_type = run_type if run_type else ''
36+
self._run_id = run_id if run_id else ''
37+
38+
39+
def is_robot(self) -> bool:
40+
return self._is_robot
41+
42+
def is_regression(self) -> bool:
43+
return self._is_regression
44+
45+
def get_run_id(self) -> str:
46+
return self._run_id
47+
48+
def get_run_type(self) -> str:
49+
return self._run_type
50+
51+
def get_repository_shorthand(self) -> str:
52+
return self._repository_shorthand
53+
54+
def _to_dict(self) -> dict:
55+
return {
56+
'raw_tag': self._raw_tag,
57+
'is_integration': self._is_integration,
58+
'is_release': self._is_release,
59+
'is_hotfix': self._is_hotfix,
60+
'is_prerelease': self._is_prerelease,
61+
'is_build': self._is_build,
62+
'is_invalid': self._is_invalid,
63+
'is_regression': self._is_regression,
64+
'is_robot': self._is_robot,
65+
'repository_shorthand': self.get_repository_shorthand(),
66+
'run_type': self._run_type,
67+
'run_id': self._run_id,
68+
}
69+
70+
def json(self) -> str:
71+
return json.dumps(self._to_dict())
72+
73+
74+
class _TagParser(object):
75+
76+
_BUILD_ONLY_TAG_PATTERN = re.compile(r'^build-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
77+
_BUILD_RELEASE_TAG_PATTERN = re.compile(r'^build-release-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
78+
_SCENARIO_ONLY_TAG_PATTERN = re.compile(r'^scenario-(?P<run_id>[^-]*)-(?P<run_type>[^-]*)-(?P<repository_shorthand>[^-]*)$')
79+
_ROBOT_TAG_PATTERN = re.compile(r'^robot-(?P<run_id>[^-]*)-(?P<run_type>[^-]*)-(?P<repository_shorthand>[^-]*)-(?P<seq>\d+)$')
80+
_ROBOT_ONLY_TAG_PATTERN = re.compile(r'^robot-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
81+
_REGRESSION_ONLY_TAG_PATTERN = re.compile(r'^regression-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
82+
83+
def __init__(
84+
self,
85+
raw_tag: str,
86+
permitted_types: List[str]
87+
):
88+
self._raw_tag = raw_tag
89+
self._permitted_types = permitted_types
90+
91+
def _parse_build_tag(self) -> _Tag:
92+
match = self._BUILD_ONLY_TAG_PATTERN.match(self._raw_tag)
93+
if match:
94+
return _Tag(raw_tag=self._raw_tag, is_build=True)
95+
match = self._BUILD_RELEASE_TAG_PATTERN.match(self._raw_tag)
96+
if match:
97+
return _Tag(raw_tag=self._raw_tag, is_build=True)
98+
return None
99+
100+
def _parse_scenario_tag(self) -> _Tag:
101+
match = self._SCENARIO_ONLY_TAG_PATTERN.match(self._raw_tag)
102+
run_id = match.group('run_id')
103+
run_type = match.group('run_type')
104+
repository_shorthand = match.group('repository_shorthand')
105+
if run_id and run_type:
106+
return _Tag(raw_tag=self._raw_tag, is_integration=True, run_id=run_id, run_type=run_type, repository_shorthand=repository_shorthand)
107+
return None
108+
109+
def _parse_robot_tag(self) -> _Tag:
110+
match = self._ROBOT_TAG_PATTERN.match(self._raw_tag)
111+
if match:
112+
return _Tag(raw_tag=self._raw_tag, is_robot=True)
113+
match = self._ROBOT_ONLY_TAG_PATTERN.match(self._raw_tag)
114+
if match:
115+
return _Tag(raw_tag=self._raw_tag, is_robot=True)
116+
return None
117+
118+
def _parse_regression_tag(self) -> _Tag:
119+
match = self._REGRESSION_ONLY_TAG_PATTERN.match(self._raw_tag)
120+
if match:
121+
return _Tag(raw_tag=self._raw_tag, is_regression=True)
122+
return None
123+
124+
def parse(self) -> _Tag:
125+
if not self._permitted_types:
126+
raise ValueError('No permitted types specified')
127+
for t in self._permitted_types:
128+
if t == 'build':
129+
tag = self._parse_build_tag()
130+
if tag:
131+
return tag
132+
elif t == 'scenario':
133+
tag = self._parse_scenario_tag()
134+
if tag:
135+
return tag
136+
elif t == 'robot':
137+
tag = self._parse_robot_tag()
138+
if tag:
139+
return tag
140+
elif t == 'regression':
141+
tag = self._parse_regression_tag()
142+
if tag:
143+
return tag
144+
raise ValueError(f'Raw tag: "{self._raw_tag}" is incompatible with permitted types: {" ".join(self._permitted_types)}')
145+
146+
def _parse_args() -> argparse.Namespace:
147+
"""
148+
Parse the arguments.
149+
"""
150+
parser = argparse.ArgumentParser(description='Handle and interpret git tags.')
151+
parser.add_argument('tag', help='The tag to parse', type=str)
152+
parser.add_argument('--parse-registry-tag', help='Opt-in to parse a registry tag', action=argparse.BooleanOptionalAction)
153+
parser.add_argument('--parse-scenario-tag', help='Opt-in to parse a scenario tag', action=argparse.BooleanOptionalAction)
154+
return parser.parse_args()
155+
156+
def main():
157+
args = _parse_args()
158+
if args.parse_registry_tag:
159+
tag_parser = _TagParser(args.tag, ['robot', 'regression'])
160+
tag = tag_parser.parse()
161+
if tag.is_robot():
162+
print(tag.json())
163+
return
164+
elif tag.is_regression():
165+
print(tag.json())
166+
return
167+
if args.parse_scenario_tag:
168+
tag_parser = _TagParser(args.tag, ['scenario'])
169+
tag = tag_parser.parse()
170+
if tag.get_run_id() and tag.get_run_type():
171+
print(tag.json())
172+
return
173+
else:
174+
raise ValueError(f'Invalid scenario tag inferred: {tag.json()}')
175+
raise ValueError('No action specified')
176+
177+
if __name__ == '__main__':
178+
main()
179+

test/registry/src/aws/v0.1.0/services/ec2.yaml

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45729,6 +45729,62 @@ components:
4572945729
update:
4573045730
- $ref: '#/components/x-stackQL-resources/volumes/methods/volume_Modify'
4573145731
title: volumes
45732+
volumes_presented:
45733+
id: aws.ec2.volumes_presented
45734+
name: volumes_presented
45735+
title: volumes_presented
45736+
methods:
45737+
describeVolumes:
45738+
config:
45739+
queryParamTranspose:
45740+
algorithm: AWSCanonical
45741+
requestTranslate:
45742+
algorithm: get_query_to_post_form_utf_8
45743+
operation:
45744+
$ref: '#/paths/~1?Action=DescribeVolumes&Version=2016-11-15/get'
45745+
response:
45746+
mediaType: application/xml
45747+
overrideMediaType: application/json
45748+
openAPIDocKey: '200'
45749+
objectKey: '$.line_items'
45750+
schema_override:
45751+
$ref: '#/components/schemas/DisplayVolumesSchema'
45752+
transform:
45753+
body: >
45754+
{
45755+
"next_page_token": {{with index . "DescribeVolumesResponse" "nextToken"}}{{printf "%q" .}}{{else}}null{{end}},
45756+
"line_items": [
45757+
{{- $items := index . "DescribeVolumesResponse" "volumeSet" "item" -}}
45758+
{{- if eq (printf "%T" $items) "map[string]interface {}" }}
45759+
{{template "volume" $items}}
45760+
{{- else }}
45761+
{{- range $i, $v := $items }}
45762+
{{- if $i}},{{end}}
45763+
{{template "volume" $v}}
45764+
{{- end }}
45765+
{{- end }}
45766+
]
45767+
}
45768+
{{define "volume"}}
45769+
{
45770+
"volume_id": {{printf "%q" (index . "volumeId")}},
45771+
"size": {{toInt (index . "size")}},
45772+
"snapshot_id": {{with index . "snapshotId"}}{{printf "%q" .}}{{else}}null{{end}},
45773+
"availability_zone": {{printf "%q" (index . "availabilityZone")}},
45774+
"status": {{printf "%q" (index . "status")}},
45775+
"create_time": {{printf "%q" (index . "createTime")}},
45776+
"volume_type": {{printf "%q" (index . "volumeType")}},
45777+
"encrypted": {{toBool (index . "encrypted")}},
45778+
"multi_attach_enabled": {{toBool (index . "multiAttachEnabled")}}
45779+
}
45780+
{{end}}
45781+
type: 'golang_template_mxj_v0.1.0'
45782+
sqlVerbs:
45783+
select:
45784+
- $ref: '#/components/x-stackQL-resources/volumes/methods/describeVolumes'
45785+
insert: []
45786+
update: []
45787+
delete: []
4573245788
volumes_modifications:
4573345789
name: volumes_modifications
4573445790
methods:
@@ -46413,6 +46469,42 @@ components:
4641346469
description: Amazon Signature authorization v4
4641446470
x-amazon-apigateway-authtype: awsSigv4
4641546471
schemas:
46472+
DisplayVolumesSchema:
46473+
title: Key Display
46474+
type: object
46475+
properties:
46476+
next_page_token:
46477+
type: string
46478+
description: The <code>NextToken</code> value to include in a future <code>DescribeVolumes</code> request. When the results of a <code>DescribeVolumes</code> request exceed <code>MaxResults</code>, this value can be used to retrieve the next page of results. This value is <code>null</code> when there are no more results to return.
46479+
line_items:
46480+
type: array
46481+
items:
46482+
type: object
46483+
properties:
46484+
volume_type:
46485+
type: string
46486+
description: The volume type
46487+
example: gp3
46488+
volume_id:
46489+
type: string
46490+
example: vol-00aaaccc111000000
46491+
snapshot_id:
46492+
type: string
46493+
status:
46494+
type: string
46495+
availability_zone:
46496+
type: string
46497+
create_time:
46498+
type: string
46499+
description: Textual datetime representation of the volume's creation time.
46500+
example: '2024-08-20T05:47:06.409Z'
46501+
size:
46502+
type: integer
46503+
example: 8
46504+
encrypted:
46505+
type: bool
46506+
multi_attach_enabled:
46507+
type: bool
4641646508
AcceptReservedInstancesExchangeQuoteResult:
4641746509
type: object
4641846510
properties:

0 commit comments

Comments
 (0)