Skip to content

Commit da403bb

Browse files
xml-http-response-transform (#543)
Summary: - Support for: - Textual transformation of `xml` responses. - Presentation through schema overrrides. - Added robot test `Select Star From Transformed XML Response Body`. - Added robot test `Select Projection From Transformed XML Response Body`. - Added robot test `Describe Transformed XML Response Body`. - Inaugurate version `v0.8`.
1 parent faab96b commit da403bb

File tree

14 files changed

+403
-33
lines changed

14 files changed

+403
-33
lines changed

.vscode/launch.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,10 @@
171171
"insert into local_openssl.keys.rsa(config_file, key_out_file, cert_out_file, days) select '${workspaceFolder}/test/server/mtls/openssl.cnf', '${workspaceFolder}/test/tmp/manual_key.pem', '${workspaceFolder}/test/tmp/manuial_cert.pem', 90;",
172172
"describe local_openssl.keys.x509;",
173173
"select * from local_openssl.keys.x509 where cert_file = '/Users/admin/stackql/stackql-devel/test/tmp/manual_cert.pem';",
174+
"select * from aws.ec2.volumes_presented where region = 'ap-southeast-2';",
175+
"select BackupId, BackupState from aws.cloudhsm.backups where region = 'rubbish-region' order by BackupId;",
176+
"select rings.projectsId as project, rings.locationsId as locale, split_part(rings.name, '/', -1) as key_ring_name, split_part(keys.name, '/', -1) as key_name, json_extract(keys.\"versionTemplate\", '$.algorithm') as key_algorithm, json_extract(keys.\"versionTemplate\", '$.protectionLevel') as key_protection_level from google.cloudkms.key_rings rings inner join google.cloudkms.crypto_keys keys on keys.keyRingsId = split_part(rings.name, '/', -1) and keys.projectsId = rings.projectsId and keys.locationsId = rings.locationsId where rings.projectsId in ('testing-project', 'testing-project-two', 'testing-project-three') and rings.locationsId in ('global', 'australia-southeast1', 'australia-southeast2') order by project, locale, key_name ;",
177+
"delete from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::Logs::LogGroup' and data__Identifier = 'LogGroupResourceExampleThird' ;",
174178
],
175179
"default": "show providers;"
176180
},

cicd/version.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
MajorVersion=0
2-
MinorVersion=7
2+
MinorVersion=8

docs/developer_guide.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ go test -timeout 1200s --tags "sqlite_stackql" ./...
7777
**Note**: this requires the local build (above) to have been completed successfully, which builds a binary in `./build/`.
7878

7979
```bash
80-
robot -d test/robot/functional test/robot/functional
80+
env PYTHONPATH="$PYTHONPATH:$(pwd)/test/python" robot -d test/robot/functional test/robot/functional
8181
```
8282

8383
Or better yet, if you have docker desktop and the `postgres` image cited in the docker compose files:

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ require (
1919
github.com/spf13/cobra v1.4.0
2020
github.com/spf13/pflag v1.0.5
2121
github.com/spf13/viper v1.10.1
22-
github.com/stackql/any-sdk v0.1.2-beta02
22+
github.com/stackql/any-sdk v0.1.3-alpha13
2323
github.com/stackql/go-suffix-map v0.0.1-alpha01
2424
github.com/stackql/psql-wire v0.1.1-beta23
2525
github.com/stackql/stackql-parser v0.0.14-alpha05

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -484,8 +484,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
484484
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
485485
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
486486
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
487-
github.com/stackql/any-sdk v0.1.2-beta02 h1:+XXSHnnKlGYWIDgBcVEiiC9HEFq2iTXGwKmHkFsBv8Q=
488-
github.com/stackql/any-sdk v0.1.2-beta02/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
487+
github.com/stackql/any-sdk v0.1.3-alpha13 h1:IJLnZojR7JU6f3cixL0U8KdJva+qociPb7Uu3Vctq9w=
488+
github.com/stackql/any-sdk v0.1.3-alpha13/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
489489
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
490490
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
491491
github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pguvTFuUhhVQ=

internal/stackql/dependencyplanner/dependencyplanner.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,7 @@ func (dp *standardDependencyPlanner) orchestrate(
494494
return idx, err
495495
}
496496

497+
//nolint:gocognit // live with it
497498
func (dp *standardDependencyPlanner) processAcquire(
498499
sqlNode sqlparser.SQLNode,
499500
annotationCtx taxonomy.AnnotationCtx,
@@ -531,6 +532,9 @@ func (dp *standardDependencyPlanner) processAcquire(
531532
return util.NewAnnotatedTabulation(nil, nil, "", ""), nil, err
532533
}
533534
selectItemsKey := annotationCtx.GetTableMeta().GetSelectItemsKey()
535+
if selectItemsKey == "" {
536+
selectItemsKey = m.GetSelectItemsKey()
537+
}
534538
var defaultColName string
535539
if selectItemsKey != "" {
536540
defaultColName = util.TrimSelectItemsKey(selectItemsKey)

internal/stackql/execution/mono_valent_execution.go

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ type monoValentExecution struct {
7878
isReadOnly bool //nolint:unused // TODO: build out
7979
isSkipResponse bool
8080
isMutation bool
81+
isAwait bool
8182
}
8283

8384
func NewMonoValentExecutorFactory(
@@ -90,6 +91,7 @@ func NewMonoValentExecutorFactory(
9091
stream streaming.MapStream,
9192
isSkipResponse bool,
9293
isMutation bool,
94+
isAwait bool,
9395
) MonoValentExecutorFactory {
9496
var tcc internaldto.TxnControlCounters
9597
if insertCtx != nil {
@@ -110,6 +112,7 @@ func NewMonoValentExecutorFactory(
110112
stream: stream,
111113
isSkipResponse: isSkipResponse,
112114
isMutation: isMutation,
115+
isAwait: isAwait,
113116
}
114117
}
115118

@@ -586,6 +589,7 @@ type AgnosticatePayload interface {
586589
GetInsertPreparator() InsertPreparator
587590
IsSkipResponse() bool
588591
IsMutation() bool
592+
IsAwait() bool
589593
}
590594

591595
type httpAgnosticatePayload struct {
@@ -604,6 +608,7 @@ type httpAgnosticatePayload struct {
604608
insertPreparator InsertPreparator
605609
isSkipResponse bool
606610
isMutation bool
611+
isAwait bool
607612
}
608613

609614
func newHTTPAgnosticatePayload(
@@ -622,6 +627,7 @@ func newHTTPAgnosticatePayload(
622627
insertPreparator InsertPreparator,
623628
isSkipResponse bool,
624629
isMutation bool,
630+
isAwait bool,
625631
) AgnosticatePayload {
626632
return &httpAgnosticatePayload{
627633
tableMeta: tableMeta,
@@ -639,13 +645,18 @@ func newHTTPAgnosticatePayload(
639645
insertPreparator: insertPreparator,
640646
isSkipResponse: isSkipResponse,
641647
isMutation: isMutation,
648+
isAwait: isAwait,
642649
}
643650
}
644651

645652
func (ap *httpAgnosticatePayload) GetPolyHandler() PolyHandler {
646653
return ap.polyHandler
647654
}
648655

656+
func (ap *httpAgnosticatePayload) IsAwait() bool {
657+
return ap.isAwait
658+
}
659+
649660
func (ap *httpAgnosticatePayload) GetInsertPreparator() InsertPreparator {
650661
return ap.insertPreparator
651662
}
@@ -748,6 +759,7 @@ func agnosticate(
748759
insertPreparator := agPayload.GetInsertPreparator()
749760
isSkipResponse := agPayload.IsNilResponseAcceptable()
750761
isMutation := agPayload.IsMutation()
762+
isAwait := agPayload.IsAwait()
751763
// TODO: TCC setup
752764
armoury, armouryErr := agPayload.GetArmoury()
753765
if armouryErr != nil {
@@ -797,7 +809,7 @@ func agnosticate(
797809
insertPreparator,
798810
isSkipResponse,
799811
false,
800-
false,
812+
isAwait,
801813
false,
802814
isMutation,
803815
"",
@@ -1155,6 +1167,10 @@ func (sp *standardProcessor) Process() ProcessorResponse {
11551167
polyHandler.LogHTTPResponseMap(res.GetProcessedBody())
11561168
logging.GetLogger().Infoln(fmt.Sprintf("monoValentExecution.Execute() response = %v", res))
11571169

1170+
if selectItemsKey == "" {
1171+
selectItemsKey = method.GetSelectItemsKey()
1172+
}
1173+
11581174
itemisationResult := itemise(res.GetProcessedBody(), resErr, selectItemsKey)
11591175

11601176
if itemisationResult.IsNilPayload() {
@@ -1189,12 +1205,12 @@ func (sp *standardProcessor) Process() ProcessorResponse {
11891205
)
11901206
housekeepingDone = insertPrepResult.IsHousekeepingDone()
11911207
insertPrepErr, hasInsertPrepErr := insertPrepResult.GetError()
1208+
if !isAwait && isSkipResponse && isMutation && httpResponse.StatusCode < 300 {
1209+
return newHTTPProcessorResponse(
1210+
nil, reversalStream, false, nil,
1211+
).WithSuccessMessages([]string{"The operation was despatched successfully"})
1212+
}
11921213
if hasInsertPrepErr {
1193-
if isSkipResponse && isMutation && httpResponse.StatusCode < 300 {
1194-
return newHTTPProcessorResponse(
1195-
nil, reversalStream, false, nil,
1196-
).WithSuccessMessages([]string{"The operation was despatched successfully"})
1197-
}
11981214
return newHTTPProcessorResponse(nil, reversalStream, false, insertPrepErr)
11991215
}
12001216

@@ -1384,6 +1400,7 @@ func (mv *monoValentExecution) GetExecutor() (func(pc primitive.IPrimitiveCtx) i
13841400
mv,
13851401
mv.isSkipResponse,
13861402
mv.isMutation,
1403+
mv.isAwait,
13871404
)
13881405
processorResponse, agnosticErr := agnosticate(agnosticatePayload)
13891406
if agnosticErr != nil {

internal/stackql/primitivebuilder/delete.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ func (ss *Delete) Build() error {
8080
streaming.NewNopMapStream(),
8181
!ss.isAwait,
8282
true,
83+
ss.isAwait,
8384
)
8485
ex, exErr := mvb.GetExecutor()
8586
if exErr != nil {

internal/stackql/primitivebuilder/mono_valent_builder.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ func newMonoValentBuilder(
7272
stream,
7373
isSkipResponse,
7474
isMutation,
75+
false,
7576
),
7677
}
7778
}
Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
2+
import argparse
3+
import json
4+
import re
5+
6+
from typing import List, Optional
7+
8+
class _Tag(object):
9+
10+
def __init__(
11+
self,
12+
raw_tag: str,
13+
is_integration: bool = False,
14+
is_release: bool = False,
15+
is_hotfix: bool = False,
16+
is_prerelease: bool = False,
17+
is_build: bool = False,
18+
is_invalid: bool = False,
19+
is_regression: bool = False,
20+
is_robot: bool = False,
21+
repository_shorthand: Optional[str] = None,
22+
run_type: Optional[str] = None,
23+
run_id: Optional[str] = None,
24+
):
25+
self._raw_tag = raw_tag
26+
self._is_integration = is_integration
27+
self._is_release = is_release
28+
self._is_hotfix = is_hotfix
29+
self._is_prerelease = is_prerelease
30+
self._is_build = is_build
31+
self._is_invalid = is_invalid
32+
self._is_regression = is_regression
33+
self._is_robot = is_robot
34+
self._repository_shorthand = repository_shorthand if repository_shorthand else ''
35+
self._run_type = run_type if run_type else ''
36+
self._run_id = run_id if run_id else ''
37+
38+
39+
def is_robot(self) -> bool:
40+
return self._is_robot
41+
42+
def is_regression(self) -> bool:
43+
return self._is_regression
44+
45+
def get_run_id(self) -> str:
46+
return self._run_id
47+
48+
def get_run_type(self) -> str:
49+
return self._run_type
50+
51+
def get_repository_shorthand(self) -> str:
52+
return self._repository_shorthand
53+
54+
def _to_dict(self) -> dict:
55+
return {
56+
'raw_tag': self._raw_tag,
57+
'is_integration': self._is_integration,
58+
'is_release': self._is_release,
59+
'is_hotfix': self._is_hotfix,
60+
'is_prerelease': self._is_prerelease,
61+
'is_build': self._is_build,
62+
'is_invalid': self._is_invalid,
63+
'is_regression': self._is_regression,
64+
'is_robot': self._is_robot,
65+
'repository_shorthand': self.get_repository_shorthand(),
66+
'run_type': self._run_type,
67+
'run_id': self._run_id,
68+
}
69+
70+
def json(self) -> str:
71+
return json.dumps(self._to_dict())
72+
73+
74+
class _TagParser(object):
75+
76+
_BUILD_ONLY_TAG_PATTERN = re.compile(r'^build-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
77+
_BUILD_RELEASE_TAG_PATTERN = re.compile(r'^build-release-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
78+
_SCENARIO_ONLY_TAG_PATTERN = re.compile(r'^scenario-(?P<run_id>[^-]*)-(?P<run_type>[^-]*)-(?P<repository_shorthand>[^-]*)$')
79+
_ROBOT_TAG_PATTERN = re.compile(r'^robot-(?P<run_id>[^-]*)-(?P<run_type>[^-]*)-(?P<repository_shorthand>[^-]*)-(?P<seq>\d+)$')
80+
_ROBOT_ONLY_TAG_PATTERN = re.compile(r'^robot-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
81+
_REGRESSION_ONLY_TAG_PATTERN = re.compile(r'^regression-(?P<descriptor>[^-]*)-(?P<seq>\d+).*$')
82+
83+
def __init__(
84+
self,
85+
raw_tag: str,
86+
permitted_types: List[str]
87+
):
88+
self._raw_tag = raw_tag
89+
self._permitted_types = permitted_types
90+
91+
def _parse_build_tag(self) -> _Tag:
92+
match = self._BUILD_ONLY_TAG_PATTERN.match(self._raw_tag)
93+
if match:
94+
return _Tag(raw_tag=self._raw_tag, is_build=True)
95+
match = self._BUILD_RELEASE_TAG_PATTERN.match(self._raw_tag)
96+
if match:
97+
return _Tag(raw_tag=self._raw_tag, is_build=True)
98+
return None
99+
100+
def _parse_scenario_tag(self) -> _Tag:
101+
match = self._SCENARIO_ONLY_TAG_PATTERN.match(self._raw_tag)
102+
run_id = match.group('run_id')
103+
run_type = match.group('run_type')
104+
repository_shorthand = match.group('repository_shorthand')
105+
if run_id and run_type:
106+
return _Tag(raw_tag=self._raw_tag, is_integration=True, run_id=run_id, run_type=run_type, repository_shorthand=repository_shorthand)
107+
return None
108+
109+
def _parse_robot_tag(self) -> _Tag:
110+
match = self._ROBOT_TAG_PATTERN.match(self._raw_tag)
111+
if match:
112+
return _Tag(raw_tag=self._raw_tag, is_robot=True)
113+
match = self._ROBOT_ONLY_TAG_PATTERN.match(self._raw_tag)
114+
if match:
115+
return _Tag(raw_tag=self._raw_tag, is_robot=True)
116+
return None
117+
118+
def _parse_regression_tag(self) -> _Tag:
119+
match = self._REGRESSION_ONLY_TAG_PATTERN.match(self._raw_tag)
120+
if match:
121+
return _Tag(raw_tag=self._raw_tag, is_regression=True)
122+
return None
123+
124+
def parse(self) -> _Tag:
125+
if not self._permitted_types:
126+
raise ValueError('No permitted types specified')
127+
for t in self._permitted_types:
128+
if t == 'build':
129+
tag = self._parse_build_tag()
130+
if tag:
131+
return tag
132+
elif t == 'scenario':
133+
tag = self._parse_scenario_tag()
134+
if tag:
135+
return tag
136+
elif t == 'robot':
137+
tag = self._parse_robot_tag()
138+
if tag:
139+
return tag
140+
elif t == 'regression':
141+
tag = self._parse_regression_tag()
142+
if tag:
143+
return tag
144+
raise ValueError(f'Raw tag: "{self._raw_tag}" is incompatible with permitted types: {" ".join(self._permitted_types)}')
145+
146+
def _parse_args() -> argparse.Namespace:
147+
"""
148+
Parse the arguments.
149+
"""
150+
parser = argparse.ArgumentParser(description='Handle and interpret git tags.')
151+
parser.add_argument('tag', help='The tag to parse', type=str)
152+
parser.add_argument('--parse-registry-tag', help='Opt-in to parse a registry tag', action=argparse.BooleanOptionalAction)
153+
parser.add_argument('--parse-scenario-tag', help='Opt-in to parse a scenario tag', action=argparse.BooleanOptionalAction)
154+
return parser.parse_args()
155+
156+
def main():
157+
args = _parse_args()
158+
if args.parse_registry_tag:
159+
tag_parser = _TagParser(args.tag, ['robot', 'regression'])
160+
tag = tag_parser.parse()
161+
if tag.is_robot():
162+
print(tag.json())
163+
return
164+
elif tag.is_regression():
165+
print(tag.json())
166+
return
167+
if args.parse_scenario_tag:
168+
tag_parser = _TagParser(args.tag, ['scenario'])
169+
tag = tag_parser.parse()
170+
if tag.get_run_id() and tag.get_run_type():
171+
print(tag.json())
172+
return
173+
else:
174+
raise ValueError(f'Invalid scenario tag inferred: {tag.json()}')
175+
raise ValueError('No action specified')
176+
177+
if __name__ == '__main__':
178+
main()
179+

0 commit comments

Comments
 (0)