Skip to content

Commit 0e5d4fd

Browse files
general-kroll-4-lifegeneral-kroll-4-life
authored
indirect-column-routing (#518)
Summary: - Support for nested functions on indirect columns. - Added robot test `AWS Materialized View And Multiple Function Query on Resource Costs Exemplifies Multiple Functions on Materialized Views`. - Function convention corrected. - Deterministic, informative query. - Naming corrrection.
1 parent 87e57d9 commit 0e5d4fd

File tree

3 files changed

+181
-2
lines changed

3 files changed

+181
-2
lines changed

‎internal/stackql/astvisit/query_rewriting.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@ import (
2525
)
2626

2727
var (
28-
_ QueryRewriteAstVisitor = &standardQueryRewriteAstVisitor{}
29-
isJSONEachCompatibleRegexp *regexp.Regexp = regexp.MustCompile(`^.*\.value$`) //nolint:revive // acceptable
28+
_ QueryRewriteAstVisitor = &standardQueryRewriteAstVisitor{}
29+
//nolint:revive // acceptable
30+
isJSONEachCompatibleRegexp *regexp.Regexp = regexp.MustCompile(
31+
`^(.*\.value|value)$`)
3032
)
3133

3234
type QueryRewriteAstVisitor interface {

‎test/assets/expected/aws/ce/ce-nested-function-materialized-view.txt‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
|------------|------------|--------------------------------|---------------|------|
2+
| beginning | ending | keyz | amount | unit |
3+
|------------|------------|--------------------------------|---------------|------|
4+
| 2024-11-01 | 2024-11-30 | ["AWS CloudFormation"] | 0 | USD |
5+
|------------|------------|--------------------------------|---------------|------|
6+
| 2024-08-01 | 2024-09-01 | ["AWS Glue"] | 0 | USD |
7+
|------------|------------|--------------------------------|---------------|------|
8+
| 2024-09-01 | 2024-10-01 | ["AWS Glue"] | 0 | USD |
9+
|------------|------------|--------------------------------|---------------|------|
10+
| 2024-10-01 | 2024-11-01 | ["AWS Glue"] | 0 | USD |
11+
|------------|------------|--------------------------------|---------------|------|
12+
| 2024-11-01 | 2024-11-30 | ["AWS Glue"] | 0 | USD |
13+
|------------|------------|--------------------------------|---------------|------|
14+
| 2024-11-01 | 2024-11-30 | ["AWS Lambda"] | 0 | USD |
15+
|------------|------------|--------------------------------|---------------|------|
16+
| 2024-08-01 | 2024-09-01 | ["AWS Secrets Manager"] | 0 | USD |
17+
|------------|------------|--------------------------------|---------------|------|
18+
| 2024-09-01 | 2024-10-01 | ["AWS Secrets Manager"] | 0 | USD |
19+
|------------|------------|--------------------------------|---------------|------|
20+
| 2024-10-01 | 2024-11-01 | ["AWS Secrets Manager"] | 0 | USD |
21+
|------------|------------|--------------------------------|---------------|------|
22+
| 2024-11-01 | 2024-11-30 | ["AWS Secrets Manager"] | 0 | USD |
23+
|------------|------------|--------------------------------|---------------|------|
24+
| 2024-08-01 | 2024-09-01 | ["Amazon Simple Notification | 0 | USD |
25+
| | | Service"] | | |
26+
|------------|------------|--------------------------------|---------------|------|
27+
| 2024-09-01 | 2024-10-01 | ["Amazon Simple Notification | 0 | USD |
28+
| | | Service"] | | |
29+
|------------|------------|--------------------------------|---------------|------|
30+
| 2024-10-01 | 2024-11-01 | ["Amazon Simple Notification | 0 | USD |
31+
| | | Service"] | | |
32+
|------------|------------|--------------------------------|---------------|------|
33+
| 2024-11-01 | 2024-11-30 | ["Amazon Simple Notification | 0 | USD |
34+
| | | Service"] | | |
35+
|------------|------------|--------------------------------|---------------|------|
36+
| 2024-08-01 | 2024-09-01 | ["Amazon Simple Queue | 0 | USD |
37+
| | | Service"] | | |
38+
|------------|------------|--------------------------------|---------------|------|
39+
| 2024-09-01 | 2024-10-01 | ["Amazon Simple Queue | 0 | USD |
40+
| | | Service"] | | |
41+
|------------|------------|--------------------------------|---------------|------|
42+
| 2024-10-01 | 2024-11-01 | ["Amazon Simple Queue | 0 | USD |
43+
| | | Service"] | | |
44+
|------------|------------|--------------------------------|---------------|------|
45+
| 2024-11-01 | 2024-11-30 | ["Amazon Simple Queue | 0 | USD |
46+
| | | Service"] | | |
47+
|------------|------------|--------------------------------|---------------|------|
48+
| 2024-08-01 | 2024-09-01 | ["AmazonCloudWatch"] | 0 | USD |
49+
|------------|------------|--------------------------------|---------------|------|
50+
| 2024-09-01 | 2024-10-01 | ["AmazonCloudWatch"] | 0 | USD |
51+
|------------|------------|--------------------------------|---------------|------|
52+
| 2024-10-01 | 2024-11-01 | ["AmazonCloudWatch"] | 0 | USD |
53+
|------------|------------|--------------------------------|---------------|------|
54+
| 2024-11-01 | 2024-11-30 | ["AmazonCloudWatch"] | 0 | USD |
55+
|------------|------------|--------------------------------|---------------|------|
56+
| 2024-11-01 | 2024-11-30 | ["AWS CloudShell"] | 0.0000002152 | USD |
57+
|------------|------------|--------------------------------|---------------|------|
58+
| 2024-11-01 | 2024-11-30 | ["Amazon Location Service"] | 0.001455 | USD |
59+
|------------|------------|--------------------------------|---------------|------|
60+
| 2024-09-01 | 2024-10-01 | ["Amazon Location Service"] | 0.001554 | USD |
61+
|------------|------------|--------------------------------|---------------|------|
62+
| 2024-08-01 | 2024-09-01 | ["Amazon Location Service"] | 0.001584 | USD |
63+
|------------|------------|--------------------------------|---------------|------|
64+
| 2024-10-01 | 2024-11-01 | ["Amazon Location Service"] | 0.001653 | USD |
65+
|------------|------------|--------------------------------|---------------|------|
66+
| 2024-08-01 | 2024-09-01 | ["Amazon DynamoDB"] | 0.0051069546 | USD |
67+
|------------|------------|--------------------------------|---------------|------|
68+
| 2024-08-01 | 2024-09-01 | ["Amazon Simple Storage | 0.0473099157 | USD |
69+
| | | Service"] | | |
70+
|------------|------------|--------------------------------|---------------|------|
71+
| 2024-11-01 | 2024-11-30 | ["Amazon EC2 Container | 0.0611570328 | USD |
72+
| | | Registry (ECR)"] | | |
73+
|------------|------------|--------------------------------|---------------|------|
74+
| 2024-08-01 | 2024-09-01 | ["Amazon EC2 Container | 0.0632658912 | USD |
75+
| | | Registry (ECR)"] | | |
76+
|------------|------------|--------------------------------|---------------|------|
77+
| 2024-10-01 | 2024-11-01 | ["Amazon EC2 Container | 0.0632658912 | USD |
78+
| | | Registry (ECR)"] | | |
79+
|------------|------------|--------------------------------|---------------|------|
80+
| 2024-09-01 | 2024-10-01 | ["Amazon EC2 Container | 0.063265896 | USD |
81+
| | | Registry (ECR)"] | | |
82+
|------------|------------|--------------------------------|---------------|------|
83+
| 2024-11-01 | 2024-11-30 | ["Amazon DynamoDB"] | 0.066691505 | USD |
84+
|------------|------------|--------------------------------|---------------|------|
85+
| 2024-09-01 | 2024-10-01 | ["Amazon DynamoDB"] | 0.0787472044 | USD |
86+
|------------|------------|--------------------------------|---------------|------|
87+
| 2024-10-01 | 2024-11-01 | ["Amazon DynamoDB"] | 0.1278523226 | USD |
88+
|------------|------------|--------------------------------|---------------|------|
89+
| 2024-09-01 | 2024-10-01 | ["Amazon Simple Storage | 0.3545428953 | USD |
90+
| | | Service"] | | |
91+
|------------|------------|--------------------------------|---------------|------|
92+
| 2024-10-01 | 2024-11-01 | ["Amazon Simple Storage | 0.3908839969 | USD |
93+
| | | Service"] | | |
94+
|------------|------------|--------------------------------|---------------|------|
95+
| 2024-11-01 | 2024-11-30 | ["Amazon Simple Storage | 0.6006374463 | USD |
96+
| | | Service"] | | |
97+
|------------|------------|--------------------------------|---------------|------|
98+
| 2024-08-01 | 2024-09-01 | ["AWS Key Management Service"] | 0.999999984 | USD |
99+
|------------|------------|--------------------------------|---------------|------|
100+
| 2024-11-01 | 2024-11-30 | ["AWS Config"] | 1.679 | USD |
101+
|------------|------------|--------------------------------|---------------|------|
102+
| 2024-09-01 | 2024-10-01 | ["AWS Config"] | 1.759 | USD |
103+
|------------|------------|--------------------------------|---------------|------|
104+
| 2024-10-01 | 2024-11-01 | ["AWS Config"] | 1.806 | USD |
105+
|------------|------------|--------------------------------|---------------|------|
106+
| 2024-08-01 | 2024-09-01 | ["EC2 - Other"] | 1.8325932937 | USD |
107+
|------------|------------|--------------------------------|---------------|------|
108+
| 2024-08-01 | 2024-09-01 | ["AWS Config"] | 1.982 | USD |
109+
|------------|------------|--------------------------------|---------------|------|
110+
| 2024-09-01 | 2024-10-01 | ["Amazon Virtual Private | 10.8008 | USD |
111+
| | | Cloud"] | | |
112+
|------------|------------|--------------------------------|---------------|------|
113+
| 2024-10-01 | 2024-11-01 | ["Amazon Virtual Private | 11.16 | USD |
114+
| | | Cloud"] | | |
115+
|------------|------------|--------------------------------|---------------|------|
116+
| 2024-11-01 | 2024-11-30 | ["Tax"] | 14.04 | USD |
117+
|------------|------------|--------------------------------|---------------|------|
118+
| 2024-10-01 | 2024-11-01 | ["EC2 - Other"] | 2.3042661799 | USD |
119+
|------------|------------|--------------------------------|---------------|------|
120+
| 2024-09-01 | 2024-10-01 | ["EC2 - Other"] | 2.3043115302 | USD |
121+
|------------|------------|--------------------------------|---------------|------|
122+
| 2024-08-01 | 2024-09-01 | ["Amazon Elastic Compute Cloud | 20.445402 | USD |
123+
| | | - Compute"] | | |
124+
|------------|------------|--------------------------------|---------------|------|
125+
| 2024-09-01 | 2024-10-01 | ["Amazon Elastic Compute Cloud | 26.2821332206 | USD |
126+
| | | - Compute"] | | |
127+
|------------|------------|--------------------------------|---------------|------|
128+
| 2024-10-01 | 2024-11-01 | ["Amazon Elastic Compute Cloud | 27.1268 | USD |
129+
| | | - Compute"] | | |
130+
|------------|------------|--------------------------------|---------------|------|
131+
| 2024-11-01 | 2024-11-30 | ["Amazon Elastic Compute Cloud | 27.1546234373 | USD |
132+
| | | - Compute"] | | |
133+
|------------|------------|--------------------------------|---------------|------|
134+
| 2024-08-01 | 2024-09-01 | ["Tax"] | 3.42 | USD |
135+
|------------|------------|--------------------------------|---------------|------|
136+
| 2024-09-01 | 2024-10-01 | ["AWS Key Management Service"] | 4.163155008 | USD |
137+
|------------|------------|--------------------------------|---------------|------|
138+
| 2024-09-01 | 2024-10-01 | ["Tax"] | 4.58 | USD |
139+
|------------|------------|--------------------------------|---------------|------|
140+
| 2024-10-01 | 2024-11-01 | ["Tax"] | 4.94 | USD |
141+
|------------|------------|--------------------------------|---------------|------|
142+
| 2024-11-01 | 2024-11-30 | ["Amazon Virtual Private | 47.5634987792 | USD |
143+
| | | Cloud"] | | |
144+
|------------|------------|--------------------------------|---------------|------|
145+
| 2024-11-01 | 2024-11-30 | ["EC2 - Other"] | 50.9307417991 | USD |
146+
|------------|------------|--------------------------------|---------------|------|
147+
| 2024-10-01 | 2024-11-01 | ["AWS Key Management Service"] | 6.383010812 | USD |
148+
|------------|------------|--------------------------------|---------------|------|
149+
| 2024-11-01 | 2024-11-30 | ["AWS Key Management Service"] | 6.4058586744 | USD |
150+
|------------|------------|--------------------------------|---------------|------|
151+
| 2024-08-01 | 2024-09-01 | ["Amazon Virtual Private | 8.857988885 | USD |
152+
| | | Cloud"] | | |
153+
|------------|------------|--------------------------------|---------------|------|

‎test/robot/functional/stackql_mocked_from_cmd_line.robot‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7496,3 +7496,27 @@ AWS Materialized View And Query on Resource Costs Exemplifies Functions On Mater
74967496
... DDL Execution Completed
74977497
... stdout=${CURDIR}/tmp/AWS-Materialized-View-And-Query-on-Resource-Costs-Exemplifies-Functions-On-Materialized-Views.tmp
74987498
... stderr=${CURDIR}/tmp/AWS-Materialized-View-And-Query-on-Resource-Costs-Exemplifies-Functions-On-Materialized-Views-stderr.tmp
7499+
7500+
AWS Materialized View And Multiple Function Query on Resource Costs Exemplifies Multiple Functions on Materialized Views
7501+
[Tags] registry tls_proxied
7502+
${sqliteInputStr} = Catenate
7503+
... create or replace materialized view e1 as select json_extract(json_each.value, '$.TimePeriod.Start') as beginning, json_extract(json_each.value, '$.TimePeriod.End') as ending, json_extract(json_each.value, '$.Groups') as rez from aws.ce_native.cost_and_usage, json_each(ResultsByTime) where data__Granularity = 'MONTHLY' and data__Metrics = '["UnblendedCost"]' and data__TimePeriod = '{"Start": "2024-08-01", "End": "2024-11-30"}' and data__GroupBy = '[{"Type":"DIMENSION","Key":"SERVICE"}]' and region = 'us-east-1';
7504+
... select beginning, ending, json_extract(json_each.value, '$.Keys') as keyz, json_extract(json_each.value, '$.Metrics.UnblendedCost.Amount') as amount, json_extract(json_each.value, '$.Metrics.UnblendedCost.Unit') as unit from e1, json_each(e1.rez) order by amount, keyz, ending;
7505+
${postgresInputStr} = Catenate
7506+
... create or replace materialized view e1 as select json_extract_path_text(rd.value, 'TimePeriod', 'Start') as beginning, json_extract_path_text(rd.value, 'TimePeriod', 'End') as ending, json_extract_path_text(rd.value, 'Groups') as rez from aws.ce_native.cost_and_usage, json_array_elements_text(ResultsByTime) as rd where data__Granularity = 'MONTHLY' and data__Metrics = '["UnblendedCost"]' and data__TimePeriod = '{"Start": "2024-08-01", "End": "2024-11-30"}' and data__GroupBy = '[{"Type":"DIMENSION","Key":"SERVICE"}]' and region = 'us-east-1';
7507+
... select beginning, ending, json_extract_path_text(rd.value, 'Keys') as keyz, json_extract_path_text(rd.value, 'Metrics', 'UnblendedCost', 'Amount') as amount, json_extract_path_text(rd.value, 'Metrics', 'UnblendedCost', 'Unit') as unit from e1, json_array_elements_text(e1.rez) as rd order by amount, keyz, ending;
7508+
${inputStr} = Set Variable If "${SQL_BACKEND}" == "postgres_tcp" ${postgresInputStr} ${sqliteInputStr}
7509+
${outputStr} = Get File ${REPOSITORY_ROOT}${/}test${/}assets${/}expected${/}aws${/}ce${/}ce-nested-function-materialized-view.txt
7510+
Should Stackql Exec Inline Equal Both Streams
7511+
... ${STACKQL_EXE}
7512+
... ${OKTA_SECRET_STR}
7513+
... ${GITHUB_SECRET_STR}
7514+
... ${K8S_SECRET_STR}
7515+
... ${REGISTRY_NO_VERIFY_CFG_STR}
7516+
... ${AUTH_CFG_STR}
7517+
... ${SQL_BACKEND_CFG_STR_CANONICAL}
7518+
... ${inputStr}
7519+
... ${outputStr}
7520+
... DDL Execution Completed
7521+
... stdout=${CURDIR}/tmp/AWS-Materialized-View-And-Multiple-Function-Query-on-Resource-Costs-Exemplifies-Multiple-Functions-on-Materialized-Views.tmp
7522+
... stderr=${CURDIR}/tmp/AWS-Materialized-View-And-Multiple-Function-Query-on-Resource-Costs-Exemplifies-Multiple-Functions-on-Materialized-Views-stderr.tmp

0 commit comments

Comments
 (0)