Skip to content

Commit ad9409b

Browse files
general-kroll-4-lifegeneral-kroll-4-life
authored
broader-aws-support (#608)
Summary: - Support for `aws.s3` parameter mobility. - Support for fine grained host parameters. - Support for `context` parameters, a convenient abstraction loosely aligned to `golang` `context` at this point. - Added handler unit testing. - Added context variable unit testing.
1 parent 094cf44 commit ad9409b

File tree

21 files changed

+28390
-14
lines changed

21 files changed

+28390
-14
lines changed

‎.golangci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,13 +206,17 @@ linters:
206206
- bodyclose
207207
- dupl
208208
- errcheck
209+
- errorlint
209210
- funlen
210211
- goconst
211212
- gosec
212213
- noctx
213214
- revive
214215
- wrapcheck
215216
- govet
217+
- lll
218+
- whitespace
219+
- cyclop
216220
path: _test\.go
217221
paths:
218222
- third_party$

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ require (
1818
github.com/spf13/cobra v1.4.0
1919
github.com/spf13/pflag v1.0.5
2020
github.com/spf13/viper v1.10.1
21-
github.com/stackql/any-sdk v0.4.2-beta08
21+
github.com/stackql/any-sdk v0.4.3-alpha07
2222
github.com/stackql/go-suffix-map v0.0.1-alpha01
2323
github.com/stackql/psql-wire v0.1.2-alpha01
2424
github.com/stackql/stackql-parser v0.0.16-alpha01

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -467,8 +467,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
467467
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
468468
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
469469
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
470-
github.com/stackql/any-sdk v0.4.2-beta08 h1:LyCWuUAnis8mukHc+2el7shqwAZxalSEOcXkv+b6ZQQ=
471-
github.com/stackql/any-sdk v0.4.2-beta08/go.mod h1:ZRTGrcDKFLaC+5yWo4OqXVs1HTNxgYM9nQsQDlq0Fe0=
470+
github.com/stackql/any-sdk v0.4.3-alpha07 h1:TJaVO63sE49q4KSl7RCQtfX82mLB3FANH3FbwuUcj1o=
471+
github.com/stackql/any-sdk v0.4.3-alpha07/go.mod h1:ZRTGrcDKFLaC+5yWo4OqXVs1HTNxgYM9nQsQDlq0Fe0=
472472
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
473473
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
474474
github.com/stackql/psql-wire v0.1.2-alpha01 h1:RMBRURGspmSNqm2/sgoEc+D6Sri2y/3drjl4nKlOOi4=

‎internal/stackql/docparser/doc_parser.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ func OpenapiStackQLTabulationsPersistor(
7171
for _, tblt := range tabluationsAnnotated {
7272
ddl, ddlErr := drmCfg.GenerateDDL(tblt, prov, svc, resource, m, isAwait, discoveryGenerationID, false, true)
7373
if ddlErr != nil {
74-
displayErr := fmt.Errorf("error generating DDL: %w", err)
74+
displayErr := fmt.Errorf("error generating DDL: %w", ddlErr)
7575
logging.GetLogger().Infoln(displayErr.Error())
7676
txn.Rollback() //nolint:errcheck // TODO: investigate
7777
return discoveryGenerationID, displayErr

‎internal/stackql/driver/driver_defaulted_integration_test.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package driver_test
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net"
8+
"net/http"
9+
"net/http/httptest"
10+
"runtime"
11+
"testing"
12+
13+
"github.com/stackql/any-sdk/pkg/dto"
14+
. "github.com/stackql/stackql/internal/stackql/driver"
15+
"github.com/stackql/stackql/internal/stackql/util"
16+
"github.com/stretchr/testify/assert"
17+
18+
"github.com/stackql/stackql/internal/stackql/handler"
19+
20+
"github.com/stackql/stackql/internal/test/stackqltestutil"
21+
"github.com/stackql/stackql/internal/test/testobjects"
22+
23+
lrucache "github.com/stackql/stackql-parser/go/cache"
24+
)
25+
26+
func getRuntimeCtx(providerStr string, outputFmtStr string, testName string) (*dto.RuntimeCtx, error) {
27+
saKeyPath, err := util.GetFilePathFromRepositoryRoot("internal/stackql/driver/testdata/dummy_credentials/dummy-sa-key.json")
28+
if err != nil {
29+
return nil, fmt.Errorf("test failed on %s: %v", saKeyPath, err)
30+
}
31+
oktaSaKeyPath, err := util.GetFilePathFromRepositoryRoot("internal/stackql/driver/testdata/dummy_credentials/okta-api-key.txt")
32+
if err != nil {
33+
return nil, fmt.Errorf("test failed on %s: %v", oktaSaKeyPath, err)
34+
}
35+
appRoot, err := util.GetFilePathFromRepositoryRoot("internal/stackql/driver/testdata/.stackql")
36+
if err != nil {
37+
return nil, fmt.Errorf("test failed on %s: %v", appRoot, err)
38+
}
39+
dbInitFilePath, err := util.GetFilePathFromRepositoryRoot("test/db/sqlite/setup.sql")
40+
if err != nil {
41+
return nil, fmt.Errorf("test failed on %s: %v", dbInitFilePath, err)
42+
}
43+
registryRoot, err := util.GetFilePathFromRepositoryRoot("internal/stackql/driver/testdata/registry")
44+
if err != nil {
45+
return nil, fmt.Errorf("test failed on %s: %v", registryRoot, err)
46+
}
47+
return &dto.RuntimeCtx{
48+
Delimiter: ",",
49+
ProviderStr: providerStr,
50+
LogLevelStr: "warn",
51+
ApplicationFilesRootPath: appRoot,
52+
AuthRaw: fmt.Sprintf(`{ "google": { "credentialsfilepath": "%s" }, "okta": { "credentialsfilepath": "%s", "type": "api_key" } }`, saKeyPath, oktaSaKeyPath),
53+
RegistryRaw: fmt.Sprintf(`{ "url": "file://%s", "localDocRoot": "%s", "useEmbedded": false, "verifyConfig": { "nopVerify": true } }`, registryRoot, registryRoot),
54+
OutputFormat: outputFmtStr,
55+
SQLBackendCfgRaw: fmt.Sprintf(`{ "dbInitFilepath": "%s", "dsn": "file:%s?mode=memory&cache=shared" }`, dbInitFilePath, testName),
56+
ExecutionConcurrencyLimit: 1,
57+
VarList: []string{"test_var=test_value"},
58+
IndirectDepthMax: 5, //nolint:mnd // test config value
59+
}, nil
60+
}
61+
62+
//nolint:lll // legacy test
63+
func TestDefaultedHttpClientIntegration(t *testing.T) {
64+
if runtime.GOOS == "windows" {
65+
t.Skip("skipping test on Windows")
66+
}
67+
t.Setenv("AWS_SECRET_ACCESS_KEY", "some-junk")
68+
t.Setenv("AWS_ACCESS_KEY_ID", "some-other-junk")
69+
runtimeCtx, err := getRuntimeCtx(testobjects.GetGoogleProviderString(), "text", "TestSimpleSelectGoogleComputeInstanceDriver")
70+
if err != nil {
71+
t.Fatalf("Test failed: %v", err)
72+
}
73+
runtimeCtx.AllowInsecure = true
74+
75+
inputBundle, err := stackqltestutil.BuildInputBundle(*runtimeCtx)
76+
if err != nil {
77+
t.Fatalf("Test failed: %v", err)
78+
}
79+
80+
expectedHost := "my-test-bucket.s3-ap-southeast-2.amazonaws.com"
81+
82+
reponseCounter := 0
83+
84+
tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
85+
defer r.Body.Close()
86+
io.Copy(io.Discard, r.Body)
87+
assert.Equal(t, r.Host, expectedHost, "expected host does not match actual host")
88+
w.WriteHeader(http.StatusOK)
89+
reponseCounter++
90+
}))
91+
t.Cleanup(tlsServer.Close)
92+
93+
baseTransport := tlsServer.Client().Transport.(*http.Transport)
94+
dummyClient := &http.Client{
95+
Transport: &http.Transport{
96+
TLSClientConfig: baseTransport.TLSClientConfig,
97+
DisableKeepAlives: true,
98+
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
99+
return net.Dial("tcp", tlsServer.Listener.Addr().String())
100+
},
101+
},
102+
}
103+
104+
testingQuery := `SELECT * FROM aws.s3.bucket_acls WHERE Bucket = 'my-test-bucket' AND created_date = '2024-01-01T00:00:00Z' AND region = 'ap-southeast-2';`
105+
106+
handlerCtx, err := handler.NewHandlerCtx(
107+
testingQuery,
108+
*runtimeCtx,
109+
lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)),
110+
inputBundle,
111+
"v0.1.0",
112+
)
113+
114+
handlerCtx.SetDefaultHTTPClient(dummyClient)
115+
116+
dr, _ := NewStackQLDriver(handlerCtx)
117+
118+
if err != nil {
119+
t.Fatalf("Test failed: %v", err)
120+
}
121+
122+
dr.ProcessQuery(handlerCtx.GetRawQuery())
123+
124+
if reponseCounter != 1 {
125+
t.Fatalf("Test failed: expected 1 request to test server, got %d", reponseCounter)
126+
}
127+
128+
t.Logf("simple select driver integration test passed")
129+
}

‎internal/stackql/driver/testdata/dummy_credentials/dummy-sa-key.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"type": "service_account",
3+
"project_id": "silly-project-01",
4+
"private_key_id": "silly-key-id",
5+
"private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEAsTfIs92qRg+srgviU9EHJfcgjn/38QFJTa0xYPSNSLSj4hXw\niyPyZ4ih6nIfW2NP4tvwK3piQrmHx/pQesrp5exKF3b97tf+Cvf9gM+i/3wJGhEb\nE24Py91+/5aXf6TfkuhNutZ3v5qzHYRtWcr3AXcD2WAThF6hxmvQTFTYqcyUb6vl\nt7evG6OleifU5rAKmD8Odk8Hs1skwAmlWjWT1uouWVegFv53a8eUilNiBSnYy/PA\nYchL2fGI8Os+CFyju+ns6dG9H7bAOywvZZaTzlVNvJ42gop03l5xzEwHkUKlcukg\nhyDtP5FTeD5liYUJ/Xqq6Xn6Rjck/AgTwqYePwIDAQABAoIBAQCdS08X3oJ4hwcU\nwDWVgW1f1DYQZSLzxdmDWVr/nHAefT8Mt752MWTBYnOcfMi6O663Q9GrNYgrgzMy\nW0m9g4cRbaXhp9sBeLLil3RpNWKOc1A807v9he39W86SGt7DC9rpMMl1MVC+PxgF\n9fl8/no40aMX+H+6OKhMTntmlNRt+E1WUIGBk9cScb37e+oAStfS0Z6c8kQ9CkOZ\nHwydCtq8gQDpkRYMM3m7j3j9mr2Rma1o8xcBbgbomYn6y2xPQf1B4rs9fb8txngv\nGxA+QSRSXNMQmQFLmoc6d3XHFZQ5LX7atRhHsnt4blb8GNqJg8FsTFED68cWKoNp\n0Lk7c/9xAoGBAOLplpGdg3ezAMtFR5gAi8pV9vGJairJsuBij22uK0rItNkaff4G\nYyciybCKJZfXTcOzYq/CGh8KwxhN6RECmHm+bHfnxFDdHXS8e0bs8oTlR7w3yHs4\nhrplBMCwyAgSsma7s8aNBfdkDlPMxYmvytWdHf3T4mj6C5LGlFzpTLmpAoGBAMfv\naHNPLCoJjvQmNlzboN0sLZYhRl2YgW86mM0MoW7pO4cY4On5twbdFD7AtHIKF4j5\nomsMVfTDvKZ+jbHoJFuvvhozq0T/LpEx5b6d27A4pfacmG0zNaLaopoDYQ6wBvYD\n4L9OScFRrtf/X3ZgLzsmNpEVlqyUU2tbCAJVa5mnAoGBAI7ODVmVNPD3Mc+7ySPr\nbA6p7WDzZ2KIT9ARl0yiqVJGYDKmDpb5NBukNCSrvJ8D/EfmtHwCf2f74O6B0eVH\nqegspJ0NuqpdjjUyja8EXliu52eX/880su3Jt6UBXNJf2fD3vlt90zxvtuicXdGa\nVd/8IqzlVX9VpkT4PtT+arAJAoGAZkmknYG+7Y7QVTaLj3xJ0327oNhLQK06YyaO\ncDFrEew/KUHgJ7Q7IEbRCb3bU5C4M7rLjorUGxJdHK0YXxGOMF48GvmeQQFw2JW3\nnYrzjzecKQw6q3uMkFHc6ICcEkCafxjCzf0GnOHmWtlrBIv2/gLx3c42tPp5py3+\nbfs3vncCgYEA0x6tF67q3zt+daUMUox6LimkkDJLsfqnmKjvHFAhKQaqOObWaody\nfGOzlf/KLX15uicYWeCwRrs03OjBfk6LEtVddsZOrjJEp8+6gA8KEf58MJhNuRVj\no3qSe9po1TMj0/hvzd32klXslpqQjXFjm9U3lAxfcRdgWsG7SciOAAs=\n-----END RSA PRIVATE KEY-----\n",
6+
"client_email": "[email protected]",
7+
"client_id": "11",
8+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
9+
"token_uri": "https://oauth2.googleapis.com/token",
10+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
11+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/silly-sa%40silly-project-01.iam.gserviceaccount.com"
12+
}

‎internal/stackql/driver/testdata/dummy_credentials/okta-api-key.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
some-dummy-api-key

‎internal/stackql/driver/testdata/registry/src/aws/v0.1.0/provider.yaml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
id: aws
2+
name: aws
3+
version: v0.1.0
4+
providerServices:
5+
acmpca:
6+
description: acmpca
7+
id: acmpca:v0.1.0
8+
name: acmpca
9+
preferred: true
10+
service:
11+
$ref: aws/v0.1.0/services/acmpca.yaml
12+
title: acmpca API
13+
version: v0.1.0
14+
cloud_control:
15+
description: cloud_control
16+
id: cloud_control:v0.1.0
17+
name: cloud_control
18+
preferred: true
19+
service:
20+
$ref: aws/v0.1.0/services/cloud_control.yaml
21+
title: Cloud Control API
22+
version: v0.1.0
23+
ce_native:
24+
description: ce_native
25+
id: ce_native:v0.1.0
26+
name: ce_native
27+
preferred: true
28+
service:
29+
$ref: aws/v0.1.0/services/ce_native.yaml
30+
title: ce_native API
31+
version: v0.1.0
32+
cloud_control_legacy:
33+
description: cloud_control_legacy
34+
id: cloud_control_legacy:v0.1.0
35+
name: cloud_control_legacy
36+
preferred: true
37+
service:
38+
$ref: aws/v0.1.0/services/cloud_control_legacy.yaml
39+
title: Cloud Control Legacy API
40+
version: v0.1.0
41+
cloudhsm:
42+
description: cloud_hsm
43+
id: cloud_hsm:v2.0.0
44+
name: cloudhsm
45+
preferred: true
46+
service:
47+
$ref: aws/v0.1.0/services/cloudhsm.yaml
48+
title: Cloud HSM API
49+
version: v2.0.0
50+
cloud_watch:
51+
description: cloud_watch
52+
id: cloud_watch:v0.1.0
53+
name: cloud_watch
54+
preferred: true
55+
service:
56+
$ref: aws/v0.1.0/services/cloudwatch.yaml
57+
title: Cloud Control API
58+
version: v0.1.0
59+
ec2:
60+
description: ec2
61+
id: ec2:v0.1.0
62+
name: ec2
63+
preferred: true
64+
service:
65+
$ref: aws/v0.1.0/services/ec2.yaml
66+
title: EC2
67+
version: v0.1.0
68+
ec2_nextgen:
69+
description: ec2_nextgen
70+
id: ec2_nextgen:v0.1.0
71+
name: ec2_nextgen
72+
preferred: true
73+
service:
74+
$ref: aws/v0.1.0/services/ec2_nextgen.yaml
75+
title: EC2 NextGen
76+
iam:
77+
description: iam
78+
id: iam:v0.1.0
79+
name: ec2
80+
preferred: true
81+
service:
82+
$ref: aws/v0.1.0/services/iam.yaml
83+
title: IAM
84+
version: v0.1.0
85+
pseudo_s3:
86+
description: pseudo_s3
87+
id: pseudo_s3:v0.1.0
88+
name: pseudo_s3
89+
preferred: true
90+
service:
91+
$ref: aws/v0.1.0/services/pseudo_s3.yaml
92+
title: Pseudo S3 API
93+
version: v0.1.0
94+
route53:
95+
description: route53
96+
id: route53:v0.1.0
97+
name: route53
98+
preferred: true
99+
service:
100+
$ref: aws/v0.1.0/services/route53.yaml
101+
title: Route 53 API
102+
version: v0.1.0
103+
s3:
104+
description: s3
105+
id: s3:v0.1.0
106+
name: s3
107+
preferred: true
108+
service:
109+
$ref: aws/v0.1.0/services/s3.yaml
110+
title: S3
111+
version: v0.1.0
112+
transfer:
113+
description: transfer
114+
id: transfer:v0.1.0
115+
name: transfer
116+
preferred: true
117+
service:
118+
$ref: aws/v0.1.0/services/transfer.yaml
119+
title: Transfer
120+
version: v0.1.0
121+
openapi: 3.0.0
122+
config:
123+
auth:
124+
type: "aws_signing_v4"
125+
credentialsenvvar: "AWS_SECRET_ACCESS_KEY"
126+
keyIDenvvar: "AWS_ACCESS_KEY_ID"

0 commit comments

Comments
 (0)