Skip to content

Commit e4cfc6c

Browse files
general-kroll-4-lifegeneral-kroll-4-life
authored
any-sdk-protocol-agnostic (#535)
Summary: - Next gen `any-sdk` with protocol abstraction. - Stream processing moved to `any-sdk`. - GraphQL migrated. - Encapsulation of call logic. - Tests passing.
1 parent 1608439 commit e4cfc6c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+2286
-1844
lines changed

‎.golangci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,7 @@ issues:
318318
- gosec
319319
- noctx
320320
- revive
321+
- typecheck
321322
- wrapcheck
322323
output:
323324
formats:

‎.vscode/launch.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,14 @@
159159
"SELECT instance_id FROM aws.ec2.instances WHERE region IN ('us-east-1', 'ap-southeast-2');",
160160
"SELECT instance_id FROM aws.ec2_solid_gold.instances WHERE region IN ('us-east-1', 'ap-southeast-2');",
161161
"SELECT region FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id;",
162+
"begin; INSERT INTO digitalocean.droplets.droplets( data__name, data__region, data__size, data__image, data__backups, data__ipv6, data__monitoring, data__tags ) SELECT 'some.example.com', 'nyc3', 's-1vcpu-1gb', 'ubuntu-20-04-x64', true, true, true, '[\"env:prod\", \"web\"]' ; INSERT INTO digitalocean.droplets.droplets( data__name, data__region, data__size, data__image, data__backups, data__ipv6, data__monitoring, data__tags ) SELECT 'some.example.com', 'nyc3', 's-1vcpu-1gb', 'ubuntu-20-04-x64', true, true, true, '[\"env:prod\", \"web\"]' ; INSERT INTO digitalocean.droplets.droplets( data__name, data__region, data__size, data__image, data__backups, data__ipv6, data__monitoring, data__tags ) SELECT 'error.example.com', 'nyc3', 's-1vcpu-1gb', 'ubuntu-20-04-x64', true, true, true, '[\"env:prod\", \"web\"]' ; commit;",
163+
"select keyz.name as key_name, keyz.tags as key_tags, json_extract(detail.properties, '$.kty') as key_class, json_extract(detail.properties, '$.keySize') as key_size, json_extract(detail.properties, '$.keyOps') as key_ops, keyz.type as key_type from azure.key_vault.vaults vaultz inner join azure.key_vault.keys keyz on keyz.vaultName = split_part(vaultz.id, '/', -1) and keyz.subscriptionId = vaultz.subscriptionId and keyz.resourceGroupName = split_part(vaultz.id, '/', 5) inner join azure.key_vault.keys detail on detail.vaultName = split_part(vaultz.id, '/', -1) and detail.subscriptionId = '000000-0000-0000-0000-000000000022' and detail.resourceGroupName = split_part(vaultz.id, '/', 5) and detail.keyName = split_part(keyz.id, '/', -1) where vaultz.subscriptionId = '000000-0000-0000-0000-000000000022' order by key_name ;",
164+
"delete from aws.transfer.users where data__ServerId = 's-0000000001' and data__UserName = '[email protected]' and region = 'ap-southeast-2';",
165+
"select * from google.cloudkms.key_rings where projectsId = 'testing-project' and locationsId = 'australia-southeast1';",
166+
"delete from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::Logs::LogGroup' and data__Identifier = 'LogGroupResourceExampleThird' ;",
167+
"EXEC aws.transfer.servers.stop_server @region = 'ap-southeast-2' @@json='{ \"ServerId\": \"s-0000000001\" }' ;",
168+
"exec aws.ec2.instances.instances_Stop @region = 'ap-southeast-2', @InstanceId = 'id-001' ;",
169+
"begin; insert into googleadmin.directory.users(data__primaryEmail) values ('[email protected]'); rollback;",
162170
],
163171
"default": "show providers;"
164172
},
@@ -190,7 +198,7 @@
190198
"{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
191199
"{ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" },s \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/netlify/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/k8s/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/sumologic/sumologic-token.txt\", \"type\": \"basic\" } }",
192200
"{ \"pgi\": { \"type\": \"sql_data_source::postgres\", \"sqlDataSource\": { \"dsn\": \"postgres://stackql:[email protected]:8432\" } }, \"azure\": { \"type\": \"azure_default\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
193-
"{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }",
201+
"{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" }, \"azure\": {\"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsenvvar\": \"AZ_ACCESS_TOKEN\"} }",
194202
"{}"
195203
]
196204
},

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ require (
1919
github.com/spf13/cobra v1.4.0
2020
github.com/spf13/pflag v1.0.5
2121
github.com/spf13/viper v1.10.1
22-
github.com/stackql/any-sdk v0.0.5-beta01
22+
github.com/stackql/any-sdk v0.1.1-alpha11
2323
github.com/stackql/go-suffix-map v0.0.1-alpha01
2424
github.com/stackql/psql-wire v0.1.1-alpha07
2525
github.com/stackql/stackql-parser v0.0.14-alpha05

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -482,8 +482,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
482482
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
483483
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
484484
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
485-
github.com/stackql/any-sdk v0.0.5-beta01 h1:1KEAJC/IjOlxVLh385SecmLyDiERW+E3fFN2Rl7HMcY=
486-
github.com/stackql/any-sdk v0.0.5-beta01/go.mod h1:4jYKpPoX2GWEK+qBnlGLvr8SUfndiYwHMIkg1dn9tFM=
485+
github.com/stackql/any-sdk v0.1.1-alpha11 h1:jaJ4vjQ1Vm+1zUGBQ9HTWjWAqzCIJ9qdmuSMi3AQoFM=
486+
github.com/stackql/any-sdk v0.1.1-alpha11/go.mod h1:4jYKpPoX2GWEK+qBnlGLvr8SUfndiYwHMIkg1dn9tFM=
487487
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
488488
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
489489
github.com/stackql/psql-wire v0.1.1-alpha07 h1:LQWVUlx4Bougk6dztDNG5tmXxpIVeeTSsInTj801xCs=

‎internal/stackql/data_staging/output_data_staging/packet_preparator.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"github.com/lib/pq/oid"
77
"github.com/stackql/any-sdk/pkg/dto"
88
"github.com/stackql/any-sdk/pkg/logging"
9+
"github.com/stackql/any-sdk/pkg/streaming"
910
"github.com/stackql/stackql/internal/stackql/drm"
10-
"github.com/stackql/stackql/internal/stackql/streaming"
1111
"github.com/stackql/stackql/internal/stackql/typing"
1212
)
1313

‎internal/stackql/dependencyplanner/dependencyplanner.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/stackql/any-sdk/pkg/constants"
99
"github.com/stackql/any-sdk/pkg/logging"
1010
"github.com/stackql/any-sdk/pkg/media"
11+
"github.com/stackql/any-sdk/pkg/streaming"
1112
"github.com/stackql/stackql-parser/go/vt/sqlparser"
1213
"github.com/stackql/stackql/internal/stackql/astanalysis/annotatedast"
1314
"github.com/stackql/stackql/internal/stackql/astindirect"
@@ -22,7 +23,6 @@ import (
2223
"github.com/stackql/stackql/internal/stackql/primitivecomposer"
2324
"github.com/stackql/stackql/internal/stackql/sqlrewrite"
2425
"github.com/stackql/stackql/internal/stackql/sqlstream"
25-
"github.com/stackql/stackql/internal/stackql/streaming"
2626
"github.com/stackql/stackql/internal/stackql/tableinsertioncontainer"
2727
"github.com/stackql/stackql/internal/stackql/taxonomy"
2828
"github.com/stackql/stackql/internal/stackql/util"
@@ -407,7 +407,7 @@ func (dp *standardDependencyPlanner) processOrphan(
407407
} else {
408408
// Persist SQL mirror table here prior to generating insert DML
409409
drmCfg := dp.handlerCtx.GetDrmConfig()
410-
ddl, ddlErr := drmCfg.GenerateDDL(anTab, opStore, 0, false)
410+
ddl, ddlErr := drmCfg.GenerateDDL(anTab, opStore, 0, false, false)
411411
if ddlErr != nil {
412412
return nil, nil, ddlErr
413413
}
@@ -416,7 +416,7 @@ func (dp *standardDependencyPlanner) processOrphan(
416416
return nil, nil, err
417417
}
418418
}
419-
insPsc, err := dp.primitiveComposer.GetDRMConfig().GenerateInsertDML(anTab, opStore, tcc)
419+
insPsc, err := dp.primitiveComposer.GetDRMConfig().GenerateInsertDML(anTab, opStore, tcc, false)
420420
return insPsc, tcc, err
421421
}
422422

‎internal/stackql/dependencyplanner/stream_collection.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package dependencyplanner
22

33
import (
4-
"github.com/stackql/stackql/internal/stackql/streaming"
4+
"github.com/stackql/any-sdk/pkg/streaming"
55
)
66

77
type StreamDependecyCollection interface {

‎internal/stackql/docparser/doc_parser.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ func OpenapiStackQLTabulationsPersistor(
6262
return discoveryGenerationID, err
6363
}
6464
for _, tblt := range tabluationsAnnotated {
65-
ddl, ddlErr := drmCfg.GenerateDDL(tblt, m, discoveryGenerationID, false)
65+
ddl, ddlErr := drmCfg.GenerateDDL(tblt, m, discoveryGenerationID, false, true)
6666
if ddlErr != nil {
6767
displayErr := fmt.Errorf("error generating DDL: %w", err)
6868
logging.GetLogger().Infoln(displayErr.Error())

‎internal/stackql/drm/drm_cfg.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ import (
1111
"github.com/stackql/any-sdk/anysdk"
1212
"github.com/stackql/any-sdk/pkg/constants"
1313
"github.com/stackql/any-sdk/pkg/logging"
14+
"github.com/stackql/any-sdk/pkg/streaming"
1415
"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
1516
"github.com/stackql/stackql/internal/stackql/internal_data_transfer/relationaldto"
1617
"github.com/stackql/stackql/internal/stackql/sql_system"
1718
"github.com/stackql/stackql/internal/stackql/sqlcontrol"
1819
"github.com/stackql/stackql/internal/stackql/sqlengine"
1920
"github.com/stackql/stackql/internal/stackql/sqlmachinery"
20-
"github.com/stackql/stackql/internal/stackql/streaming"
2121
"github.com/stackql/stackql/internal/stackql/tablenamespace"
2222
"github.com/stackql/stackql/internal/stackql/typing"
2323
"github.com/stackql/stackql/internal/stackql/util"
@@ -44,7 +44,7 @@ type Config interface {
4444
) (map[string]map[string]interface{}, map[int]map[int]interface{})
4545
GetCurrentTable(internaldto.HeirarchyIdentifiers) (internaldto.DBTable, error)
4646
GetRelationalType(string) string
47-
GenerateDDL(util.AnnotatedTabulation, anysdk.OperationStore, int, bool) ([]string, error)
47+
GenerateDDL(util.AnnotatedTabulation, anysdk.OperationStore, int, bool, bool) ([]string, error)
4848
GetControlAttributes() sqlcontrol.ControlAttributes
4949
GetGolangValue(string) interface{}
5050
GetGolangSlices([]typing.ColumnMetadata) ([]interface{}, []string)
@@ -56,6 +56,7 @@ type Config interface {
5656
util.AnnotatedTabulation,
5757
anysdk.OperationStore,
5858
internaldto.TxnControlCounters,
59+
bool,
5960
) (PreparedStatementCtx, error)
6061
GenerateSelectDML(
6162
util.AnnotatedTabulation,
@@ -443,6 +444,7 @@ func (dc *staticDRMConfig) genRelationalTable(
443444
tabAnn util.AnnotatedTabulation,
444445
m anysdk.OperationStore,
445446
discoveryGenerationID int,
447+
isNilResponseAlloed bool,
446448
) (relationaldto.RelationalTable, error) {
447449
tableName, err := dc.getTableName(tabAnn.GetHeirarchyIdentifiers(), discoveryGenerationID)
448450
if err != nil {
@@ -458,7 +460,7 @@ func (dc *staticDRMConfig) genRelationalTable(
458460
tableName,
459461
tabAnn.GetInputTableName(),
460462
)
461-
schemaAnalyzer := util.NewTableSchemaAnalyzer(tabAnn.GetTabulation().GetSchema(), m)
463+
schemaAnalyzer := util.NewTableSchemaAnalyzer(tabAnn.GetTabulation().GetSchema(), m, isNilResponseAlloed)
462464
tableColumns, err := schemaAnalyzer.GetColumns()
463465
if err != nil {
464466
return nil, err
@@ -480,8 +482,9 @@ func (dc *staticDRMConfig) GenerateDDL(
480482
m anysdk.OperationStore,
481483
discoveryGenerationID int,
482484
dropTable bool,
485+
isNilResponseAlloed bool,
483486
) ([]string, error) {
484-
relationalTable, err := dc.genRelationalTable(tabAnn, m, discoveryGenerationID)
487+
relationalTable, err := dc.genRelationalTable(tabAnn, m, discoveryGenerationID, isNilResponseAlloed)
485488
if err != nil {
486489
return nil, err
487490
}
@@ -493,6 +496,7 @@ func (dc *staticDRMConfig) GenerateInsertDML(
493496
tabAnnotated util.AnnotatedTabulation,
494497
method anysdk.OperationStore,
495498
tcc internaldto.TxnControlCounters,
499+
isNilResponseAlloed bool,
496500
) (PreparedStatementCtx, error) {
497501
var columns []typing.ColumnMetadata
498502
_, isSQLDataSource := tabAnnotated.GetSQLDataSource()
@@ -536,9 +540,9 @@ func (dc *staticDRMConfig) GenerateInsertDML(
536540
relationalTable.PushBackColumn(col)
537541
}
538542
} else {
539-
schemaAnalyzer := util.NewTableSchemaAnalyzer(tabAnnotated.GetTabulation().GetSchema(), method)
543+
schemaAnalyzer := util.NewTableSchemaAnalyzer(tabAnnotated.GetTabulation().GetSchema(), method, isNilResponseAlloed)
540544
tableColumns, err := schemaAnalyzer.GetColumnDescriptors(tabAnnotated)
541-
if err != nil {
545+
if err != nil && !isNilResponseAlloed {
542546
return nil, err
543547
}
544548
for _, col := range tableColumns {

0 commit comments

Comments
 (0)