Skip to content

Commit b6b8683

Browse files
local-execution (#538)
Summary: - New and refined `any-sdk` API. - Local execution functionality in place. - `insert`, `select`, `describe`, `show` all covered. - Added robot test `Local Execution Openssl RSA Show Methods`. - Added robot test `Local Execution Openssl Create RSA Key Pair`. - Added robot test `Local Execution Openssl x509 Describe`. - Added robot test `Local Execution Openssl x509 Select`.
1 parent 07fce6c commit b6b8683

File tree

11 files changed

+752
-153
lines changed

11 files changed

+752
-153
lines changed

.vscode/launch.json

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,10 @@
167167
"EXEC aws.transfer.servers.stop_server @region = 'ap-southeast-2' @@json='{ \"ServerId\": \"s-0000000001\" }' ;",
168168
"exec aws.ec2.instances.instances_Stop @region = 'ap-southeast-2', @InstanceId = 'id-001' ;",
169169
"begin; insert into googleadmin.directory.users(data__primaryEmail) values ('[email protected]'); rollback;",
170+
"show methods in local_openssl.keys.rsa;",
171+
"insert into local_openssl.keys.rsa(config_file, key_out_file, cert_out_file, days) select '${workspaceFolder}/test/server/mtls/openssl.cnf', '${workspaceFolder}/test/tmp/manual_key.pem', '${workspaceFolder}/test/tmp/manuial_cert.pem', 90;",
172+
"describe local_openssl.keys.x509;",
173+
"select * from local_openssl.keys.x509 where cert_file = '/Users/admin/stackql/stackql-devel/test/tmp/manual_cert.pem';",
170174
],
171175
"default": "show providers;"
172176
},
@@ -195,7 +199,7 @@
195199
"description": "Auth Input arg String",
196200
"default": "{}",
197201
"options": [
198-
"{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
202+
"{ \"local_openssl\": { \"type\": \"null_auth\"}, \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
199203
"{ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" },s \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/netlify/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/k8s/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/sumologic/sumologic-token.txt\", \"type\": \"basic\" } }",
200204
"{ \"pgi\": { \"type\": \"sql_data_source::postgres\", \"sqlDataSource\": { \"dsn\": \"postgres://stackql:[email protected]:8432\" } }, \"azure\": { \"type\": \"azure_default\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
201205
"{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" }, \"azure\": {\"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsenvvar\": \"AZ_ACCESS_TOKEN\"} }",

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ ENV PATH="${APP_DIR}:${PATH}"
161161
COPY --from=integration ${TEST_ROOT_DIR}/build/stackql ${APP_DIR}/
162162

163163
RUN apt-get update \
164-
&& apt-get install -y ca-certificates \
164+
&& apt-get install -y ca-certificates openssl \
165165
&& update-ca-certificates
166166

167167
EXPOSE ${STACKQL_PG_PORT}/tcp

docker-compose.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ services:
4747
- ./test/assets/credentials/dummy:/opt/stackql/credentials/dummy:ro
4848
- ./test/assets/input:/opt/stackql/input:ro
4949
- ./test/tmp:/opt/test/tmp:rw
50+
- ./test/server:/opt/test/server:ro
5051
- ${DB_SETUP_SRC:-./test/db/sqlite}:/opt/stackql/db:ro
5152
- ${REGISTRY_SRC:-./test/registry-mocked}:/opt/stackql/registry:ro
5253
- ./cicd/vol/stackql/config:/opt/stackql/.stackql:rw

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ require (
1919
github.com/spf13/cobra v1.4.0
2020
github.com/spf13/pflag v1.0.5
2121
github.com/spf13/viper v1.10.1
22-
github.com/stackql/any-sdk v0.1.1-alpha11
22+
github.com/stackql/any-sdk v0.1.2-alpha37
2323
github.com/stackql/go-suffix-map v0.0.1-alpha01
2424
github.com/stackql/psql-wire v0.1.1-alpha07
2525
github.com/stackql/stackql-parser v0.0.14-alpha05

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -482,8 +482,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
482482
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
483483
github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
484484
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
485-
github.com/stackql/any-sdk v0.1.1-alpha11 h1:jaJ4vjQ1Vm+1zUGBQ9HTWjWAqzCIJ9qdmuSMi3AQoFM=
486-
github.com/stackql/any-sdk v0.1.1-alpha11/go.mod h1:4jYKpPoX2GWEK+qBnlGLvr8SUfndiYwHMIkg1dn9tFM=
485+
github.com/stackql/any-sdk v0.1.2-alpha37 h1:bvU576YCwqtjztMD4wSpgj//StNH8qGoBRBnN6uOH1I=
486+
github.com/stackql/any-sdk v0.1.2-alpha37/go.mod h1:4jYKpPoX2GWEK+qBnlGLvr8SUfndiYwHMIkg1dn9tFM=
487487
github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
488488
github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
489489
github.com/stackql/psql-wire v0.1.1-alpha07 h1:LQWVUlx4Bougk6dztDNG5tmXxpIVeeTSsInTj801xCs=

internal/stackql/execution/mono_valent_execution.go

Lines changed: 146 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package execution
22

33
import (
4+
"bytes"
5+
"encoding/json"
46
"fmt"
57
"io"
68
"net/http"
@@ -11,8 +13,10 @@ import (
1113
"github.com/stackql/any-sdk/pkg/client"
1214
"github.com/stackql/any-sdk/pkg/dto"
1315
"github.com/stackql/any-sdk/pkg/httpelement"
16+
"github.com/stackql/any-sdk/pkg/local_template_executor"
1417
"github.com/stackql/any-sdk/pkg/logging"
1518
"github.com/stackql/any-sdk/pkg/response"
19+
"github.com/stackql/any-sdk/pkg/stream_transform"
1620
"github.com/stackql/any-sdk/pkg/streaming"
1721
"github.com/stackql/stackql-parser/go/vt/sqlparser"
1822
"github.com/stackql/stackql/internal/stackql/acid/binlog"
@@ -1232,7 +1236,7 @@ func (sp *standardProcessor) Process() ProcessorResponse {
12321236
return newHTTPProcessorResponse(nil, reversalStream, false, nil)
12331237
}
12341238

1235-
//nolint:revive // TODO: investigate
1239+
//nolint:revive,nestif,funlen,gocognit // TODO: investigate
12361240
func (mv *monoValentExecution) GetExecutor() (func(pc primitive.IPrimitiveCtx) internaldto.ExecutorOutput, error) {
12371241
prov, err := mv.tableMeta.GetProvider()
12381242
if err != nil {
@@ -1261,47 +1265,150 @@ func (mv *monoValentExecution) GetExecutor() (func(pc primitive.IPrimitiveCtx) i
12611265
polyHandler := NewStandardPolyHandler(
12621266
mv.handlerCtx,
12631267
)
1264-
agnosticatePayload := newHTTPAgnosticatePayload(
1265-
mv.tableMeta,
1266-
provider,
1267-
m,
1268-
tableName,
1269-
authCtx,
1270-
mv.handlerCtx.GetRuntimeContext(),
1271-
mv.handlerCtx.GetOutErrFile(),
1272-
mr,
1273-
mv.elideActionIfPossible(
1274-
currentTcc,
1275-
tableName,
1276-
"", // late binding, should remove AOT reference
1277-
),
1278-
true,
1279-
polyHandler,
1280-
mv.tableMeta.GetSelectItemsKey(),
1281-
mv,
1282-
mv.isSkipResponse,
1283-
mv.isMutation,
1284-
)
1285-
processorResponse, agnosticErr := agnosticate(agnosticatePayload)
1286-
if agnosticErr != nil {
1287-
return internaldto.NewErroneousExecutorOutput(agnosticErr)
1288-
}
1289-
messages := polyHandler.GetMessages()
1290-
var castMessages internaldto.BackendMessages
1291-
if len(messages) > 0 {
1292-
castMessages = internaldto.NewBackendMessages(messages)
1268+
protocolType, protocolTypeErr := provider.GetProtocolType()
1269+
if protocolTypeErr != nil {
1270+
return internaldto.NewErroneousExecutorOutput(protocolTypeErr)
12931271
}
1294-
if processorResponse != nil && len(processorResponse.GetSuccessMessages()) > 0 {
1295-
if len(messages) == 0 {
1296-
castMessages = internaldto.NewBackendMessages(processorResponse.GetSuccessMessages())
1297-
} else {
1298-
castMessages.AppendMessages(processorResponse.GetSuccessMessages())
1272+
//nolint:exhaustive // acceptable for now
1273+
switch protocolType {
1274+
case client.LocalTemplated:
1275+
inlines := m.GetInline()
1276+
if len(inlines) == 0 {
1277+
return internaldto.NewErroneousExecutorOutput(fmt.Errorf("no inlines found"))
12991278
}
1279+
executor := local_template_executor.NewLocalTemplateExecutor(
1280+
inlines[0],
1281+
inlines[1:],
1282+
nil,
1283+
)
1284+
armoury, armouryErr := mv.tableMeta.GetHTTPArmoury()
1285+
if armouryErr != nil {
1286+
return internaldto.NewErroneousExecutorOutput(armouryErr)
1287+
}
1288+
requestParams := armoury.GetRequestParams()
1289+
logging.GetLogger().Infoln(fmt.Sprintf("requestParams = %v", requestParams))
1290+
flatInlineParams := make(map[string]interface{})
1291+
for _, p := range requestParams {
1292+
foundInlineParams, loopErr := p.GetParameters().GetInlineParameterFlatMap()
1293+
if loopErr == nil {
1294+
flatInlineParams = foundInlineParams
1295+
}
1296+
break //nolint:staticcheck // acceptable for now
1297+
}
1298+
// if mapsErr != nil {
1299+
// return internaldto.NewErroneousExecutorOutput(mapsErr)
1300+
// }
1301+
// paramMap := interestingMaps.getParameterMap()
1302+
// params := paramMap[0]
1303+
resp, exErr := executor.Execute(
1304+
map[string]any{"parameters": flatInlineParams},
1305+
)
1306+
if exErr != nil {
1307+
return internaldto.NewErroneousExecutorOutput(exErr)
1308+
}
1309+
var backendMessages []string
1310+
stdOut, stdOutExists := resp.GetStdOut()
1311+
var stdoutStr string
1312+
if stdOutExists {
1313+
stdoutStr = stdOut.String()
1314+
expectedResponse, isExpectedResponse := m.GetResponse()
1315+
if isExpectedResponse {
1316+
responseTransform, responseTransformExists := expectedResponse.GetTransform()
1317+
if responseTransformExists && responseTransform.GetType() == "golang_template_v0.1.0" {
1318+
input := stdoutStr
1319+
tmpl := responseTransform.GetBody()
1320+
inStream := stream_transform.NewTextReader(bytes.NewBufferString(input))
1321+
outStream := bytes.NewBuffer(nil)
1322+
tfm, setupErr := stream_transform.NewTemplateStreamTransformer(tmpl, inStream, outStream)
1323+
if setupErr != nil {
1324+
return internaldto.NewErroneousExecutorOutput(fmt.Errorf("template stream transform error: %w", setupErr))
1325+
}
1326+
if tfErr := tfm.Transform(); tfErr != nil {
1327+
return internaldto.NewErroneousExecutorOutput(fmt.Errorf("failed to transform: %w", tfErr))
1328+
}
1329+
outputStr := outStream.String()
1330+
stdoutStr = outputStr
1331+
}
1332+
}
1333+
var res []map[string]interface{}
1334+
resErr := json.Unmarshal([]byte(stdoutStr), &res)
1335+
itemisationResult := itemise(res, resErr, "")
1336+
insertPrepResult := mv.ActionInsertPreparation(
1337+
newHTTPActionInsertPayload(
1338+
itemisationResult,
1339+
false,
1340+
tableName,
1341+
flatInlineParams,
1342+
"",
1343+
),
1344+
)
1345+
insertErr, hasErr := insertPrepResult.GetError()
1346+
if hasErr {
1347+
return internaldto.NewErroneousExecutorOutput(insertErr)
1348+
}
1349+
// fmt.Fprintf(os.Stdout, "%s", stdoutStr)
1350+
}
1351+
// if stdOutExists {
1352+
// backendMessages = append(backendMessages, stdOut.String())
1353+
// }
1354+
stdErr, stdErrExists := resp.GetStdErr()
1355+
if stdErrExists {
1356+
backendMessages = append(backendMessages, stdErr.String())
1357+
}
1358+
backendMessages = append(backendMessages, "OK")
1359+
return internaldto.NewExecutorOutput(
1360+
nil,
1361+
nil,
1362+
nil,
1363+
internaldto.NewBackendMessages(backendMessages),
1364+
nil,
1365+
)
1366+
case client.HTTP:
1367+
agnosticatePayload := newHTTPAgnosticatePayload(
1368+
mv.tableMeta,
1369+
provider,
1370+
m,
1371+
tableName,
1372+
authCtx,
1373+
mv.handlerCtx.GetRuntimeContext(),
1374+
mv.handlerCtx.GetOutErrFile(),
1375+
mr,
1376+
mv.elideActionIfPossible(
1377+
currentTcc,
1378+
tableName,
1379+
"", // late binding, should remove AOT reference
1380+
),
1381+
true,
1382+
polyHandler,
1383+
mv.tableMeta.GetSelectItemsKey(),
1384+
mv,
1385+
mv.isSkipResponse,
1386+
mv.isMutation,
1387+
)
1388+
processorResponse, agnosticErr := agnosticate(agnosticatePayload)
1389+
if agnosticErr != nil {
1390+
return internaldto.NewErroneousExecutorOutput(agnosticErr)
1391+
}
1392+
messages := polyHandler.GetMessages()
1393+
var castMessages internaldto.BackendMessages
1394+
if len(messages) > 0 {
1395+
castMessages = internaldto.NewBackendMessages(messages)
1396+
}
1397+
if processorResponse != nil && len(processorResponse.GetSuccessMessages()) > 0 {
1398+
if len(messages) == 0 {
1399+
castMessages = internaldto.NewBackendMessages(processorResponse.GetSuccessMessages())
1400+
} else {
1401+
castMessages.AppendMessages(processorResponse.GetSuccessMessages())
1402+
}
1403+
}
1404+
if processorResponse == nil {
1405+
return internaldto.NewExecutorOutput(nil, nil, nil, castMessages, nil)
1406+
}
1407+
return internaldto.NewExecutorOutput(nil, processorResponse.GetSingletonBody(), nil, castMessages, err)
1408+
default:
1409+
return internaldto.NewErroneousExecutorOutput(
1410+
fmt.Errorf("unsupported protocol type '%v'", protocolType))
13001411
}
1301-
if processorResponse == nil {
1302-
return internaldto.NewExecutorOutput(nil, nil, nil, castMessages, nil)
1303-
}
1304-
return internaldto.NewExecutorOutput(nil, processorResponse.GetSingletonBody(), nil, castMessages, err)
13051412
}
13061413
return ex, nil
13071414
}

0 commit comments

Comments
 (0)