Merge branch 'main' into 470

Author: general-kroll-4-life

.github/workflows/build.yml

@@ -1461,6 +1461,8 @@ jobs:
 
 
   dockertest:
+    env:
+      IS_DOCKER: 'true'
     name: Docker Test
     needs:
       - dockerbuild

Dockerfile

@@ -162,7 +162,7 @@ ENV PATH="${APP_DIR}:${PATH}"
 COPY --from=integration ${TEST_ROOT_DIR}/build/stackql ${APP_DIR}/
 
 RUN apt-get update \
-    && apt-get install -y ca-certificates openssl \
+    && apt-get install -y ca-certificates openssl netcat-traditional \
     && update-ca-certificates
 
 EXPOSE ${STACKQL_PG_PORT}/tcp

docs/GC_cache_concurrency.md docs/ACID_and_GC.mddocs/GC_cache_concurrency.md renamed to docs/ACID_and_GC.md

@@ -111,3 +111,11 @@ Please **watch this space** on all items which are **TBD**-inclusive.
   - List active Txns.
   - Halt / Allow new Txns.
   - Cancel Txns (filtered / unfiltered).
+
+## ACID
+
+
+### When RDBMS ACID fails
+
+Despite all efforts to enforce ACID, there are times when it can fail and require admin intervention, even for local disk backed RDBMs, [per `postgres` corruption documentation](https://wiki.postgresql.org/wiki/Corruption).
+

docs/developer_guide.md

@@ -268,3 +268,13 @@ time ./stackql exec --cpuprofile=./select-disks-improved-05.profile --auth='{ "g
 ## AWS HTTP request signing
 
 https://docs.aws.amazon.com/sdk-for-go/api/aws/signer/v4/
+
+## Selection from non-select DML
+
+`INSERT RETURNING` can function in two mechanisms:
+
+- Synchronous responses, such as [`google.storage.buckets`](https://cloud.google.com/storage/docs/json_api/v1/buckets/insert).  The returning clause is a projection on the immediately available reponse body.
+- Asynchronous responses, such as [`google.compute.instances`](https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert) and [`google.compute.networks`](https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert).  The returning clause is a projection on the reponse body **after** the await flow has concluded.
+
+Future use cases for `UPDATE RETURNING`, `REPLACE RETURNING` and `DELETE RETURNING` will function the same observable fashion.
+

go.mod

@@ -12,17 +12,17 @@ require (
 	github.com/jackc/pgx/v5 v5.0.4
 	github.com/lib/pq v1.10.4
 	github.com/magiconair/properties v1.8.6
-	github.com/mattn/go-sqlite3 v1.0.3-stackql
+	github.com/mattn/go-sqlite3 v1.0.4-stackql
 	github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4
 	github.com/sirupsen/logrus v1.9.0
 	github.com/snowflakedb/gosnowflake v1.6.16
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.10.1
-	github.com/stackql/any-sdk v0.1.4-alpha01
+	github.com/stackql/any-sdk v0.1.4-alpha06
 	github.com/stackql/go-suffix-map v0.0.1-alpha01
 	github.com/stackql/psql-wire v0.1.1-beta23
-	github.com/stackql/stackql-parser v0.0.14-alpha05
+	github.com/stackql/stackql-parser v0.0.15-alpha06
 	github.com/stretchr/testify v1.10.0
 	golang.org/x/sync v0.10.0
 	gonum.org/v1/gonum v0.11.0
@@ -131,4 +131,4 @@ require (
 
 replace github.com/chzyer/readline => github.com/stackql/readline v0.0.2-alpha05
 
-replace github.com/mattn/go-sqlite3 => github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql
+replace github.com/mattn/go-sqlite3 => github.com/stackql/stackql-go-sqlite3 v1.0.4-stackql

go.sum

@@ -484,18 +484,18 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk=
 github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
-github.com/stackql/any-sdk v0.1.4-alpha01 h1:GQmsDcqCgnjPBq8IUcGSIFb8xlVyb9zlMayjH1SdaWQ=
-github.com/stackql/any-sdk v0.1.4-alpha01/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
+github.com/stackql/any-sdk v0.1.4-alpha06 h1:QJPf3ehPrRqmYZR+TmD897AsmsaOamHErLhaE5B9v/w=
+github.com/stackql/any-sdk v0.1.4-alpha06/go.mod h1:AKS/g28y7m4SWL/YW8veE9MCNy8XJgaicVibemVE9e8=
 github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8=
 github.com/stackql/go-suffix-map v0.0.1-alpha01/go.mod h1:QAi+SKukOyf4dBtWy8UMy+hsXXV+yyEE4vmBkji2V7g=
 github.com/stackql/psql-wire v0.1.1-beta23 h1:1ayYMjZArfDcIMyEOKnm+Bp1zRCISw8pguvTFuUhhVQ=
 github.com/stackql/psql-wire v0.1.1-beta23/go.mod h1:a44Wd8kDC3irFLpGutarKDBqhJ/aqXlj1aMzO5bVJYg=
 github.com/stackql/readline v0.0.2-alpha05 h1:ID4QzGdplFBsrSnTuz8pvKzWw96JbrJg8fsLry2UriU=
 github.com/stackql/readline v0.0.2-alpha05/go.mod h1:OFAYOdXk/X4+5GYiDXFfaGrk+bCN6Qv0SYY5HNzD2E0=
-github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql h1:j0yt6T5thZuz5+HIr81PXz2AClAtCko0vzr5tm8C1g8=
-github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql/go.mod h1:HemqCrcMK2xyhMMMt6oZ7ERDtoSmyyDsw5LBBcTZ+Rk=
-github.com/stackql/stackql-parser v0.0.14-alpha05 h1:DLcsaeTypH5p1T+g/9NqaGYdC9uIQ+7pZetnkjHi0G0=
-github.com/stackql/stackql-parser v0.0.14-alpha05/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ=
+github.com/stackql/stackql-go-sqlite3 v1.0.4-stackql h1:fp70Vdw+PCVEoPrAhkyqPuAlrIiHT79mght/0rlR4oY=
+github.com/stackql/stackql-go-sqlite3 v1.0.4-stackql/go.mod h1:HemqCrcMK2xyhMMMt6oZ7ERDtoSmyyDsw5LBBcTZ+Rk=
+github.com/stackql/stackql-parser v0.0.15-alpha06 h1:bdaudybbEmrR9m88CO1HpX/eKda2+gjiy4Kf6ZjwPoQ=
+github.com/stackql/stackql-parser v0.0.15-alpha06/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ=
 github.com/stackql/stackql-provider-registry v0.0.1-rc06 h1:MgroWOr0bSqjSTDGnXB0UoZGFXpW3SRtN0EFkzB8Rpo=
 github.com/stackql/stackql-provider-registry v0.0.1-rc06/go.mod h1:87rVxnS2aRASK20lBQgoYA0o7FSJTZBGGRaWFR7IDm4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

…tackql/primitivebuilder/async_compose.go …al/stackql/asynccompose/async_compose.gointernal/stackql/primitivebuilder/async_compose.go renamed to internal/stackql/asynccompose/async_compose.go internal/stackql/primitivebuilder/async_compose.go renamed to internal/stackql/asynccompose/async_compose.go

@@ -1,22 +1,26 @@
-package primitivebuilder
+package asynccompose
 
 import (
 	"github.com/stackql/any-sdk/anysdk"
 	"github.com/stackql/stackql-parser/go/vt/sqlparser"
+	"github.com/stackql/stackql/internal/stackql/drm"
 	"github.com/stackql/stackql/internal/stackql/handler"
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
 	"github.com/stackql/stackql/internal/stackql/primitive"
 	"github.com/stackql/stackql/internal/stackql/provider"
 )
 
-func composeAsyncMonitor(
+func ComposeAsyncMonitor(
 	handlerCtx handler.HandlerContext,
 	precursor primitive.IPrimitive,
 	prov provider.IProvider,
 	method anysdk.OperationStore,
 	commentDirectives sqlparser.CommentDirectives,
+	isReturning bool,
+	insertCtx drm.PreparedStatementCtx,
+	drmCfg drm.Config,
 ) (primitive.IPrimitive, error) {
-	asm, err := NewAsyncMonitor(handlerCtx, prov, method)
+	asm, err := NewAsyncMonitor(handlerCtx, prov, method, isReturning)
 	if err != nil {
 		return nil, err
 	}
@@ -31,7 +35,8 @@ func composeAsyncMonitor(
 		handlerCtx.GetOutfile(),
 		handlerCtx.GetOutErrFile(),
 	)
-	primitive, err := asm.GetMonitorPrimitive(prov, method, precursor, pl, commentDirectives)
+	primitive, err := asm.GetMonitorPrimitive(
+		prov, method, precursor, pl, commentDirectives, isReturning, insertCtx, drmCfg)
 	if err != nil {
 		return nil, err
 	}

…stackql/primitivebuilder/asyncmonitor.go …nal/stackql/asynccompose/asyncmonitor.gointernal/stackql/primitivebuilder/asyncmonitor.go renamed to internal/stackql/asynccompose/asyncmonitor.go internal/stackql/primitivebuilder/asyncmonitor.go renamed to internal/stackql/asynccompose/asyncmonitor.go

@@ -1,11 +1,12 @@
-package primitivebuilder
+package asynccompose
 
 import (
 	"fmt"
 	"strings"
 
 	"github.com/stackql/any-sdk/anysdk"
 	"github.com/stackql/stackql/internal/stackql/acid/binlog"
+	"github.com/stackql/stackql/internal/stackql/drm"
 	"github.com/stackql/stackql/internal/stackql/execution"
 	"github.com/stackql/stackql/internal/stackql/handler"
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
@@ -22,6 +23,9 @@ type IAsyncMonitor interface {
 		precursor primitive.IPrimitive,
 		initialCtx primitive.IPrimitiveCtx,
 		comments sqlparser.CommentDirectives,
+		isReturning bool,
+		insertCtx drm.PreparedStatementCtx,
+		drmCfg drm.Config,
 	) (primitive.IPrimitive, error)
 }
 
@@ -120,11 +124,12 @@ func NewAsyncMonitor(
 	handlerCtx handler.HandlerContext,
 	prov provider.IProvider,
 	op anysdk.OperationStore,
+	isReturning bool,
 ) (IAsyncMonitor, error) {
 	//nolint:gocritic //TODO: refactor
 	switch prov.GetProviderString() {
 	case "google":
-		return newGoogleAsyncMonitor(handlerCtx, prov, op, prov.GetVersion())
+		return newGoogleAsyncMonitor(handlerCtx, prov, op, prov.GetVersion(), isReturning)
 	}
 	return nil, fmt.Errorf(
 		"async operation monitor for provider = '%s', api version = '%s' currently not supported",
@@ -136,6 +141,7 @@ func newGoogleAsyncMonitor(
 	prov provider.IProvider,
 	op anysdk.OperationStore,
 	version string, //nolint:unparam // TODO: refactor
+	isReturning bool, //nolint:unparam,revive // TODO: refactor
 ) (IAsyncMonitor, error) {
 	//nolint:gocritic //TODO: refactor
 	switch version {
@@ -160,11 +166,14 @@ func (gm *DefaultGoogleAsyncMonitor) GetMonitorPrimitive(
 	precursor primitive.IPrimitive,
 	initialCtx primitive.IPrimitiveCtx,
 	comments sqlparser.CommentDirectives,
+	isReturning bool,
+	insertCtx drm.PreparedStatementCtx,
+	drmCfg drm.Config,
 ) (primitive.IPrimitive, error) {
 	//nolint:gocritic,staticcheck //TODO: refactor
 	switch strings.ToLower(prov.GetVersion()) {
 	default:
-		return gm.getV1Monitor(prov, op, precursor, initialCtx, comments)
+		return gm.getV1Monitor(prov, op, precursor, initialCtx, comments, isReturning, insertCtx, drmCfg)
 	}
 }
 
@@ -174,6 +183,9 @@ func (gm *DefaultGoogleAsyncMonitor) getV1Monitor(
 	precursor primitive.IPrimitive,
 	initialCtx primitive.IPrimitiveCtx,
 	comments sqlparser.CommentDirectives,
+	isReturning bool,
+	insertCtx drm.PreparedStatementCtx,
+	drmCfg drm.Config,
 ) (primitive.IPrimitive, error) {
 	provider, providerErr := prov.GetProvider()
 	if providerErr != nil {
@@ -186,6 +198,9 @@ func (gm *DefaultGoogleAsyncMonitor) getV1Monitor(
 		precursor,
 		initialCtx,
 		comments,
+		isReturning,
+		insertCtx,
+		drmCfg,
 	)
 	if exPrepErr != nil {
 		return nil, exPrepErr

internal/stackql/dependencyplanner/dependencyplanner.go

@@ -485,6 +485,7 @@ func (dp *standardDependencyPlanner) orchestrate(
 			insPsc,
 			nil,
 			outStream,
+			false, // returning hardcoded to false for now
 		)
 	}
 	dp.execSlice = append(dp.execSlice, builder)

internal/stackql/driver/dependent_simple_integration_test.go

@@ -38,7 +38,7 @@ func TestSimpleInsertDependentGoogleComputeDiskAsync(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test failed: %v", err)
 		}
-		handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdOut(outFile), true)
+		handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdErr(outFile), true)
 		if err != nil {
 			t.Fatalf("Test failed: %v", err)
 		}
@@ -76,7 +76,7 @@ func TestSimpleInsertDependentGoogleComputeDiskAsyncReversed(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test failed: %v", err)
 		}
-		handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdOut(outFile), true)
+		handlerCtx, err := entryutil.BuildHandlerContext(*runtimeCtx, rdr, lrucache.NewLRUCache(int64(runtimeCtx.QueryCacheSize)), inputBundle.WithStdErr(outFile), true)
 		if err != nil {
 			t.Fatalf("Test failed: %v", err)
 		}