enable adaptive encoding with an env var

.github/workflows/ci-bats-unix-adaptive.yaml

@@ -0,0 +1,135 @@
+name: Test Bats Unix (Adaptive Encoding)
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'go/**'
+      - 'integration-tests/**'
+      - '.github/workflows/ci-bats-unix-adaptive.yaml'
+
+concurrency:
+  group: ci-bats-unix-adaptive-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Bats tests (adaptive encoding)
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-22.04
+    env:
+      DOLT_USE_ADAPTIVE_ENCODING: "true"
+      use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }}
+    steps:
+      - name: Free disk space
+        run: |
+          NAME="DISK-CLEANUP"
+          echo "[${NAME}] Starting background cleanup..."
+          [ -d /usr/share/dotnet ] && sudo rm -rf /usr/share/dotnet &
+          [ -d /usr/local/lib/android ] && sudo rm -rf /usr/local/lib/android &
+          [ -d /opt/ghc ] && sudo rm -rf /opt/ghc &
+          [ -d /usr/local/share/boost ] && sudo rm -rf /usr/local/share/boost &
+
+      - name: Conditionally Set ENV VARS for AWS tests
+        run: |
+          if [[ $use_credentials == true ]]; then
+            echo "AWS_SDK_LOAD_CONFIG=1" >> $GITHUB_ENV
+            echo "AWS_REGION=us-west-2" >> $GITHUB_ENV
+            echo "DOLT_BATS_AWS_TABLE=dolt-ci-bats-manifests-us-west-2" >> $GITHUB_ENV
+            echo "DOLT_BATS_AWS_BUCKET=dolt-ci-bats-chunks-us-west-2" >> $GITHUB_ENV
+            echo "DOLT_BATS_AWS_EXISTING_REPO=aws_remote_bats_tests__dolt__" >> $GITHUB_ENV
+          fi
+      - name: Configure filter tags for lambda bats
+        if: ${{ env.use_credentials == 'true' }}
+        run: |
+          echo "BATS_FILTER=--filter-tags no_lambda" >> $GITHUB_ENV
+      - name: Configure AWS Credentials
+        if: ${{ env.use_credentials == 'true' }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-duration-seconds: 10800 # 3 hours D:
+      - uses: actions/checkout@v6
+      - name: Setup Go 1.x
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go/go.mod
+        id: go
+      - name: Setup Python 3.x
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ^16
+      - name: Create CI Bin
+        run: |
+          mkdir -p ./.ci_bin
+          echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
+      - name: Install Bats
+        run: |
+          npm i bats
+          echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH
+        working-directory: ./.ci_bin
+      - name: Install Python Deps
+        run: |
+          pip install mysql-connector-python
+          pip install pandas
+          pip install pyarrow
+      - name: Install Dolt
+        working-directory: ./go
+        run: |
+          go build -mod=readonly -o ../.ci_bin/dolt ./cmd/dolt/.
+
+          go build -mod=readonly -o ../.ci_bin/remotesrv ./utils/remotesrv/.
+          go build -mod=readonly -o ../.ci_bin/noms ./store/cmd/noms/.
+      - name: Setup Dolt Config
+        run: |
+          dolt config --global --add user.name 'Dolthub Actions'
+          dolt config --global --add user.email 'actions@dolthub.com'
+      - name: Install expect
+        run: sudo apt-get install -y expect
+      - name: Install pcre2grep
+        run: sudo apt-get install -y pcre2-utils
+      - name: Install Maven
+        run: sudo apt-get install -y maven
+      - name: Install Hadoop
+        working-directory: ./.ci_bin
+        run: |
+          curl -LO https://downloads.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz
+          tar xvf hadoop-3.3.6.tar.gz
+          echo "$(pwd)/hadoop-3.3.6/bin" >> $GITHUB_PATH
+      - name: Install parquet-cli
+        id: parquet_cli
+        working-directory: ./.ci_bin
+        run: |
+          curl -OL https://github.com/apache/parquet-mr/archive/refs/tags/apache-parquet-1.12.3.tar.gz
+          tar zxvf apache-parquet-1.12.3.tar.gz
+          cd parquet-java-apache-parquet-1.12.3/parquet-cli
+          mvn clean install -DskipTests
+          runtime_jar="$(pwd)"/target/parquet-cli-1.12.3-runtime.jar
+          echo "runtime_jar=$runtime_jar" >> $GITHUB_OUTPUT
+      - name: Check expect
+        run: expect -v
+      - name: Test all Unix
+        env:
+          SQL_ENGINE: "local-engine"
+          PARQUET_RUNTIME_JAR: ${{ steps.parquet_cli.outputs.runtime_jar }}
+          BATS_TEST_RETRIES: "3"
+        run: |
+          bats --print-output-on-failure --tap $BATS_FILTER .
+        working-directory: ./integration-tests/bats
+      - name: Test all Unix, SQL_ENGINE=remote-engine
+        if: ${{ env.use_credentials == 'true' }}
+        env:
+          SQL_ENGINE: "remote-engine"
+          PARQUET_RUNTIME_JAR: ${{ steps.parquet_cli.outputs.runtime_jar }}
+          BATS_TEST_RETRIES: "3"
+        run: |
+          bats --print-output-on-failure --tap $BATS_FILTER .
+        working-directory: ./integration-tests/bats

.github/workflows/ci-binlog-tests.yaml

@@ -22,6 +22,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-22.04]
+        # TODO: bin log doesn't yet handle adapative encoding
+#        adaptive_encoding: ["false", "true"]
+        adaptive_encoding: ["false"]
     steps:
     - uses: actions/checkout@v6
     - name: Set up Go 1.x
@@ -36,12 +39,20 @@ jobs:
         go test -vet=off -timeout 60m ./libraries/doltcore/sqle/binlogreplication/...
       env:
         MATRIX_OS: ${{ matrix.os }}
+        DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }}
   binlog-race-test:
     name: Binlog tests - race
     defaults:
       run:
         shell: bash
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04]
+        # TODO: bin log doesn't yet handle adapative encoding
+#        adaptive_encoding: ["false", "true"]
+        adaptive_encoding: ["false"]
     steps:
     - uses: actions/checkout@v6
     - name: Set up Go 1.x
@@ -55,4 +66,5 @@ jobs:
         # Test binlog packages with race detector
         go test -vet=off -timeout 60m -race ./libraries/doltcore/sqle/binlogreplication/...
       env:
-        MATRIX_OS: ubuntu-22.04
+        MATRIX_OS: ${{ matrix.os }}
+        DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }}

.github/workflows/ci-compatibility-tests.yaml

@@ -19,6 +19,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ ubuntu-22.04 ]
+        adaptive_encoding: [ "false", "true" ]
     steps:
       - uses: actions/checkout@v6
       - name: Setup Go 1.x
@@ -49,3 +50,5 @@ jobs:
       - name: Test all
         run: ./runner.sh
         working-directory: ./integration-tests/compatibility
+        env:
+          DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }}

.github/workflows/ci-go-race-tests.yaml

@@ -19,6 +19,7 @@ jobs:
       matrix:
         os: [ ubuntu-22.04 ]
         dolt_fmt: [ "__DOLT__" ]
+        adaptive_encoding: [ "false", "true" ]
     steps:
     - uses: actions/checkout@v6
     - name: Set up Go 1.x
@@ -32,7 +33,10 @@ jobs:
         DOLT_SKIP_PREPARED_ENGINETESTS=1 go test -vet=off -v -race -timeout 30m github.com/dolthub/dolt/go/libraries/doltcore/sqle/enginetest
       env:
         DOLT_DEFAULT_BIN_FORMAT: ${{ matrix.dolt_fmt }}
+        DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }}
     - name: Test concurrentmap
       working-directory: ./go
       run: |
         go test -vet=off -v -race -timeout 1m github.com/dolthub/dolt/go/libraries/utils/concurrentmap
+      env:
+        DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }}

.github/workflows/ci-go-tests-adaptive.yaml

@@ -0,0 +1,80 @@
+name: Test Go (Adaptive Encoding)
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'go/**'
+      - '.github/workflows/ci-go-tests-adaptive.yaml'
+  workflow_dispatch:
+
+concurrency:
+  group: ci-go-tests-adaptive-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Go tests (adaptive encoding)
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-22.04
+    env:
+      DOLT_USE_ADAPTIVE_ENCODING: "true"
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go/go.mod
+      id: go
+    - name: Test All
+      working-directory: ./go
+      run: |
+        files=$(go list ./...)
+        SAVEIFS=$IFS
+        IFS=$'\n'
+        file_arr=($files)
+        IFS=$SAVEIFS
+
+        for (( i=0; i<${#file_arr[@]}; i++ ))
+        do
+          # Skip binlog tests as they run in a separate CI job
+          if [[ "${file_arr[$i]}" == *binlogreplication* ]]; then
+            echo "Skipping binlog package: ${file_arr[$i]} (runs in separate CI)"
+            continue
+          fi
+
+          echo "Testing Package: ${file_arr[$i]}"
+          if [[ "${file_arr[$i]}" != *enginetest* ]]; then
+            go test -vet=off -timeout 45m -race "${file_arr[$i]}"
+          else
+            echo "skipping enginetests for -race"
+          fi
+          succeeded=$(echo "$?")
+          if [ "$succeeded" -ne 0 ]; then
+            echo "Testing failed in package ${file_arr[$i]}"
+            exit 1;
+          fi
+        done
+  noracetest:
+    name: Go tests - no race (adaptive encoding)
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-22.04
+    env:
+      DOLT_USE_ADAPTIVE_ENCODING: "true"
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go/go.mod
+      id: go
+    - name: Test All
+      working-directory: ./go
+      run: |
+        go test -vet=off -timeout 30m ./libraries/doltcore/sqle/integration_test
+      env:
+        DOLT_TEST_RUN_NON_RACE_TESTS: "true"

.github/workflows/ci-sql-server-integration-tests.yaml

@@ -22,6 +22,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-22.04]
+        adaptive_encoding: ["false", "true"]
     steps:
       - uses: actions/checkout@v6
       - name: Setup Go 1.x
@@ -41,3 +42,5 @@ jobs:
           export DOLT_BIN_PATH="$(pwd)/../../.ci_bin/dolt"
           go test . -v
         working-directory: ./integration-tests/go-sql-server-driver
+        env:
+          DOLT_USE_ADAPTIVE_ENCODING: ${{ matrix.adaptive_encoding }}

go/libraries/doltcore/schema/encoding/serialization.go

@@ -269,6 +269,15 @@ func serializeSchemaColumns(b *fb.Builder, sch schema.Schema) fb.UOffsetT {
 		if onUpdateVal != "" {
 			serial.ColumnAddOnUpdateValue(b, ou)
 		}
+
+		// Only write the adaptive encoding field if the column uses adaptive encoding. This will force older clients that
+		// don't know about this field to update in order to read it. Older versions of Dolt ignored the serialized
+		// |encoding| field and inferred the encoding based on column type, which means they would try to interpret an
+		// adaptive encoded field as a literal value.
+		if usesAdaptiveEncoding(col) {
+			serial.ColumnAddUsesAdaptiveEncoding(b, true)
+		}
+
 		serial.ColumnAddHidden(b, false)
 		offs[i] = serial.ColumnEnd(b)
 	}
@@ -281,6 +290,16 @@ func serializeSchemaColumns(b *fb.Builder, sch schema.Schema) fb.UOffsetT {
 	return b.EndVector(len(offs))
 }
 
+func usesAdaptiveEncoding(col schema.Column) bool {
+	switch col.TypeInfo.Encoding() {
+	// val.ExtendedAdaptiveEnc is absent from this list because the extended types have their own ser / deser logic
+	case val.BytesAdaptiveEnc, val.StringAdaptiveEnc:
+		return true
+	default:
+		return false
+	}
+}
+
 func serializeHiddenKeylessColumns(b *fb.Builder) (id, card fb.UOffsetT) {
 	// cardinality column
 	no := b.CreateString(keylessCardCol)

go/libraries/doltcore/schema/typeinfo/typeinfo.go

@@ -16,6 +16,7 @@ package typeinfo
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/dolthub/go-mysql-server/sql"
 	gmstypes "github.com/dolthub/go-mysql-server/sql/types"
@@ -30,6 +31,12 @@ import (
 // columns. Extended types will always use adaptive encoding for TEXT and BLOB types regardless of this value.
 var UseAdaptiveEncoding = false
 
+func init() {
+	if envVal, ok := os.LookupEnv("DOLT_USE_ADAPTIVE_ENCODING"); ok && envVal == "true" {
+		UseAdaptiveEncoding = true
+	}
+}
+
 // TypeInfo is an interface used for encoding type information.
 type TypeInfo interface {
 	// Equals returns whether the given TypeInfo is equivalent to this TypeInfo.

go/serial/schema.fbs

@@ -68,6 +68,10 @@ table Column {
 
   // sql on update value
   on_update_value:string;
+
+  // a marker value only written when the column uses adaptive encoding, which will force older clients to upgrade
+  // before reading this data.
+  uses_adaptive_encoding:bool;
 }
 
 table Index {