fix(go): debug stuck ingestion job

Author: lidavidm

.github/workflows/go_test.yaml

@@ -52,111 +52,6 @@ defaults:
     shell: bash
 
 jobs:
-  test:
-    name: "Test/${{ matrix.platform }}_${{ matrix.arch }}"
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - { platform: linux, arch: amd64, runner: ubuntu-latest }
-          - { platform: macos, arch: arm64, runner: macos-latest }
-          - { platform: win, arch: amd64, runner: windows-latest }
-    environment: BigQuery CI
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - name: free up disk space
-        if: runner.os != 'Windows'
-        run: |
-          # Preinstalled tools use a lot of disk space, free up some space
-          # https://github.com/actions/runner-images/issues/2840
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c  # v6.1.0
-        with:
-          cache-dependency-path: go/go.sum
-          check-latest: true
-          go-version-file: go/go.mod
-
-      - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7  # v0.9.3
-        with:
-          pixi-version: v0.50.2
-          run-install: false
-
-      - name: Google Cloud Login
-        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093  # v3.0.0
-        with:
-          service_account: ${{ secrets.gcloud_service_account }}
-          workload_identity_provider: ${{ secrets.gcloud_workload_identity_provider }}
-
-      - name: Build
-        working-directory: go
-        run: |
-          go build ./...
-
-      - name: Start Test Dependencies
-        # Can't use Docker on macOS AArch64 runners, and Windows containers
-        # work but often the container doesn't support Windows
-        if: runner.os == 'Linux'
-        working-directory: go
-        run: |
-          if [[ -f compose.yaml ]]; then
-            if ! docker compose up --detach --wait test-service; then
-              echo "Service failed to start"
-              echo "Logs:"
-              docker compose logs test-service
-              exit 1
-            fi
-          fi
-
-      - name: Test
-        if: runner.os == 'Linux'
-        working-directory: go
-        run: |
-          set -a
-          if [[ -f .env ]]; then
-            source .env
-          fi
-          if [[ -f .env.ci ]]; then
-            source .env.ci
-          fi
-          set +a
-
-          if [[ -n "${{ secrets.environment }}" ]]; then
-            echo "Loading secret environment variables"
-            eval "${{ secrets.environment }}"
-          fi
-
-          if [[ -f ci/scripts/pre-test.sh ]]; then
-            echo "Loading pre-test"
-            ./ci/scripts/pre-test.sh
-          fi
-
-          go test -tags assert -v ./...
-
-          if [[ -f ci/scripts/post-test.sh ]]; then
-            ./ci/scripts/post-test.sh
-          fi
-
-      - name: go mod tidy
-        if: runner.os == 'Linux'
-        working-directory: go
-        run: |
-          go mod tidy --diff
-
-      - name: Test
-        if: runner.os != 'Linux'
-        working-directory: go
-        run: |
-          go test -tags assert -v ./...
-
   validate:
     name: "Validate/${{ matrix.platform }}_${{ matrix.arch }}"
     runs-on: ${{ matrix.runner }}
@@ -252,6 +147,7 @@ jobs:
           fi
 
           docker ps
+          export ADBC_DRIVER_BIGQUERY_LOG_LEVEL=debug
           pixi run validate
 
           if [[ -f ci/scripts/post-test.sh ]]; then
@@ -268,176 +164,3 @@ jobs:
           name: docs
           path: "go/generated/bigquery.md"
           retention-days: 2
-
-  build:
-    name: "Build bigquery/${{ matrix.platform }}_${{ matrix.arch }}"
-    needs: test
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - { platform: linux, arch: amd64, runner: ubuntu-latest }
-          - { platform: linux, arch: arm64, runner: ubuntu-24.04-arm }
-          - { platform: macos, arch: arm64, runner: macos-latest }
-          - { platform: windows, arch: amd64, runner: windows-latest }
-    permissions:
-      contents: read
-      packages: read
-
-    steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c  # v6.1.0
-        with:
-          cache-dependency-path: go/go.sum
-          check-latest: true
-          go-version-file: go/go.mod
-
-      - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7  # v0.9.3
-        with:
-          pixi-version: v0.50.2
-          run-install: false
-
-      - name: Install dev tools
-        working-directory: go
-        run: |
-          pixi install
-
-      - name: Log in to ghcr.io
-        if: runner.os == 'Linux'
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Build Library
-        working-directory: go
-        run: |
-          if [[ -f ci/scripts/pre-build.sh ]]; then
-            ./ci/scripts/pre-build.sh release ${{ matrix.platform }} ${{ matrix.arch }}
-          fi
-          set -a
-          if [[ -f .env.release ]]; then
-            source .env.release
-          fi
-          set +a
-          pixi run adbc-make check CI=true VERBOSE=true DRIVER=bigquery
-
-      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
-        with:
-          name: drivers-${{ matrix.platform }}-${{ matrix.arch }}
-          path: "go/build/libadbc_driver_bigquery.*"
-          retention-days: 2
-
-  package:
-    name: "Generate Packages"
-    runs-on: ubuntu-latest
-    needs: build
-    permissions:
-      contents: read
-
-    steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c  # v6.1.0
-        with:
-          check-latest: true
-          go-version: "stable"
-
-      - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7  # v0.9.3
-        with:
-          pixi-version: v0.50.2
-          run-install: false
-
-      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131  # v7.0.0
-        with:
-          pattern: "drivers-*"
-          path: "~/drivers"
-
-      - name: Install tools
-        working-directory: go
-        run: |
-          # XXX: can't install go-licenses under go 1.25
-          # https://github.com/google/go-licenses/issues/312
-          git clone --depth=1 https://github.com/google/go-licenses
-          git config --global --add 'url.https://github.com/.insteadOf' ssh://git@github.com/
-          pushd go-licenses
-          go get -u
-          # Silent break: https://github.com/spf13/cobra/pull/2303
-          # Manually edit the go.mod to get around this for now
-          sed -i 's|github.com/spf13/pflag v1.0.8|github.com/spf13/pflag v1.0.7|g' go.mod
-          go mod tidy
-          go install .
-          popd
-
-      - name: Generate packages
-        working-directory: go
-        run: |
-          pixi install
-
-          pixi run adbc-gen-package \
-            --name bigquery \
-            --root $(pwd) \
-            --manifest-template manifest.toml \
-            ${{ (inputs.release && '--release') || '' }}\
-            -o ~/packages \
-            ~/drivers/drivers-*-*/
-
-          ls ~/packages
-
-      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
-        with:
-          name: all-packages
-          path: ~/packages
-          retention-days: 7
-
-  release:
-    name: "Release (Dry Run)"
-    runs-on: ubuntu-latest
-    needs:
-      - package
-      - validate
-    permissions:
-      contents: read
-
-    steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c  # v6.1.0
-        with:
-          check-latest: true
-          go-version: "stable"
-
-      - uses: prefix-dev/setup-pixi@82d477f15f3a381dbcc8adc1206ce643fe110fb7  # v0.9.3
-        with:
-          pixi-version: v0.50.2
-          run-install: false
-
-      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131  # v7.0.0
-        with:
-          name: "all-packages"
-          path: "~/packages"
-
-      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131  # v7.0.0
-        with:
-          name: "docs"
-          path: "~/packages"
-
-      - name: Release (dry-run)
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        working-directory: go
-        run: |
-          git tag go/v1000.0.0
-          tag=go/v1000.0.0
-
-          pixi run release --dry-run $(pwd) $tag
-          echo gh release upload $tag $(find ~/packages -name '*.tar.gz') $(find ~/packages -name 'manifest.yaml') $(find ~/packages -name '*.md')

go/bulk_ingest.go

@@ -21,7 +21,9 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	// "runtime/pprof"
 	"strings"
+	// "time"
 
 	"cloud.google.com/go/bigquery"
 	"github.com/adbc-drivers/driverbase-go/driverbase"
@@ -30,6 +32,15 @@ import (
 	"github.com/google/uuid"
 )
 
+// func init() {
+// 	go func() {
+// 		for {
+// 			time.Sleep(time.Minute)
+// 			_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 2)
+// 		}
+// 	}()
+// }
+
 type bigqueryBulkIngestSink struct {
 	f    *os.File
 	path string
@@ -86,6 +97,7 @@ func (bi *bigqueryBulkIngestImpl) Close() {
 
 func (bi *bigqueryBulkIngestImpl) Copy(ctx context.Context, chunk driverbase.BulkIngestPendingCopy) error {
 	pendingFile := chunk.(*bigqueryBulkIngestSink)
+	bi.logger.DebugContext(ctx, "prepare COPY", "file", pendingFile.path)
 
 	source := bigquery.NewReaderSource(pendingFile.f)
 	source.ParquetOptions = &bigquery.ParquetOptions{
@@ -115,7 +127,9 @@ func (bi *bigqueryBulkIngestImpl) Copy(ctx context.Context, chunk driverbase.Bul
 	if err != nil {
 		return errToAdbcErr(adbc.StatusIO, err, "run loader")
 	}
-	status, err := safeWaitForJob(ctx, job)
+	bi.logger.DebugContext(ctx, "created COPY job", "file", pendingFile.path)
+	status, err := safeWaitForJob(ctx, bi.logger, job)
+	bi.logger.DebugContext(ctx, "finished COPY job", "file", pendingFile.path, "error", err)
 	if err != nil {
 		return err
 	}
@@ -149,12 +163,15 @@ func (bi *bigqueryBulkIngestImpl) CreateTable(ctx context.Context, schema *arrow
 		if err != nil {
 			return err
 		}
-		js, err := safeWaitForJob(ctx, job)
+		js, err := safeWaitForJob(ctx, bi.logger, job)
 		if err != nil {
+			bi.logger.Debug("failed to create table", "table", bi.options.TableName, "stmt", stmt, "error", err)
 			return err
 		} else if err = js.Err(); err != nil {
+			bi.logger.Debug("failed to create table", "table", bi.options.TableName, "stmt", stmt, "error", err)
 			return errToAdbcErr(adbc.StatusInternal, err, "create table")
 		} else if !js.Done() {
+			bi.logger.Debug("failed to create table", "table", bi.options.TableName, "stmt", stmt, "error", "did not complete")
 			return adbc.Error{
 				Code: adbc.StatusInternal,
 				Msg:  "[bq] CREATE TABLE query did not complete",

go/connection.go

@@ -326,7 +326,7 @@ func (c *connectionImpl) exec(ctx context.Context, stmt string, config func(*big
 	if err != nil {
 		return nil, err
 	}
-	status, err := safeWaitForJob(ctx, job)
+	status, err := safeWaitForJob(ctx, c.Logger, job)
 	if err != nil {
 		return nil, err
 	} else if err := status.Err(); err != nil {