ci: Only distribute the tool as a container image (#56)

* Only distribute the tool as a container image

We can not build static binaries with Chroma.

* fix: Docker build with CC

Signed-off-by: Anush008 <anushshetty90@gmail.com>

* Fix trivy scan

* Fix tests

* Fix tests

---------

Signed-off-by: Anush008 <anushshetty90@gmail.com>
Co-authored-by: Anush008 <anushshetty90@gmail.com>

Author: bashofmann

.github/workflows/pr-workflow.yaml

@@ -30,16 +30,6 @@ jobs:
           version: v2.1.2 # Specify the golangci-lint version, so we are stable
           args: --timeout 10m # Increase the timeout to 10 minutes
 
-      - name: Run Trivy vulnerability scanner in repo mode
-        uses: aquasecurity/trivy-action@master
-        with:
-          scan-type: 'fs'
-          exit-code: '1'
-          ignore-unfixed: true
-          format: 'table'
-          vuln-type: 'os,library'
-          severity: 'CRITICAL,HIGH'
-
   tests:
     name: Tests
     runs-on: ubuntu-latest
@@ -56,6 +46,52 @@ jobs:
         run: |
           make test_unit
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract build info
+        id: extract_build_info
+        run: |
+          echo "tag=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
+          echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Set Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            registry.cloud.qdrant.io/library/qdrant-migration
+          tags: |
+            type=raw,value=dev
+
+      - name: Build and push container image
+        uses: docker/build-push-action@v6
+        id: build-and-push
+        with:
+          context: .
+          push: false
+          load: true
+          sbom: false
+          provenance: false
+          platforms: linux/amd64
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          build-args: |
+            VERSION=dev
+            BUILD=${{ steps.extract_build_info.outputs.commit_short }}
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: registry.cloud.qdrant.io/library/qdrant-migration:dev
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+
       - name: Setup BATS
         uses: mig4/setup-bats@v1
         with:

.github/workflows/release-workflow.yaml

@@ -24,15 +24,6 @@ jobs:
           echo "tag=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
           echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '1.24'
-          cache: false
-
-      - name: Build Binaries
-        run: |
-          make release VERSION="${{ steps.extract_build_info.outputs.tag }}" BUILD="${{ steps.extract_build_info.outputs.commit_short }}"
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -95,11 +86,3 @@ jobs:
           publish: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload Release Artifacts
-        run: |
-          for file in dist/*; do
-              gh release upload --clobber ${{ steps.extract_build_info.outputs.tag }} $file
-          done
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
 ARG BUILDKIT_SBOM_SCAN_CONTEXT=true
 ARG BUILDKIT_SBOM_SCAN_STAGE=builder
-FROM --platform=${BUILDPLATFORM:-linux/amd64} registry.suse.com/bci/golang:1.24 AS builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.24 AS builder
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -10,11 +10,21 @@ ARG TARGETARCH
 ARG VERSION
 ARG BUILD
 
+RUN apt-get update && case "${TARGETARCH}" in \
+    amd64) apt-get install -y gcc-x86-64-linux-gnu ;; \
+    arm64) apt-get install -y gcc-aarch64-linux-gnu ;; \
+esac && rm -rf /var/lib/apt/lists/*
+
 COPY . /app
 
 WORKDIR /app
 
-RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags "-X 'main.projectVersion=${VERSION:-0.0.0}' -X 'main.projectBuild=${BUILD:-dev}'" -o bin/qdrant-migration main.go
+RUN case "${TARGETARCH}" in \
+      amd64) CC=x86_64-linux-gnu-gcc ;; \
+      arm64) CC=aarch64-linux-gnu-gcc ;; \
+    esac && \
+    CGO_ENABLED=1 GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=$CC \
+    go build -ldflags "-X 'main.projectVersion=${VERSION:-0.0.0}' -X 'main.projectBuild=${BUILD:-dev}'" -o bin/qdrant-migration main.go
 
 FROM --platform=${TARGETPLATFORM:-linux/amd64} registry.suse.com/bci/bci-minimal:15.6
 

Makefile

@@ -48,13 +48,3 @@ test_unit:
 lint:
 	golangci-lint run
 
-release: $(PLATFORMS)
-
-temp = $(subst /, ,$@)
-os = $(word 1, $(temp))
-arch = $(word 2, $(temp))
-
-$(PLATFORMS):
-	BUILD_DATE=$(date +%F-%T) GOOS=$(os) GOARCH=$(arch) CGO_ENABLED=0 go build -o $(DIST_DIR)/qdrant-migrate-$(os)-$(arch) \
-		-ldflags "-s -w -extldflags \"-static\" -X 'main.projectVersion=$(VERSION)' -X 'main.projectBuild=$(BUILD)'" main.go
-

README.md

@@ -18,13 +18,9 @@ CLI tool for migrating data to [Qdrant](http://qdrant.tech) with support for res
 
 You can run this tool on any machine with connectivity to both the source and the Qdrant database. For best performance, use a machine with a fast network and minimal latency to both endpoints.
 
-#### Binaries
+The tool is distributed as a container image, which can be run on any system with Docker, Podman or similar container runtimes. The following examples use Docker.
 
-Each release includes **precompiled binaries** for all major OS and CPU architectures. Download the latest one from the [Releases Page](https://github.com/qdrant/migration/releases).
-
-#### Docker
-
-To get the latest Docker image run the following command.
+To get the latest container image run the following command:
 
 ```bash
 docker pull registry.cloud.qdrant.io/library/qdrant-migration
@@ -43,21 +39,13 @@ Migrate data from a **Chroma** database to **Qdrant**:
 ### 📥 Example
 
 ```bash
-migration chroma \
+docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration chroma \
     --chroma.url=http://localhost:8000
     --chroma.collection 'collection-name' \
     --qdrant.url 'https://example.cloud-region.cloud-provider.cloud.qdrant.io:6334' \
     --qdrant.api-key 'optional-qdrant-api-key' \
     --qdrant.collection 'target-collection' \
     --migration.batch-size 64
-````
-
-With Docker:
-
-```bash
-docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration chroma \
-    --chroma.url=http://localhost:8000
-    ...
 ```
 
 ### Chroma Options
@@ -102,22 +90,14 @@ Migrate data from a **Pinecone** database to **Qdrant**:
 ### 📥 Example
 
 ```bash
-migration pinecone \
+docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration pinecone \
     --pinecone.index-host 'https://example-index.svc.region.pinecone.io' \
     --pinecone.index-name 'example-index' \
     --pinecone.api-key 'optional-pinecone-api-key' \
     --qdrant.url 'https://example.cloud-region.cloud-provider.cloud.qdrant.io:6334' \
     --qdrant.api-key 'optional-qdrant-api-key' \
     --qdrant.collection 'target-collection' \
     --migration.batch-size 64
-````
-
-With Docker:
-
-```bash
-docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration pinecone \
-    --pinecone.index-host 'https://example-index.svc.region.pinecone.io' \
-    ...
 ```
 
 #### Pinecone Options
@@ -154,7 +134,7 @@ Migrate data from a **Milvus** database to **Qdrant**:
 ### 📥 Example
 
 ```bash
-migration milvus \
+docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration milvus \
     --milvus.url 'https://example.gcp-us-west1.cloud.zilliz.com' \
     --milvus.enable-tls-auth \
     --milvus.collection 'example-collection' \
@@ -165,14 +145,6 @@ migration milvus \
     --migration.batch-size 64
 ```
 
-With Docker:
-
-```bash
-docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration milvus \
-    --milvus.url 'https://example.gcp-us-west1.cloud.zilliz.com' \
-    ...
-```
-
 #### Milvus Options
 
 | Flag                       | Description                                             |
@@ -214,7 +186,7 @@ Migrate data from a **Weaviate** database to **Qdrant**:
  > Ensure that the **vector dimensions in Qdrant exactly match** those used in Weaviate.
 
 ```bash
-migration weaviate \
+docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration weaviate \
     --weaviate.host 'example.c0.asia-southeast1.gcp.weaviate.cloud' \
     --weaviate.scheme 'https' \
     --weaviate.auth-type 'apiKey' \
@@ -225,14 +197,6 @@ migration weaviate \
     --migration.batch-size 64
 ```
 
-With Docker:
-
-```bash
-docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration weaviate \
-    --weaviate.host 'example.c0.asia-southeast1.gcp.weaviate.cloud' \
-    ...
-```
-
 #### Weaviate Options
 
 | Flag                       | Description                                                                                      |
@@ -277,22 +241,14 @@ Migrate data from a **Redis** database to **Qdrant**:
 ### 📥 Example
 
 ```bash
-migration redis \
+docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration milvus \
     --redis.index 'index_name' \
     --redis.addr 'localhost:6379' \
     --qdrant.url 'http://localhost:6334' \
     --qdrant.collection 'target-collection' \
     --migration.batch-size 100
 ```
 
-With Docker:
-
-```bash
-docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration milvus \
-    --redis.index 'index_name' \
-    ...
-```
-
 #### Redis Options
 
 | Flag                  | Description                                                             |
@@ -327,7 +283,7 @@ Migrate data from one **Qdrant** instance to another.
 ### 📥 Example
 
 ```bash
-migration qdrant \
+docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration qdrant \
     --source.url 'http://localhost:6334' \
     --source.collection 'source-collection' \
     --target.url 'https://example.cloud-region.cloud-provider.cloud.qdrant.io:6334' \
@@ -336,14 +292,6 @@ migration qdrant \
     --migration.batch-size 64
 ```
 
-With Docker:
-
-```bash
-docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration qdrant \
-    --source.url 'http://localhost:6334' \
-    ...
-```
-
 NOTE: If the target collection already exists, its vector size and dimensions must match the source. Other settings like replication, shards can differ.
 
 #### Source Qdrant Options

integration_tests/milvus_to_qdrant.bats

@@ -48,7 +48,7 @@ teardown() {
   echo "Wait for a few seconds to load the vectors"
   sleep 5
 
-  run go run main.go milvus \
+  run docker run --net=host --rm registry.cloud.qdrant.io/library/qdrant-migration:dev milvus \
     --milvus.url 'http://localhost:19530' \
     --milvus.collection 'migrate_collection' \
     --qdrant.url 'http://localhost:6334' \

integration_tests/qdrant_to_qdrant.bats

@@ -47,7 +47,7 @@
 
   echo $source_result
 
-  run go run main.go qdrant --source.url http://localhost:7334 --source.collection source_collection --target.url http://localhost:8334 --target.collection target_collection --migration.batch-size 1
+  run docker run --net=host --rm registry.cloud.qdrant.io/library/qdrant-migration:dev qdrant --source.url http://localhost:7334 --source.collection source_collection --target.url http://localhost:8334 --target.collection target_collection --migration.batch-size 1
   [ $status -eq 0 ]
 
   run curl -s -X POST http://localhost:8333/collections/target_collection/points/scroll \
@@ -74,13 +74,13 @@
   [ $status -eq 0 ]
 
 
-  run go run main.go qdrant --source.url http://localhost:7334 --source.collection source_collection --target.url http://localhost:7334 --target.collection source_collection --migration.batch-size 1
+  run docker run --net=host --rm registry.cloud.qdrant.io/library/qdrant-migration:dev qdrant --source.url http://localhost:7334 --source.collection source_collection --target.url http://localhost:7334 --target.collection source_collection --migration.batch-size 1
   [ $status -ne 0 ]
   [[ "$output" =~ "source and target collections must be different" ]]
 }
 
 @test "Migrating with invalid port should fail" {
-  run go run main.go qdrant --source.url http://localhost:invalid --source.collection source_collection --target.url http://localhost:8334 --target.collection source_collection --migration.batch-size 1
+  run docker run --net=host --rm registry.cloud.qdrant.io/library/qdrant-migration:dev qdrant --source.url http://localhost:invalid --source.collection source_collection --target.url http://localhost:8334 --target.collection source_collection --migration.batch-size 1
   [ $status -ne 0 ]
   [[ "$output" =~ "invalid port \":invalid\" after host" ]]
 }