Add docker release to the full release process for final releases (#1170)

* add docker release to release pipeline
* add docker to dependabot
* update Dockerfile to match new release pipeline
* update release pipeline to match new release pipeline
* update readme to match the new container
* update make recipe
* update dev docker image
* add permissions for nightly release
* don’t release docker when testing, allow to release only docker
* Update docker/Dockerfile
* remove unused test script

---------

Co-authored-by: Emily Rockman <[email protected]>

Author: mikealfare

.github/dependabot.yml

@@ -10,3 +10,8 @@ updates:
     schedule:
       interval: "weekly"
     rebase-strategy: "disabled"
+  - package-ecosystem: "docker"
+    directory: "/docker"
+    schedule:
+      interval: "weekly"
+    rebase-strategy: "disabled"

.github/workflows/nightly-release.yml

@@ -20,6 +20,7 @@ on:
 
 permissions:
   contents: write # this is the permission that allows creating a new release
+  packages: write # allows creating a Docker release as a GitHub package on GHCR
 
 defaults:
   run:

.github/workflows/release.yml

@@ -13,7 +13,8 @@
 #
 # **when?**
 # This workflow can be run manually on demand or can be called by other workflows
-name: Release to GitHub and PyPI
+name: "Release to GitHub, PyPI, and Docker"
+run-name: "Release ${{ inputs.version_number }} to GitHub, PyPI, and Docker"
 
 on:
   workflow_dispatch:
@@ -60,6 +61,11 @@ on:
         type: boolean
         default: false
         required: false
+      only_docker:
+        description: "Only release Docker image, skip GitHub & PyPI"
+        type: boolean
+        default: false
+        required: false
   workflow_call:
     inputs:
       sha:
@@ -128,30 +134,25 @@ jobs:
           echo Package test command:               ${{ inputs.package_test_command }}
           echo Test run:                           ${{ inputs.test_run }}
           echo Nightly release:                    ${{ inputs.nightly_release }}
+          echo Only Docker:                        ${{ inputs.only_docker }}
 
   bump-version-generate-changelog:
     name: Bump package version, Generate changelog
-
     uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
-
     with:
       sha: ${{ inputs.sha }}
       version_number: ${{ inputs.version_number }}
       target_branch: ${{ inputs.target_branch }}
       env_setup_script_path: ${{ inputs.env_setup_script_path }}
       test_run: ${{ inputs.test_run }}
       nightly_release: ${{ inputs.nightly_release }}
-
     secrets: inherit
 
   log-outputs-bump-version-generate-changelog:
     name: "[Log output] Bump package version, Generate changelog"
-    if: ${{ !failure() && !cancelled() }}
-
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
     needs: [bump-version-generate-changelog]
-
     runs-on: ubuntu-latest
-
     steps:
       - name: Print variables
         run: |
@@ -160,11 +161,9 @@ jobs:
 
   build-test-package:
     name: Build, Test, Package
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
     needs: [bump-version-generate-changelog]
-
     uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
-
     with:
       sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
       version_number: ${{ inputs.version_number }}
@@ -174,19 +173,15 @@ jobs:
       package_test_command: ${{ inputs.package_test_command }}
       test_run: ${{ inputs.test_run }}
       nightly_release: ${{ inputs.nightly_release }}
-
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   github-release:
     name: GitHub Release
-    if: ${{ !failure() && !cancelled() }}
-
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
     needs: [bump-version-generate-changelog, build-test-package]
-
     uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
-
     with:
       sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
       version_number: ${{ inputs.version_number }}
@@ -195,34 +190,41 @@ jobs:
 
   pypi-release:
     name: PyPI Release
-
-    needs: [github-release]
-
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
+    needs: [bump-version-generate-changelog, build-test-package]
     uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
-
     with:
       version_number: ${{ inputs.version_number }}
       test_run: ${{ inputs.test_run }}
-
     secrets:
       PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
       TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
 
+  docker-release:
+    name: "Docker Release"
+    # We cannot release to docker on a test run because it uses the tag in GitHub as
+    # what we need to release but draft releases don't actually tag the commit so it
+    # finds nothing to release
+    if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }}
+    needs: [bump-version-generate-changelog, build-test-package, github-release]
+    permissions:
+      packages: write
+    uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main
+    with:
+      version_number: ${{ inputs.version_number }}
+      test_run: ${{ inputs.test_run }}
+
   slack-notification:
     name: Slack Notification
     if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}
-
     needs:
       [
-        bump-version-generate-changelog,
-        build-test-package,
         github-release,
         pypi-release,
+        docker-release,
       ]
-
     uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
     with:
       status: "failure"
-
     secrets:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }}

Makefile

@@ -11,17 +11,11 @@ dev-uninstall: ## Uninstalls all packages while maintaining the virtual environm
 	pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y
 	pip uninstall -y dbt-bigquery
 
-.PHONY: ubuntu-py311
-ubuntu-py311: ## Builds and runs an Ubuntu Python 3.11 development container
-	docker build -f docker_dev/ubuntu.Dockerfile -t dbt-bigquery-ubuntu-py311 .
-	docker run --rm -it --name dbt-bigquery-ubuntu-py311 -v $(shell pwd):/opt/code dbt-bigquery-ubuntu-py311
+.PHONY: docker-dev
+docker-dev:
+	docker build -f docker/dev.Dockerfile -t dbt-bigquery-dev .
+	docker run --rm -it --name dbt-bigquery-dev -v $(shell pwd):/opt/code dbt-bigquery-dev
 
-.PHONY: ubuntu-py39
-ubuntu-py39: ## Builds and runs an Ubuntu Python 3.9 development container
-	docker build -f docker_dev/ubuntu.Dockerfile -t dbt-bigquery-ubuntu-py39 . --build-arg version=3.9
-	docker run --rm -it --name dbt-bigquery-ubuntu-py39 -v $(shell pwd):/opt/code dbt-bigquery-ubuntu-py39
-
-.PHONY: ubuntu-py38
-ubuntu-py38: ## Builds and runs an Ubuntu Python 3.8 development container
-	docker build -f docker_dev/ubuntu.Dockerfile -t dbt-bigquery-ubuntu-py38 . --build-arg version=3.8
-	docker run --rm -it --name dbt-bigquery-ubuntu-py38 -v $(shell pwd):/opt/code dbt-bigquery-ubuntu-py38
+.PHONY: docker-prod
+docker-prod:
+	docker build -f docker/Dockerfile -t dbt-bigquery .

docker/Dockerfile

@@ -0,0 +1,37 @@
+# this image gets published to GHCR for production use
+ARG py_version=3.11.2
+
+FROM python:$py_version-slim-bullseye as base
+
+RUN apt-get update \
+  && apt-get dist-upgrade -y \
+  && apt-get install -y --no-install-recommends \
+    build-essential=12.9 \
+    ca-certificates=20210119 \
+    git=1:2.30.2-1+deb11u2 \
+    libpq-dev=13.14-0+deb11u1 \
+    make=4.3-4.1 \
+    openssh-client=1:8.4p1-5+deb11u3 \
+    software-properties-common=0.96.20.2-2.1 \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
+ENV PYTHONIOENCODING=utf-8
+ENV LANG=C.UTF-8
+
+RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir
+
+
+FROM base as dbt-bigquery
+
+ARG commit_ref=main
+
+HEALTHCHECK CMD dbt --version || exit 1
+
+WORKDIR /usr/app/dbt/
+ENTRYPOINT ["dbt"]
+
+RUN python -m pip install --no-cache-dir "dbt-bigquery @ git+https://github.com/dbt-labs/dbt-bigquery@${commit_ref}"

docker/README.md

@@ -0,0 +1,58 @@
+# Docker for dbt
+This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry.
+
+
+## Building an image:
+This Dockerfile can create images for the following target: `dbt-bigquery`
+
+In order to build a new image, run the following docker command.
+```shell
+docker build --tag <your_image_name> --target dbt-bigquery <path/to/dockerfile>
+```
+---
+> **Note:**  Docker must be configured to use [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) in order for images to build properly!
+
+---
+
+By default the image will be populated with the latest version of `dbt-bigquery` on `main`.
+If you need to use a different version you can specify it by git ref using the `--build-arg` flag:
+```shell
+docker build --tag <your_image_name> \
+  --target dbt-bigquery \
+  --build-arg commit_ref=<commit_ref> \
+  <path/to/dockerfile>
+```
+
+### Examples:
+To build an image named "my-dbt" that supports Snowflake using the latest releases:
+```shell
+cd dbt-core/docker
+docker build --tag my-dbt --target dbt-bigquery .
+```
+
+To build an image named "my-other-dbt" that supports Snowflake using the adapter version 1.0.0b1:
+```shell
+cd dbt-core/docker
+docker build \
+  --tag my-other-dbt \
+  --target dbt-bigquery \
+  --build-arg commit_ref=v1.0.0b1 \
+ .
+```
+
+## Running an image in a container:
+The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal:
+```shell
+docker run \
+  --network=host \
+  --mount type=bind,source=path/to/project,target=/usr/app \
+  --mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/profiles.yml \
+  my-dbt \
+  ls
+```
+---
+**Notes:**
+* Bind-mount sources _must_ be an absolute path
+* You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host.
+
+---

docker/dev.Dockerfile

@@ -0,0 +1,50 @@
+# this image does not get published, it is intended for local development only, see `Makefile` for usage
+FROM ubuntu:22.04 as base
+
+# prevent python installation from asking for time zone region
+ARG DEBIAN_FRONTEND=noninteractive
+
+# add python repository
+RUN apt-get update \
+  && apt-get install -y software-properties-common=0.99.22.9 \
+  && add-apt-repository -y ppa:deadsnakes/ppa \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
+# install python
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    build-essential=12.9ubuntu3 \
+    git-all=1:2.34.1-1ubuntu1.10 \
+    python3.8=3.8.19-1+jammy1 \
+    python3.8-dev=3.8.19-1+jammy1 \
+    python3.8-distutils=3.8.19-1+jammy1 \
+    python3.8-venv=3.8.19-1+jammy1 \
+    python3-pip=22.0.2+dfsg-1ubuntu0.4 \
+    python3-wheel=0.37.1-2ubuntu0.22.04.1 \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
+# update the default system interpreter to the newly installed version
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
+
+
+FROM base as dbt-bigquery-dev
+
+HEALTHCHECK CMD python3 --version || exit 1
+
+# send stdout/stderr to terminal
+ENV PYTHONUNBUFFERED=1
+
+# setup mount for local code
+WORKDIR /opt/code
+VOLUME /opt/code
+
+# create a virtual environment
+RUN python3 -m venv /opt/venv