diff --git a/.github/workflows/release_prepare.yml b/.github/workflows/release_prepare.yml index 619db0218cc..9c083468cf9 100644 --- a/.github/workflows/release_prepare.yml +++ b/.github/workflows/release_prepare.yml @@ -100,10 +100,6 @@ jobs: fi done git add examples/quickstart - - name: Regenerate quickstart cloud requirements - run: | - scripts/generate_cloud_requirements.sh --zenml-version "${{ needs.fetch-versions.outputs.new_version }}" - git add examples/quickstart/requirements_*.txt # Generate and append release notes - name: Generate release notes # Only run this part once when the release branch is created @@ -231,11 +227,8 @@ jobs: matrix: include: - cloud: aws - parent_image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws - cloud: azure - parent_image: zenmldocker/prepare-release:quickstart-azure - cloud: gcp - parent_image: zenmldocker/prepare-release:quickstart-gcp steps: # Check out the code - name: Checkout code @@ -253,6 +246,4 @@ jobs: run: | scripts/qs_run_release_flow.sh \ --cloud "${{ matrix.cloud }}" \ - --parent-image "${{ matrix.parent_image }}" \ - --new-version "${{ needs.fetch-versions.outputs.new_version }}" \ - --branch-ref "${{ github.ref }}" + --new-version "${{ needs.fetch-versions.outputs.new_version }}" diff --git a/docker/zenml-quickstart-dev.Dockerfile b/docker/zenml-quickstart-dev.Dockerfile deleted file mode 100644 index c7f6399a009..00000000000 --- a/docker/zenml-quickstart-dev.Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -ARG BASE_IMAGE - -FROM $BASE_IMAGE AS base - -# Set the working directory -WORKDIR /app - -ARG ZENML_BRANCH -ARG CLOUD_PROVIDER - -# Install the Python requirements -RUN pip install uv - -RUN echo "Cloud Provider: $CLOUD_PROVIDER"; -# Install cloud-specific ZenML integrations -RUN if [ "$CLOUD_PROVIDER" = "aws" ]; then \ - zenml integration install aws s3 --uv -y; \ - elif [ "$CLOUD_PROVIDER" = "azure" ]; then \ - zenml integration install azure --uv -y; \ - elif [ "$CLOUD_PROVIDER" = "gcp" ]; then \ - zenml integration install gcp --uv -y; \ - else \ - echo "No specific cloud integration installed"; \ - fi - -ENV ZENML_REQUIRES_CODE_DOWNLOAD=True \ No newline at end of file diff --git a/docker/zenml-quickstart.Dockerfile b/docker/zenml-quickstart.Dockerfile deleted file mode 100644 index fb67ae0dac6..00000000000 --- a/docker/zenml-quickstart.Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -ARG PYTHON_VERSION=3.11 -ARG ZENML_VERSION=latest - -FROM zenmldocker/zenml:${ZENML_VERSION}-py${PYTHON_VERSION} as base - -# Set the working directory -WORKDIR /app - -# Redeclare ARGs -ARG PYTHON_VERSION -ARG ZENML_VERSION -ARG CLOUD_PROVIDER - -# Install the Python requirements -RUN pip install uv - -RUN echo "Cloud Provider: $CLOUD_PROVIDER"; -# Install cloud-specific ZenML integrations -RUN if [ "$CLOUD_PROVIDER" = "aws" ]; then \ - zenml integration install aws s3 --uv -y; \ - elif [ "$CLOUD_PROVIDER" = "azure" ]; then \ - zenml integration install azure --uv -y; \ - elif [ "$CLOUD_PROVIDER" = "gcp" ]; then \ - zenml integration install gcp --uv -y; \ - else \ - echo "No specific cloud integration installed"; \ - fi - -ENV ZENML_REQUIRES_CODE_DOWNLOAD=True \ No newline at end of file diff --git a/examples/quickstart/pipelines/simple_pipeline.py b/examples/quickstart/pipelines/simple_pipeline.py index 1e5cccad0ee..82ea2049add 100644 --- a/examples/quickstart/pipelines/simple_pipeline.py +++ b/examples/quickstart/pipelines/simple_pipeline.py @@ -5,11 +5,7 @@ from steps.simple_step import simple_step from zenml import pipeline -from zenml.config import CORSConfig, DeploymentSettings, DockerSettings - -docker_settings = DockerSettings( - requirements="requirements.txt", -) +from zenml.config import CORSConfig, DeploymentSettings deployment_settings = DeploymentSettings( app_title="Simple Pipeline", @@ -19,7 +15,6 @@ @pipeline( settings={ - "docker": docker_settings, "deployment": deployment_settings, }, enable_cache=False, diff --git a/release-cloudbuild-preparation.yaml b/release-cloudbuild-preparation.yaml index 9dbcfb8d0f0..b7a44702dbe 100644 --- a/release-cloudbuild-preparation.yaml +++ b/release-cloudbuild-preparation.yaml @@ -10,35 +10,6 @@ steps: - USERNAME - PASSWORD - # Build base image - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker build . \ - --platform linux/amd64 \ - -f docker/zenml-dev.Dockerfile \ - -t $$USERNAME/prepare-release:base-${_ZENML_NEW_VERSION} - - id: build-base - waitFor: ['-'] - entrypoint: bash - secretEnv: - - USERNAME - - # Push base image - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - docker push $$USERNAME/prepare-release:base-${_ZENML_NEW_VERSION} - id: push-base - waitFor: - - docker-login - - build-base - entrypoint: bash - secretEnv: - - USERNAME - # Build server image - name: gcr.io/cloud-builders/docker args: @@ -68,82 +39,6 @@ steps: secretEnv: - USERNAME - # Build Quickstart GCP Image - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker build . \ - --platform linux/amd64 \ - --build-arg BASE_IMAGE=$$USERNAME/prepare-release:base-${_ZENML_NEW_VERSION} \ - --build-arg CLOUD_PROVIDER=gcp \ - --build-arg ZENML_BRANCH=${_ZENML_BRANCH} \ - -f docker/zenml-quickstart-dev.Dockerfile \ - -t $$USERNAME/prepare-release:quickstart-gcp-${_ZENML_NEW_VERSION} - - id: build-quickstart-gcp - waitFor: - - push-base - entrypoint: bash - secretEnv: - - USERNAME - - # Build Quickstart AWS image - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker build . \ - --platform linux/amd64 \ - --build-arg BASE_IMAGE=$$USERNAME/prepare-release:base-${_ZENML_NEW_VERSION} \ - --build-arg CLOUD_PROVIDER=aws \ - --build-arg ZENML_BRANCH=${_ZENML_BRANCH} \ - -f docker/zenml-quickstart-dev.Dockerfile \ - -t $$USERNAME/prepare-release:quickstart-aws-${_ZENML_NEW_VERSION} - id: build-quickstart-aws - waitFor: - - push-base - entrypoint: bash - secretEnv: - - USERNAME - - # Build Quickstart Azure image - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker build . \ - --platform linux/amd64 \ - --build-arg BASE_IMAGE=$$USERNAME/prepare-release:base-${_ZENML_NEW_VERSION} \ - --build-arg CLOUD_PROVIDER=azure \ - --build-arg ZENML_BRANCH=${_ZENML_BRANCH} \ - -f docker/zenml-quickstart-dev.Dockerfile \ - -t $$USERNAME/prepare-release:quickstart-azure-${_ZENML_NEW_VERSION} - id: build-quickstart-azure - waitFor: - - push-base - entrypoint: bash - secretEnv: - - USERNAME - - # Push Quickstart images - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker push $$USERNAME/prepare-release:quickstart-aws-${_ZENML_NEW_VERSION} - docker push $$USERNAME/prepare-release:quickstart-azure-${_ZENML_NEW_VERSION} - docker push $$USERNAME/prepare-release:quickstart-gcp-${_ZENML_NEW_VERSION} - id: push-quickstart - waitFor: - - docker-login - - build-quickstart-gcp - - build-quickstart-aws - - build-quickstart-azure - entrypoint: bash - secretEnv: - - USERNAME - timeout: 3600s availableSecrets: secretManager: diff --git a/release-cloudbuild.yaml b/release-cloudbuild.yaml index 661ad563fde..a5b2372b79d 100644 --- a/release-cloudbuild.yaml +++ b/release-cloudbuild.yaml @@ -156,75 +156,6 @@ steps: secretEnv: - USERNAME - # build client quickstart gcp image - python 3.11 - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker buildx build \ - --platform linux/amd64 \ - --build-arg ZENML_VERSION=$TAG_NAME \ - --build-arg PYTHON_VERSION=3.11 \ - --build-arg CLOUD_PROVIDER=gcp \ - -f docker/zenml-quickstart.Dockerfile . \ - -t $$USERNAME/zenml-public-pipelines:quickstart-$TAG_NAME-py3.11-gcp - id: build-quickstart-3.11-gcp - waitFor: [ 'push-base' ] - entrypoint: bash - secretEnv: - - USERNAME - - # build client quickstart aws image - python 3.11 - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker buildx build \ - --platform linux/amd64 \ - --build-arg ZENML_VERSION=$TAG_NAME \ - --build-arg PYTHON_VERSION=3.11 \ - --build-arg CLOUD_PROVIDER=aws \ - -f docker/zenml-quickstart.Dockerfile . \ - -t $$USERNAME/zenml-public-pipelines:quickstart-$TAG_NAME-py3.11-aws - id: build-quickstart-3.11-aws - waitFor: [ 'push-base' ] - entrypoint: bash - secretEnv: - - USERNAME - - # build client quickstart azure image - python 3.11 - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - | - docker buildx build \ - --platform linux/amd64 \ - --build-arg ZENML_VERSION=$TAG_NAME \ - --build-arg PYTHON_VERSION=3.11 \ - --build-arg CLOUD_PROVIDER=azure \ - -f docker/zenml-quickstart.Dockerfile . \ - -t $$USERNAME/zenml-public-pipelines:quickstart-$TAG_NAME-py3.11-azure - id: build-quickstart-3.11-azure - waitFor: [ 'push-base' ] - entrypoint: bash - secretEnv: - - USERNAME - - # push quickstart images - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - docker push --all-tags $$USERNAME/zenml-public-pipelines - id: push-quickstart - waitFor: - - build-quickstart-3.11-gcp - - build-quickstart-3.11-aws - - build-quickstart-3.11-azure - - docker-login - entrypoint: bash - secretEnv: - - USERNAME - timeout: 3600s availableSecrets: secretManager: diff --git a/scripts/generate_cloud_requirements.sh b/scripts/generate_cloud_requirements.sh deleted file mode 100755 index 8e80a8fdaeb..00000000000 --- a/scripts/generate_cloud_requirements.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env bash -# Generate Quickstart Cloud Requirements -# -# Regenerates examples/quickstart/requirements_{aws,azure,gcp}.txt from: -# - a base requirements file, and -# - ZenML integration metadata (via `zenml integration export-requirements`). -# -# Why this exists: -# - Keeps provider-specific requirements aligned with ZenML integration definitions. -# - Ensures the pinned ZenML version is consistent across release preparations. -# - Produces deterministic, deduplicated outputs written atomically for CI stability. - -set -euo pipefail - -usage() { - cat <<'USAGE' -Usage: - scripts/generate_cloud_requirements.sh - [--base ] # Default: examples/quickstart/requirements.txt - [--out-dir ] # Default: examples/quickstart - --zenml-version # Required - [--clouds ] # Default: aws,azure,gcp - [-h|--help] - -Description: - Generates provider-specific requirements files for the Quickstart example: - - requirements_aws.txt - - requirements_azure.txt - - requirements_gcp.txt - - Each file contains: - 1) A header noting the file is auto-generated. - 2) Pinned zenml[server]== as the first requirement. - 3) Base requirements (excluding any line that mentions 'zenml[server]'), preserving order. - 4) A section with the exported integration requirements for the provider. - 5) Global de-duplication while preserving the first occurrence order. - -Notes: - - Output files are written atomically via a temp file + mv. - - The script fails if the integration export is empty or if the base file is missing. - -Examples: - scripts/generate_cloud_requirements.sh --zenml-version 0.90.0 - scripts/generate_cloud_requirements.sh --base path/to/base.txt --out-dir examples/quickstart --zenml-version 0.90.0 --clouds aws,gcp -USAGE -} - -# Defaults -BASE="examples/quickstart/requirements.txt" -OUT_DIR="examples/quickstart" -ZENML_VERSION="" -CLOUDS="aws,azure,gcp" - -# Parse arguments -while [[ $# -gt 0 ]]; do - case "$1" in - --base) - BASE="${2:-}"; shift 2;; - --out-dir) - OUT_DIR="${2:-}"; shift 2;; - --zenml-version) - ZENML_VERSION="${2:-}"; shift 2;; - --clouds) - CLOUDS="${2:-}"; shift 2;; - -h|--help) - usage; exit 0;; - *) - echo "Error: Unknown argument: $1" >&2 - usage - exit 1;; - esac -done - -# Validate required arguments and inputs -if [[ -z "${ZENML_VERSION}" ]]; then - echo "Error: --zenml-version is required (e.g., --zenml-version 0.90.0)." >&2 - usage - exit 1 -fi - -if [[ ! -f "${BASE}" ]]; then - echo "Error: Base requirements file not found: ${BASE}" >&2 - exit 1 -fi - -mkdir -p "${OUT_DIR}" - -# Normalize clouds CSV to an array -IFS=',' read -r -a CLOUD_ARRAY <<< "${CLOUDS}" - -# Helper to export integration requirements as one requirement per line. -# Some exports can be whitespace-separated; we normalize to newline-separated tokens. -_export_integration_requirements() { - local cloud="$1" - local raw_output - if ! raw_output="$(zenml integration export-requirements "${cloud}" 2>&1)"; then - echo "Error: Failed to export requirements for integration '${cloud}'." >&2 - echo "Output:" >&2 - echo "${raw_output}" >&2 - return 1 - fi - - # Convert any whitespace (spaces/tabs/newlines) into single newlines, strip blanks. - # This ensures we get one requirement per line regardless of the export format. - # Note: Pip requirement spec rarely contains spaces; if present, this would split them. - # If that ever becomes an issue, adjust this normalization accordingly. - printf "%s" "${raw_output}" | tr -s '[:space:]' '\n' | sed -E '/^[[:space:]]*$/d' -} - -for cloud in "${CLOUD_ARRAY[@]}"; do - cloud_trimmed="$(printf "%s" "${cloud}" | sed -E 's/^[[:space:]]+|[[:space:]]+$//g')" - if [[ -z "${cloud_trimmed}" ]]; then - continue - fi - - out_file="${OUT_DIR}/requirements_${cloud_trimmed}.txt" - tmp_file="$(mktemp "${OUT_DIR}/requirements_${cloud_trimmed}.tmp.XXXXXX")" - tmp_dedup="$(mktemp "${OUT_DIR}/requirements_${cloud_trimmed}.dedup.XXXXXX")" - - echo "=== Generating ${out_file} ===" - echo "-> Exporting integration requirements for '${cloud_trimmed}'..." - export_lines="$(_export_integration_requirements "${cloud_trimmed}")" || { - echo "Error: Could not retrieve requirements for '${cloud_trimmed}'. Aborting." >&2 - rm -f "${tmp_file}" "${tmp_dedup}" || true - exit 1 - } - - if [[ -z "${export_lines}" ]]; then - echo "Error: Integration '${cloud_trimmed}' returned no requirements. Failing to avoid empty output." >&2 - rm -f "${tmp_file}" "${tmp_dedup}" || true - exit 1 - fi - - { - echo "# Auto-generated by scripts/generate_cloud_requirements.sh - DO NOT EDIT" - echo "# Cloud integration: ${cloud_trimmed}" - echo "zenml[server]==${ZENML_VERSION}" - # Append base requirements while excluding any line that mentions 'zenml[server]' anywhere - # to avoid duplicate pins or git-egg variants. Order for the remaining lines is preserved. - grep -E -v 'zenml\[server\]' "${BASE}" || true - echo "" - echo "# ${cloud_trimmed} integration requirements" - # Write the exported requirements, one per line - printf "%s\n" "${export_lines}" - } > "${tmp_file}" - - # Deduplicate globally while preserving the first occurrence order. - # This avoids duplicates that may arise when base reqs already include some integration deps. - awk '!seen[$0]++' "${tmp_file}" > "${tmp_dedup}" - - # Atomic move into place - mv -f "${tmp_dedup}" "${out_file}" - rm -f "${tmp_file}" || true - - echo "-> Wrote ${out_file}" -done - -echo "=== Done generating cloud requirements for: ${CLOUDS} ===" \ No newline at end of file diff --git a/scripts/qs_run_release_flow.sh b/scripts/qs_run_release_flow.sh index e3bbd31b2fd..aa1c6aca912 100755 --- a/scripts/qs_run_release_flow.sh +++ b/scripts/qs_run_release_flow.sh @@ -9,39 +9,26 @@ usage() { Usage: scripts/qs_run_release_flow.sh \ --cloud \ - --parent-image \ - --new-version \ - [--branch-ref ] + --new-version Description: Runs the Quickstart release validation flow: - - Patches the example config's parent_image and requirements to use the branch ref - - Installs dependencies - Phase 1: Runs the pipeline locally - Phase 2: Runs the pipeline on the cloud stack -Notes: - --branch-ref is optional. If not provided, the script falls back to $GITHUB_REF. - If neither is available, the script exits with an error. USAGE } # Simple argument parser CLOUD="" -PARENT_PREFIX="" NEW_VERSION="" -BRANCH_REF="${BRANCH_REF:-}" # allow environment override if set externally while [[ $# -gt 0 ]]; do case "$1" in --cloud) CLOUD="${2:-}"; shift 2;; - --parent-image) - PARENT_PREFIX="${2:-}"; shift 2;; --new-version) NEW_VERSION="${2:-}"; shift 2;; - --branch-ref) - BRANCH_REF="${2:-}"; shift 2;; -h|--help) usage; exit 0;; *) @@ -51,36 +38,20 @@ while [[ $# -gt 0 ]]; do done # Validate required args -if [[ -z "${CLOUD}" || -z "${PARENT_PREFIX}" || -z "${NEW_VERSION}" ]]; then - echo "Error: --cloud, --parent-image, and --new-version are required." >&2 +if [[ -z "${CLOUD}" || -z "${NEW_VERSION}" ]]; then + echo "Error: --cloud and --new-version are required." >&2 usage exit 1 fi -# Determine branch ref -if [[ -z "${BRANCH_REF}" ]]; then - if [[ -n "${GITHUB_REF:-}" ]]; then - BRANCH_REF="${GITHUB_REF}" - else - echo "Error: --branch-ref not provided and GITHUB_REF is not set. Please supply one of them." >&2 - exit 1 - fi -fi echo "=== Quickstart Release Flow ===" echo "Cloud stack: ${CLOUD}" -echo "Parent image prefix: ${PARENT_PREFIX}" -echo "New version: ${NEW_VERSION}" -echo "Git branch ref: ${BRANCH_REF}" echo "================================" # Minimal variables needed for the simplified flow DIR_PUSHED=0 CLOUD_STACK="${CLOUD}" -CONFIG="configs/training_${CLOUD}.yaml" -REQS="requirements_${CLOUD}.txt" -PARENT_IMAGE="${PARENT_PREFIX}-${NEW_VERSION}" -ZENML_GIT_SPEC="git+https://github.com/zenml-io/zenml.git@${BRANCH_REF}#egg=zenml[server]" _cleanup() { # Preserve original exit code from the point of trap invocation @@ -106,29 +77,8 @@ echo "=== Switching to examples/quickstart directory ===" pushd examples/quickstart >/dev/null DIR_PUSHED=1 -# Validate required files -if [[ ! -f "${CONFIG}" ]]; then - echo "Error: Config file not found: ${CONFIG}" >&2 - exit 1 -fi -if [[ ! -f "${REQS}" ]]; then - echo "Error: Requirements file not found: ${REQS}" >&2 - exit 1 -fi - -# Patch config parent_image -echo "=== Patching parent_image in ${CONFIG} to: ${PARENT_IMAGE} ===" -# Replace the entire parent_image line, preserving indentation -sed -i -E "s|^([[:space:]]*parent_image:).*|\1 \"${PARENT_IMAGE}\"|g" "${CONFIG}" - -# Patch requirements to install zenml from the provided branch ref -echo "=== Updating ${REQS}: pinning zenml to branch ref ${BRANCH_REF} ===" -# Replace any line that starts with 'zenml[server]' to make the pin robust to different version operators. -sed -i -E "s|^zenml\\[server\\].*|${ZENML_GIT_SPEC}|g" "${REQS}" - -# Install dependencies -echo "=== Installing dependencies from ${REQS} ===" -pip install -r "${REQS}" +# Build parent image +docker build -f docker/zenml-dev.Dockerfile -t zenmldocker/zenml:${NEW_VERSION}-py3.12 . # Phase 1: Run the pipeline locally echo "=== Phase 1: Run pipeline locally ===" @@ -136,6 +86,15 @@ python run.py # Phase 2: Run on the cloud stack echo "=== Phase 2: Run pipeline on cloud stack ===" + +if [[ "${CLOUD}" == "aws" ]]; then + zenml integration install aws s3 --uv -y +elif [[ "${CLOUD}" == "azure" ]]; then + zenml integration install azure --uv -y +elif [[ "${CLOUD}" == "gcp" ]]; then + zenml integration install gcp --uv -y +fi + zenml stack set "${CLOUD_STACK}" python run.py