From 3e9ce619bd45dc58608e2055589188a260487719 Mon Sep 17 00:00:00 2001 From: xeniape Date: Fri, 11 Oct 2024 15:45:12 +0200 Subject: [PATCH] ci: Add workflows for pyspark images --- ...dev_jupyter-pyspark-with-alibi-detect.yaml | 89 +++++++++++++++++++ .../dev_pyspark-k8s-with-scikit-learn.yaml | 89 +++++++++++++++++++ .../Dockerfile | 2 - demos/signal-processing/Dockerfile-jupyter | 2 - 4 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml create mode 100644 .github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml diff --git a/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml b/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml new file mode 100644 index 00000000..ac842416 --- /dev/null +++ b/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml @@ -0,0 +1,89 @@ +--- +name: Build and publish jupyter-pyspark-with-alibi-detect + +env: + IMAGE_NAME: jupyter-pyspark-with-alibi-detect + IMAGE_VERSION: python-3.9 + REGISTRY_PATH: stackable + DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-jupyter" + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - demos/signal-processing/Dockerfile-jupyter + - demos/signal-processing/requirements.txt + - .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml + +jobs: + build: + name: Build/Publish ${{ matrix.runner.arch }} Image + permissions: + id-token: write + runs-on: ${{ matrix.runner.name }} + strategy: + matrix: + runner: + - {name: "ubuntu-latest", arch: "amd64"} + - {name: "ubicloud-standard-8-arm", arch: "arm64"} + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Build image + id: build + uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-name: ${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} + container-file: ${{ env.DOCKERFILE_PATH }} + + - name: Publish Container Image on docker.stackable.tech + uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: docker.stackable.tech + image-registry-username: github + image-registry-password: ${{ secrets.NEXUS_PASSWORD }} + image-repository: demos/${{ env.IMAGE_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + + - name: Publish Container Image on oci.stackable.tech + uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$stackable+github-action-build + image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} + image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + + publish_manifests: + name: Build/Publish Manifest + needs: [build] + permissions: + id-token: write + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Publish and Sign Image Index Manifest to docker.stackable.tech + uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: docker.stackable.tech + image-registry-username: github + image-registry-password: ${{ secrets.NEXUS_PASSWORD }} + image-repository: demos/${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} + + - name: Publish and Sign Image Index Manifest to oci.stackable.tech + uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$stackable+github-action-build + image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} + image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} diff --git a/.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml b/.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml new file mode 100644 index 00000000..f9b497e5 --- /dev/null +++ b/.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml @@ -0,0 +1,89 @@ +--- +name: Build and publish pyspark-k8s-with-scikit-learn + +env: + IMAGE_NAME: pyspark-k8s-with-scikit-learn + IMAGE_VERSION: 3.4.0-stackable0.0.0-dev + REGISTRY_PATH: stackable + DOCKERFILE_PATH: "demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile" + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile + - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt + - .github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml + +jobs: + build: + name: Build/Publish ${{ matrix.runner.arch }} Image + permissions: + id-token: write + runs-on: ${{ matrix.runner.name }} + strategy: + matrix: + runner: + - {name: "ubuntu-latest", arch: "amd64"} + - {name: "ubicloud-standard-8-arm", arch: "arm64"} + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Build image + id: build + uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-name: ${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} + container-file: ${{ env.DOCKERFILE_PATH }} + + - name: Publish Container Image on docker.stackable.tech + uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: docker.stackable.tech + image-registry-username: github + image-registry-password: ${{ secrets.NEXUS_PASSWORD }} + image-repository: demos/${{ env.IMAGE_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + + - name: Publish Container Image on oci.stackable.tech + uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$stackable+github-action-build + image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} + image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + + publish_manifests: + name: Build/Publish Manifest + needs: [build] + permissions: + id-token: write + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Publish and Sign Image Index Manifest to docker.stackable.tech + uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: docker.stackable.tech + image-registry-username: github + image-registry-password: ${{ secrets.NEXUS_PASSWORD }} + image-repository: demos/${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} + + - name: Publish and Sign Image Index Manifest to oci.stackable.tech + uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$stackable+github-action-build + image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} + image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile index 04c23b43..ad6efe86 100644 --- a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile +++ b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile @@ -1,5 +1,3 @@ -# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.4.0-stackable0.0.0-dev . - FROM docker.stackable.tech/stackable/pyspark-k8s:3.4.0-stackable23.7.0 COPY requirements.txt . diff --git a/demos/signal-processing/Dockerfile-jupyter b/demos/signal-processing/Dockerfile-jupyter index bc25e7df..62abee7d 100644 --- a/demos/signal-processing/Dockerfile-jupyter +++ b/demos/signal-processing/Dockerfile-jupyter @@ -1,5 +1,3 @@ -# docker build -f ./Dockerfile-jupyter -t docker.stackable.tech/demos/jupyter-pyspark-with-alibi-detect:python-3.9 . - FROM jupyter/pyspark-notebook:python-3.9 COPY requirements.txt .