Merge pull request #20 from linuxfoundation/ems/meltano-containers

Support meltano container build [LFXV2-870]

Author: emsearcy

.dockerignore

@@ -4,22 +4,22 @@
 .git
 .gitignore
 .dockerignore
-/bin/
-*.sh
-*.pem
-.env
-*.env
+**/bin
 
 # Helm chart dependencies
-/charts/*/charts/
-*.tgz
+charts/*/charts
+**/*.tgz
 
 # Local and temporary files
-.DS_Store
-.idea/
-.vscode/
-*.swp
-*~
+**/.DS_Store
+.idea
+.vscode
+**/*.swp
+**/*.out
+**/*~
+**/*.pem
+**/.env
+**/*.env
 
 # Rendered templates
 **/templates/*.rendered
@@ -30,19 +30,27 @@
 /megalinter-reports/
 
 # Python environment for meltano
-.venv/
-__pycache__/
-.mypy_cache/
-.ruff_cache/
+**/.venv
+**/venv
+**/.meltano
+**/__pycache__
+**/.mypy_cache
+**/.ruff_cache
+
+# More potential Meltano paths
+meltano/ui.cfg
+meltano/output
+meltano/transform/target
+meltano/transform/dbt_modules
+meltano/transform/logs
 
 # Documentation
-README.md
-*.md
+**/README.md
+**/*.md
 LICENSE*
 
 # Test files
-*_test.go
-testdata/
+**/*_test.go
 
 # Development files
 Makefile

.github/workflows/mega-linter.yml

@@ -33,7 +33,7 @@ jobs:
       - name: MegaLinter
         id: ml
         # Use the Cupcake flavor (Go and Python).
-        uses: oxsecurity/megalinter/flavors/cupcake@5a91fb06c83d0e69fbd23756d47438aa723b4a5a  # 8.7.0
+        uses: oxsecurity/megalinter/flavors/cupcake@62c799d895af9bcbca5eacfebca29d527f125a57  # 9.1.0
         env:
           # All available variables are described in documentation
           # https://megalinter.io/latest/configuration/

.github/workflows/ko-build-main.yaml .github/workflows/publish-main.yaml.github/workflows/ko-build-main.yaml renamed to .github/workflows/publish-main.yaml

@@ -13,8 +13,8 @@ permissions:
   contents: read
 
 jobs:
-  publish:
-    name: Publish Main
+  publish-v1-sync-helper:
+    name: Publish v1-sync-helper Main
     runs-on: ubuntu-latest
     permissions:
       contents: read

.github/workflows/ko-build-tag.yaml .github/workflows/publish-release.yaml.github/workflows/ko-build-tag.yaml renamed to .github/workflows/publish-release.yaml

@@ -1,7 +1,7 @@
 # Copyright The Linux Foundation and each contributor to LFX.
 # SPDX-License-Identifier: MIT
 ---
-name: Publish Tagged Release
+name: Publish Release
 
 "on":
   push:
@@ -10,14 +10,15 @@ name: Publish Tagged Release
 
 env:
   COSIGN_VERSION: v3.0.2
-  HELM_VERSION: 4.0.1
+  HELM_VERSION: 4.0.4
+  DOCKER_REGISTRY: ghcr.io/linuxfoundation/lfx-v1-sync-helper
 
 permissions:
   contents: read
 
 jobs:
-  publish:
-    name: Publish Tagged Release
+  publish-v1-sync-helper:
+    name: Publish v1-sync-helper Tagged Release
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -62,8 +63,80 @@ jobs:
             -t latest \
             --sbom spdx
 
+  publish-meltano:
+    name: Publish Meltano Tagged Release
+    runs-on: ubuntu-latest
+    needs: [publish-v1-sync-helper]
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+    outputs:
+      image_name: ${{ steps.build.outputs.image_name }}
+      digest: ${{ steps.build.outputs.digest }}
+      app_version: ${{ steps.prepare.outputs.app_version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4
+
+      - name: Prepare versions
+        id: prepare
+        run: |
+          set -euo pipefail
+          APP_VERSION=$(echo ${{ github.ref_name }} | sed 's/v//g')
+          echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435  # v3.11.1
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef  # v3.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051  # v5.10.0
+        with:
+          images: ${{ env.DOCKER_REGISTRY }}/meltano
+          tags: |
+            type=ref,event=tag
+            type=semver,pattern={{version}}
+            type=raw,value=latest
+
+      - name: Build and push Meltano Docker image
+        id: build
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83  # v6.18.0
+        with:
+          context: .
+          file: docker/Dockerfile.meltano
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          provenance: mode=max
+          sbom: true
+
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad  # v4.0.0
+        with:
+          cosign-release: "${{ env.COSIGN_VERSION }}"
+
+      - name: Sign the Meltano container image
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          cosign sign --yes '${{ env.DOCKER_REGISTRY }}/meltano@${{ steps.build.outputs.digest }}'
+
   release-helm-chart:
-    needs: publish
+    needs:
+      - publish-v1-sync-helper
+      - publish-meltano
     runs-on: ubuntu-24.04
     permissions:
       contents: write
@@ -81,10 +154,10 @@ jobs:
         uses: >-  # main
           linuxfoundation/lfx-public-workflows/.github/actions/helm-chart-oci-publisher@c465d6571fa0b8be9d551d902955164ea04a00af
         with:
-          name: ${{ needs.publish.outputs.chart_name }}
+          name: ${{ needs.publish-v1-sync-helper.outputs.chart_name }}
           repository: ${{ github.repository }}/chart
-          chart_version: ${{ needs.publish.outputs.chart_version }}
-          app_version: ${{ needs.publish.outputs.app_version }}
+          chart_version: ${{ needs.publish-v1-sync-helper.outputs.chart_version }}
+          app_version: ${{ needs.publish-v1-sync-helper.outputs.app_version }}
           helm_version: "${{ env.HELM_VERSION }}"
           registry: ghcr.io
           registry_username: ${{ github.actor }}
@@ -96,7 +169,7 @@ jobs:
           cosign-release: "${{ env.COSIGN_VERSION }}"
 
       - name: Login to GitHub
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v3.4.0
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef  # v3.6.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -126,3 +199,21 @@ jobs:
       registry-username: ${{ github.actor }}
     secrets:
       registry-password: ${{ secrets.GITHUB_TOKEN }}
+
+  create-meltano-provenance:
+    needs:
+      - publish-meltano
+    permissions:
+      actions: read
+      id-token: write
+      packages: write
+    # Note, this action *cannot* be pinned to a ref: see the project's
+    # explanation at "Referencing SLSA builders and generators" in their
+    # README.
+    uses: slsa-framework/slsa-github-generator/.github/workflows/[email protected]
+    with:
+      image: ${{ needs.publish-meltano.outputs.image_name }}
+      digest: ${{ needs.publish-meltano.outputs.digest }}
+      registry-username: ${{ github.actor }}
+    secrets:
+      registry-password: ${{ secrets.GITHUB_TOKEN }}

.lycheeignore

@@ -1,2 +1,2 @@
 # Ignore local Kubernetes paths.
-^https?://[a-zA-Z0-9.-]+\.svc\.cluster\.local
+^(https?|nats)://[a-zA-Z0-9.-]+\.svc\.cluster\.local

.secretlintignore

@@ -0,0 +1 @@
+meltano/plugins/*/*.lock