Implemented exit gate

Komal Yadav · Komal Yadav · commit 15a1cb4fed81 · 2026-02-10T05:59:07.000Z
diff --git a/.github/workflows/build-and-unit-test.yml b/.github/workflows/build-and-unit-test.yml
@@ -1,4 +1,4 @@
-# Copyright © 2022 Cask Data, Inc.
+# Copyright © 2025 Cask Data, Inc.
 #  Licensed under the Apache License, Version 2.0 (the "License"); you may not
 #  use this file except in compliance with the License. You may obtain a copy of
 #  the License at
@@ -85,10 +85,8 @@ jobs:
         if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
         with:
           secrets: |-
-            CDAP_OSSRH_USERNAME:cdapio-github-builds/CDAP_OSSRH_USERNAME
-            CDAP_OSSRH_PASSWORD:cdapio-github-builds/CDAP_OSSRH_PASSWORD
-            CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE
-            CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY
+            artifacts_bucket:cdapio-github-builds/artifacts_bucket
+            secure_publish_bucket:cdapio-github-builds/publish_bucket
 
       - name: Recursively Checkout Repository
         uses: actions/checkout@v4
@@ -151,18 +149,6 @@ jobs:
           commit: ${{ github.sha }}
           check_name: Test Report - ${{ env.ARTIFACT_NAME }}
 
-      - name: Build Standalone
-        # Pinned version 2.8.2
-        uses: nick-fields/retry@3e91a01664abd3c5cd539100d10d33b9c5b68482
-        with:
-          timeout_minutes: 60
-          max_attempts: 3
-          retry_on: error
-          on_retry_command: echo "Build Standalone failed in this attempt, retrying ..."
-          command: |
-              cd cdap-build
-              MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
-
       - name: Find Build Version
         working-directory: cdap-build/cdap
         run:  |
@@ -180,74 +166,43 @@ jobs:
             echo "Release will be overwritten if exists."
           fi
 
-      - name: Upload CDAP Standalone
-        if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
-        uses: actions/upload-artifact@v4 # https://github.com/actions/upload-artifact#zipped-artifact-downloads
-        with:
-          name: cdap-sandbox-${{env.CDAP_VERSION}}.zip
-          path: cdap-build/cdap/cdap-standalone/target/cdap-sandbox-${{env.CDAP_VERSION}}.zip
-
-      - name: Set up GPG conf
-        if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
+      - name: Submit Build to GCB
+        id: gcb
+        working-directory: cdap-build
         run: |
-          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
+          BUILD_OUTPUT=$(gcloud builds submit . \
+          --config=cloudbuild-release.yaml \
+          --project='${{ env.GCP_PROJECT_ID }}' \
+          --substitutions="_ARTIFACT_ID=cdap-build,_SECURE_PUBLISH_BUCKET_NAME=${{ steps.gcp_secrets.outputs.secure_publish_bucket }},_CDAP_VERSION=${{ env.CDAP_VERSION }}" 2>&1)
+          echo "$BUILD_OUTPUT"
+          BUILD_ID=$(echo "$BUILD_OUTPUT" | grep -oP 'ID: \K[a-f0-9-]+' | head -n 1)
+          if [ -z "$BUILD_ID" ]; then
+          echo "Failed to extract BUILD_ID from gcloud output." >&2
+          # Fallback: Get the last build submitted by this service account.
+          # This assumes no other concurrent builds are started by this SA.
+          BUILD_ID=$(gcloud builds list --project='${{ env.GCP_PROJECT_ID }}' --limit=1 --format='value(ID)' --filter="buildTriggerId='' AND status!=WORKING AND status!=QUEUED" --sort-by=~CREATE_TIME)
+          if [ -z "$BUILD_ID" ]; then
+          echo "Fallback failed to get BUILD_ID." >&2
+          exit 1
+          fi
+          echo "Fallback BUILD_ID: $BUILD_ID"
+          fi
+          echo "build_id=$BUILD_ID" >> $GITHUB_OUTPUT
+          echo "GCB Build ID: $BUILD_ID"
 
-      - name: Import GPG key
+      - name: Create Download Directory
         if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
-        run: |
-          echo "$GPG_PRIVATE_KEY" > private.key
-          gpg --import --batch private.key
-        env:
-          GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }}
+        run: mkdir -p downloads
 
-      - name: Maven Deploy
+      - name: Download CDAP Standalone from GCS
+        if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
         run: |
-          cd cdap-build
-          if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]];
-          then
-            retry_count=0
-            failed_module=""
-            while [ $retry_count -lt 3 ]; do
-              if [ -n "$failed_module" ];
-              then
-                build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true)
-              else
-                build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true)
-              fi
-              echo "$build_output"
-              if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ];
-              then
-                echo "[WARNING] Deployment failed, retrying..."
-                failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p')
-                echo "[INFO] FAILED MODULE = $failed_module"
-                retry_count=$((retry_count + 1))
-              else
-                echo "[INFO] Deployment successful"
-                break
-              fi
-            done
-            if [ $retry_count -ge 3 ];
-            then
-              echo "[ERROR] Max retries reached..., deployment failed"
-              exit 1
-            fi
-          else
-            mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
-          fi
-        env:
-          CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }}
-          CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }}
-          CDAP_GPG_PASSPHRASE: ${{ steps.secrets.outputs.CDAP_GPG_PASSPHRASE }}
-          MAVEN_OPTS: "-Xmx12G"
+          gsutil cp gs://${{ steps.gcp_secrets.outputs.secure_publish_bucket }}/cdap/cdap/${{ steps.gcb.outputs.build_id }}/cdap-sandbox-${{ env.CDAP_VERSION }}.zip downloads/
 
-      - name: Build DEB Bundle
-        working-directory: cdap-build/cdap
+      - name: Download CDAP DEB Bundle from GCS
+        if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
         run: |
-          mkdir -p cdap-distributions/target/deb-bundle-tmp
-          cd cdap-distributions/target/deb-bundle-tmp
-          cp ../../../*/target/*.deb .
-          tar zcf ../cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz *.deb
+          gsutil cp gs://${{ steps.gcp_secrets.outputs.secure_publish_bucket }}/cdap/cdap/${{ steps.gcb.outputs.build_id }}/cdap-distributed-deb-bundle-${{ env.CDAP_VERSION }}.tgz downloads/
 
       - name: Set Up Tag
         working-directory: cdap-build
@@ -270,6 +225,17 @@ jobs:
             git push -f origin refs/tags/${{ env.TAG_NAME }}:refs/tags/${{ env.TAG_NAME }}
           fi
 
+      - name: 'Download Artifacts from Cloud Storage'
+        if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
+        working-directory: cdap-build
+        run: |
+          GCS_STAGING_PATH="${{ env.ARTIFACTS_BUCKET }}"
+          CDAP_VERSION="${{ env.CDAP_VERSION }}"
+          mkdir -p cdap/cdap-standalone/target
+          mkdir -p cdap/cdap-distributions/target
+          gsutil cp "${GCS_STAGING_PATH}/cdap-sandbox-${CDAP_VERSION}.zip" cdap/cdap-standalone/target/
+          gsutil cp "${GCS_STAGING_PATH}/cdap-distributed-deb-bundle-${CDAP_VERSION}.tgz" cdap/cdap-distributions/target/
+
       - name: Upload CDAP Standalone and CDAP DEB Bundle
         # Pinned 1.14.0 version
         uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5
diff --git a/.github/workflows/cloudbuild.yaml b/.github/workflows/cloudbuild.yaml
@@ -0,0 +1,152 @@
+# Copyright © 2025 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+steps:
+  - name: 'gcr.io/cdapio-github-builds/runner'
+    id: build-standalone
+    entrypoint: 'bash'
+    env:
+      - 'MAVEN_OPTS=-Xmx12G'
+    args:
+      - '-c'
+      - |
+        set -ex
+
+        yarn global add node-gyp
+        
+        echo ">>> Running Maven build..."
+        MVN_CMD="mvn -e -T2 -U clean package \
+          -Dgpg.skip=true \
+          -DskipTests=true \
+          -Ddocker.skip=true \
+          -Dadditional.artifacts.dir=/workspace/app-artifacts \
+          -Dsecurity.extensions.dir=/workspace/security-extensions \
+          -nsu -am -amd \
+          -Ptemplates,dist,release \
+          -Dmaven.wagon.http.retryHandler.count=5 \
+          -Dmaven.wagon.httpconnectionManager.ttlSeconds=30"
+        $$MVN_CMD
+
+
+  - name: 'gcr.io/cdapio-github-builds/runner'
+    id: build-distributions
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        set -ex
+        echo ">>> Building Distribution Artifacts (DEB, RPM, TGZ)..."
+        # Run package on the distributions module, 'package' phase without 'clean'
+        # Assumes dependencies are available from the previous build-standalone step
+        MAVEN_OPTS="-Xmx12G" mvn -B -V -e -U package \
+          -DskipTests=true -Dgpg.skip=true -Ddocker.skip=true -nsu \
+          -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz \
+          -Dadditional.artifacts.dir=/workspace/app-artifacts \
+          -Dsecurity.extensions.dir=/workspace/security-extensions
+        echo ">>> Distribution Builds Complete."
+    waitFor: [ 'build-standalone' ]
+
+  - name: 'bash'
+    id: create-deb-bundle
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        set -e
+        VERSION=${_CDAP_VERSION}
+        echo "Packaging DEB bundle for version $$VERSION"
+        mkdir -p cdap/cdap-distributions/target/deb-bundle-tmp
+        cd cdap/cdap-distributions/target/deb-bundle-tmp
+        find /workspace/cdap -name "*.deb" -exec cp {} . \;
+        if [ -z "$(ls -A *.deb 2>/dev/null)" ]; then
+          echo "ERROR: No .deb files found to bundle!"
+          exit 1
+        fi
+        tar zcf ../cdap-distributed-deb-bundle-$${VERSION}.tgz *.deb
+        echo "Created DEB bundle: $(pwd)/../cdap-distributed-deb-bundle-$${VERSION}.tgz"
+        cd /workspace
+    waitFor: [ 'build-distributions' ]
+
+  - name: 'anchore/syft:v1.5.0'
+    id: generate-sbom
+    args:
+      - 'packages'
+      - '-o'
+      - 'spdx-json=/workspace/attestations/project-sbom.spdx.json'
+      - '.'
+    waitFor: [ 'build-distributions' ]
+
+  - name: 'bash'
+    id: stage-artifacts
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        set -e
+        mkdir -p /workspace/staging
+        mkdir -p /workspace/attestations
+        VERSION=${_CDAP_VERSION}
+
+        echo "Copying Maven artifacts..."
+        find . -type f \( -name "*.jar" -o -name "*.pom" -o -name "*.war" \) \
+          ! -name "original-*.jar" ! -name "*-tests.jar" \
+          ! -path "*/target/site/*" ! -path "*/target/apidocs/*" \
+          ! -path "*/target/dependency-cache/*" ! -path "*/target/maven-archiver/*" \
+          ! -path "*/target/generated-sources/*" ! -path "*/target/maven-status/*" \
+          -exec cp --parents {} /workspace/staging/ \;
+
+        echo "Copying Standalone ZIP..."
+        cp cdap/cdap-standalone/target/cdap-sandbox-$${VERSION}.zip /workspace/staging/
+
+        echo "Copying DEB bundle..."
+        cp cdap/cdap-distributions/target/cdap-distributed-deb-bundle-$${VERSION}.tgz /workspace/staging/
+
+        echo "Copying SBOM..."
+        cp /workspace/attestations/project-sbom.spdx.json /workspace/staging/
+
+        echo "Staged files (top level):"
+        ls -l /workspace/staging
+    waitFor: [ 'create-deb-bundle', 'generate-sbom' ]
+
+  - name: 'bash'
+    id: create-manifest
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        set -e
+        echo "Creating manifest.json..."
+        cd /workspace/staging
+        printf '{\n  "artifacts": [\n' > manifest.json
+        find . -type f ! -name "manifest.json" | sed 's|./||' | sort | sed 's/.*/    "&",/' >> manifest.json
+        sed -i '$ s/,$//' manifest.json
+        printf '\n  ]\n}\n' >> manifest.json
+        echo "Generated manifest.json:"
+        cat manifest.json
+        cd /workspace
+    waitFor: [ 'stage-artifacts' ]
+
+  - name: 'gcr.io/cloud-builders/gsutil'
+    id: upload-to-staging-bucket
+    args:
+      - '-m'
+      - 'cp'
+      - '-r'
+      - '/workspace/staging/*'
+      - 'gs://${_SECURE_PUBLISH_BUCKET_NAME}/${_ARTIFACT_ID}/${BUILD_ID}/'
+    waitFor: [ 'create-manifest' ]
+
+options:
+  requestedVerifyOption: VERIFIED
+  machineType: 'E2_HIGHCPU_32'
diff --git a/pom.xml b/pom.xml
@@ -53,6 +53,30 @@
     <tag>HEAD</tag>
   </scm>
 
+  <distributionManagement>
+    <snapshotRepository>
+      <id>artifact-registry</id>
+      <url>artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral</url>
+    </snapshotRepository>
+    <repository>
+      <id>artifact-registry</id>
+      <url>artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral</url>
+    </repository>
+  </distributionManagement>
+
+  <repositories>
+    <repository>
+      <id>artifact-registry</id>
+      <url>artifactregistry:us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral</url>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>
+
   <modules>
     <module>app-artifacts/bigquery-delta-plugins</module>
     <module>app-artifacts/database-delta-plugins</module>
@@ -72,6 +96,16 @@
     <module>cdap</module>
   </modules>
 
+  <build>
+    <extensions>
+      <extension>
+        <groupId>com.google.cloud.artifactregistry</groupId>
+        <artifactId>artifactregistry-maven-wagon</artifactId>
+        <version>2.2.5</version>
+      </extension>
+    </extensions>
+  </build>
+
   <profiles>
     <profile>
       <id>release</id>
