Merge branch 'develop' of https://github.com/rudderlabs/rudder-transformer into technocrats/ONCEHUB-94984

Author: sumitsheokand

.eslintignore

@@ -26,3 +26,5 @@ scripts/skipPrepareScript.js
 .eslintignore
 .prettierignore
 *.json
+Dockerfile*
+*.properties

.github/workflows/allure-test-reporter.yml

@@ -0,0 +1,114 @@
+name: Allure Test Reporter
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+permissions:
+  id-token: write
+  contents: write # Required for gh-pages deployment
+
+jobs:
+  test_and_publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.2.1
+
+      - name: Setup Node
+        uses: actions/setup-node@v4.0.4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Run Tests and Generate Report
+        run: |
+          npm run test:ts:silent
+
+      - name: Install Allure
+        run: npm install -g allure-commandline
+
+      - name: Generate Allure Report
+        run: |
+          REPO_NAME=$(basename ${{ github.repository }})
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          REPORT_FOLDER="${REPO_NAME}/${PR_NUMBER}"
+          allure generate allure-results --clean -o "${REPORT_FOLDER}"
+          echo "REPORT_FOLDER=${REPORT_FOLDER}" >> $GITHUB_ENV  # Persist this variable
+
+      - name: Checkout Reports Repository
+        uses: actions/checkout@v4
+        with:
+          repository: rudderlabs/test-reports
+          token: ${{ secrets.PAT }}
+          path: test-reports
+
+      - name: Copy Allure Report to Reports Repository
+        run: |
+          mkdir -p "test-reports/$REPORT_FOLDER"
+          cp -r $REPORT_FOLDER/* test-reports/$REPORT_FOLDER/
+
+      - name: Cleanup Old Reports (Keep Only Last 50)
+        run: |
+          REPO_NAME=$(basename ${{ github.repository }})
+          cd test-reports/${REPO_NAME}
+          ls -t | tail -n +51 | xargs rm -rf || echo "No old reports to delete"
+
+      - name: Commit and Push Report
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd test-reports
+          git config --global user.name "github-actions"
+          git config --global user.email "github-actions@github.com"
+
+          # Stash any unstaged changes before pulling
+          git add .
+          git stash || echo "No changes to stash"
+
+          # Pull latest changes safely
+          git pull --rebase origin main
+
+          # Apply the stashed changes back
+          git stash pop || echo "No stash to apply"
+
+          # Commit new report if there are changes
+          git add .
+          git commit -m "chore: add allure report for $REPORT_FOLDER" || echo "No changes to commit"
+
+          # Push changes with retry logic
+          for i in {1..5}; do
+            git push origin main && break || sleep 5
+            git pull --rebase origin main
+          done
+
+      - name: Add Test Report Link as Comment on PR
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.PAT }}
+          script: |
+            const { owner, repo } = context.repo;
+            const prNumber = context.payload.pull_request.number;
+
+            const reportFolder = process.env.REPORT_FOLDER;  // Read from environment variable
+            const commentBody = `Allure Test reports for this run are available at:
+            - Allure Report: [View Report](https://rudderlabs.github.io/test-reports/${reportFolder}/index.html)`;
+
+            // Comment on the pull request
+            await github.rest.issues.createComment({
+              owner,
+              repo,
+              issue_number: prNumber,
+              body: commentBody
+            });

.github/workflows/build-push-docker-image.yml

@@ -33,7 +33,7 @@ on:
       workflow_url:
         type: string
     secrets:
-      DOCKERHUB_PROD_TOKEN:
+      DOCKERHUB_TOKEN:
         required: true
       DOCKERHUB_USERNAME:
         required: true
@@ -101,7 +101,7 @@ jobs:
         with:
           ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
-      
+
       - name: Login to DockerHub
         uses: docker/login-action@v3.3.0
         with:
@@ -154,12 +154,12 @@ jobs:
         with:
           ref: ${{ needs.get_sha.outputs.sha }}
           fetch-depth: 1
-      
+
       - name: Login to DockerHub
         uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v3.7.1
@@ -207,7 +207,7 @@ jobs:
         uses: docker/login-action@v3.3.0
         with:
           username: ${{ env.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3.7.1

.github/workflows/component-test-report.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow these users to create new hotfix branch from 'main'
-    if: github.ref == 'refs/heads/main' && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'koladilip' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'utsabc') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc')
+    if: github.ref == 'refs/heads/main' && (github.actor == 'vinayteki95' || github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'koladilip' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'utsabc') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' ||  github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc' || github.triggering_actor == 'vinayteki95')
     steps:
       - name: Create Branch
         uses: peterjgrainger/action-create-branch@v2.4.0

.github/workflows/create-hotfix-branch.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # Only allow release stakeholders to initiate releases
-    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823'  || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'utsabc') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc')
+    if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020'  || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'utsabc' || github.actor == 'vinayteki95') &&  (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc')
     steps:
       - name: Checkout
         uses: actions/checkout@v4.2.1
@@ -79,10 +79,11 @@ jobs:
           git push
 
       - name: Create Pull Request
-        uses: repo-sync/pull-request@v2.12.1
-        with:
-          source_branch: ${{ steps.create-release.outputs.branch_name }}
-          destination_branch: 'main'
-          github_token: ${{ secrets.PAT }}
-          pr_title: 'chore(release): pull ${{ steps.create-release.outputs.branch_name }} into main'
-          pr_body: ':crown: *An automated PR*'
+        run: |
+          gh pr create \
+            --base main \
+            --head "${{ steps.create-release.outputs.branch_name }}" \
+            --title "chore(release): pull ${{ steps.create-release.outputs.branch_name }} into main" \
+            --body ":crown: *An automated PR*"
+        env:
+          GH_TOKEN: ${{ secrets.PAT }}

.github/workflows/draft-new-release.yml

@@ -77,4 +77,4 @@ jobs:
     secrets:
       DOCKERHUB_PROD_TOKEN: ${{ secrets.DOCKERHUB_PROD_TOKEN }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/prepare-for-dev-deploy.yml

@@ -161,29 +161,29 @@ jobs:
 
           cd customer-objects
 
-          declare -a enabled_ut_customers=()
+          declare -a enabled_dt_customers=()
           declare -a sub_directories=('enterprise-us' 'enterprise-eu')
 
           # identify the customers enabled in sub-directories
           for directory in "${sub_directories[@]}"; do
             for f in "./$directory"/*; do
               [[ -f $f ]] || continue
 
-              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              enabled="$(yq e '.spec.transformer.enabled' $f)"
               if [ $enabled == "true" ]; then
-                  enabled_ut_customers+=( $f )
+                  enabled_dt_customers+=( $f )
               fi
             done
           done
 
           # bump up the customers version and repository information
-          for customer in "${enabled_ut_customers[@]}"; do
-            yq eval -i ".spec.user_transformer.image.version=\"$TAG_NAME\"" $customer
-            yq eval -i ".spec.user_transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" $customer
+          for customer in "${enabled_dt_customers[@]}"; do
+            yq eval -i ".spec.transformer.image.version=\"$TAG_NAME\"" $customer
+            yq eval -i ".spec.transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" $customer
             git add $customer
           done
 
-          git commit -m "chore: upgrade dedicated transformers to $TAG_NAME"
+          git commit -m "chore: upgrade dedicated dt transformers to $TAG_NAME"
           git push -u origin dedicated-transformer-$TAG_NAME
 
           gh pr create --fill

.github/workflows/prepare-for-prod-dt-deploy.yml

@@ -29,6 +29,7 @@ jobs:
     if: ((startsWith(github.event.pull_request.head.ref, 'release/') || startsWith(github.event.pull_request.head.ref, 'hotfix-release/') || startsWith(github.event.pull_request.head.ref, 'master')) && github.event.pull_request.merged == true)
     outputs:
       tag_name_ut: ${{ steps.gen_tag_names.outputs.tag_name_ut }}
+      tag_name: ${{ steps.gen_tag_names.outputs.tag_name }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4.2.1
@@ -73,6 +74,7 @@ jobs:
     needs: [generate-tag-names, build-user-transformer-image]
     env:
       UT_TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name_ut }}
+      TAG_NAME: ${{ needs.generate-tag-names.outputs.tag_name }}
       TF_IMAGE_REPOSITORY: rudderstack/rudder-transformer
     steps:
       - name: Checkout
@@ -136,3 +138,39 @@ jobs:
           git push -u origin hosted-user-transformer-$UT_TAG_NAME
 
           gh pr create --fill
+
+      - name: Update helm charts and raise pull request for enterprise customers on dedicated transformers
+        env:
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          cd rudder-devops
+          git checkout -b dedicated-user-transformer-$TAG_NAME
+
+          cd customer-objects
+
+          declare -a enabled_ut_customers=()
+          declare -a sub_directories=('enterprise-us' 'enterprise-eu')
+
+          # identify the customers enabled in sub-directories
+          for directory in "${sub_directories[@]}"; do
+            for f in "./$directory"/*; do
+              [[ -f $f ]] || continue
+
+              enabled="$(yq e '.spec.user_transformer.enabled' $f)"
+              if [ $enabled == "true" ]; then
+                  enabled_ut_customers+=( $f )
+              fi
+            done
+          done
+
+          # bump up the customers version and repository information
+          for customer in "${enabled_ut_customers[@]}"; do
+            yq eval -i ".spec.user_transformer.image.version=\"$TAG_NAME\"" $customer
+            yq eval -i ".spec.user_transformer.image.repository=\"$TF_IMAGE_REPOSITORY\"" $customer
+            git add $customer
+          done
+
+          git commit -m "chore: upgrade dedicated user transformers to $TAG_NAME"
+          git push -u origin dedicated-user-transformer-$TAG_NAME
+
+          gh pr create --fill