Merge branch 'main' into release-rs-gilboa

Author: rrelledge

.github/workflows/autocomment.yaml

@@ -49,11 +49,10 @@ jobs:
             -H "X-GitHub-Api-Version: 2022-11-28" \
             -H "Authorization: Bearer ${GITHUB_TOKEN}" \
             $FILES_URL \
-              | jq -r --arg prefix $BRANCH_NAME/ '.[] | select(((.filename | test("content\/.+\\.md")) and .status != "removed")) | ($prefix + .filename)' \
+              | jq -r --arg prefix $BRANCH_NAME/ '.[] | select(((.filename | test("content/(?!.*embed).*\\.md")) and .status != "removed")) | ($prefix + .filename)' \
               | sed -E -e 's|(^[^/]+/)([^/]+/)|\1|' -e 's|^|https://redis.io/docs/staging/|' -e 's|(_?index)?\.md||' \
-              | uniq \
-              | xargs \
-              | sed 's/ /<br>/g')
+              | sort \
+              | uniq)
 
           # Get all changed image files
           CHANGED_IMAGE_FILES=$(curl -Ls \
@@ -63,23 +62,51 @@ jobs:
             $FILES_URL \
               | jq -r '.[] | select(.filename | test("^static/images\/.+(.png|.svg|.gif|.)")) | .filename | sub("^static/";"")')
 
+          # Get all changed embeds files
+          CHANGED_EMBED_FILES=$(curl -Ls \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+            $FILES_URL \
+              | jq -r '.[] | select(.filename | test("^content/embeds\/.+.md")) | .filename | split("/")[-1]')
+
           if [[ -n "$CHANGED_IMAGE_FILES" ]]
           then
             # For each image, work out in which README it is present
             MD_FILES_WITH_IMAGE=()
-            for CHANGED_IMAGE_FILE in $CHANGED_IMAGE_FILES; do
+            for CHANGED_IMAGE_FILE in "${CHANGED_IMAGE_FILES[@]}"; do
               MD_FILE_WITH_IMAGE=$(grep -ro "$CHANGED_IMAGE_FILE" content \
                 | sed -E -e 's|:.+||' -e "s|^content/|https://redis.io/docs/staging/$BRANCH_NAME/|" -e 's|(_?index)?\.md||' \
+                | grep -v "https://redis.io/docs/staging/$BRANCH_NAME/embeds/" \
+                | sort \
                 | uniq)
               MD_FILES_WITH_IMAGE+=($MD_FILE_WITH_IMAGE)
             done
-              CHANGED_MD_FILES_WITH_IMAGE=$(printf "%s\n" "${MD_FILES_WITH_IMAGE[@]}" \
-                | uniq \
-                | xargs \
-                | sed 's/ /<br>/g')
-              CHANGED_MD_FILES="${CHANGED_MD_FILES}<br>${CHANGED_MD_FILES_WITH_IMAGE}"
+              CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" "${MD_FILES_WITH_IMAGE[@]}" \
+                | sort \
+                | uniq)
+          fi
+
+          if [[ -n "$CHANGED_EMBED_FILES" ]]
+          then
+            # For each embed, work out in which README it is present
+            MD_FILES_WITH_EMBED=()
+            for CHANGED_EMBED_FILE in "${CHANGED_EMBED_FILES[@]}"; do
+              MD_FILE_WITH_EMBED=$(grep -ro "< embed-md \"${CHANGED_EMBED_FILE}\" >" content \
+                | sed -E -e 's|:.+||' -e "s|^content/|https://redis.io/docs/staging/$BRANCH_NAME/|" -e 's|(_?index)?\.md||' \
+                | sort \
+                | uniq)
+              MD_FILES_WITH_EMBED+=($MD_FILE_WITH_EMBED)
+            done
+              CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" "${MD_FILES_WITH_EMBED[@]}" \
+                | sort \
+                | uniq)
           fi
 
+          CHANGED_MD_FILES=$(printf "%s\n" "${CHANGED_MD_FILES}" \
+            | xargs \
+            | sed 's/ /<br>/g')
+
           if [[ -z "$CHANGED_MD_FILES" ]]
           then
             if [[ -z "$EXISTING_COMMENT_ID" ]]

.github/workflows/cleanup_pr_folders.yaml

@@ -0,0 +1,79 @@
+name: cleanup_pr_folders
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  staging:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    env:
+      STAGING_BUCKET: docs-staging-learn-redis-com
+      STAGING_PROJECT_ID: ${{ secrets.GCP_PROJECT_STAGING }}
+      STAGING_SERVICE_ACCOUNT: ${{ secrets.STAGING_SERVICE_ACCOUNT }}
+      STAGING_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.STAGING_WORKLOAD_IDENTITY_PROVIDER }}
+
+    steps:
+    - name: 'Google auth'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        project_id: '${{ env.STAGING_PROJECT_ID }}'
+        service_account: '${{ env.STAGING_SERVICE_ACCOUNT }}'
+        workload_identity_provider: '${{ env.STAGING_WORKLOAD_IDENTITY_PROVIDER }}'
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        project_id: '${{ env.STAGING_PROJECT_ID }}'
+        version: '>= 363.0.0'
+
+    - name: Remove pull request staging folder 
+      run: |
+        BRANCH_TO_CLEANUP="${{ github.event.pull_request.head.ref }}"
+        STAGING_BUCKET="${{ env.STAGING_BUCKET }}"
+
+        if gsutil -q stat gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*; then
+          gsutil -q -m rm -r gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*
+          echo "Removed gs://${STAGING_BUCKET}/staging/${BRANCH_TO_CLEANUP}"
+        fi
+
+  prod:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    env:
+      PROD_BUCKET: docs-prod-learn-redis-com
+      PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }}
+      PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }}
+      PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }}
+
+    steps:
+    - name: 'Google auth'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        project_id: '${{ env.PROD_PROJECT_ID }}'
+        service_account: '${{ env.PROD_SERVICE_ACCOUNT }}'
+        workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}'
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        project_id: '${{ env.PROD_PROJECT_ID }}'
+        version: '>= 363.0.0'
+
+    - name: Remove pull request staging folder
+      run: |
+        BRANCH_TO_CLEANUP="${{ github.event.pull_request.head.ref }}"
+        PROD_BUCKET="${{ env.PROD_BUCKET }}"
+
+        if gsutil -q stat gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*; then
+          gsutil -q -m rm -r gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}/*
+          echo "Removed gs://${PROD_BUCKET}/staging/${BRANCH_TO_CLEANUP}"
+        fi

.github/workflows/main-staging.yml

@@ -6,13 +6,17 @@ jobs:
   build_docs:
     name: Build the staging documentation site
     runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
     env:
-      HUGO_VERSION: 0.111.2
+      HUGO_VERSION: 0.143.1
       GCLOUD_VERSION: 458.0.1-linux-x86_64
       BUCKET: docs-staging-learn-redis-com
-      BUCKET_SERVICE_ACCOUNT: ${{ secrets.BUCKET_DOCUMENTATION_SA_STAGING }}
-      BUCKET_SECRET: ${{ secrets.BUCKET_DOCUMENTATION_STAGING }}
-      GCP_PROJECT: ${{ secrets.GCP_PROJECT_STAGING }}
+      STAGING_PROJECT_ID: ${{ secrets.GCP_PROJECT_STAGING }}
+      STAGING_SERVICE_ACCOUNT: ${{ secrets.STAGING_SERVICE_ACCOUNT }}
+      STAGING_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.STAGING_WORKLOAD_IDENTITY_PROVIDER }}
+
     steps:
     - name: Start
       run: echo "The name of the branch is ${{ github.ref }} on ${{ github.repository }}"
@@ -60,6 +64,7 @@ jobs:
         setBaseUrl
         kubernetes_versions=($(find content/operate/kubernetes/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}'))
         rs_versions=($(find content/operate/rs/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}'))
+        rdi_versions=($(find content/integrate/redis-data-integration/ -maxdepth 1 -type d -regex ".*[0-9-]" | awk -F/ '{print $NF}'))
 
         # build latest
         for version in "${kubernetes_versions[@]}"; do
@@ -68,6 +73,9 @@ jobs:
         for version in "${rs_versions[@]}"; do
           rm -r "content/operate/rs/${version}"
         done
+        for version in "${rdi_versions[@]}"; do
+          rm -r "content/integrate/redis-data-integration/${version}"
+        done
         make all
 
         git checkout .
@@ -83,13 +91,17 @@ jobs:
             rm -r "content/operate/kubernetes/${version_to_remove}"
           done
 
-          cp -r "content/operate/kubernetes/${version}"/* content/operate/kubernetes/
-          rm -r "content/operate/kubernetes/${version}"
+          # relrefs should not include version
+          find "content/operate/kubernetes/${version}" -type f | while read file; do
+              awk '{gsub(/\(\{\{< ?relref "\/operate\/kubernetes\/'${version}'/, "({{< relref \"/operate/kubernetes") }1' "$file" > tmpfile && mv tmpfile "$file"
+          done
+
+          rsync -a --delete-after "content/operate/kubernetes/${version}"/ content/operate/kubernetes/
           sed -i 's/id="versionSelectorKubernetesValue" class="version-selector-control">latest/id="versionSelectorKubernetesValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html
           sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis for Kubernetes/' content/operate/kubernetes/_index.md
 
           # inject replace command in meta-links to make sure editURL and issuesURL point to right version
-          sed -i "11i \{\{ \$gh_file = replaceRE \`\^operate\/kubernetes\/\` \"operate\/kubernetes\/$version\/\" \$gh_file }}" layouts/partials/meta-links.html
+          sed -i "12i \{\{ \$gh_path = replaceRE \`\^operate\/kubernetes\/\` \"operate\/kubernetes\/$version\/\" \$gh_path }}" layouts/partials/meta-links.html
 
           hugo -d "kubernetes-${version}"
 
@@ -106,34 +118,68 @@ jobs:
             rm -r "content/operate/rs/${version_to_remove}"
           done
 
-          cp -r "content/operate/rs/${version}"/* content/operate/rs/
-          rm -r "content/operate/rs/${version}"
+          # relrefs should not include version
+          find "content/operate/rs/${version}" -type f | while read file; do
+              awk '{gsub(/\(\{\{< ?relref "\/operate\/rs\/'${version}'/, "({{< relref \"/operate/rs") }1' "$file" > tmpfile && mv tmpfile "$file"
+          done
+
+          rsync -a --delete-after "content/operate/rs/${version}"/ content/operate/rs/
           sed -i 's/id="versionSelectorRsValue" class="version-selector-control">latest/id="versionSelectorRsValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html
           sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis Software/' content/operate/rs/_index.md
 
           # inject replace command in meta-links to make sure editURL and issuesURL point to right version
-          sed -i "11i \{\{ \$gh_file = replaceRE \`\^operate\/rs\/\` \"operate\/rs\/$version\/\" \$gh_file }}" layouts/partials/meta-links.html
+          sed -i "12i \{\{ \$gh_path = replaceRE \`\^operate\/rs\/\` \"operate\/rs\/$version\/\" \$gh_path }}" layouts/partials/meta-links.html
 
           hugo -d "rs-${version}"
 
           git checkout .
         done
 
+        for version in "${rdi_versions[@]}"; do
+
+          setBaseUrl
+
+          # for each version, remove all other versions before building
+          versions_to_remove=($(echo "${rdi_versions[@]}" "${version}" | tr ' ' '\n' | sort | uniq -u))
+          for version_to_remove in "${versions_to_remove[@]}"; do
+            rm -r "content/integrate/redis-data-integration/${version_to_remove}"
+          done
+
+          # relrefs should not include version
+          find "content/operate/redis-data-integration/${version}" -type f | while read file; do
+              awk '{gsub(/\(\{\{< ?relref "\/operate\/redis-data-integration\/'${version}'/, "({{< relref \"/operate/redis-data-integration") }1' "$file" > tmpfile && mv tmpfile "$file"
+          done
+
+          rsync -a --delete-after "content/operate/redis-data-integration/${version}"/ content/operate/redis-data-integration/
+          sed -i 's/id="versionSelectorRedis-Data-IntegrationValue" class="version-selector-control">latest/id="versionSelectorRedis-Data-IntegrationValue" class="version-selector-control">v'"${version}"'/' layouts/partials/docs-nav.html
+          sed -i 's/linkTitle: '"${version}"'/linkTitle: Redis Data Integration/' content/integrate/redis-data-integration/_index.md
+
+          # inject replace command in meta-links to make sure editURL and issuesURL point to right version
+          sed -i "12i \{\{ \$gh_path = replaceRE \`\^integrate\/redis-data-integration\/\` \"integrate\/redis-data-integration\/$version\/\" \$gh_path }}" layouts/partials/meta-links.html
+
+          hugo -d "redis-data-integration-${version}"
+
+          git checkout .
+        done
+
     - name: List client examples
       run: ls "${{ github.workspace }}/examples"
     - name: List files to be published
       run: ls "${{ github.workspace }}/public"    
-    - name: Install the Google Cloud CLI
-      run: |
-          wget -O ${{ github.workspace }}/google-cloud-cli.tar.gz "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-$GCLOUD_VERSION.tar.gz" \
-          && tar -xvf google-cloud-cli.tar.gz -C ${{ github.workspace }}\
-          && ${{ github.workspace }}/google-cloud-sdk/install.sh --quiet
-    - name: Prepare bucket authentication
-      run: echo $BUCKET_SECRET | base64 --decode > service_account.json && ls -a service_account.json
-    - name: Authenticate to the bucket
-      run: |
-          ./google-cloud-sdk/bin/gcloud auth activate-service-account $BUCKET_SERVICE_ACCOUNT --key-file=./service_account.json --project=$GCP_PROJECT \
-          && ./google-cloud-sdk/bin/gcloud auth list
+
+    - name: 'Google auth'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        project_id: '${{ env.STAGING_PROJECT_ID }}'
+        service_account: '${{ env.STAGING_SERVICE_ACCOUNT }}'
+        workload_identity_provider: '${{ env.STAGING_WORKLOAD_IDENTITY_PROVIDER }}'
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        project_id: '${{ env.STAGING_PROJECT_ID }}'
+        version: '>= 363.0.0'
+
     - name: Sync the branch to the bucket
       run: |
           if [[ "${{ github.ref_name }}" == "main" ]]
@@ -148,12 +194,16 @@ jobs:
           else
             bucket_path=staging/${{ github.ref_name }}
           fi \
-          && ./google-cloud-sdk/bin/gsutil -m rsync -r -c -j html -d ${{ github.workspace }}/public gs://$BUCKET/$bucket_path
+          && gsutil -m rsync -r -c -j html -d ${{ github.workspace }}/public gs://$BUCKET/$bucket_path
 
           versioned_builds=($(find . -type d -regex ".*[0-9-]" -maxdepth 1 | sed -E 's/^.\///'))
           for versioned_build in "${versioned_builds[@]}"; do
-            product=$(grep -Eo '[a-zA-Z]+' <<< $versioned_build)
-            gsutil -m rsync -r -c -j html -d "${{ github.workspace }}/${versioned_build}/operate/${product}" "gs://${BUCKET}/${versioned_build}/operate/${product}"
+            product=$(awk 'BEGIN{FS=OFS="-"}{NF--; print}' <<< $versioned_build)
+            if [[ "${product}" == "redis-data-integration" ]]; then
+              gsutil -m rsync -r -c -j html -d "${{ github.workspace }}/${versioned_build}/integrate/${product}/" "gs://${BUCKET}/${bucket_path}/${versioned_build}/integrate/${product}"
+            else
+              gsutil -m rsync -r -c -j html -d "${{ github.workspace }}/${versioned_build}/operate/${product}/" "gs://${BUCKET}/${bucket_path}/${versioned_build}/operate/${product}"
+            fi
           done
 
     - name: End