feat: Down merge from main to dev

.env.sample

@@ -14,6 +14,9 @@ AZURE_SEARCH_FIELDS_TAG=tag
 AZURE_SEARCH_FIELDS_METADATA=metadata
 AZURE_SEARCH_FILENAME_COLUMN=filepath
 AZURE_SEARCH_TITLE_COLUMN=title
+AZURE_SEARCH_SOURCE_COLUMN=source
+AZURE_SEARCH_TEXT_COLUMN=text
+AZURE_SEARCH_LAYOUT_TEXT_COLUMN=layoutText
 AZURE_SEARCH_URL_COLUMN=url
 AZURE_SEARCH_CONVERSATIONS_LOG_INDEX=conversations-log
 AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION=false
@@ -60,6 +63,8 @@ AZURE_SPEECH_SERVICE_REGION=
 AZURE_AUTH_TYPE=keys
 USE_KEY_VAULT=true
 AZURE_KEY_VAULT_ENDPOINT=
+# Application environment (e.g., dev, prod)
+APP_ENV="dev"
 # Chat conversation type to decide between custom or byod (bring your own data) conversation type
 CONVERSATION_FLOW=
 # Chat History CosmosDB Integration Settings

.github/workflows/broken-links-checker.yml

@@ -20,34 +20,38 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Get Added/Modified Markdown Files (PR only)
-        id: changed-files
+      # For PR : Get only changed markdown files
+      - name: Get changed markdown files (PR only)
+        id: changed-markdown-files
         if: github.event_name == 'pull_request'
-        run: |
-          git fetch origin ${{ github.base_ref }}
-          files=$(git diff --name-only origin/${{ github.base_ref }}...HEAD | grep '\.md$' || true)
-          echo "md_files<<EOF" >> $GITHUB_OUTPUT
-          echo "$files" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-      - name: Check Broken Links in Added/Modified Files (PR)
-        if: github.event_name == 'pull_request' && steps.changed-files.outputs.md_files != ''
+        uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46
+        with:
+          files: |
+            **/*.md
+
+
+      # For PR: Check broken links only in changed files
+      - name: Check Broken Links in Changed Markdown Files
+        id: lychee-check-pr
+        if: github.event_name == 'pull_request' && steps.changed-markdown-files.outputs.any_changed == 'true'
         uses: lycheeverse/[email protected]
         with:
           args: >
             --verbose --exclude-mail --no-progress --exclude ^https?://
-            ${{ steps.changed-files.outputs.md_files }}
+            ${{ steps.changed-markdown-files.outputs.all_changed_files }}
           failIfEmpty: false
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Check Broken Links in Entire Repo (Manual)
+      # For manual trigger: Check all markdown files in repo
+      - name: Check Broken Links in All Markdown Files in Entire Repo (Manual Trigger)
+        id: lychee-check-manual
         if: github.event_name == 'workflow_dispatch'
         uses: lycheeverse/[email protected]
         with:
           args: >
             --verbose --exclude-mail --no-progress --exclude ^https?://
             '**/*.md'
           failIfEmpty: false
-          output: lychee/out.md
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-docker-images.yml

@@ -32,9 +32,7 @@ jobs:
             dockerfile: docker/Frontend.Dockerfile
     uses: ./.github/workflows/build-docker.yml
     with:
-      old_registry: ${{ github.ref_name == 'main' && 'fruoccopublic.azurecr.io' }}
       new_registry: 'cwydcontainerreg.azurecr.io'
-      old_username: ${{ github.ref_name == 'main' && 'fruoccopublic' }}
       new_username: 'cwydcontainerreg'
       app_name: ${{ matrix.app_name }}
       dockerfile: ${{ matrix.dockerfile }}

.github/workflows/build-docker.yml

@@ -3,12 +3,6 @@ name: Reusable Docker build and push workflow
 on:
   workflow_call:
     inputs:
-      old_registry:
-        required: true
-        type: string
-      old_username:
-        required: true
-        type: string
       new_registry:
         required: true
         type: string
@@ -37,15 +31,6 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
-    # Login for 'main' branch to both registries
-    - name: Docker Login to fruoccopublic (Main)
-      if: ${{ inputs.push == true && github.ref_name == 'main' }}
-      uses: docker/login-action@v3
-      with:
-        registry: ${{ inputs.old_registry }}
-        username: ${{ inputs.old_username }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
     - name: Docker Login to cwydcontainerreg (Main)
       if: ${{ inputs.push == true && github.ref_name == 'main' }}
       uses: docker/login-action@v3
@@ -70,18 +55,6 @@ jobs:
       id: date
       run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
 
-    - name: Build Docker Image and optionally push (Old Registry)
-      if: ${{ github.ref_name == 'main' }}
-      uses: docker/build-push-action@v6
-      with:
-        context: .
-        file: ${{ inputs.dockerfile }}
-        push: ${{ inputs.push }}
-        cache-from: type=registry,ref=${{ inputs.old_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest' || github.head_ref || github.ref_name }}
-        tags: |
-          ${{ inputs.old_registry }}/${{ inputs.app_name }}:${{ github.ref_name == 'main' && 'latest' || github.head_ref || 'default' }}
-          ${{ inputs.old_registry }}/${{ inputs.app_name }}:${{ steps.date.outputs.date }}_${{ github.run_number }}
-
     - name: Build Docker Image and optionally push (New Registry)
       if: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo'|| github.ref_name == 'dependabotchanges' }}
       uses: docker/build-push-action@v6

.github/workflows/group_dependabot_security_updates.yml

@@ -0,0 +1,267 @@
+# Workflow: Group Dependabot PRs
+# Description:
+# This GitHub Actions workflow automatically groups open Dependabot PRs by ecosystem (pip, npm).
+# It cherry-picks individual PR changes into grouped branches, resolves merge conflicts automatically, and opens consolidated PRs.
+# It also closes the original Dependabot PRs and carries over their labels and metadata.
+# Improvements:
+# - Handles multiple conflicting files during cherry-pick
+# - Deduplicates entries in PR description
+# - Avoids closing original PRs unless grouped PR creation succeeds
+# - More efficient retry logic
+# - Ecosystem grouping is now configurable via native YAML map
+# - Uses safe namespaced branch naming (e.g. actions/grouped-...) to avoid developer conflict
+# - Ensures PR body formatting uses real newlines for better readability
+# - Adds strict error handling for script robustness
+# - Accounts for tool dependencies (jq, gh) and race conditions
+# - Optimized PR metadata lookup by preloading into associative array
+# - Supports --dry-run mode for validation/testing without side effects
+# - Note: PRs created during workflow execution will be picked up in the next scheduled run.
+
+name: Group Dependabot PRs
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Run daily at midnight UTC
+  workflow_dispatch:
+    inputs:
+      group_config_pip:
+        description: "Group name for pip ecosystem"
+        required: false
+        default: "backend"
+      group_config_npm:
+        description: "Group name for npm ecosystem"
+        required: false
+        default: "frontend"
+      group_config_yarn:
+        description: "Group name for yarn ecosystem"
+        required: false
+        default: "frontend"
+      dry_run:
+        description: "Run in dry-run mode (no changes will be pushed or PRs created/closed)"
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  group-dependabot-prs:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      TARGET_BRANCH: "main"
+      DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
+      GROUP_CONFIG_PIP: ${{ github.event.inputs.group_config_pip || 'backend' }}
+      GROUP_CONFIG_NPM: ${{ github.event.inputs.group_config_npm || 'frontend' }}
+      GROUP_CONFIG_YARN: ${{ github.event.inputs.group_config_yarn || 'frontend' }}
+    steps:
+      - name: Checkout default branch
+        uses: actions/checkout@v4
+
+      - name: Set up Git
+        run: |
+          git config --global user.name "github-actions"
+          git config --global user.email "[email protected]"
+
+      - name: Install required tools
+        uses: awalsh128/[email protected]
+        with:
+          packages: "jq gh"
+
+      - name: Enable strict error handling
+        shell: bash
+        run: |
+          set -euo pipefail
+
+      - name: Fetch open Dependabot PRs targeting main
+        id: fetch_prs
+        run: |
+          gh pr list \
+            --search "author:dependabot[bot] base:$TARGET_BRANCH is:open" \
+            --limit 100 \
+            --json number,title,headRefName,labels,files,url \
+            --jq '[.[] | {number, title, url, ref: .headRefName, labels: [.labels[].name], files: [.files[].path]}]' > prs.json
+          cat prs.json
+
+      - name: Validate prs.json
+        run: |
+          jq empty prs.json 2> jq_error.log || { echo "Malformed JSON in prs.json: $(cat jq_error.log)"; exit 1; }
+
+      - name: Check if any PRs exist
+        id: check_prs
+        run: |
+          count=$(jq length prs.json)
+          echo "Found $count PRs"
+          if [ "$count" -eq 0 ]; then
+            echo "No PRs to group. Exiting."
+            echo "skip=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Exit early if no PRs
+        if: steps.check_prs.outputs.skip == 'true'
+        run: exit 0
+
+      - name: Dry-run validation (CI/test only)
+        if: env.DRY_RUN == 'true'
+        run: |
+          echo "Running in dry-run mode. No changes will be pushed or PRs created/closed."
+          # Optionally, add more validation logic here (e.g., check grouped files, print planned actions).
+
+      - name: Group PRs by ecosystem and cherry-pick with retry
+        run: |
+          declare -A GROUP_CONFIG=(
+            [pip]="${GROUP_CONFIG_PIP:-backend}"
+            [npm]="${GROUP_CONFIG_NPM:-frontend}"
+            [yarn]="${GROUP_CONFIG_YARN:-frontend}"
+          )
+          mkdir -p grouped
+          jq -c '.[]' prs.json | while read pr; do
+            ref=$(echo "$pr" | jq -r '.ref')
+            number=$(echo "$pr" | jq -r '.number')
+            group="misc"
+            for key in "${!GROUP_CONFIG[@]}"; do
+              if [[ "$ref" == *"$key"* ]]; then
+                group="${GROUP_CONFIG[$key]}"
+                break
+              fi
+            done
+            echo "$number $ref $group" >> grouped/$group.txt
+          done
+
+          shopt -s nullglob
+          grouped_files=(grouped/*.txt)
+
+          if [ ${#grouped_files[@]} -eq 0 ]; then
+            echo "No groups were formed. Exiting."
+            exit 0
+          fi
+
+          declare -A pr_metadata_map
+          while IFS=$'\t' read -r number title url labels; do
+            pr_metadata_map["$number"]="$title|$url|$labels"
+          done < <(jq -r '.[] | "\(.number)\t\(.title)\t\(.url)\t\(.labels | join(","))"' prs.json)
+
+          for file in "${grouped_files[@]}"; do
+            group_name=$(basename "$file" .txt)
+            # Sanitize group_name: allow only alphanum, dash, underscore
+            safe_group_name=$(echo "$group_name" | tr -c '[:alnum:]_-' '-')
+            branch_name="security/grouped-${safe_group_name}-updates"
+            git checkout -B "$branch_name"
+
+            while read -r number ref group; do
+              git fetch origin "$ref"
+              if ! git cherry-pick FETCH_HEAD; then
+                echo "Conflict found in $ref. Attempting to resolve."
+                conflict_files=($(git diff --name-only --diff-filter=U))
+                if [ ${#conflict_files[@]} -gt 0 ]; then
+                  echo "Resolving conflicts in files: ${conflict_files[*]}"
+                  for conflict_file in "${conflict_files[@]}"; do
+                    echo "Resolving conflict in $conflict_file"
+                    git checkout --theirs "$conflict_file"
+                    git add "$conflict_file"
+                  done
+                  git cherry-pick --continue || {
+                    echo "Failed to continue cherry-pick. Aborting."
+                    git cherry-pick --abort
+                    continue 2
+                  }
+                else
+                  echo "No conflicting files found. Aborting."
+                  git cherry-pick --abort
+                  continue 2
+                fi
+              fi
+            done < "$file"
+
+            # Non-destructive push: check for drift before force-pushing
+            if [ "$DRY_RUN" == "true" ]; then
+              echo "[DRY-RUN] Skipping git push for $branch_name"
+            else
+              remote_hash=$(git ls-remote origin "$branch_name" | awk '{print $1}')
+              local_hash=$(git rev-parse "$branch_name")
+              if [ -n "$remote_hash" ] && [ "$remote_hash" != "$local_hash" ]; then
+                echo "Remote branch $branch_name has diverged. Skipping force-push to avoid overwriting changes."
+                continue
+              fi
+              git push --force-with-lease origin "$branch_name"
+            fi
+
+            new_lines=""
+            while read -r number ref group; do
+              IFS="|" read -r title url _ <<< "${pr_metadata_map["$number"]}"
+              new_lines+="$title - [#$number]($url)\n"
+            done < "$file"
+
+            pr_title="chore(deps): bump grouped $group_name Dependabot updates"
+            # Add --state open to ensure only open PRs are considered
+            existing_url=$(gh pr list --head "$branch_name" --base "$TARGET_BRANCH" --state open --json url --jq '.[0].url // empty')
+
+            if [ -n "$existing_url" ]; then
+              echo "PR already exists: $existing_url"
+              pr_url="$existing_url"
+              current_body=$(gh pr view "$pr_url" --json body --jq .body)
+              # Simplified duplicate-detection using Bash array
+              IFS=$'\n' read -d '' -r -a current_lines < <(printf '%s\0' "$current_body")
+              IFS=$'\n' read -d '' -r -a new_lines_arr < <(printf '%b\0' "$new_lines")
+              declare -A seen
+              for line in "${current_lines[@]}"; do
+                seen["$line"]=1
+              done
+              filtered_lines=""
+              for line in "${new_lines_arr[@]}"; do
+                if [[ -n "$line" && -z "${seen["$line"]}" ]]; then
+                  filtered_lines+="$line\n"
+                fi
+              done
+              # Ensure a newline separator between the existing body and new lines
+              if [ -n "$filtered_lines" ]; then
+                new_body="$current_body"$'\n'"$filtered_lines"
+              else
+                new_body="$current_body"
+              fi
+              if [ "$DRY_RUN" == "true" ]; then
+                echo "[DRY-RUN] Would update PR body for $pr_url"
+              else
+                tmpfile=$(mktemp)
+                printf '%s' "$new_body" > "$tmpfile"
+                gh pr edit "$pr_url" --body-file "$tmpfile"
+                rm -f "$tmpfile"
+              fi
+            else
+              pr_body=$(printf "This PR groups multiple open PRs by Dependabot for %s.\n\n%b" "$group_name" "$new_lines")
+              if [ "$DRY_RUN" == "true" ]; then
+                echo "[DRY-RUN] Would create PR titled: $pr_title"
+                echo "$pr_body"
+                pr_url=""
+              else
+                pr_url=$(gh pr create \
+                  --title "$pr_title" \
+                  --body "$pr_body" \
+                  --base "$TARGET_BRANCH" \
+                  --head "$branch_name")
+              fi
+            fi
+
+            if [ -n "$pr_url" ]; then
+              for number in $(cut -d ' ' -f1 "$file"); do
+                IFS="|" read -r _ _ labels <<< "${pr_metadata_map["$number"]}"
+                IFS="," read -ra label_arr <<< "$labels"
+                for label in "${label_arr[@]}"; do
+                  if [ "$DRY_RUN" == "true" ]; then
+                    echo "[DRY-RUN] Would add label $label to $pr_url"
+                  else
+                    gh pr edit "$pr_url" --add-label "$label"
+                  fi
+                done
+                if [ "$DRY_RUN" == "true" ]; then
+                  echo "[DRY-RUN] Would close PR #$number"
+                else
+                  gh pr close "$number" --comment "Grouped into $pr_url."
+                fi
+              done
+              echo "Grouped PR created. Leaving branch $branch_name for now."
+            else
+              echo "Grouped PR was not created. Skipping closing of original PRs."
+            fi
+          done