From 3cf886062dc6ddd64395c88f1800f36cb4cadff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Simard?= Date: Thu, 20 Feb 2025 11:37:26 -0500 Subject: [PATCH 01/26] Moved git auth after auth is set --- .github/workflows/add_new_or_updated_feeds.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 1e19ad0a6..1c2f2d77e 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -116,10 +116,6 @@ jobs: run: | # Check GitHub CLI authentication gh auth status - - # Check Git authentication - git config --get user.name - git config --get user.email env: GH_TOKEN: ${{ env.CREDENTIALS }} GITHUB_TOKEN: ${{ env.CREDENTIALS }} @@ -141,6 +137,10 @@ jobs: # Configure git git config --global user.name "${{ env.USERNAME }}" git config --global user.email "${{ env.USERNAME_EMAIL }}" + + # Check Git authentication + git config --get user.name + git config --get user.email # Add changes git add . From 96310dcb592af87725f54117684a372c8db6e309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Simard?= Date: Thu, 20 Feb 2025 11:56:56 -0500 Subject: [PATCH 02/26] Added permissions for the bot --- .github/workflows/add_new_or_updated_feeds.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 1c2f2d77e..46f76ced1 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -5,6 +5,11 @@ on: # schedule: # - cron: '55 3 * * *' # Run every night +permissions: + contents: write # Permissions to read/write content like code + issues: write # Permissions to create issues or comment on issues + pull-requests: write # Permissions to create/merge pull requests + env: DATE_FORMAT: "[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}" # this is the format we need to compare dates between the CSV and the local system. DATE_FORMAT_DESIRED: "MM/dd/yyyy" From 44d2b6ecb35a05aa46dd26f1d32c438d4265ee06 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 20 Feb 2025 12:00:16 -0500 Subject: [PATCH 03/26] test adding permissions --- .github/workflows/add_new_or_updated_feeds.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 46f76ced1..9fa221d96 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -21,10 +21,14 @@ env: REVIEWERS_JSON: "[\"emmambd\"]" # List of GitHub usernames of the reviewers, in a JSON array : ["username1", "username2"] + jobs: add-new-updated-feeds: runs-on: ubuntu-22.04 - + permissions: + contents: write # Allows pushing to branches + pull-requests: write # Allows creating and managing PRs + steps: - name: Setup global variables id: global_vars From 643028434656941637b0bc9bb78364e8b46f9d23 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 20 Feb 2025 12:16:12 -0500 Subject: [PATCH 04/26] remove credentials from workflow --- .../workflows/add_new_or_updated_feeds.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 9fa221d96..73dc071ee 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -25,9 +25,6 @@ env: jobs: add-new-updated-feeds: runs-on: ubuntu-22.04 - permissions: - contents: write # Allows pushing to branches - pull-requests: write # Allows creating and managing PRs steps: - name: Setup global variables @@ -52,7 +49,7 @@ jobs: export-env: true # Export loaded secrets as environment variables env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential" + # CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential" # The URL to obtain the csv file with the data of the google form. Taken from the "GiHub Actions — URLs" entry in 1password CSV_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/qkn5esttmtojawglm4l6t2bqaa/al2gjfhiuddabkp7o26hszuvia" SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/rdpfgrmnbxqaelgi5oky3lryz4/internal-add-feeds" @@ -63,7 +60,7 @@ jobs: with: ref: ${{ env.BASE }} fetch-depth: 0 - token: ${{ env.CREDENTIALS }} + # token: ${{ env.CREDENTIALS }} - name: Create new branch shell: bash @@ -125,9 +122,9 @@ jobs: run: | # Check GitHub CLI authentication gh auth status - env: - GH_TOKEN: ${{ env.CREDENTIALS }} - GITHUB_TOKEN: ${{ env.CREDENTIALS }} + # env: + # GH_TOKEN: ${{ env.CREDENTIALS }} + # GITHUB_TOKEN: ${{ env.CREDENTIALS }} shell: bash @@ -181,9 +178,9 @@ jobs: # Set the PR URL as an output echo "pr-url=$PR_URL" >> $GITHUB_OUTPUT echo "new-file-count=$NEW_FILE_COUNT" >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ env.CREDENTIALS }} - GITHUB_TOKEN: ${{ env.CREDENTIALS }} + # env: + # GH_TOKEN: ${{ env.CREDENTIALS }} + # GITHUB_TOKEN: ${{ env.CREDENTIALS }} shell: bash - name: Post notification in Slack channel of update From b6f869584c846d024b6dc23e49fcea9f03c6cf7d Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 20 Feb 2025 12:22:51 -0500 Subject: [PATCH 05/26] use default GH token --- .github/workflows/add_new_or_updated_feeds.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 73dc071ee..e4f22f6b9 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -60,7 +60,6 @@ jobs: with: ref: ${{ env.BASE }} fetch-depth: 0 - # token: ${{ env.CREDENTIALS }} - name: Create new branch shell: bash @@ -122,9 +121,9 @@ jobs: run: | # Check GitHub CLI authentication gh auth status - # env: - # GH_TOKEN: ${{ env.CREDENTIALS }} - # GITHUB_TOKEN: ${{ env.CREDENTIALS }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash @@ -178,9 +177,9 @@ jobs: # Set the PR URL as an output echo "pr-url=$PR_URL" >> $GITHUB_OUTPUT echo "new-file-count=$NEW_FILE_COUNT" >> $GITHUB_OUTPUT - # env: - # GH_TOKEN: ${{ env.CREDENTIALS }} - # GITHUB_TOKEN: ${{ env.CREDENTIALS }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash - name: Post notification in Slack channel of update From ee070f113f33fdbbc3915e4ec7c60d7656df65eb Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 20 Feb 2025 12:30:15 -0500 Subject: [PATCH 06/26] add echos --- .github/workflows/add_new_or_updated_feeds.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index e4f22f6b9..a67114978 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -134,14 +134,14 @@ jobs: BRANCH_NAME=${{ steps.create_branch_name.outputs.BRANCH }} # Fetch the latest changes from main - #git fetch origin main + git fetch origin main # Create or reset the branch to the current main git checkout -B "$BRANCH_NAME" origin/main # Configure git - git config --global user.name "${{ env.USERNAME }}" - git config --global user.email "${{ env.USERNAME_EMAIL }}" + # git config --global user.name "${{ env.USERNAME }}" + # git config --global user.email "${{ env.USERNAME_EMAIL }}" # Check Git authentication git config --get user.name @@ -153,13 +153,13 @@ jobs: # Count new files NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") - # Commit changes + echo "Commit changes: $NEW_FILE_COUNT" git commit -m "Automated commit — New/Updated feed(s)" - # Force push changes to the branch + echo "Force push changes to the branch" git push -f origin "$BRANCH_NAME" || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) - # Check if a PR already exists + echo "Check if a PR already exists" EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') if [ -n "$EXISTING_PR" ]; then From 074caf3e192eb305421732e5ad43eed9a975ca88 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 20 Feb 2025 12:37:35 -0500 Subject: [PATCH 07/26] change push command --- .github/workflows/add_new_or_updated_feeds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index a67114978..fe629aac9 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -157,7 +157,7 @@ jobs: git commit -m "Automated commit — New/Updated feed(s)" echo "Force push changes to the branch" - git push -f origin "$BRANCH_NAME" || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) + git push -u origin "$BRANCH_NAME" || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) echo "Check if a PR already exists" EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') @@ -226,7 +226,7 @@ jobs: SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - name: Post notification in Slack in case of failure - if: failure() + # if: failure() uses: slackapi/slack-github-action@v1.26.0 with: payload: | From d2dc573b8fe1467356b88af859100d3e6e480251 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 20 Feb 2025 13:23:30 -0500 Subject: [PATCH 08/26] stop tracking main branch from new branch --- .github/workflows/add_new_or_updated_feeds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index fe629aac9..1a2f265d8 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -134,10 +134,10 @@ jobs: BRANCH_NAME=${{ steps.create_branch_name.outputs.BRANCH }} # Fetch the latest changes from main - git fetch origin main + # git fetch origin main # Create or reset the branch to the current main - git checkout -B "$BRANCH_NAME" origin/main + git checkout -B "$BRANCH_NAME" # Configure git # git config --global user.name "${{ env.USERNAME }}" From 94ff6d69dec2361a76b0621c5c49fcd4c4dfa508 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 20 Feb 2025 13:28:58 -0500 Subject: [PATCH 09/26] change push flow --- .github/workflows/add_new_or_updated_feeds.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 1a2f265d8..b64b5cfaa 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -66,8 +66,8 @@ jobs: run: | git pull git checkout -b ${{ steps.create_branch_name.outputs.BRANCH }} - git reset --hard ${{ env.BASE }} - git clean -df + # git reset --hard ${{ env.BASE }} + # git clean -df - name: Download CSV and process each lines id: process-csv @@ -137,7 +137,7 @@ jobs: # git fetch origin main # Create or reset the branch to the current main - git checkout -B "$BRANCH_NAME" + git checkout "$BRANCH_NAME" # Configure git # git config --global user.name "${{ env.USERNAME }}" @@ -157,7 +157,7 @@ jobs: git commit -m "Automated commit — New/Updated feed(s)" echo "Force push changes to the branch" - git push -u origin "$BRANCH_NAME" || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) + git push origin HEAD || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) echo "Check if a PR already exists" EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') From 42bdc72b82c6ce35d2d02193c8f1338f9f0d9411 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 13:22:25 -0500 Subject: [PATCH 10/26] =?UTF-8?q?Automated=20commit=20=E2=80=94=20New/Upda?= =?UTF-8?q?ted=20feed(s)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/create_csv_pr.sh | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 scripts/create_csv_pr.sh diff --git a/scripts/create_csv_pr.sh b/scripts/create_csv_pr.sh new file mode 100755 index 000000000..d6f5ca132 --- /dev/null +++ b/scripts/create_csv_pr.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e # Exit immediately if a command exits with a non-zero status + +BRANCH_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}" + +# Checkout or create the branch +git checkout "$BRANCH_NAME" + +# Configure git +git config --get user.name +git config --get user.email + +# Add changes +git add . + +# Count new files +NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A" || echo "0") + +echo "Commit changes: $NEW_FILE_COUNT" +git commit -m "Automated commit — New/Updated feed(s)" + +echo "Force push changes to the branch" +git push origin HEAD || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) + +echo "Check if a PR already exists" +EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') + +if [ -n "$EXISTING_PR" ]; then + echo "Pull request already exists. Updating PR #$EXISTING_PR" + # PR_URL=$(gh pr view $EXISTING_PR --json url -q '.url') +else + echo "Creating new pull request" + # PR_URL=$(gh pr create --title "Automated Pull Request — New/Updated feed(s) [SOURCES]" \ + # --body "This pull request contains new or updated feed(s)" \ + # --base main \ + # --head "$BRANCH_NAME") +fi + +# Output PR URL and new file count +echo "pr-url=$PR_URL" +echo "new-file-count=$NEW_FILE_COUNT" From 4468c41661ebfea133c95dad23ef43bccd4ac5d2 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 13:31:00 -0500 Subject: [PATCH 11/26] =?UTF-8?q?Automated=20commit=20=E2=80=94=20New/Upda?= =?UTF-8?q?ted=20feed(s)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/create_csv_pr.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/scripts/create_csv_pr.sh b/scripts/create_csv_pr.sh index d6f5ca132..533b6acf3 100755 --- a/scripts/create_csv_pr.sh +++ b/scripts/create_csv_pr.sh @@ -4,8 +4,18 @@ set -e # Exit immediately if a command exits with a non-zero status BRANCH_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}" -# Checkout or create the branch -git checkout "$BRANCH_NAME" +echo "Using GitHub token: ${GH_TOKEN:0:4}********" + +# Set GitHub CLI authentication +export GH_TOKEN="${GH_TOKEN}" +export GITHUB_TOKEN="${GH_TOKEN}" + +echo "Checking out branch: $BRANCH_NAME" +git fetch origin "$BRANCH_NAME" + +# Reset local branch to remote HEAD (removes local commits) +# echo "Resetting local branch to match remote..." +# git reset --hard "origin/$BRANCH_NAME" # Configure git git config --get user.name @@ -24,17 +34,17 @@ echo "Force push changes to the branch" git push origin HEAD || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) echo "Check if a PR already exists" -EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') +EXISTING_PR=$(GH_TOKEN="${GH_TOKEN}" gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') if [ -n "$EXISTING_PR" ]; then echo "Pull request already exists. Updating PR #$EXISTING_PR" - # PR_URL=$(gh pr view $EXISTING_PR --json url -q '.url') + PR_URL=$(GH_TOKEN="${GH_TOKEN}" gh pr view $EXISTING_PR --json url -q '.url') else echo "Creating new pull request" - # PR_URL=$(gh pr create --title "Automated Pull Request — New/Updated feed(s) [SOURCES]" \ - # --body "This pull request contains new or updated feed(s)" \ - # --base main \ - # --head "$BRANCH_NAME") + PR_URL=$(GH_TOKEN="${GH_TOKEN}" gh pr create --title "Automated Pull Request — New/Updated feed(s) [SOURCES]" \ + --body "This pull request contains new or updated feed(s)" \ + --base main \ + --head "$BRANCH_NAME") fi # Output PR URL and new file count From 5f17605b8b09401a8b1e971fa6766fe1e059f174 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 13:48:42 -0500 Subject: [PATCH 12/26] test only push --- .github/workflows/add_new_or_updated_feeds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index b64b5cfaa..ebc519395 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -157,7 +157,7 @@ jobs: git commit -m "Automated commit — New/Updated feed(s)" echo "Force push changes to the branch" - git push origin HEAD || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) + git push origin HEAD echo "Check if a PR already exists" EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') From b42a33d094c16cfe6b5caff1da7ce568b31920cc Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:12:00 -0500 Subject: [PATCH 13/26] add upstream and logs --- .github/workflows/add_new_or_updated_feeds.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index ebc519395..ed4fe170b 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -20,6 +20,8 @@ env: BASE: "main" REVIEWERS_JSON: "[\"emmambd\"]" # List of GitHub usernames of the reviewers, in a JSON array : ["username1", "username2"] + ACTIONS_STEP_DEBUG: true + ACTIONS_RUNNER_DEBUG: true jobs: @@ -156,6 +158,8 @@ jobs: echo "Commit changes: $NEW_FILE_COUNT" git commit -m "Automated commit — New/Updated feed(s)" + git push --set-upstream origin "$BRANCH_NAME" + echo "Force push changes to the branch" git push origin HEAD From 41c6910cf6a8770de69d60b7e4d62bcff4cd3a5d Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:44:10 -0500 Subject: [PATCH 14/26] add permissions --- .github/workflows/add_new_or_updated_feeds.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index ed4fe170b..32075f792 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -27,7 +27,9 @@ env: jobs: add-new-updated-feeds: runs-on: ubuntu-22.04 - + permissions: + contents: write # Permissions to read/write content like code + pull-requests: write # Permissions to create/merge pull requests steps: - name: Setup global variables id: global_vars From 530733fe9d37ec6f49019dd726ea9fedfc31d5d9 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:55:57 -0500 Subject: [PATCH 15/26] revert changes --- .github/workflows/add_new_or_updated_feeds.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 32075f792..a599680d9 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -70,8 +70,8 @@ jobs: run: | git pull git checkout -b ${{ steps.create_branch_name.outputs.BRANCH }} - # git reset --hard ${{ env.BASE }} - # git clean -df + git reset --hard ${{ env.BASE }} + git clean -df - name: Download CSV and process each lines id: process-csv @@ -138,18 +138,18 @@ jobs: BRANCH_NAME=${{ steps.create_branch_name.outputs.BRANCH }} # Fetch the latest changes from main - # git fetch origin main + git fetch origin main # Create or reset the branch to the current main - git checkout "$BRANCH_NAME" + git checkout -B "$BRANCH_NAME" origin/main # Configure git # git config --global user.name "${{ env.USERNAME }}" # git config --global user.email "${{ env.USERNAME_EMAIL }}" # Check Git authentication - git config --get user.name - git config --get user.email + # git config --get user.name + # git config --get user.email # Add changes git add . From 87c1a033e1bc5aa2bf5d227e0bf1087ee84d662c Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:03:31 -0500 Subject: [PATCH 16/26] more logs --- .github/workflows/add_new_or_updated_feeds.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index a599680d9..2b01deaf8 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -138,6 +138,7 @@ jobs: BRANCH_NAME=${{ steps.create_branch_name.outputs.BRANCH }} # Fetch the latest changes from main + echo "fetching from origin" git fetch origin main # Create or reset the branch to the current main @@ -151,7 +152,8 @@ jobs: # git config --get user.name # git config --get user.email - # Add changes + git diff + echo "Add changes" git add . # Count new files From b55cdd64607182c2eba47a546abed0abeb377ba1 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:04:54 -0500 Subject: [PATCH 17/26] more logs --- .github/workflows/add_new_or_updated_feeds.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 2b01deaf8..06c970ef7 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -141,7 +141,7 @@ jobs: echo "fetching from origin" git fetch origin main - # Create or reset the branch to the current main + echo "Create or reset the branch to the current main" git checkout -B "$BRANCH_NAME" origin/main # Configure git @@ -152,6 +152,7 @@ jobs: # git config --get user.name # git config --get user.email + echo "Diff" git diff echo "Add changes" git add . From 2e68a503c1e24472efac0af831f1751d332191a5 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:17:15 -0500 Subject: [PATCH 18/26] more logs --- .github/workflows/add_new_or_updated_feeds.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 06c970ef7..03c1b1b2e 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -152,10 +152,17 @@ jobs: # git config --get user.name # git config --get user.email + printf "Status:\" + git status + printf "----------\n" echo "Diff" git diff + echo "Listing" + ls -al + printf "Listed-----" echo "Add changes" git add . + echo "Files added--------" # Count new files NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") From b6522050211ed92b178e65034cfd86210a87f835 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:20:24 -0500 Subject: [PATCH 19/26] more logs --- .github/workflows/add_new_or_updated_feeds.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 03c1b1b2e..2df8ea6f4 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -152,17 +152,17 @@ jobs: # git config --get user.name # git config --get user.email - printf "Status:\" + printf "Status:\n" git status printf "----------\n" echo "Diff" git diff echo "Listing" ls -al - printf "Listed-----" + printf "----------\n" echo "Add changes" git add . - echo "Files added--------" + printf "----------\n" # Count new files NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") From a46008fe18e392ea86b805add7d72c7485470a9a Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:37:20 -0500 Subject: [PATCH 20/26] more logs --- .github/workflows/add_new_or_updated_feeds.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 2df8ea6f4..c68c13c23 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -152,17 +152,17 @@ jobs: # git config --get user.name # git config --get user.email - printf "Status:\n" + echo "Status" git status - printf "----------\n" + echo "----------" echo "Diff" git diff echo "Listing" ls -al - printf "----------\n" + echo "----------" echo "Add changes" git add . - printf "----------\n" + echo "----------" # Count new files NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") From 5f52d8633bea98c54e46aa6cb7a260c7616e9e97 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:44:47 -0500 Subject: [PATCH 21/26] more logs --- .github/workflows/add_new_or_updated_feeds.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index c68c13c23..357897287 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -164,17 +164,25 @@ jobs: git add . echo "----------" - # Count new files - NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") + echo "Check commit" + git status --porcelain + echo "----------" + echo "Count new files" + NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") + echo "----------" + echo "Commit changes: $NEW_FILE_COUNT" git commit -m "Automated commit — New/Updated feed(s)" - + echo "----------" + echo "Set upstream to $BRANCH_NAME" git push --set-upstream origin "$BRANCH_NAME" + echo "----------" echo "Force push changes to the branch" git push origin HEAD - + echo "----------" + echo "Check if a PR already exists" EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') From e2f19976b6aac41bef30e93beb14d8ffa6d767c7 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:55:29 -0500 Subject: [PATCH 22/26] more logs --- .../workflows/add_new_or_updated_feeds.yml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 357897287..4ea861afe 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -130,6 +130,28 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash + - name: Ensure grep is installed + run: | + echo "Checking if grep is installed..." + + if ! command -v grep &> /dev/null; then + echo "Error: grep is not installed! Attempting to install..." + + # Install grep on Ubuntu + sudo apt-get update && sudo apt-get install -y grep + + # Verify installation + if ! command -v grep &> /dev/null; then + echo "Installation failed. grep is still missing!" + exit 1 + else + echo "grep installed successfully ✅" + fi + else + echo "grep is already installed ✅" + fi + shell: bash + - name: Commit, push, and create PR id: commit-push-create-pr @@ -167,9 +189,10 @@ jobs: echo "Check commit" git status --porcelain echo "----------" - + echo "Count new files" NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") + # NEW_FILE_COUNT=$(git status --porcelain | awk '$1 == "A" {count++} END {print count+0}') echo "----------" echo "Commit changes: $NEW_FILE_COUNT" From 2d75b8465a3d1cc076f54a18625a6fc5a27d3abb Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 16:00:18 -0500 Subject: [PATCH 23/26] test awk --- .../workflows/add_new_or_updated_feeds.yml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 4ea861afe..2c27d86c6 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -130,27 +130,27 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash - - name: Ensure grep is installed - run: | - echo "Checking if grep is installed..." + # - name: Ensure grep is installed + # run: | + # echo "Checking if grep is installed..." - if ! command -v grep &> /dev/null; then - echo "Error: grep is not installed! Attempting to install..." + # if ! command -v grep &> /dev/null; then + # echo "Error: grep is not installed! Attempting to install..." - # Install grep on Ubuntu - sudo apt-get update && sudo apt-get install -y grep + # # Install grep on Ubuntu + # sudo apt-get update && sudo apt-get install -y grep - # Verify installation - if ! command -v grep &> /dev/null; then - echo "Installation failed. grep is still missing!" - exit 1 - else - echo "grep installed successfully ✅" - fi - else - echo "grep is already installed ✅" - fi - shell: bash + # # Verify installation + # if ! command -v grep &> /dev/null; then + # echo "Installation failed. grep is still missing!" + # exit 1 + # else + # echo "grep installed successfully ✅" + # fi + # else + # echo "grep is already installed ✅" + # fi + # shell: bash - name: Commit, push, and create PR @@ -191,8 +191,8 @@ jobs: echo "----------" echo "Count new files" - NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") - # NEW_FILE_COUNT=$(git status --porcelain | awk '$1 == "A" {count++} END {print count+0}') + # NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") + NEW_FILE_COUNT=$(git status --porcelain | awk '$1 == "A" {count++} END {print count+0}') echo "----------" echo "Commit changes: $NEW_FILE_COUNT" From 71150de8fab6ba6d5549f776ea829851c2fa2b43 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 16:09:57 -0500 Subject: [PATCH 24/26] test awk --- .../workflows/add_new_or_updated_feeds.yml | 39 +++---------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 2c27d86c6..29affde71 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -7,7 +7,6 @@ on: permissions: contents: write # Permissions to read/write content like code - issues: write # Permissions to create issues or comment on issues pull-requests: write # Permissions to create/merge pull requests env: @@ -26,10 +25,7 @@ env: jobs: add-new-updated-feeds: - runs-on: ubuntu-22.04 - permissions: - contents: write # Permissions to read/write content like code - pull-requests: write # Permissions to create/merge pull requests + runs-on: ubuntu-latest steps: - name: Setup global variables id: global_vars @@ -53,7 +49,6 @@ jobs: export-env: true # Export loaded secrets as environment variables env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - # CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential" # The URL to obtain the csv file with the data of the google form. Taken from the "GiHub Actions — URLs" entry in 1password CSV_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/qkn5esttmtojawglm4l6t2bqaa/al2gjfhiuddabkp7o26hszuvia" SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/rdpfgrmnbxqaelgi5oky3lryz4/internal-add-feeds" @@ -130,29 +125,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash - # - name: Ensure grep is installed - # run: | - # echo "Checking if grep is installed..." - - # if ! command -v grep &> /dev/null; then - # echo "Error: grep is not installed! Attempting to install..." - - # # Install grep on Ubuntu - # sudo apt-get update && sudo apt-get install -y grep - - # # Verify installation - # if ! command -v grep &> /dev/null; then - # echo "Installation failed. grep is still missing!" - # exit 1 - # else - # echo "grep installed successfully ✅" - # fi - # else - # echo "grep is already installed ✅" - # fi - # shell: bash - - - name: Commit, push, and create PR id: commit-push-create-pr if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != '' @@ -167,8 +139,8 @@ jobs: git checkout -B "$BRANCH_NAME" origin/main # Configure git - # git config --global user.name "${{ env.USERNAME }}" - # git config --global user.email "${{ env.USERNAME_EMAIL }}" + git config --global user.name "${{ env.USERNAME }}" + git config --global user.email "${{ env.USERNAME_EMAIL }}" # Check Git authentication # git config --get user.name @@ -191,7 +163,6 @@ jobs: echo "----------" echo "Count new files" - # NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A") NEW_FILE_COUNT=$(git status --porcelain | awk '$1 == "A" {count++} END {print count+0}') echo "----------" @@ -225,8 +196,8 @@ jobs: echo "pr-url=$PR_URL" >> $GITHUB_OUTPUT echo "new-file-count=$NEW_FILE_COUNT" >> $GITHUB_OUTPUT env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash - name: Post notification in Slack channel of update From 4339a3b5cf75fc962030fa87fb97846797a5f5d2 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 16:17:53 -0500 Subject: [PATCH 25/26] cleanup --- .../workflows/add_new_or_updated_feeds.yml | 32 ++++-------- scripts/create_csv_pr.sh | 52 ------------------- 2 files changed, 9 insertions(+), 75 deletions(-) delete mode 100755 scripts/create_csv_pr.sh diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 29affde71..2501da400 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -19,9 +19,6 @@ env: BASE: "main" REVIEWERS_JSON: "[\"emmambd\"]" # List of GitHub usernames of the reviewers, in a JSON array : ["username1", "username2"] - ACTIONS_STEP_DEBUG: true - ACTIONS_RUNNER_DEBUG: true - jobs: add-new-updated-feeds: @@ -120,9 +117,13 @@ jobs: run: | # Check GitHub CLI authentication gh auth status + + # Check Git authentication + git config --get user.name + git config --get user.email env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash - name: Commit, push, and create PR @@ -141,19 +142,7 @@ jobs: # Configure git git config --global user.name "${{ env.USERNAME }}" git config --global user.email "${{ env.USERNAME_EMAIL }}" - - # Check Git authentication - # git config --get user.name - # git config --get user.email - - echo "Status" - git status - echo "----------" - echo "Diff" - git diff - echo "Listing" - ls -al - echo "----------" + echo "Add changes" git add . echo "----------" @@ -169,12 +158,9 @@ jobs: echo "Commit changes: $NEW_FILE_COUNT" git commit -m "Automated commit — New/Updated feed(s)" echo "----------" - echo "Set upstream to $BRANCH_NAME" - git push --set-upstream origin "$BRANCH_NAME" - echo "----------" echo "Force push changes to the branch" - git push origin HEAD + git push -f origin "$BRANCH_NAME" || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) echo "----------" echo "Check if a PR already exists" @@ -244,7 +230,7 @@ jobs: SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - name: Post notification in Slack in case of failure - # if: failure() + if: failure() uses: slackapi/slack-github-action@v1.26.0 with: payload: | diff --git a/scripts/create_csv_pr.sh b/scripts/create_csv_pr.sh deleted file mode 100755 index 533b6acf3..000000000 --- a/scripts/create_csv_pr.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -e # Exit immediately if a command exits with a non-zero status - -BRANCH_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}" - -echo "Using GitHub token: ${GH_TOKEN:0:4}********" - -# Set GitHub CLI authentication -export GH_TOKEN="${GH_TOKEN}" -export GITHUB_TOKEN="${GH_TOKEN}" - -echo "Checking out branch: $BRANCH_NAME" -git fetch origin "$BRANCH_NAME" - -# Reset local branch to remote HEAD (removes local commits) -# echo "Resetting local branch to match remote..." -# git reset --hard "origin/$BRANCH_NAME" - -# Configure git -git config --get user.name -git config --get user.email - -# Add changes -git add . - -# Count new files -NEW_FILE_COUNT=$(git status --porcelain | grep -c "^A" || echo "0") - -echo "Commit changes: $NEW_FILE_COUNT" -git commit -m "Automated commit — New/Updated feed(s)" - -echo "Force push changes to the branch" -git push origin HEAD || (echo "Git push failed!" && git remote -v && git branch -vv && exit 1) - -echo "Check if a PR already exists" -EXISTING_PR=$(GH_TOKEN="${GH_TOKEN}" gh pr list --head "$BRANCH_NAME" --json number -q '.[0].number') - -if [ -n "$EXISTING_PR" ]; then - echo "Pull request already exists. Updating PR #$EXISTING_PR" - PR_URL=$(GH_TOKEN="${GH_TOKEN}" gh pr view $EXISTING_PR --json url -q '.url') -else - echo "Creating new pull request" - PR_URL=$(GH_TOKEN="${GH_TOKEN}" gh pr create --title "Automated Pull Request — New/Updated feed(s) [SOURCES]" \ - --body "This pull request contains new or updated feed(s)" \ - --base main \ - --head "$BRANCH_NAME") -fi - -# Output PR URL and new file count -echo "pr-url=$PR_URL" -echo "new-file-count=$NEW_FILE_COUNT" From 95bbf65e87c977b00729d38a535998fcd7a43c81 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 24 Feb 2025 16:24:42 -0500 Subject: [PATCH 26/26] delete check gh --- .github/workflows/add_new_or_updated_feeds.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 2501da400..488fa8304 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -113,19 +113,6 @@ jobs: echo -e "\n\n" git status - - name: Checking GitHub CLI and Git Authentication - run: | - # Check GitHub CLI authentication - gh auth status - - # Check Git authentication - git config --get user.name - git config --get user.email - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - - name: Commit, push, and create PR id: commit-push-create-pr if: steps.process-csv.outputs.PYTHON_SCRIPT_ARGS != ''