Address review feedback: stale context, injection warning, failure visibility, detection robustness

- Use webhook payload for triggering comment instead of relying on
  API eventual consistency; deduplicate against API results by comment ID
- Add explicit warning in agent instructions that jq --arg pattern
  is mandatory (never use shell string interpolation for JSON body)
- Add agent-outcome output and confused reaction on failure so
  developers know their reply wasn't processed
- Add user.type == Bot check on root comment author as defense-in-depth
  alongside the HTML marker detection

Author: derekmisler

.github/workflows/review-pr.yml

@@ -325,9 +325,15 @@ jobs:
 
           parent=$(gh api "repos/$REPO/pulls/comments/$PARENT_ID" 2>/dev/null || echo "{}")
           body=$(echo "$parent" | jq -r '.body // ""')
-
-          # Match <!-- cagent-review --> but NOT <!-- cagent-review-reply --> (substring overlap)
-          if echo "$body" | grep -q "<!-- cagent-review -->" && ! echo "$body" | grep -q "<!-- cagent-review-reply -->"; then
+          parent_user_type=$(echo "$parent" | jq -r '.user.type // ""')
+
+          # Defense-in-depth: verify the root comment was posted by a Bot (agent) AND
+          # contains the review marker but NOT the reply marker (substring overlap).
+          # The user.type check prevents matching human comments that happen to contain
+          # the marker text (e.g., in discussions about the review system).
+          if [ "$parent_user_type" = "Bot" ] && \
+             echo "$body" | grep -q "<!-- cagent-review -->" && \
+             ! echo "$body" | grep -q "<!-- cagent-review-reply -->"; then
             echo "is_agent=true" >> $GITHUB_OUTPUT
             echo "root_comment_id=$PARENT_ID" >> $GITHUB_OUTPUT
 
@@ -369,6 +375,11 @@ jobs:
           REPO: ${{ github.repository }}
           FILE_PATH: ${{ steps.check.outputs.file_path }}
           LINE: ${{ steps.check.outputs.line }}
+          # The triggering comment from the webhook payload — guaranteed fresh,
+          # unlike the API which may have eventual consistency lag.
+          TRIGGER_COMMENT_BODY: ${{ github.event.comment.body }}
+          TRIGGER_COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+          TRIGGER_COMMENT_ID: ${{ github.event.comment.id }}
         run: |
           # Fetch the root comment
           root=$(gh api "repos/$REPO/pulls/comments/$ROOT_ID")
@@ -377,13 +388,14 @@ jobs:
           # Fetch all review comments on this PR and filter to this thread.
           # Uses --paginate to handle PRs with >100 review comments.
           # Each page is processed by jq independently, then merged with jq -s.
+          # Note: the triggering comment may not appear here due to eventual
+          # consistency, so we append it from the webhook payload below.
           all_comments=$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/comments" \
             --jq "[.[] | select(.in_reply_to_id == $ROOT_ID)]" | jq -s 'add // [] | sort_by(.created_at)')
 
           # Build the thread context and save as step output.
           # Use a randomized delimiter to prevent comment body content from
           # colliding with the GITHUB_OUTPUT heredoc terminator.
-          reply_count=$(echo "$all_comments" | jq 'length')
           DELIM="THREAD_CONTEXT_$(openssl rand -hex 8)"
 
           {
@@ -402,18 +414,31 @@ jobs:
             echo "$root_body"
             echo ""
 
-            # Add each reply in chronological order
+            # Add earlier replies from the API (excludes the triggering comment
+            # to avoid duplication if the API already has it)
+            reply_count=$(echo "$all_comments" | jq 'length')
             for i in $(seq 0 $((reply_count - 1))); do
+              comment_id=$(echo "$all_comments" | jq -r ".[$i].id")
+              # Skip the triggering comment — we append it from the payload below
+              if [ "$comment_id" = "$TRIGGER_COMMENT_ID" ]; then
+                continue
+              fi
               author=$(echo "$all_comments" | jq -r ".[$i].user.login")
               body=$(echo "$all_comments" | jq -r ".[$i].body")
-              echo "[REPLY $((i + 1)) by @$author]"
+              echo "[REPLY by @$author]"
               echo "$body"
               echo ""
             done
+
+            # Always append the triggering comment last — sourced directly from
+            # the webhook payload so it's guaranteed to be present.
+            echo "[REPLY by @$TRIGGER_COMMENT_AUTHOR] ← this is the reply you are responding to"
+            echo "$TRIGGER_COMMENT_BODY"
+            echo ""
             echo "$DELIM"
           } >> $GITHUB_OUTPUT
 
-          echo "✅ Built thread context with $reply_count replies"
+          echo "✅ Built thread context with replies (triggering comment from webhook payload)"
 
       # Safe to checkout PR head because the reply agent only READS files (no code execution)
       - name: Checkout PR head
@@ -439,6 +464,7 @@ jobs:
         uses: docker/cagent-action/review-pr/reply@latest
         with:
           thread-context: ${{ steps.thread.outputs.prompt }}
+          comment-id: ${{ github.event.comment.id }}
           anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
           openai-api-key: ${{ secrets.OPENAI_API_KEY }}
           google-api-key: ${{ secrets.GOOGLE_API_KEY }}

.github/workflows/self-review-pr.yml

@@ -229,9 +229,15 @@ jobs:
 
           parent=$(gh api "repos/$REPO/pulls/comments/$PARENT_ID" 2>/dev/null || echo "{}")
           body=$(echo "$parent" | jq -r '.body // ""')
-
-          # Match <!-- cagent-review --> but NOT <!-- cagent-review-reply --> (substring overlap)
-          if echo "$body" | grep -q "<!-- cagent-review -->" && ! echo "$body" | grep -q "<!-- cagent-review-reply -->"; then
+          parent_user_type=$(echo "$parent" | jq -r '.user.type // ""')
+
+          # Defense-in-depth: verify the root comment was posted by a Bot (agent) AND
+          # contains the review marker but NOT the reply marker (substring overlap).
+          # The user.type check prevents matching human comments that happen to contain
+          # the marker text (e.g., in discussions about the review system).
+          if [ "$parent_user_type" = "Bot" ] && \
+             echo "$body" | grep -q "<!-- cagent-review -->" && \
+             ! echo "$body" | grep -q "<!-- cagent-review-reply -->"; then
             echo "is_agent=true" >> $GITHUB_OUTPUT
             echo "root_comment_id=$PARENT_ID" >> $GITHUB_OUTPUT
 
@@ -273,6 +279,11 @@ jobs:
           REPO: ${{ github.repository }}
           FILE_PATH: ${{ steps.check.outputs.file_path }}
           LINE: ${{ steps.check.outputs.line }}
+          # The triggering comment from the webhook payload — guaranteed fresh,
+          # unlike the API which may have eventual consistency lag.
+          TRIGGER_COMMENT_BODY: ${{ github.event.comment.body }}
+          TRIGGER_COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+          TRIGGER_COMMENT_ID: ${{ github.event.comment.id }}
         run: |
           # Fetch the root comment
           root=$(gh api "repos/$REPO/pulls/comments/$ROOT_ID")
@@ -281,13 +292,14 @@ jobs:
           # Fetch all review comments on this PR and filter to this thread.
           # Uses --paginate to handle PRs with >100 review comments.
           # Each page is processed by jq independently, then merged with jq -s.
+          # Note: the triggering comment may not appear here due to eventual
+          # consistency, so we append it from the webhook payload below.
           all_comments=$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/comments" \
             --jq "[.[] | select(.in_reply_to_id == $ROOT_ID)]" | jq -s 'add // [] | sort_by(.created_at)')
 
           # Build the thread context and save as step output.
           # Use a randomized delimiter to prevent comment body content from
           # colliding with the GITHUB_OUTPUT heredoc terminator.
-          reply_count=$(echo "$all_comments" | jq 'length')
           DELIM="THREAD_CONTEXT_$(openssl rand -hex 8)"
 
           {
@@ -306,18 +318,31 @@ jobs:
             echo "$root_body"
             echo ""
 
-            # Add each reply in chronological order
+            # Add earlier replies from the API (excludes the triggering comment
+            # to avoid duplication if the API already has it)
+            reply_count=$(echo "$all_comments" | jq 'length')
             for i in $(seq 0 $((reply_count - 1))); do
+              comment_id=$(echo "$all_comments" | jq -r ".[$i].id")
+              # Skip the triggering comment — we append it from the payload below
+              if [ "$comment_id" = "$TRIGGER_COMMENT_ID" ]; then
+                continue
+              fi
               author=$(echo "$all_comments" | jq -r ".[$i].user.login")
               body=$(echo "$all_comments" | jq -r ".[$i].body")
-              echo "[REPLY $((i + 1)) by @$author]"
+              echo "[REPLY by @$author]"
               echo "$body"
               echo ""
             done
+
+            # Always append the triggering comment last — sourced directly from
+            # the webhook payload so it's guaranteed to be present.
+            echo "[REPLY by @$TRIGGER_COMMENT_AUTHOR] ← this is the reply you are responding to"
+            echo "$TRIGGER_COMMENT_BODY"
+            echo ""
             echo "$DELIM"
           } >> $GITHUB_OUTPUT
 
-          echo "✅ Built thread context with $reply_count replies"
+          echo "✅ Built thread context with replies (triggering comment from webhook payload)"
 
       # Safe to checkout PR head because the reply agent only READS files (no code execution)
       - name: Checkout PR head
@@ -343,6 +368,7 @@ jobs:
         uses: ./review-pr/reply
         with:
           thread-context: ${{ steps.thread.outputs.prompt }}
+          comment-id: ${{ github.event.comment.id }}
           anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
           openai-api-key: ${{ secrets.OPENAI_API_KEY }}
           google-api-key: ${{ secrets.GOOGLE_API_KEY }}

review-pr/agents/pr-review-reply.yaml

@@ -51,7 +51,9 @@ agents:
       ## Posting Your Reply
 
       After formulating your response, post it using `gh api` with JSON input piped via stdin.
-      This avoids shell quoting issues with special characters in the response body:
+      IMPORTANT: You MUST use the `jq -n --arg` pattern shown below. NEVER construct the JSON
+      body using shell string interpolation (e.g., `echo "{\"body\": \"$text\"}"`) — this creates
+      a command injection vulnerability if the response contains quotes or special characters.
 
       ```bash
       jq -n \

review-pr/reply/action.yml

@@ -6,6 +6,9 @@ inputs:
   thread-context:
     description: "Thread context (original comment + replies) to respond to"
     required: true
+  comment-id:
+    description: "ID of the triggering comment (for failure notification reaction)"
+    required: false
   anthropic-api-key:
     description: "Anthropic API key"
     required: false
@@ -31,6 +34,11 @@ inputs:
     description: "GitHub token for API access"
     required: false
 
+outputs:
+  agent-outcome:
+    description: "Outcome of the reply agent (success/failure/skipped)"
+    value: ${{ steps.run-reply.outcome }}
+
 runs:
   using: "composite"
   steps:
@@ -72,3 +80,18 @@ runs:
       with:
         path: ${{ github.workspace }}/.cache/pr-review-memory.db
         key: pr-review-memory-${{ github.repository }}-reply-${{ github.run_id }}
+
+    # Add a "confused" reaction to the triggering comment if the agent failed,
+    # so the developer knows their reply wasn't processed.
+    - name: React on failure
+      if: always() && steps.run-reply.outcome == 'failure' && inputs.comment-id != ''
+      continue-on-error: true
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.github-token }}
+        REPO: ${{ github.repository }}
+        COMMENT_ID: ${{ inputs.comment-id }}
+      run: |
+        gh api "repos/$REPO/pulls/comments/$COMMENT_ID/reactions" \
+          -f content="confused" --silent || true
+        echo "😕 Added reaction to indicate reply failed"