Skip to content

Commit 91d9349

Browse files
committed
feat: add genai-assisted label
Co-authored-by: Claude
1 parent 17a832b commit 91d9349

File tree

5 files changed

+691
-1
lines changed

5 files changed

+691
-1
lines changed

README.md

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
This action validates that the various Mobsuccess policies are enforced when
66
changes are made to the repository. It checks, for example, the title of the
7-
pull request, the names of the branch
7+
pull request, the names of the branch, and detects GenAI-assisted contributions.
88

99
# Install the workflow in repository
1010

@@ -29,3 +29,35 @@ Usage:
2929
max-releases: 10 #(optional, default to 10)
3030
unreleased-tag: 0.0.90289303809 # newly created unreleased tag (optional)
3131
```
32+
33+
## GenAI Detection
34+
35+
The `validate-pr` action automatically detects GenAI-assisted PRs and adds the `genai-assisted` label.
36+
37+
### Detection Triggers
38+
39+
The action checks for AI assistance in:
40+
41+
- **PR Author**: Bot accounts (dependabot, renovate, copilot, etc.)
42+
- **PR Body**: Mentions of AI tools (Claude, Copilot, ChatGPT, Cursor, etc.)
43+
- **Existing Labels**: AI-related labels (ai-assisted, copilot, claude, etc.)
44+
- **Commits**: Messages containing AI signatures or co-authored-by AI
45+
- **Comments**: Bot comments or user mentions of AI usage
46+
47+
### Supported AI Tools
48+
49+
Claude, Claude Code, ChatGPT, GPT-4, Copilot, GitHub Copilot, Gemini, Cursor, Windsurf, Codeium, Tabnine, Cody, Aider, Devin, Amazon Q, CodeWhisperer, and 50+ other tools.
50+
51+
### Patterns Detected
52+
53+
```
54+
- Co-Authored-By: Claude <[email protected]>
55+
- Generated with [Claude Code](https://claude.ai/code)
56+
- AI-assisted / AI-generated
57+
- [Copilot] / [Claude] / [ChatGPT] in commit messages
58+
- "Created using Copilot" / "With help from Claude"
59+
```
60+
61+
### Customization
62+
63+
To add new patterns, edit `lib/genaiPatterns.js`. The detection is permissive: a single match triggers the label.

lib/actions/pullRequest.js

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const {
44
isPullRequestTitleValid: isPullRequestTitleValid,
55
} = require("../pullRequest");
66
const { getAmplifyURIs } = require("./amplify");
7+
const { detectGenAI, GENAI_LABEL } = require("../genaiDetection");
78

89
exports.validatePR = async function validatePR({ pullRequest, issue }) {
910
const octokit = getOctokit();
@@ -59,6 +60,34 @@ exports.validatePR = async function validatePR({ pullRequest, issue }) {
5960
// ignore error
6061
console.log(`Could not apply label: ${e}`);
6162
}
63+
64+
// GenAI detection: add label if AI assistance is detected
65+
try {
66+
const genaiResult = await detectGenAI(octokit, {
67+
owner,
68+
repo,
69+
pullNumber,
70+
pullRequest,
71+
});
72+
73+
if (genaiResult.detected) {
74+
console.log(`GenAI detected: ${genaiResult.reason}`);
75+
await octokit.issues.addLabels({
76+
owner,
77+
repo,
78+
issue_number: pullNumber,
79+
labels: [GENAI_LABEL],
80+
});
81+
console.log(`Added "${GENAI_LABEL}" label to PR #${pullNumber}`);
82+
} else if (genaiResult.alreadyLabeled) {
83+
console.log(`PR #${pullNumber} already has "${GENAI_LABEL}" label`);
84+
} else {
85+
console.log(`No GenAI assistance detected in PR #${pullNumber}`);
86+
}
87+
} catch (e) {
88+
// Don't fail the action if GenAI detection fails
89+
console.log(`Warning: GenAI detection failed: ${e.message}`);
90+
}
6291
}
6392

6493
// do we have an AWS Amplify URI? If so, make sure that at least one comment

lib/genaiDetection.js

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
/**
2+
* GenAI Detection Logic
3+
*
4+
* Detects AI assistance in PRs by checking:
5+
* - Existing labels
6+
* - PR author (bot detection)
7+
* - PR body/description
8+
* - Commit messages and co-authors
9+
* - PR comments
10+
*/
11+
12+
const {
13+
BOT_PATTERNS,
14+
COMMIT_PATTERNS,
15+
PR_BODY_PATTERNS,
16+
COMMENT_PATTERNS,
17+
AI_LABELS,
18+
GENAI_LABEL,
19+
} = require("./genaiPatterns");
20+
21+
/**
22+
* Check if a string matches any pattern in an array
23+
* @param {string} text - Text to check
24+
* @param {RegExp[]} patterns - Array of regex patterns
25+
* @returns {RegExp|null} - The matching pattern or null
26+
*/
27+
function matchesAnyPattern(text, patterns) {
28+
if (!text) return null;
29+
for (const pattern of patterns) {
30+
if (pattern.test(text)) {
31+
return pattern;
32+
}
33+
}
34+
return null;
35+
}
36+
37+
/**
38+
* Check if author is a bot
39+
* @param {string} author - Author login or name
40+
* @returns {RegExp|null} - The matching pattern or null
41+
*/
42+
function isBot(author) {
43+
return matchesAnyPattern(author, BOT_PATTERNS);
44+
}
45+
46+
/**
47+
* Check if any existing label indicates AI assistance
48+
* @param {Array<{name: string}>} labels - Array of label objects
49+
* @returns {string|null} - The matching label or null
50+
*/
51+
function hasAILabel(labels) {
52+
if (!labels || !Array.isArray(labels)) return null;
53+
for (const label of labels) {
54+
const labelName = (label.name || label).toLowerCase();
55+
if (AI_LABELS.includes(labelName)) {
56+
return labelName;
57+
}
58+
}
59+
return null;
60+
}
61+
62+
/**
63+
* Check if the genai-assisted label already exists
64+
* @param {Array<{name: string}>} labels - Array of label objects
65+
* @returns {boolean}
66+
*/
67+
function hasGenAILabel(labels) {
68+
if (!labels || !Array.isArray(labels)) return false;
69+
return labels.some(
70+
(label) => (label.name || label).toLowerCase() === GENAI_LABEL
71+
);
72+
}
73+
74+
/**
75+
* Detect GenAI assistance in a PR
76+
* @param {Object} octokit - GitHub API client
77+
* @param {Object} params - Detection parameters
78+
* @param {string} params.owner - Repository owner
79+
* @param {string} params.repo - Repository name
80+
* @param {number} params.pullNumber - PR number
81+
* @param {Object} params.pullRequest - PR object from GitHub context
82+
* @returns {Promise<{detected: boolean, reason: string|null}>}
83+
*/
84+
async function detectGenAI(octokit, { owner, repo, pullNumber, pullRequest }) {
85+
// 1. Check if already labeled with genai-assisted
86+
if (hasGenAILabel(pullRequest.labels)) {
87+
console.log("PR already has genai-assisted label, skipping detection");
88+
return { detected: false, reason: null, alreadyLabeled: true };
89+
}
90+
91+
// 2. Check PR author (bot detection)
92+
const prAuthor = pullRequest.user?.login || pullRequest.user?.name;
93+
const botMatch = isBot(prAuthor);
94+
if (botMatch) {
95+
return {
96+
detected: true,
97+
reason: `PR author "${prAuthor}" matches bot pattern: ${botMatch}`,
98+
};
99+
}
100+
101+
// 3. Check PR body/description
102+
const bodyMatch = matchesAnyPattern(pullRequest.body, PR_BODY_PATTERNS);
103+
if (bodyMatch) {
104+
return {
105+
detected: true,
106+
reason: `PR body matches GenAI pattern: ${bodyMatch}`,
107+
};
108+
}
109+
110+
// 4. Check existing labels for AI-related labels
111+
const aiLabel = hasAILabel(pullRequest.labels);
112+
if (aiLabel) {
113+
return {
114+
detected: true,
115+
reason: `PR has AI-related label: "${aiLabel}"`,
116+
};
117+
}
118+
119+
// 5. Fetch and check commits
120+
try {
121+
const commits = await octokit.paginate(
122+
"GET /repos/{owner}/{repo}/pulls/{pull_number}/commits",
123+
{ owner, repo, pull_number: pullNumber }
124+
);
125+
126+
for (const commit of commits) {
127+
// Check commit message
128+
const message = commit.commit?.message || "";
129+
const messageMatch = matchesAnyPattern(message, COMMIT_PATTERNS);
130+
if (messageMatch) {
131+
return {
132+
detected: true,
133+
reason: `Commit "${commit.sha.slice(0, 7)}" message matches GenAI pattern: ${messageMatch}`,
134+
};
135+
}
136+
137+
// Check commit author
138+
const commitAuthor =
139+
commit.author?.login || commit.commit?.author?.name || "";
140+
const commitAuthorMatch = isBot(commitAuthor);
141+
if (commitAuthorMatch) {
142+
return {
143+
detected: true,
144+
reason: `Commit "${commit.sha.slice(0, 7)}" author "${commitAuthor}" matches bot pattern: ${commitAuthorMatch}`,
145+
};
146+
}
147+
148+
// Check committer
149+
const committer =
150+
commit.committer?.login || commit.commit?.committer?.name || "";
151+
const committerMatch = isBot(committer);
152+
if (committerMatch) {
153+
return {
154+
detected: true,
155+
reason: `Commit "${commit.sha.slice(0, 7)}" committer "${committer}" matches bot pattern: ${committerMatch}`,
156+
};
157+
}
158+
}
159+
} catch (error) {
160+
console.log(`Warning: Could not fetch commits: ${error.message}`);
161+
}
162+
163+
// 6. Fetch and check PR comments
164+
try {
165+
const comments = await octokit.paginate(
166+
"GET /repos/{owner}/{repo}/issues/{issue_number}/comments",
167+
{ owner, repo, issue_number: pullNumber }
168+
);
169+
170+
for (const comment of comments) {
171+
// Check comment author
172+
const commentAuthor = comment.user?.login || "";
173+
const authorMatch = isBot(commentAuthor);
174+
if (authorMatch) {
175+
return {
176+
detected: true,
177+
reason: `Comment author "${commentAuthor}" matches bot pattern: ${authorMatch}`,
178+
};
179+
}
180+
181+
// Check comment body
182+
const commentMatch = matchesAnyPattern(comment.body, COMMENT_PATTERNS);
183+
if (commentMatch) {
184+
return {
185+
detected: true,
186+
reason: `Comment by "${commentAuthor}" matches GenAI pattern: ${commentMatch}`,
187+
};
188+
}
189+
}
190+
} catch (error) {
191+
console.log(`Warning: Could not fetch comments: ${error.message}`);
192+
}
193+
194+
// No GenAI detected
195+
return { detected: false, reason: null };
196+
}
197+
198+
module.exports = {
199+
detectGenAI,
200+
matchesAnyPattern,
201+
isBot,
202+
hasAILabel,
203+
hasGenAILabel,
204+
GENAI_LABEL,
205+
};

0 commit comments

Comments
 (0)