Skip to content

Commit 655cc07

Browse files
init
0 parents  commit 655cc07

File tree

17 files changed

+7105
-0
lines changed

17 files changed

+7105
-0
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
**.env
2+
node_modules
3+
coverage

README.md

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Docs Assertion Tester
2+
3+
Do you love unit tests for your code? Do you wish you could write unit tests for your documentation? Well, now you can!
4+
5+
<!-- TODO: Add gif -->
6+
7+
## Prerequisites
8+
9+
- An OpenAI API key
10+
- A GitHub token with `repo` scope
11+
12+
We recommend storing these in GitHub secrets. You can find instructions on how to do that
13+
[here](https://docs.github.com/en/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository).
14+
15+
## Installation
16+
17+
In any workflow for PRs, add the following step:
18+
19+
```yaml
20+
- name: Docs Assertion Tester
21+
uses: hasura/docs-assertion-tester@v1
22+
with:
23+
openai_api_key: ${{ secrets.OPENAI_API_KEY }}
24+
github_token: ${{ secrets.GITHUB_TOKEN }}
25+
org: ${{ github.repository_owner }}
26+
repo: ${{ github.repository }}
27+
pr_number: ${{ github.event.number }}
28+
```
29+
30+
## Usage
31+
32+
In the description of any PR, add the following comments:
33+
34+
```html
35+
<!-- DX:Assertion-start -->
36+
<!-- DX:Assertion-end -->
37+
```
38+
39+
Then, between the start and end comments, add your assertions in the following format:
40+
41+
```html
42+
<!-- DX:Assertion-start -->
43+
A user should be able to easily add a comment to their PR's description.
44+
<!-- DX:Assertion-end -->
45+
```
46+
47+
The assertion tester will then run your assertions against the documentation in your PR's description. It will check
48+
over two scopes:
49+
50+
- `Diff`
51+
- `Integrated`
52+
53+
The `Diff` scope will check the assertions against the diff (i.e., only what the author contributed). The `Integrated`
54+
scope will check the assertions against the entire set of files changed, including the author's changes.
55+
56+
Upon completion, the assertion tester will output the analysis in markdown format. You can add a comment to your PR
57+
using our handy [GitHub Action](https://github.com/marketplace/actions/comment-progress).
58+
59+
Using our `comment-progress` action, the output looks like this after running [the sample workflow](#) in the `/samples`
60+
folder:
61+
62+
<!-- TODO: Add screenshot -->
63+
64+
## Contributing
65+
66+
Before opening a PR, please create an issue to discuss the proposed change.

__tests__/github.test.ts

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import {
2+
testConnection,
3+
getRepo,
4+
getPullRequests,
5+
getSinglePR,
6+
getAssertion,
7+
getDiff,
8+
getChangedFiles,
9+
getFileContent,
10+
} from '../src/github';
11+
12+
describe('GitHub Functionality', () => {
13+
it('Should be able to connect to GitHub', () => {
14+
expect(testConnection()).resolves.toBe(true);
15+
});
16+
it('Should be able to access a repo in the Hasura org', () => {
17+
expect(getRepo('hasura', 'v3-docs')).resolves.toHaveProperty('name', 'v3-docs');
18+
});
19+
it('Should be able to get the PRs for the repo', async () => {
20+
const pullRequests = await getPullRequests('hasura', 'v3-docs');
21+
expect(pullRequests?.length).toBeGreaterThanOrEqual(0);
22+
});
23+
it('Should be able to get the contents of a single PR', () => {
24+
const prNumber: number = 262;
25+
expect(getSinglePR('hasura', 'v3-docs', prNumber)).resolves.toHaveProperty('number', prNumber);
26+
});
27+
it('Should be able to get the assertion from the description of a PR', async () => {
28+
// test PR with 262 about PATs
29+
const prNumber: number = 262;
30+
const PR = await getSinglePR('hasura', 'v3-docs', prNumber);
31+
const assertion = await getAssertion(PR?.body || '');
32+
// the last I checked, this was the text — if it's failing, check th PR 🤷‍♂️
33+
expect(assertion).toContain('understand how to log in using the PAT with VS Code.');
34+
});
35+
it('Should be able to return the diff of a PR', async () => {
36+
const diffUrl: number = 262;
37+
const diff = await getDiff(diffUrl);
38+
expect(diff).toContain('diff');
39+
});
40+
it('Should be able to determine which files were changed in a PR', async () => {
41+
const diffUrl: number = 262;
42+
const diff = await getDiff(diffUrl);
43+
const files = getChangedFiles(diff);
44+
expect(files).toContain('docs/ci-cd/projects.mdx');
45+
});
46+
it('Should be able to get the contents of a file', async () => {
47+
const contents = await getFileContent(['README.md']);
48+
expect(contents).toContain('Website');
49+
});
50+
});

__tests__/openai.test.ts

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { testConnection, generatePrompt, testAssertion, writeAnalysis } from '../src/open_ai';
2+
import { getDiff, getSinglePR, getAssertion, getChangedFiles, getFileContent } from '../src/github';
3+
4+
describe('OpenAI Functionality', () => {
5+
it('Should be able to connect to OpenAI', async () => {
6+
await expect(testConnection()).resolves.toBe(true);
7+
});
8+
it('Should be able to generate a prompt using the diff, assertion, and whole file', async () => {
9+
const diff: string = await getDiff(262);
10+
const assertion: string = 'The documentation is easy to read and understand.';
11+
const file: string = 'This is a test file.';
12+
const prompt: string = generatePrompt(diff, assertion, file);
13+
expect(prompt).toContain(assertion);
14+
expect(prompt).toContain(diff);
15+
expect(prompt).toContain(file);
16+
});
17+
it.todo('Should return null when an error is thrown');
18+
it('Should be able to generate a response using the prompt and a sample diff', async () => {
19+
// This should produce somewhat regularly happy results 👇
20+
// const prNumber: number = 243;
21+
// This should produce somewhat regularly unhappy results 👇
22+
const prNumber: number = 262;
23+
const PR = await getSinglePR('hasura', 'v3-docs', prNumber);
24+
const assertion = await getAssertion(PR?.body || '');
25+
const diff: string = await getDiff(prNumber);
26+
const changedFiles = getChangedFiles(diff);
27+
const file: any = await getFileContent(changedFiles);
28+
const prompt: string = generatePrompt(diff, assertion, file);
29+
const response = await testAssertion(prompt);
30+
expect(response).toBeTruthy();
31+
}, 50000);
32+
it('Should create a nicely formatted message using the response', async () => {
33+
expect(
34+
writeAnalysis(
35+
`[{"satisfied": "\u2705", "scope": "diff", "feedback": "You did a great job!"}, {"satisfied": "\u2705", "scope": "wholeFile", "feedback": "Look. At. You. Go!"}]`
36+
)
37+
).toContain('You did a great job!');
38+
});
39+
});

action.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: 'Docs Assertion Tester'
2+
description: 'Test user-facing assertions about documentation changes using OpenAI.'
3+
branding:
4+
icon: user-check
5+
color: white
6+
inputs:
7+
GITHUB_TOKEN:
8+
description: 'GitHub token'
9+
required: true
10+
OPENAI_API_KEY:
11+
description: 'OpenAI API key'
12+
required: true
13+
GITHUB_ORG:
14+
description: 'The owner of the GitHub repository'
15+
required: true
16+
GITHUB_REPOSITORY:
17+
description: 'The name of the GitHub repository'
18+
required: true
19+
PR_NUMBER:
20+
description: 'Pull Request number'
21+
required: true
22+
outputs:
23+
analysis:
24+
description: 'The analysis of the PR from OpenAI.'
25+
runs:
26+
using: 'node18'
27+
main: 'dist/index.js'

bable.config.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
module.exports = {
2+
presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
3+
};

dist/github/config.js

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
"use strict";
2+
Object.defineProperty(exports, "__esModule", { value: true });
3+
exports.github = void 0;
4+
const rest_1 = require("@octokit/rest");
5+
exports.github = new rest_1.Octokit({
6+
auth: process.env.GITHUB_TOKEN,
7+
});
8+
console.log(exports.github);

dist/github/index.js

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
"use strict";
2+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3+
if (k2 === undefined) k2 = k;
4+
var desc = Object.getOwnPropertyDescriptor(m, k);
5+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6+
desc = { enumerable: true, get: function() { return m[k]; } };
7+
}
8+
Object.defineProperty(o, k2, desc);
9+
}) : (function(o, m, k, k2) {
10+
if (k2 === undefined) k2 = k;
11+
o[k2] = m[k];
12+
}));
13+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14+
Object.defineProperty(o, "default", { enumerable: true, value: v });
15+
}) : function(o, v) {
16+
o["default"] = v;
17+
});
18+
var __importStar = (this && this.__importStar) || function (mod) {
19+
if (mod && mod.__esModule) return mod;
20+
var result = {};
21+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22+
__setModuleDefault(result, mod);
23+
return result;
24+
};
25+
var __importDefault = (this && this.__importDefault) || function (mod) {
26+
return (mod && mod.__esModule) ? mod : { "default": mod };
27+
};
28+
Object.defineProperty(exports, "__esModule", { value: true });
29+
exports.getFileContent = exports.getChangedFiles = exports.getDiff = exports.getAssertion = exports.getSinglePR = exports.getPullRequests = exports.getRepo = exports.testConnection = exports.github = void 0;
30+
const dotenv_1 = __importDefault(require("dotenv"));
31+
const core = __importStar(require("@actions/core"));
32+
const rest_1 = require("@octokit/rest");
33+
dotenv_1.default.config();
34+
const token = core.getInput('GITHUB_TOKEN') || process.env.GITHUB_TOKEN;
35+
exports.github = new rest_1.Octokit({
36+
auth: token,
37+
});
38+
// We'll use this to test a connection to GitHub
39+
const testConnection = async () => {
40+
try {
41+
const { data } = await exports.github.repos.listForAuthenticatedUser();
42+
console.log(`🚀 Connection to GH established`);
43+
return true;
44+
}
45+
catch (error) {
46+
console.error(error);
47+
return false;
48+
}
49+
};
50+
exports.testConnection = testConnection;
51+
// If we can get a connection, we can get access to a repo, or we'll return null because
52+
const getRepo = async (owner, repo) => {
53+
try {
54+
const { data } = await exports.github.repos.get({ owner, repo });
55+
console.log(`🐕 Fetched the repo`);
56+
return data;
57+
}
58+
catch (error) {
59+
console.error(error);
60+
return null;
61+
}
62+
};
63+
exports.getRepo = getRepo;
64+
// If we can get a repo, we can get all the PRs associated with it
65+
const getPullRequests = async (owner, repo) => {
66+
try {
67+
const { data } = await exports.github.pulls.list({ owner, repo });
68+
console.log(`🐕 Fetched all PRs`);
69+
return data;
70+
}
71+
catch (error) {
72+
console.error(error);
73+
return null;
74+
}
75+
};
76+
exports.getPullRequests = getPullRequests;
77+
// We should be able to get a PR by its number
78+
const getSinglePR = async (owner, repo, prNumber) => {
79+
try {
80+
const { data } = await exports.github.pulls.get({ owner, repo, pull_number: prNumber });
81+
console.log(`✅ Got PR #${prNumber}`);
82+
return data;
83+
}
84+
catch (error) {
85+
console.error(error);
86+
return null;
87+
}
88+
};
89+
exports.getSinglePR = getSinglePR;
90+
// If we can get a PR, we can parse the description and isolate the assertion using the comments
91+
const getAssertion = async (description) => {
92+
// find everything in between <!-- DX:Assertion-start --> and <!-- DX:Assertion-end -->
93+
const regex = /<!-- DX:Assertion-start -->([\s\S]*?)<!-- DX:Assertion-end -->/g;
94+
const assertion = regex.exec(description);
95+
if (assertion) {
96+
console.log(`✅ Got assertion: ${assertion[1]}`);
97+
return assertion[1];
98+
}
99+
return null;
100+
};
101+
exports.getAssertion = getAssertion;
102+
// If we have a diff_url we can get the diff
103+
const getDiff = async (prNumber) => {
104+
const { data: diff } = await exports.github.pulls.get({
105+
owner: 'hasura',
106+
repo: 'v3-docs',
107+
pull_number: prNumber,
108+
mediaType: {
109+
format: 'diff',
110+
},
111+
});
112+
// We'll have to convert the diff to a string, then we can return it
113+
const diffString = diff.toString();
114+
console.log(`✅ Got diff for PR #${prNumber}`);
115+
return diffString;
116+
};
117+
exports.getDiff = getDiff;
118+
// If we have the diff, we can determine which files were changed
119+
const getChangedFiles = (diff) => {
120+
const fileLines = diff.split('\n').filter((line) => line.startsWith('diff --git'));
121+
const changedFiles = fileLines
122+
.map((line) => {
123+
const paths = line.split(' ').slice(2);
124+
return paths.map((path) => path.replace('a/', '').replace('b/', ''));
125+
})
126+
.flat();
127+
console.log(`✅ Found ${changedFiles.length} affected files`);
128+
return [...new Set(changedFiles)];
129+
};
130+
exports.getChangedFiles = getChangedFiles;
131+
// We'll also need to get the whole file using the files changed from
132+
async function getFileContent(path) {
133+
let content = '';
134+
// loop over the array of files
135+
for (let i = 0; i < path.length; i++) {
136+
// get the file content
137+
const { data } = await exports.github.repos.getContent({
138+
owner: 'hasura',
139+
repo: 'v3-docs',
140+
path: path[i],
141+
});
142+
// decode the file content
143+
const decodedContent = Buffer.from(data.content, 'base64').toString();
144+
// add the decoded content to the content string
145+
content += decodedContent;
146+
}
147+
console.log(`✅ Got file(s) contents`);
148+
return content;
149+
}
150+
exports.getFileContent = getFileContent;

0 commit comments

Comments
 (0)