Skip to content

Commit 8854a52

Browse files
EasyListHebrewEasyListHebrew
authored
Add dead domains detector
1 parent 3a35149 commit 8854a52

File tree

4 files changed

+808
-0
lines changed

4 files changed

+808
-0
lines changed

‎.github/workflows/dead-domains-check.yml‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
# Taken from easylistgermany
2+
name: Dead Domains Check
3+
4+
env:
5+
NODE_VERSION: 20
6+
7+
on:
8+
schedule:
9+
# Run the job on every Monday at 8:00 AM UTC
10+
- cron: '0 8 * * 1'
11+
push:
12+
branches:
13+
- master
14+
paths:
15+
- '.github/workflows/dead-domains-check.yml'
16+
workflow_dispatch:
17+
18+
jobs:
19+
dead-domains-check:
20+
name: Run dead domains check
21+
runs-on: ubuntu-latest
22+
steps:
23+
- name: Check out to repository
24+
uses: actions/checkout@v4
25+
26+
- name: Install pnpm
27+
uses: pnpm/action-setup@v4
28+
29+
- name: Set up Node.js
30+
uses: actions/setup-node@v4
31+
with:
32+
node-version: ${{ env.NODE_VERSION }}
33+
cache: pnpm
34+
35+
- name: Install dependencies
36+
run: pnpm install
37+
38+
- name: Export dead domains to a file
39+
run: pnpm dead-domains-linter --export dead-domains.txt
40+
41+
- name: Read dead domains from the file
42+
id: read-dead-domains
43+
uses: actions/github-script@v7
44+
with:
45+
script: |
46+
const { readFile } = require('fs').promises;
47+
48+
const deadDomains = (await readFile('dead-domains.txt', 'utf8'))
49+
.split(/\r?\n/)
50+
.filter(Boolean);
51+
52+
// Add warning to the console
53+
for (const domain of deadDomains) {
54+
core.warning(`Possible dead domain: ${domain}`);
55+
}
56+
57+
core.setOutput('has_dead_domains', deadDomains.length > 0 ? 'true' : 'false');
58+
core.setOutput('dead_domains', deadDomains);
59+
60+
- name: Close previous issue(s)
61+
uses: actions/github-script@v7
62+
with:
63+
script: |
64+
const title = 'Automated dead domains report';
65+
66+
// Close previous issues which have the same title and opened by github-actions[bot]
67+
const { data: issues } = await github.rest.issues.listForRepo({
68+
owner: context.repo.owner,
69+
repo: context.repo.repo,
70+
state: 'open',
71+
});
72+
73+
const previousIssues = issues.filter(issue => {
74+
const isBot = issue.user.login === 'github-actions[bot]';
75+
76+
return issue.title === title && isBot;
77+
});
78+
79+
for (const previousIssue of previousIssues) {
80+
await github.rest.issues.update({
81+
owner: context.repo.owner,
82+
repo: context.repo.repo,
83+
issue_number: previousIssue.number,
84+
state: 'closed',
85+
});
86+
87+
core.info(`Closed previous issue #${previousIssue.number}`);
88+
}
89+
90+
- name: Close previous pull request(s)
91+
uses: actions/github-script@v7
92+
with:
93+
script: |
94+
// Close previous pull requests which have the branch name 'fix/dead-domains-<number>'
95+
// and opened by github-actions[bot]
96+
const { data: pullRequests } = await github.rest.pulls.list({
97+
owner: context.repo.owner,
98+
repo: context.repo.repo,
99+
state: 'open',
100+
});
101+
102+
const previousPullRequests = pullRequests.filter(pullRequest => {
103+
const branchName = pullRequest.head.ref;
104+
const isAutomatedFix = branchName.startsWith('fix/dead-domains-');
105+
const isBot = pullRequest.user.login === 'github-actions[bot]';
106+
107+
return isAutomatedFix && isBot;
108+
});
109+
110+
for (const previousPullRequest of previousPullRequests) {
111+
await github.rest.pulls.update({
112+
owner: context.repo.owner,
113+
repo: context.repo.repo,
114+
pull_number: previousPullRequest.number,
115+
state: 'closed',
116+
});
117+
118+
core.info(`Closed previous pull request #${previousPullRequest.number}`);
119+
}
120+
121+
// Delete the branch of the closed pull requests
122+
for (const previousPullRequest of previousPullRequests) {
123+
await github.rest.git.deleteRef({
124+
owner: context.repo.owner,
125+
repo: context.repo.repo,
126+
ref: `heads/${previousPullRequest.head.ref}`,
127+
});
128+
129+
core.info(`Deleted branch ${previousPullRequest.head.ref}`);
130+
}
131+
132+
- name: Open an issue if there are dead domains
133+
if: steps.read-dead-domains.outputs.has_dead_domains == 'true'
134+
id: open-issue
135+
uses: actions/github-script@v7
136+
with:
137+
script: |
138+
const deadDomains = JSON.parse(${{ toJson(steps.read-dead-domains.outputs.dead_domains) }});
139+
const title = 'Automated dead domains report';
140+
141+
const { data: issue } = await github.rest.issues.create({
142+
owner: context.repo.owner,
143+
repo: context.repo.repo,
144+
title,
145+
body: [
146+
'The following domains are possibly dead:',
147+
'',
148+
'<details>',
149+
'<summary>Click to expand</summary>',
150+
'',
151+
`${deadDomains.map(domain => `- ${domain}`).join('\n')}`,
152+
'',
153+
'</details>',
154+
'',
155+
'Please note that this is an automated report and some low-traffic websites may be incorrectly marked as dead.',
156+
'For more information, see https://github.com/AdguardTeam/DeadDomainsLinter/blob/master/README.md',
157+
].join('\n'),
158+
labels: ['dead website'],
159+
});
160+
161+
core.setOutput('issue_number', issue.number);
162+
core.info(`Issue #${issue.number} opened (${title})`);
163+
164+
- name: Create a new branch for the automated fix
165+
if: steps.read-dead-domains.outputs.has_dead_domains == 'true'
166+
id: create-branch
167+
run: |
168+
branch_name="fix/dead-domains-${{ steps.open-issue.outputs.issue_number }}"
169+
git checkout -b $branch_name
170+
171+
echo "branch_name=$branch_name" >> $GITHUB_OUTPUT
172+
173+
- name: Perform the automated fix
174+
if: steps.read-dead-domains.outputs.has_dead_domains == 'true'
175+
run: |
176+
pnpm dead-domains-linter --auto --import dead-domains.txt
177+
178+
- name: Stage changes for the automated fix
179+
if: steps.read-dead-domains.outputs.has_dead_domains == 'true'
180+
run: |
181+
git config user.name "github-actions[bot]"
182+
git config user.email "github-actions[bot]@users.noreply.github.com"
183+
184+
# Add "dead-domains.txt" to the .gitignore if it's not there
185+
if ! grep -q "dead-domains.txt" .gitignore; then
186+
echo "dead-domains.txt" >> .gitignore
187+
fi
188+
189+
git add -A
190+
191+
- name: Commit and push the automated fix
192+
if: steps.read-dead-domains.outputs.has_dead_domains == 'true'
193+
run: |
194+
git commit -m "Automated dead domains fix (#${{ steps.open-issue.outputs.issue_number }})"
195+
git push --set-upstream origin ${{ steps.create-branch.outputs.branch_name }}
196+
197+
- name: Create a pull request for the automated fix
198+
if: steps.read-dead-domains.outputs.has_dead_domains == 'true'
199+
uses: actions/github-script@v7
200+
with:
201+
script: |
202+
const { data: pullRequest } = await github.rest.pulls.create({
203+
owner: context.repo.owner,
204+
repo: context.repo.repo,
205+
title: 'Automated dead domains fix',
206+
head: '${{ steps.create-branch.outputs.branch_name }}',
207+
base: context.ref.replace('refs/heads/', ''),
208+
body: [
209+
'This is an automated pull request to fix #${{ steps.open-issue.outputs.issue_number }}.',
210+
'',
211+
'Please note that this is an automated fix and some low-traffic websites may be incorrectly marked as dead.',
212+
'For more information, see https://github.com/AdguardTeam/DeadDomainsLinter/blob/master/README.md',
213+
].join('\n'),
214+
});
215+
216+
core.info(`Pull request #${pullRequest.number} opened`);
217+
218+
// Add labels to the pull request
219+
await github.rest.issues.addLabels({
220+
owner: context.repo.owner,
221+
repo: context.repo.repo,
222+
issue_number: pullRequest.number,
223+
labels: ['dead website'],
224+
});

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
/general.txt
22
/.project
33
.vscode
4+
node_modules
5+
dead-domains.txt

‎package.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"name": "easylistgermany",
3+
"devDependencies": {
4+
"@adguard/dead-domains-linter": "^1.0.19"
5+
},
6+
"packageManager": "pnpm@9.5.0+sha512.140036830124618d624a2187b50d04289d5a087f326c9edfc0ccd733d76c4f52c3a313d4fc148794a2a9d81553016004e6742e8cf850670268a7387fc220c903"
7+
}

0 commit comments

Comments
 (0)