Skip to content

Commit 5e2cf3c

Browse files
Hamza417Hamza417
committed
conflate: scraper action files into one
1 parent 628c99c commit 5e2cf3c

File tree

5 files changed

+219
-268
lines changed

5 files changed

+219
-268
lines changed

‎.github/workflows/copy_uad_file.yml‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎.github/workflows/data_sync.yml‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
name: Data Sync
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
# Every 3 days at midnight UTC
7+
- cron: "0 0 */3 * *"
8+
9+
permissions:
10+
contents: write
11+
12+
concurrency:
13+
group: data-sync-${{ github.ref }}
14+
cancel-in-progress: false
15+
16+
jobs:
17+
sync_uad_lists:
18+
name: Sync UAD lists
19+
runs-on: ubuntu-latest
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v4
23+
with:
24+
fetch-depth: 0
25+
ref: master
26+
27+
- name: Checkout UAD source repository
28+
uses: actions/checkout@v4
29+
with:
30+
repository: Universal-Debloater-Alliance/universal-android-debloater-next-generation
31+
path: source_repo
32+
33+
- name: Copy file
34+
run: |
35+
if [ ! -f source_repo/resources/assets/uad_lists.json ]; then
36+
echo "Source file does not exist"
37+
exit 1
38+
fi
39+
40+
mkdir -p app/src/github/resources
41+
cp source_repo/resources/assets/uad_lists.json app/src/github/resources/uad_lists.json
42+
43+
- name: Commit and push (UAD lists)
44+
run: |
45+
set -e
46+
git add app/src/github/resources/uad_lists.json
47+
if [ -z "$(git status --porcelain app/src/github/resources/uad_lists.json)" ]; then
48+
echo "No changes to commit."
49+
exit 0
50+
fi
51+
52+
git config user.email "actions@github.com"
53+
git config user.name "GitHub Actions"
54+
git commit -m "Update UAD Lists file from source repository"
55+
56+
git fetch origin master
57+
git push --force-with-lease origin HEAD:master
58+
59+
update_exodus_trackers:
60+
name: Update Exodus Trackers
61+
runs-on: ubuntu-latest
62+
steps:
63+
- name: Checkout repository
64+
uses: actions/checkout@v4
65+
with:
66+
fetch-depth: 0
67+
ref: master
68+
69+
- name: Download JSON file
70+
run: |
71+
wget -O trackers.json https://reports.exodus-privacy.eu.org/api/trackers
72+
73+
- name: Format JSON
74+
run: |
75+
python - <<'EOF'
76+
import json
77+
78+
json_file = 'trackers.json'
79+
80+
with open(json_file, 'r', encoding='utf8') as file:
81+
payload = json.load(file)
82+
83+
with open(json_file, 'w', encoding='utf8') as file:
84+
file.write(json.dumps(payload, indent=4, sort_keys=True))
85+
EOF
86+
87+
- name: Move Tracker JSON to Resources
88+
run: |
89+
mkdir -p ./app/src/main/resources/
90+
mv trackers.json ./app/src/main/resources/
91+
92+
- name: Commit and push (Exodus trackers)
93+
run: |
94+
set -e
95+
git add app/src/main/resources/trackers.json
96+
if [ -z "$(git status --porcelain app/src/main/resources/trackers.json)" ]; then
97+
echo "No changes to commit."
98+
exit 0
99+
fi
100+
101+
git config user.email "actions@github.com"
102+
git config user.name "GitHub Actions"
103+
git commit -m "Automated Exodus JSON data update"
104+
105+
git fetch origin master
106+
git push --force-with-lease origin HEAD:master
107+
108+
update_exodus_etip_trackers:
109+
name: Update Exodus ETIP Trackers
110+
runs-on: ubuntu-latest
111+
steps:
112+
- name: Checkout repository
113+
uses: actions/checkout@v4
114+
with:
115+
fetch-depth: 0
116+
ref: master
117+
118+
- name: Download JSON file
119+
run: |
120+
wget -O etip_trackers.json https://etip.exodus-privacy.eu.org/api/trackers/
121+
122+
- name: Format JSON
123+
run: |
124+
python - <<'EOF'
125+
import json
126+
127+
json_file = 'etip_trackers.json'
128+
129+
with open(json_file, 'r', encoding='utf8') as file:
130+
payload = json.load(file)
131+
132+
with open(json_file, 'w', encoding='utf8') as file:
133+
file.write(json.dumps(payload, indent=4, sort_keys=True))
134+
EOF
135+
136+
- name: Move Tracker JSON to Resources
137+
run: |
138+
mkdir -p ./app/src/main/resources/
139+
mv etip_trackers.json ./app/src/main/resources/
140+
141+
- name: Commit and push (Exodus ETIP trackers)
142+
run: |
143+
set -e
144+
git add app/src/main/resources/etip_trackers.json
145+
if [ -z "$(git status --porcelain app/src/main/resources/etip_trackers.json)" ]; then
146+
echo "No changes to commit."
147+
exit 0
148+
fi
149+
150+
git config user.email "actions@github.com"
151+
git config user.name "GitHub Actions"
152+
git commit -m "Automated Exodus ETIP JSON data update"
153+
154+
git fetch origin master
155+
git push --force-with-lease origin HEAD:master
156+
157+
update_fdroid_repo_data:
158+
name: Update F-Droid Repo Package Versions
159+
runs-on: ubuntu-latest
160+
steps:
161+
- name: Checkout repository
162+
uses: actions/checkout@v4
163+
with:
164+
fetch-depth: 0
165+
ref: master
166+
167+
- name: Download XML files
168+
run: |
169+
mkdir -p xml_files
170+
wget -O xml_files/fdroid_index.xml https://f-droid.org/repo/index.xml
171+
wget -O xml_files/izzyondroid_index.xml https://apt.izzysoft.de/fdroid/repo/index.xml
172+
173+
- name: Parse XML and create Android resource file
174+
run: |
175+
cd xml_files
176+
echo '<?xml version="1.0" encoding="utf-8"?>' > package_versions.xml
177+
echo '<resources>' >> package_versions.xml
178+
python - <<'EOF' >> package_versions.xml
179+
import xml.etree.ElementTree as ET
180+
181+
def parse_and_write_xml(file_path):
182+
tree = ET.parse(file_path)
183+
root = tree.getroot()
184+
for application in root.findall('.//application'):
185+
app_id = application.get('id')
186+
license_ = application.find('.//license').text if application.find('.//license') is not None else ''
187+
if app_id is None:
188+
continue
189+
print(f' <string name="{app_id}" translatable="false">{license_}</string>')
190+
191+
parse_and_write_xml('fdroid_index.xml')
192+
parse_and_write_xml('izzyondroid_index.xml')
193+
EOF
194+
echo '</resources>' >> package_versions.xml
195+
196+
- name: Move Android resource file
197+
run: |
198+
mkdir -p ./app/src/main/res/xml/
199+
mv xml_files/package_versions.xml ./app/src/main/res/xml/
200+
201+
- name: Cleanup
202+
run: |
203+
rm -rf xml_files
204+
205+
- name: Commit and push (F-Droid repo data)
206+
run: |
207+
set -e
208+
git add app/src/main/res/xml/package_versions.xml
209+
if [ -z "$(git status --porcelain app/src/main/res/xml/package_versions.xml)" ]; then
210+
echo "No changes to commit."
211+
exit 0
212+
fi
213+
214+
git config user.email "actions@github.com"
215+
git config user.name "GitHub Actions"
216+
git commit -m "Automated F-Droid repo update"
217+
218+
git fetch origin master
219+
git push --force-with-lease origin HEAD:master

‎.github/workflows/exodus_etip_json_data.yml‎

Lines changed: 0 additions & 72 deletions
This file was deleted.

0 commit comments

Comments
 (0)