Skip to content

Commit 4eec009

Browse files
dpalmeirodpalmeiro
committed
Populate with data and workflows
1 parent ec5abb3 commit 4eec009

14 files changed

+16681
-0
lines changed

‎.github/workflows/download-bugzilla-data.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Download and Merge Bugzilla Data
2+
3+
on:
4+
schedule:
5+
- cron: '00 00 * * *' # Runs every day at 12:00AM (midnight)
6+
workflow_dispatch:
7+
8+
jobs:
9+
download-and-merge-bugzilla-data:
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- name: Checkout repository
14+
uses: actions/checkout@v3
15+
with:
16+
ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
17+
18+
- name: Download Bugzilla data (medium impact)
19+
run: |
20+
curl -o bugzilla-data-medium.json "https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary,status,component,creation_time,last_change_time,keywords,priority,severity,op_sys&bug_status=__open__&classification=Client%20Software&classification=Developer%20Infrastructure&classification=Components&classification=Server%20Software&classification=Other&f1=cf_performance_impact&limit=0&o1=equals&v1=medium"
21+
22+
- name: Download Bugzilla data (low impact)
23+
run: |
24+
curl -o bugzilla-data-low.json "https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary,status,component,creation_time,last_change_time,keywords,priority,severity,op_sys&bug_status=__open__&classification=Client%20Software&classification=Developer%20Infrastructure&classification=Components&classification=Server%20Software&classification=Other&f1=cf_performance_impact&limit=0&o1=equals&v1=low"
25+
26+
- name: Download Bugzilla data (high impact)
27+
run: |
28+
curl -o bugzilla-data-high.json "https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary,status,component,creation_time,last_change_time,keywords,priority,severity,op_sys&bug_status=__open__&classification=Client%20Software&classification=Developer%20Infrastructure&classification=Components&classification=Server%20Software&classification=Other&f1=cf_performance_impact&limit=0&o1=equals&v1=high"
29+
30+
- name: Download Bugzilla data (untriaged)
31+
run: |
32+
curl -o bugzilla-data-untriaged.json "https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary,status,component,creation_time,last_change_time,keywords,priority,severity,op_sys&f1=OP&f10=OP&f11=cf_performance_impact&f12=flagtypes.name&f13=CP&f2=cf_performance_impact&f3=CP&f4=OP&f5=product&f6=component&f7=keywords&f8=cf_performance_impact&f9=CP&j_top=OR&o11=equals&o12=notsubstring&o2=equals&o5=equals&o6=equals&o7=notsubstring&o8=isempty&resolution=---&v11=pending-needinfo&v12=needinfo&v2=%3F&v5=Core&v6=Performance&v7=meta"
33+
34+
- name: Download Bugzilla data (needinfo)
35+
run: |
36+
curl -o bugzilla-data-needinfo.json "https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary,status,component,creation_time,last_change_time,keywords,priority,severity,op_sys&bug_status=__open__&classification=Client%20Software&classification=Developer%20Infrastructure&classification=Components&classification=Server%20Software&classification=Other&f1=cf_performance_impact&limit=0&o1=equals&v1=pending-needinfo"
37+
38+
- name: Download Bugzilla data (regressions)
39+
run: |
40+
curl -o bugzilla-data-regressions.json "https://bugzilla.mozilla.org/rest/bug?include_fields=id,summary,status,component,creation_time,last_change_time,keywords,priority,severity,op_sys&classification=Client%20Software&classification=Developer%20Infrastructure&classification=Components&classification=Server%20Software&classification=Other&f1=keywords&f2=keywords&o1=substring&o2=anywordssubstr&resolution=---&v1=regression&v2=perf-alert"
41+
42+
43+
- name: Merge JSON files
44+
run: |
45+
jq -n '{high: $high[], medium: $medium[], low: $low[], untriaged: $untriaged[], needinfo: $needinfo[], regressions: $regressions[]}' \
46+
--slurpfile high bugzilla-data-high.json \
47+
--slurpfile medium bugzilla-data-medium.json \
48+
--slurpfile low bugzilla-data-low.json \
49+
--slurpfile untriaged bugzilla-data-untriaged.json \
50+
--slurpfile needinfo bugzilla-data-needinfo.json \
51+
--slurpfile regressions bugzilla-data-regressions.json > bugzilla-data-all.json
52+
53+
- name: Commit and push changes
54+
run: |
55+
git config --global user.name 'github-actions[bot]'
56+
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
57+
git add bugzilla-data-medium.json bugzilla-data-low.json bugzilla-data-high.json bugzilla-data-untriaged.json bugzilla-data-needinfo.json bugzilla-data-all.json bugzilla-data-regressions.json
58+
git commit -m "Update Bugzilla data"
59+
git push

‎.github/workflows/update_js_metrics.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Update JS metrics
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: '45 23 * * *' # Runs daily at 11:45 PM
7+
8+
jobs:
9+
merge-json:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout Repository
13+
uses: actions/checkout@v2
14+
with:
15+
ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
16+
17+
- name: Set up Python
18+
uses: actions/setup-python@v2
19+
with:
20+
python-version: '3.x'
21+
22+
- name: Install Required Python Packages
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install requests pandas
26+
27+
- name: Download and Merge JSON Files
28+
run: |
29+
python <<EOF
30+
import requests
31+
import pandas as pd
32+
33+
urls = [
34+
"https://sql.telemetry.mozilla.org/api/queries/101785/results.json?api_key=5ynVkkPtGWDrzRkaj02GkFifGILHtqoqkngK6Nkw",
35+
"https://sql.telemetry.mozilla.org/api/queries/101807/results.json?api_key=mq9Z2VNBaphOVvt94YtUJVBGxxA6NpPrnG7dtHrq",
36+
"https://sql.telemetry.mozilla.org/api/queries/101771/results.json?api_key=a852JNzIiOv9Wu53e9m4FYDplIfYnNOc0cFJfDa3"
37+
]
38+
39+
all_data = []
40+
for url in urls:
41+
response = requests.get(url)
42+
data = response.json()
43+
rows = data.get("query_result", {}).get("data", {}).get("rows", [])
44+
all_data.extend(rows)
45+
46+
try:
47+
with open('js-metrics-data.json', 'r') as file:
48+
existing_data = pd.read_json(file)
49+
all_data.extend(existing_data.to_dict(orient='records'))
50+
except FileNotFoundError:
51+
raise FileNotFoundError("Error: 'js-metrics-data.json' not found.")
52+
53+
df = pd.DataFrame(all_data)
54+
df['date'] = pd.to_datetime(df['date']).dt.strftime('%Y-%m-%d')
55+
df = df.drop_duplicates(subset=['date', 'channel', 'os', 'metric'], keep='last')
56+
df = df.sort_values(by='date')
57+
df.to_json('js-metrics-data.json', orient='records')
58+
EOF
59+
60+
- name: Commit and Push Changes
61+
run: |
62+
git config --local user.email "[email protected]"
63+
git config --local user.name "GitHub Actions"
64+
git add js-metrics-data.json
65+
git commit -m "Update js-metrics-data.json with latest data"
66+
git push

‎.github/workflows/update_networking_metrics.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Update Networking metrics
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: '15 00 * * *' # Runs daily at 12:15 AM
7+
8+
jobs:
9+
merge-json:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout Repository
13+
uses: actions/checkout@v2
14+
with:
15+
ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
16+
17+
- name: Set up Python
18+
uses: actions/setup-python@v2
19+
with:
20+
python-version: '3.x'
21+
22+
- name: Install Required Python Packages
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install requests pandas
26+
27+
- name: Download and Merge JSON Files
28+
run: |
29+
python <<EOF
30+
import requests
31+
import pandas as pd
32+
33+
urls = [
34+
"https://sql.telemetry.mozilla.org/api/queries/102460/results.json?api_key=KEosvxWyFEbgQmYd2ppSc89Kig1t43zRwmQNzF03",
35+
"https://sql.telemetry.mozilla.org/api/queries/102484/results.json?api_key=CLE7ocT9aw8Q9bcCufRH0oizs3ob4CmP858Z3Cds",
36+
"https://sql.telemetry.mozilla.org/api/queries/102485/results.json?api_key=RCmKp0XiSnkyhhuqPXLK6TKxQKB3Y1qhVHdl1ADh"
37+
]
38+
39+
all_data = []
40+
for url in urls:
41+
response = requests.get(url)
42+
data = response.json()
43+
rows = data.get("query_result", {}).get("data", {}).get("rows", [])
44+
all_data.extend(rows)
45+
46+
try:
47+
with open('networking-metrics-data.json', 'r') as file:
48+
existing_data = pd.read_json(file)
49+
all_data.extend(existing_data.to_dict(orient='records'))
50+
except FileNotFoundError:
51+
raise FileNotFoundError("Error: 'networking-metrics-data.json' not found.")
52+
53+
df = pd.DataFrame(all_data)
54+
df['date'] = pd.to_datetime(df['date']).dt.strftime('%Y-%m-%d')
55+
df = df.drop_duplicates(subset=['date', 'channel', 'os', 'metric'], keep='last')
56+
df = df.sort_values(by='date')
57+
df.to_json('networking-metrics-data.json', orient='records')
58+
EOF
59+
60+
- name: Commit and Push Changes
61+
run: |
62+
git config --local user.email "[email protected]"
63+
git config --local user.name "GitHub Actions"
64+
git add networking-metrics-data.json
65+
git commit -m "Update networking-metrics-data.json with latest data"
66+
git push

‎.github/workflows/update_pageload_metrics.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Update Pageload metrics
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: '30 23 * * *' # Runs daily at 11:30 PM
7+
8+
jobs:
9+
merge-json:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout Repository
13+
uses: actions/checkout@v2
14+
with:
15+
ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
16+
17+
- name: Set up Python
18+
uses: actions/setup-python@v2
19+
with:
20+
python-version: '3.x'
21+
22+
- name: Install Required Python Packages
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install requests pandas
26+
27+
- name: Download and Merge JSON Files
28+
run: |
29+
python <<EOF
30+
import requests
31+
import pandas as pd
32+
33+
urls = [
34+
"https://sql.telemetry.mozilla.org/api/queries/101647/results.json?api_key=lTPLgqu2UNPA1XxCz0dbayVAN4rGjNvTAmLU5WBV",
35+
"https://sql.telemetry.mozilla.org/api/queries/101653/results.json?api_key=OtucZwTAEGY7SBgdLlgWtQPXvPQF4TaXGLzLUq8n",
36+
"https://sql.telemetry.mozilla.org/api/queries/101616/results.json?api_key=uc7V76QeZ3FT94y0GPeg8YZkdtaJTEKqNGUtaT19"
37+
]
38+
39+
all_data = []
40+
for url in urls:
41+
response = requests.get(url)
42+
data = response.json()
43+
rows = data.get("query_result", {}).get("data", {}).get("rows", [])
44+
all_data.extend(rows)
45+
46+
try:
47+
with open('pageload-metrics-data.json', 'r') as file:
48+
existing_data = pd.read_json(file)
49+
all_data.extend(existing_data.to_dict(orient='records'))
50+
except FileNotFoundError:
51+
raise FileNotFoundError("Error: 'pageload-metrics-data.json' not found.")
52+
53+
df = pd.DataFrame(all_data)
54+
df['date'] = pd.to_datetime(df['date']).dt.strftime('%Y-%m-%d')
55+
df = df.drop_duplicates(subset=['date', 'channel', 'os', 'metric'], keep='last')
56+
df = df.sort_values(by='date')
57+
df.to_json('pageload-metrics-data.json', orient='records')
58+
EOF
59+
60+
- name: Commit and Push Changes
61+
run: |
62+
git config --local user.email "[email protected]"
63+
git config --local user.name "GitHub Actions"
64+
git add pageload-metrics-data.json
65+
git commit -m "Update pageload-metrics-data.json with latest data"
66+
git push

0 commit comments

Comments
 (0)