Skip to content

Commit 37df4d8

Browse files
philimon-resetphilimon-reset
committed
split l10n reporting changes to another PR
1 parent 2c04c9b commit 37df4d8

File tree

11 files changed

+740
-564
lines changed

11 files changed

+740
-564
lines changed

‎.github/workflows/check-beta.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ jobs:
1414
outputs:
1515
win_reportable: ${{ steps.reportable.outputs.win }}
1616
mac_reportable: ${{ steps.reportable.outputs.mac }}
17+
mac_l10n_reportable: ${{ steps.reportable.outputs.mac_l10n }}
1718
steps:
1819
- name: Checkout repository
1920
uses: actions/checkout@v4
@@ -29,6 +30,16 @@ jobs:
2930
pipenv install
3031
echo win=$(pipenv run python -c 'from modules import testrail_integration as tri; print(tri.reportable("Windows"))') >> "$GITHUB_OUTPUT"
3132
echo mac=$(pipenv run python -c 'from modules import testrail_integration as tri; print(tri.reportable("Darwin"))') >> "$GITHUB_OUTPUT"
33+
- name: Check if the l10n run is reportable
34+
id: l10n-reportable
35+
env:
36+
TESTRAIL_REPORT: true
37+
FX_L10N: true
38+
TESTRAIL_BASE_URL: ${{ secrets.TESTRAIL_BASE_URL }}
39+
TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
40+
TESTRAIL_USERNAME: ${{ secrets.TESTRAIL_USERNAME }}
41+
run: echo mac_l10n=$(pipenv run python -c 'from modules import testrail_integration as tri; print(tri.reportable("Darwin"))') >> "$GITHUB_OUTPUT"
42+
3243
Run-Win-Smoke:
3344
needs: Check-Beta-Version
3445
if: ${{ needs.Check-Beta-Version.outputs.win_reportable == 'True' }}
@@ -44,3 +55,11 @@ jobs:
4455
with:
4556
job_to_run: Smoke-MacOS
4657
secrets: inherit
58+
59+
Run-L10N-Mac-Smoke:
60+
needs: Check-Beta-Version
61+
if: ${{ needs.Check-Beta-Version.outputs.mac_l10n_reportable == 'True' }}
62+
uses: ./.github/workflows/l10n_smoke.yml
63+
with:
64+
job_to_run: L10N-MacOS
65+
secrets: inherit

‎.github/workflows/l10n.yml

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
---
2+
name: L10N Test Execution
3+
4+
run-name: ${{ github.actor }} is running l10n tests
5+
on:
6+
pull_request:
7+
workflow_call:
8+
inputs:
9+
channel:
10+
description: "Channel to run tests against"
11+
default: "beta"
12+
type: string
13+
required: false
14+
job_to_run:
15+
required: true
16+
type: string
17+
workflow_dispatch:
18+
inputs:
19+
mac_installer_link:
20+
description: 'The link to the macOS installer for the Fx under test'
21+
required: false
22+
type: string
23+
env:
24+
FX_CHANNEL: ${{ inputs.channel }}
25+
TESTRAIL_BASE_URL: ${{ secrets.TESTRAIL_BASE_URL }}
26+
TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
27+
TESTRAIL_USERNAME: ${{ secrets.TESTRAIL_USERNAME }}
28+
SVC_ACCT_DECRYPT: ${{ secrets.SVC_ACCT_DECRYPT }}
29+
FX_L10N: 'true'
30+
31+
jobs:
32+
L10N-MacOS:
33+
if: ${{ inputs.job_to_run == 'L10N-MacOS' || github.event_name == 'pull_request' || inputs.mac_installer_link }}
34+
runs-on: macos-latest
35+
steps:
36+
- name: Create app token
37+
uses: actions/create-github-app-token@v1
38+
id: app-token
39+
with:
40+
app-id: ${{ secrets.BOT_CLIENT_ID }}
41+
private-key: ${{ secrets.BOT_PRIVATE_KEY }}
42+
- name: Checkout repository
43+
uses: actions/checkout@v4
44+
with:
45+
token: ${{ steps.app-token.outputs.token }}
46+
- name: Set up Python 3.11
47+
uses: actions/setup-python@v5
48+
with:
49+
python-version: "3.11"
50+
- name: Check test case numbers
51+
run: |
52+
python check_test_cases.py
53+
exit $?
54+
echo "Triggered by event: ${{ github.event_name }}"
55+
- name: Set Environment (Scheduled Beta)
56+
if: ${{ inputs.job_to_run == 'L10N-MacOS' }}
57+
run: |
58+
echo "TESTRAIL_REPORT='true'" >> "$GITHUB_ENV";
59+
echo "Running report for most recent Beta on MacOS";
60+
- name: Set Environment (Manual)
61+
if: ${{ inputs.mac_installer_link }}
62+
run: |
63+
echo "MANUAL='true'" >> "$GITHUB_ENV";
64+
echo "Running l10n tests on supplied executable";
65+
- name: Install dependencies
66+
run: |
67+
mkdir -p artifacts/;
68+
pip3 install 'pipenv==2023.11.15';
69+
pip3 install 'ruff>=0.4.8,<0.5';
70+
rm ./pyproject.toml;
71+
mv ./ci_pyproject.toml ./pyproject.toml;
72+
pipenv install
73+
- name: Install Fx
74+
id: setup
75+
env:
76+
MANUAL_DOWNLOAD_LINK: ${{ inputs.mac_installer_link }}
77+
run:
78+
echo app_name=$(bash ./collect_executables.sh | xargs -0 ./parse_executables.sh) >> "$GITHUB_OUTPUT"
79+
- name: Run L10N Tests in MacOS
80+
if: steps.setup.conclusion == 'success'
81+
env:
82+
FX_EXECUTABLE: /Volumes/${{ steps.setup.outputs.app_name }}/${{ steps.setup.outputs.app_name }}.app/Contents/MacOS/firefox
83+
run: |
84+
"$FX_EXECUTABLE" --version
85+
echo "0" > TEST_EXIT_CODE
86+
pipenv run python check_l10n_test_cases.py
87+
while IFS= read -r line; do
88+
echo "Running tests for: $line"
89+
pipenv run python l10n_CM/run_l10n.py --fx-executable="$FX_EXECUTABLE" $line || SCRIPT_EXIT_CODE=$?
90+
done < selected_l10n_mappings
91+
mv artifacts artifacts-mac || true
92+
EXIT_CODE=$(cat TEST_EXIT_CODE)
93+
if [ $EXIT_CODE != 0 ]; then
94+
exit $EXIT_CODE
95+
fi
96+
exit $SCRIPT_EXIT_CODE
97+
- name: Run L10N Tests in MacOS (Headed)
98+
if: steps.setup.conclusion == 'success' && always()
99+
env:
100+
FX_EXECUTABLE: /Volumes/${{ steps.setup.outputs.app_name }}/${{ steps.setup.outputs.app_name }}.app/Contents/MacOS/firefox
101+
REPORTABLE: ${{ env.TESTRAIL_REPORT == 'true' }}
102+
run: |
103+
mv ./ci_l10n_pyproject_headed.toml ./pyproject.toml;
104+
echo "0" > TEST_EXIT_CODE
105+
pipenv run python check_l10n_test_cases.py
106+
while IFS= read -r line; do
107+
echo "Running tests for: $line"
108+
pipenv run python l10n_CM/run_l10n.py --fx-executable="$FX_EXECUTABLE" $line || SCRIPT_EXIT_CODE=$?
109+
done < selected_l10n_mappings
110+
mv -n artifacts/* artifacts-mac/ || true
111+
EXIT_CODE=$(cat TEST_EXIT_CODE)
112+
if [ $EXIT_CODE != 0 ]; then
113+
exit $EXIT_CODE
114+
fi
115+
exit $SCRIPT_EXIT_CODE
116+
- name: Upload artifacts
117+
if: ${{ always() && github.event_name == 'pull_request' }}
118+
uses: actions/upload-artifact@v4
119+
with:
120+
name: artifacts-mac
121+
path: artifacts-mac
122+
Use-Artifacts:
123+
if: ${{ github.event_name != 'workflow_dispatch' }}
124+
runs-on: ubuntu-latest
125+
needs:
126+
- L10N-MacOS
127+
steps:
128+
- name: Checkout repository
129+
uses: actions/checkout@v4
130+
- name: Set up Python 3.11
131+
uses: actions/setup-python@v5
132+
with:
133+
python-version: "3.11"
134+
- name: Install dependencies
135+
run: |
136+
pip3 install 'pipenv==2023.11.15'
137+
pipenv install
138+
- name: Download MacOS artifact
139+
uses: actions/download-artifact@v4
140+
with:
141+
name: artifacts-mac
142+
path: artifacts-mac
143+
- name: List downloaded MacOS files
144+
run: ls artifacts-mac/
145+
- name: Run script with secret
146+
env:
147+
SLACK_KEY: ${{ secrets.SLACK_KEY }}
148+
GCP_CREDENTIAL: ${{ secrets.GCP_CREDENTIAL }}
149+
GITHUB_SHA: ${{ github.sha }}
150+
GITHUB_RUN_ID: ${{ github.run_id }}
151+
GITHUB_ACTOR: ${{ github.actor }}
152+
GITHUB_REF_NAME: ${{ github.ref_name }}
153+
run: |
154+
pipenv run python notifier.py

‎check_l10n_test_cases.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import json
2+
import os
3+
import sys
4+
from collections import defaultdict
5+
6+
OUTPUT_FILE = "selected_l10n_mappings"
7+
8+
9+
def valid_l10n_mappings():
10+
mapping = defaultdict(set)
11+
region_paths = [d for d in os.listdir("./l10n_CM/region/")]
12+
for region_path in region_paths:
13+
if region_path != "Unified.json":
14+
region = region_path.split(".")[0]
15+
with open(f"./l10n_CM/region/{region_path}", "r+") as f:
16+
region_file = json.load(f)
17+
if region_file.get("sites"):
18+
for site in region_file.get("sites"):
19+
mapping[site].add(region)
20+
return mapping
21+
22+
23+
if __name__ == "__main__":
24+
l10n_mappings = valid_l10n_mappings()
25+
with open(OUTPUT_FILE, "w") as file:
26+
pass # File is created or cleared
27+
for site, regions in l10n_mappings.items():
28+
with open(OUTPUT_FILE, "a+") as fh:
29+
fh.write(f"{site} {' '.join(regions)}\n")
30+
sys.exit(0)

‎ci_l10n_pyproject_headed.toml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
[tool.pytest.ini_options]
2+
generate_report_on_test = true
3+
log_cli = true
4+
log_cli_level = "warn"
5+
markers = [
6+
"audio: test is reliant on audio",
7+
"headed: test must run in headed mode (e.g. uses pynput)",
8+
"incident: incident smoke tests",
9+
"unstable: temporary mark for unstable tests",
10+
"slow: test is clocked at more than 30s on modern machines",
11+
"ci: basic tests to run in ci",
12+
"locale_de: tests run in DE locale versions",
13+
"locale_fr: tests run in FR locale versions",
14+
"locale_gb: tests run in GB locale versions",
15+
"noxvfb: tests that should not run in xvfb sessions"
16+
]
17+
18+
addopts = "-vs --ci -n auto --json-report --json-report-file artifacts/report_headed.json --reruns 3 --reruns-delay 2 --html=artifacts/report_headed.html"
19+
20+
[tool.ruff]
21+
target-version = "py310"

‎conftest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ def pytest_configure(config):
329329
raise OSError("Could not find TestRail credentials")
330330

331331
if not tri.reportable():
332-
pytest.exit("Test run is not reportable. Exiting.")
332+
pytest.exit("Test run is not reportable. Exiting.", 2)
333333

334334

335335
def pytest_sessionfinish(session):

‎l10n_CM/region/CA.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"region": "CA",
3+
"sites": ["amazon", "walmart", "demo"],
34
"tests": [
45
]
56
}

‎l10n_CM/region/DE.json

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
{
2-
"region": "DE",
3-
"tests": [
4-
]
2+
"region": "DE",
3+
"sites": [
4+
"demo"
5+
],
6+
"tests": [
7+
]
58
}

‎l10n_CM/region/FR.json

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
{
2-
"region": "FR",
3-
"tests": [
4-
]
2+
"region": "FR",
3+
"sites": [
4+
"demo"
5+
],
6+
"tests": [
7+
]
58
}

‎l10n_CM/region/US.json

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
{
2-
"region": "US",
3-
"tests": []
2+
"region": "US",
3+
"sites": [
4+
"amazon",
5+
"walmart",
6+
"demo"
7+
],
8+
"tests": []
49
}

‎l10n_CM/run_l10n.py

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,15 @@
1010
import requests
1111

1212
current_dir = os.path.dirname(__file__)
13-
valid_flags = {"--run-headless", "-n", "--reruns"}
13+
valid_flags = {"--run-headless", "-n", "--reruns", "--fx-executable", "--ci"}
1414
flag_with_parameter = {"-n", "--reruns"}
1515
valid_region = {"US", "CA", "DE", "FR"}
16-
valid_sites = {"demo", "amazon", "walmart", "mediamarkt"}
16+
valid_sites = {"demo", "amazon", "walmart"}
1717
live_sites = []
1818

1919
LOCALHOST = "127.0.0.1"
2020
PORT = 8080
21+
os.environ["TEST_EXIT_CODE"] = "0"
2122

2223

2324
class MyHttpRequestHandler(SimpleHTTPRequestHandler):
@@ -88,7 +89,13 @@ def run_tests(reg, site, flg, all_tests):
8889
else:
8990
logging.info(f"{reg} region on {site} site has no tests.")
9091
except subprocess.CalledProcessError as e:
91-
logging.warning(f"Test run failed. {e}")
92+
logging.warning(f"Test run failed with exit code: {e.returncode}")
93+
# true failure instead of run not being reportable.
94+
if e.returncode != 2:
95+
if os.environ.get("TEST_EXIT_CODE") == "0":
96+
with open("TEST_EXIT_CODE", "w") as f:
97+
f.write(str(e.returncode))
98+
os.environ["TEST_EXIT_CODE"] = str(e.returncode)
9299

93100

94101
def get_region_tests(test_region: str) -> list[str]:
@@ -128,8 +135,12 @@ def get_flags_and_sanitize(flags_arguments: list[str]) -> list[str]:
128135
"""
129136
# add workers and rerun flaky failed tests.
130137
flg = []
138+
expanded_args = [
139+
flag.split() if " " in flag else [flag] for flag in flags_arguments
140+
]
141+
flags_arguments[:] = sum(expanded_args, [])
131142
for arg in flags_arguments[:]:
132-
if arg in valid_flags:
143+
if arg.split("=")[0] in valid_flags:
133144
if arg in flag_with_parameter:
134145
try:
135146
i = flags_arguments.index(arg)
@@ -199,14 +210,14 @@ def run_unified(regions, unified_flags):
199210
logging.info(f"Running Unified Tests for {valid_region} Regions.")
200211
run_unified(list(valid_region), flags)
201212
else:
202-
# run on given region and sites.
213+
# run on a given region and sites.
203214
logging.info(f"Running Unified Tests for {arguments} Regions.")
204215
run_unified(arguments, flags)
205216
for site in live_sites:
206-
# for a given site, run all region specific tests.
217+
# for a given site, run all region-specific tests.
207218
for region in arguments:
208219
tests = get_region_tests(region)
209-
# Check if field mapping json file is present, pass test region if it isn't
220+
# Check if a field-mapping JSON file is present, pass a test region if it isn't
210221
json_path = os.path.join(current_dir, "constants", site, region)
211222
logging.info(f"Running Specific Tests for {region}.")
212223
# If the live_site is 'demo', skip starting the server

0 commit comments

Comments
 (0)