Skip to content

QA Android UI Tests #36

QA Android UI Tests

QA Android UI Tests #36

name: QA Android UI Tests
on:
workflow_dispatch:
inputs:
appBuildNumber:
description: "AppBuildNumber. Use 'latest' or build number(e.g 71389)"
required: true
default: "latest"
type: string
isUpgrade:
description: "Upgrade test? If true, oldBuildNumber is REQUIRED."
required: true
default: false
type: boolean
oldBuildNumber:
description: "For upgrade runs: type the old build number here"
required: false
default: ""
type: string
enforceAppInstall:
description: "If you want to run with a lower version than currently on the device, set this value to true."
required: true
default: false
type: boolean
flavor:
description: "App flavor."
required: true
type: choice
options:
- internal release candidate
- internal beta
- staging compat
- experimental
- column-1
- column-2
- column-3
- debug
- fdroid
- production
default: internal release candidate
buildType:
description: "Build type"
required: true
type: choice
options:
- release
- debug
- compat
default: release
TAGS:
description: "Tags: '@regression' OR '@TC-8143'."
required: false
default: ""
type: string
branch:
description: "Branch"
required: true
default: "develop"
type: string
backendType:
description: "Backend."
required: true
default: "staging"
type: string
testinyRunName:
description: "TESTINY_RUN_NAME."
required: false
default: ""
type: string
androidDeviceId:
description: "androidDeviceId. Target a specific device. Leave empty for auto."
required: false
default: ""
type: string
callingServiceEnv:
description: "Calling service environment."
required: true
type: choice
options:
- dev
- custom
- master
- avs
- qa
- edge
- staging
- prod
default: dev
callingServiceUrl:
description: "Calling service URL."
required: true
default: "loadbalanced"
type: string
deflakeCount:
description: "Rerun only failed tests."
required: true
default: 1
type: number
permissions:
contents: read
jobs:
validate-and-resolve-inputs:
name: Validate + resolve selectors (no execution)
runs-on: ubuntu-latest
outputs:
resolvedTestCaseId: ${{ steps.resolve.outputs.testCaseId }}
resolvedCategory: ${{ steps.resolve.outputs.category }}
steps:
- name: Validate upgrade inputs
shell: bash
run: |
set -euo pipefail
if [[ "${{ inputs.isUpgrade }}" == "true" && -z "${{ inputs.oldBuildNumber }}" ]]; then
echo "ERROR: oldBuildNumber is REQUIRED when isUpgrade=true"
exit 1
fi
- name: Resolve selector from TAGS
id: resolve
shell: bash
run: |
set -euo pipefail
TESTCASE_ID=""
CATEGORY=""
TAGS_RAW="${{ inputs.TAGS }}"
trim() { echo "$1" | xargs; }
if [[ -n "$(trim "${TAGS_RAW}")" ]]; then
sel=""
IFS=',' read -ra parts <<< "${TAGS_RAW}"
for p in "${parts[@]}"; do
t="$(trim "$p")"
if [[ -n "$t" ]]; then
sel="$t"
break
fi
done
sel="${sel#@}"
sel="$(trim "$sel")"
if [[ "$sel" == *:* ]]; then
echo "ERROR: TAGS format '@key:value' is not supported. Use '@TC-1234' or '@category'."
exit 1
fi
if [[ "$sel" =~ ^TC-[0-9]+$ ]]; then
TESTCASE_ID="$sel"
else
CATEGORY="$sel"
fi
fi
echo "testCaseId=$TESTCASE_ID" >> "$GITHUB_OUTPUT"
echo "category=$CATEGORY" >> "$GITHUB_OUTPUT"
- name: Print resolved selectors
shell: bash
run: |
set -euo pipefail
echo "TAGS=${{ inputs.TAGS }}"
echo "resolvedTestCaseId=${{ steps.resolve.outputs.testCaseId }}"
echo "resolvedCategory=${{ steps.resolve.outputs.category }}"
run-android-ui-tests-placeholder:
name: Runner smoke check + checkout (no test execution yet)
runs-on:
- self-hosted
- Linux
- X64
- office
- android-qa
needs: validate-and-resolve-inputs
defaults:
run:
shell: bash
steps:
- name: Checkout code (with submodules)
uses: actions/checkout@v4
with:
# FIX: ensure we checkout the SAME branch the workflow is running from (no mismatch with inputs.branch)
ref: ${{ github.ref_name }}
fetch-depth: 0
clean: true
submodules: recursive
- name: Verify Gradle contains opVault support (quick check)
run: |
set -euo pipefail
grep -n "opVault" tests/testsSupport/build.gradle.kts
# Gradle needs Java. Ensure a consistent JDK on the self-hosted runner.
- name: Set up Java
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "17"
cache: gradle
- name: Set up Android SDK (sets ANDROID_HOME + adb)
uses: android-actions/setup-android@v3
- name: Verify toolchain
run: |
set -euo pipefail
java -version
echo "ANDROID_HOME=${ANDROID_HOME:-}"
adb version
# Flavor -> S3 folder + APP_ID
- name: Resolve flavor -> S3_FOLDER + APP_ID
id: resolve_flavor
run: |
set -euo pipefail
FLAVOR="$(echo "${{ inputs.flavor }}" | xargs)"
# IMPORTANT: initialize so set -u never crashes before assignment
S3_FOLDER=""
APP_ID=""
case "$FLAVOR" in
"staging compat")
S3_FOLDER="artifacts/megazord/android/reloaded/staging/compat/"
APP_ID="com.waz.zclient.dev"
;;
"internal beta")
S3_FOLDER="artifacts/megazord/android/reloaded/beta/release/"
APP_ID="com.wire.android.internal"
;;
"internal release candidate")
S3_FOLDER="artifacts/megazord/android/reloaded/internal/compat/"
APP_ID="com.wire.internal"
;;
"experimental")
S3_FOLDER="artifacts/megazord/android/reloaded/staging/compat/"
APP_ID="com.waz.zclient.dev"
;;
"column-1")
S3_FOLDER="android/custom/bund/column1/prod/compatrelease/"
APP_ID="com.wire.android.bund"
;;
"column-2")
S3_FOLDER="android/custom/bund/column2/prod/compatrelease/"
APP_ID="com.wire.android.bund.column2"
;;
"column-3")
S3_FOLDER="android/custom/bund/column3/prod/compatrelease/"
APP_ID="com.wire.android.bund.column3"
;;
"debug")
S3_FOLDER="artifacts/megazord/android/reloaded/dev/debug/"
APP_ID="com.waz.zclient.dev.debug"
;;
"fdroid")
S3_FOLDER="artifacts/megazord/android/reloaded/fdroid/compatrelease/"
APP_ID="com.wire"
;;
"production")
S3_FOLDER="artifacts/megazord/android/reloaded/prod/compatrelease/"
APP_ID="com.wire"
;;
*)
echo "ERROR: Unknown flavor: '$FLAVOR'"
echo "Expected one of: internal release candidate, internal beta, staging compat, experimental, column-1, column-2, column-3, debug, fdroid, production"
exit 1
;;
esac
# extra safety
if [[ -z "$S3_FOLDER" || -z "$APP_ID" ]]; then
echo "ERROR: Flavor mapping produced empty values. flavor='$FLAVOR'"
exit 1
fi
echo "Resolved flavor='$FLAVOR'"
echo "S3_FOLDER=$S3_FOLDER"
echo "APP_ID=$APP_ID"
echo "S3_FOLDER=$S3_FOLDER" >> "$GITHUB_ENV"
echo "APP_ID=$APP_ID" >> "$GITHUB_ENV"
echo "S3_FOLDER=$S3_FOLDER" >> "$GITHUB_OUTPUT"
echo "APP_ID=$APP_ID" >> "$GITHUB_OUTPUT"
- name: Ensure AWS CLI is available (no sudo)
run: |
set -euo pipefail
if command -v aws >/dev/null 2>&1; then
aws --version
exit 0
fi
if ! command -v curl >/dev/null 2>&1; then
echo "ERROR: 'curl' is not available on this runner."
exit 1
fi
if ! command -v unzip >/dev/null 2>&1; then
echo "ERROR: 'unzip' is not available on this runner."
exit 1
fi
echo "aws not found. Installing AWS CLI v2 locally..."
AWS_ROOT="${RUNNER_TEMP}/awscli"
ZIP_PATH="${RUNNER_TEMP}/awscliv2.zip"
rm -rf "${AWS_ROOT}" "${ZIP_PATH}" "${RUNNER_TEMP}/aws"
mkdir -p "${AWS_ROOT}"
curl -fsSL -o "${ZIP_PATH}" "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"
unzip -oq "${ZIP_PATH}" -d "${RUNNER_TEMP}"
rm -f "${ZIP_PATH}"
# Install without sudo
"${RUNNER_TEMP}/aws/install" -i "${AWS_ROOT}" -b "${AWS_ROOT}/bin"
# Make it available for subsequent steps + also this step
echo "${AWS_ROOT}/bin" >> "${GITHUB_PATH}"
export PATH="${AWS_ROOT}/bin:${PATH}"
aws --version
- name: Configure AWS credentials (for S3)
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-1
- name: Download APK(s) from S3 (latest/build/upgrade)
id: download_apks
env:
S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
APP_BUILD_NUMBER: ${{ inputs.appBuildNumber }}
IS_UPGRADE: ${{ inputs.isUpgrade }}
OLD_BUILD_NUMBER: ${{ inputs.oldBuildNumber }}
FLAVOR: ${{ inputs.flavor }}
run: |
set -euo pipefail
if [[ -z "${S3_BUCKET:-}" ]]; then
echo "ERROR: Missing secret AWS_S3_BUCKET"
exit 1
fi
if ! command -v aws >/dev/null 2>&1; then
echo "ERROR: aws CLI not found (should have been installed earlier)."
exit 1
fi
if ! command -v python3 >/dev/null 2>&1; then
echo "ERROR: python3 not found on this runner (needed for version selection)."
exit 1
fi
# Verify AWS creds are valid (does NOT print secrets)
aws sts get-caller-identity --output json >/dev/null
: "${S3_FOLDER:?S3_FOLDER is missing}"
echo "Listing APKs in s3://${S3_BUCKET}/${S3_FOLDER} ..."
KEYS_JSON="${RUNNER_TEMP}/s3_keys.json"
aws s3api list-objects-v2 \
--bucket "${S3_BUCKET}" \
--prefix "${S3_FOLDER}" \
--query "Contents[].Key" \
--output json \
--page-size 1000 \
--max-items 10000 > "${KEYS_JSON}"
python3 - <<'PY' > "${RUNNER_TEMP}/apk_env.txt"
import json, os, re, sys
from pathlib import Path
keys_path = Path(os.environ["RUNNER_TEMP"]) / "s3_keys.json"
keys = json.loads(keys_path.read_text() or "[]")
apks = [k for k in keys if isinstance(k, str) and k.lower().endswith(".apk")]
if not apks:
print("ERROR: No .apk files found under S3_FOLDER prefix.", file=sys.stderr)
sys.exit(1)
app_build = (os.environ.get("APP_BUILD_NUMBER") or "").strip()
is_upgrade = (os.environ.get("IS_UPGRADE", "false").strip().lower() == "true")
old_input = (os.environ.get("OLD_BUILD_NUMBER") or "").strip()
def parse_version(fname: str):
m = re.search(r"-v(\d+)\.(\d+)\.(\d+)-(\d+)", fname)
if m:
return (int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)))
m = re.search(r"-v(\d+)\.(\d+)\.(\d+)-fdroid", fname)
if m:
return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 0)
m = re.search(r"-v(\d+)\.(\d+)\.(\d+)", fname)
if m:
return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 0)
return None
def build_label(fname: str):
m = re.search(r"-v(\d+\.\d+\.\d+-\d+)", fname)
if m: return m.group(1)
m = re.search(r"-v(\d+\.\d+\.\d+)-fdroid", fname)
if m: return m.group(1)
m = re.search(r"-v(\d+\.\d+\.\d+)", fname)
if m: return m.group(1)
return ""
def pick_by_substring(substr: str):
if not substr:
return None
for k in apks:
if substr in k.split("/")[-1]:
return k
return None
parsed = []
for k in apks:
fname = k.split("/")[-1]
pv = parse_version(fname)
if pv is not None:
parsed.append((pv, k))
parsed.sort(key=lambda x: x[0])
latest_key = parsed[-1][1] if parsed else None
second_latest_key = parsed[-2][1] if len(parsed) >= 2 else None
new_key = None
old_key = None
if app_build.lower().endswith(".apk"):
if app_build.startswith("s3://"):
parts = app_build.split("/", 3)
if len(parts) < 4:
print("ERROR: Invalid s3:// path in appBuildNumber", file=sys.stderr)
sys.exit(1)
new_key = parts[3]
else:
new_key = app_build.lstrip("/")
if is_upgrade:
old_key = pick_by_substring(old_input) if old_input else second_latest_key
elif app_build == "latest":
if not latest_key:
print("ERROR: Could not resolve latest build (no parseable versions).", file=sys.stderr)
sys.exit(1)
new_key = latest_key
if is_upgrade:
old_key = pick_by_substring(old_input) if old_input else second_latest_key
else:
new_key = pick_by_substring(app_build)
if is_upgrade:
if not old_input:
print("ERROR: isUpgrade=true but oldBuildNumber is empty.", file=sys.stderr)
sys.exit(1)
old_key = pick_by_substring(old_input)
if not new_key:
print(f"ERROR: Could not resolve NEW apk for appBuildNumber='{app_build}'", file=sys.stderr)
sys.exit(1)
if is_upgrade and not old_key:
print("ERROR: Upgrade requested but OLD apk could not be resolved.", file=sys.stderr)
sys.exit(1)
new_name = new_key.split("/")[-1]
old_name = old_key.split("/")[-1] if old_key else ""
print(f"NEW_S3_KEY={new_key}")
print(f"OLD_S3_KEY={old_key or ''}")
print(f"NEW_APK_NAME={new_name}")
print(f"OLD_APK_NAME={old_name}")
print(f"REAL_BUILD_NUMBER={build_label(new_name)}")
print(f"OLD_BUILD_NUMBER={build_label(old_name) if old_name else ''}")
PY
# ===== CRITICAL FIX =====
# Make vars available in THIS step (GITHUB_ENV only applies to NEXT steps)
set -a
source "${RUNNER_TEMP}/apk_env.txt"
set +a
# Persist for later steps too
cat "${RUNNER_TEMP}/apk_env.txt" >> "$GITHUB_ENV"
cat "${RUNNER_TEMP}/apk_env.txt" >> "$GITHUB_OUTPUT"
: "${NEW_S3_KEY:?NEW_S3_KEY missing after resolve}"
# ========================
NEW_APK_PATH="${RUNNER_TEMP}/Wire.apk"
echo "NEW_APK_PATH=${NEW_APK_PATH}" >> "$GITHUB_ENV"
echo "Downloading NEW apk..."
aws s3 cp "s3://${S3_BUCKET}/${NEW_S3_KEY}" "${NEW_APK_PATH}" --only-show-errors
test -s "${NEW_APK_PATH}"
if [[ "${IS_UPGRADE}" == "true" ]]; then
: "${OLD_S3_KEY:?OLD_S3_KEY missing for upgrade}"
OLD_APK_PATH="${RUNNER_TEMP}/Wire.old.apk"
echo "OLD_APK_PATH=${OLD_APK_PATH}" >> "$GITHUB_ENV"
echo "Downloading OLD apk..."
aws s3 cp "s3://${S3_BUCKET}/${OLD_S3_KEY}" "${OLD_APK_PATH}" --only-show-errors
test -s "${OLD_APK_PATH}"
fi
echo "OK: Downloaded APK(s): ${NEW_APK_NAME}${OLD_APK_NAME:+ and ${OLD_APK_NAME}}"
- name: Remove any leftover secrets files (safety)
run: |
set -euo pipefail
rm -f secrets.json
rm -f "${RUNNER_TEMP}/secrets.json"
- name: Select target device (androidDeviceId optional)
run: |
set -euo pipefail
adb devices -l
online_count="$(adb devices | awk 'NR>1 && $2=="device"{c++} END{print c+0}')"
if [[ "$online_count" -eq 0 ]]; then
echo "ERROR: No online Android devices found on this runner."
exit 1
fi
TARGET="${{ inputs.androidDeviceId }}"
if [[ -n "$TARGET" ]]; then
if ! adb devices | awk 'NR>1 && $2=="device"{print $1}' | grep -qx "$TARGET"; then
echo "ERROR: androidDeviceId '$TARGET' not found in 'adb devices'."
exit 1
fi
SELECTED="$TARGET"
else
SELECTED="$(adb devices | awk 'NR>1 && $2=="device"{print $1; exit}')"
fi
echo "ANDROID_SERIAL=$SELECTED" >> "$GITHUB_ENV"
echo "Using device: $SELECTED"
- name: Prepare device + install APK(s)
run: |
set -euo pipefail
: "${ANDROID_SERIAL:?ANDROID_SERIAL not set}"
: "${NEW_APK_PATH:?NEW_APK_PATH missing}"
if [[ ! -s "${NEW_APK_PATH}" ]]; then
echo "ERROR: NEW_APK_PATH not found or empty: ${NEW_APK_PATH}"
exit 1
fi
ADB="adb -s ${ANDROID_SERIAL}"
${ADB} wait-for-device
INSTALLED_PKGS="$(${ADB} shell pm list packages)"
PACKAGES=(
"com.wire"
"com.waz.zclient.dev"
"com.wire.internal"
"com.wire.android.internal"
"com.wire.android.bund"
"com.wire.android.bund.column2"
"com.wire.android.bund.column3"
"com.waz.zclient.dev.debug"
)
for pkg in "${PACKAGES[@]}"; do
if echo "${INSTALLED_PKGS}" | grep -qx "package:${pkg}"; then
echo "Uninstalling ${pkg}"
${ADB} uninstall "${pkg}" || true
fi
done
INSTALL_FLAGS="-r"
if [[ "${{ inputs.enforceAppInstall }}" == "true" ]]; then
INSTALL_FLAGS="-r -d"
fi
if [[ "${{ inputs.isUpgrade }}" == "true" ]]; then
: "${OLD_APK_PATH:?OLD_APK_PATH missing for upgrade}"
if [[ ! -s "${OLD_APK_PATH}" ]]; then
echo "ERROR: OLD_APK_PATH not found or empty: ${OLD_APK_PATH}"
exit 1
fi
echo "Installing OLD apk..."
${ADB} install ${INSTALL_FLAGS} "${OLD_APK_PATH}"
fi
echo "Installing NEW apk..."
${ADB} install ${INSTALL_FLAGS} "${NEW_APK_PATH}"
: "${APP_ID:?APP_ID missing}"
if ! ${ADB} shell pm list packages | grep -qx "package:${APP_ID}"; then
echo "ERROR: APP_ID '${APP_ID}' is not installed after install step."
exit 1
fi
- name: Install 1Password CLI
uses: 1password/install-cli-action@v2
- name: Fetch secrets.json from 1Password (runtime only)
env:
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
run: |
set -euo pipefail
chmod +x ./gradlew
SECRETS_FILE="${RUNNER_TEMP}/secrets.json"
export SECRETS_JSON_PATH="${SECRETS_FILE}"
echo "SECRETS_JSON_PATH=${SECRETS_FILE}" >> "$GITHUB_ENV"
# FIX: pass vault explicitly so Gradle can use it (service account requirement)
./gradlew :tests:testsSupport:fetchSecrets --no-daemon \
-PsecretsJsonPath="${SECRETS_FILE}" \
-PopVault="Test Automation"
test -s "${SECRETS_FILE}"
chmod 600 "${SECRETS_FILE}"
echo "OK: secrets.json created at ${SECRETS_FILE}"
- name: Gradle sanity check (assemble only, still no test execution)
run: |
set -euo pipefail
./gradlew :tests:testsCore:assembleDebugAndroidTest --no-daemon
- name: Cleanup secrets + workspace outputs
if: always()
run: |
set -euo pipefail
rm -f secrets.json
rm -f "${RUNNER_TEMP}/secrets.json"
rm -f "${RUNNER_TEMP}/Wire.apk" "${RUNNER_TEMP}/Wire.old.apk" || true
git clean -ffdx