From ecf014d5c986be83aefbf8b30787ecac73f9f75a Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 27 Sep 2024 16:15:19 -0700
Subject: [PATCH 01/23] Upload mobile benchmark results

---
 .github/workflows/mobile_job.yml              |  4 ++++
 .../run_on_aws_devicefarm.py                  | 23 ++++++++-----------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index afb4ee71a9..382ceaa9bf 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -273,6 +273,7 @@ jobs:
         run: |
           set -ex
 
+          echo "::group::Running iOS tests"
           ${CONDA_RUN} python run_on_aws_devicefarm.py \
             --project-arn "${PROJECT_ARN}" \
             --device-pool-arn "${DEVICE_POOL_ARN}" \
@@ -283,6 +284,7 @@ jobs:
             --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
             --workflow-id "${RUN_ID}" \
             --workflow-attempt "${RUN_ATTEMPT}"
+          echo "::endgroup::"
 
       - name: Run Android tests on devices
         if: ${{ inputs.device-type == 'android' }}
@@ -303,6 +305,7 @@ jobs:
         run: |
           set -ex
 
+          echo "::group::Running Android tests"
           ${CONDA_RUN} python run_on_aws_devicefarm.py \
             --project-arn "${PROJECT_ARN}" \
             --device-pool-arn "${DEVICE_POOL_ARN}" \
@@ -313,3 +316,4 @@ jobs:
             --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
             --workflow-id "${RUN_ID}" \
             --workflow-attempt "${RUN_ATTEMPT}"
+          echo "::endgroup::"
diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index 55596d7eba..6ea1cd1220 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#, !/usr/bin/env python3
 
 import datetime
 import logging
@@ -27,7 +27,6 @@
 AWS_GUID = "082d10e5-d7d7-48a5-ba5c-b33d66efa1f5"
 AWS_ARN_PREFIX = "arn:aws:devicefarm:"
 DEFAULT_DEVICE_POOL_ARN = f"{AWS_ARN_PREFIX}{AWS_REGION}::devicepool:{AWS_GUID}"
-TEST_SPEC_OUTPUT_HIGHLIGHT_HINT = re.compile(r"^\[PyTorch\] .*$")
 
 
 # Device Farm report type
@@ -249,20 +248,17 @@ def upload_file_to_s3(
     )
 
 
-def print_highlights(
+def print_testspec(
     file_name: str,
-    highlight: Pattern,
     indent: int = 0,
 ):
     """
     The test spec output from AWS Device Farm is the main output of the test job.
-    It's a bit too verbose, so this is a workaround to print only the relevant
-    parts
     """
+    info("::group::Test output")
     with open(file_name) as f:
-        for line in f:
-            if re.match(highlight, line):
-                info(f"{' ' * indent}{line.strip()}")
+        info(f.read())
+    info("::endgroup::")
 
 
 def print_test_artifacts(
@@ -299,10 +295,10 @@ def print_test_artifacts(
             )
             gathered_artifacts.append(artifact)
 
-            # Additional step to print highlights from the test output
+            # Additional step to print the test output
             if filetype == "TESTSPEC_OUTPUT":
-                print_highlights(
-                    local_filename, TEST_SPEC_OUTPUT_HIGHLIGHT_HINT, indent + 2
+                print_testspec(
+                    local_filename, indent + 2
                 )
 
     return gathered_artifacts
@@ -556,9 +552,10 @@ def main() -> None:
         warn(f"Failed to run {unique_prefix}: {error}")
         sys.exit(1)
     finally:
-        print_report(
+        artifacts = print_report(
             client, r.get("run"), ReportType.RUN, workflow_id, workflow_attempt
         )
+        print(artifacts)
 
     if not is_success(result):
         sys.exit(1)

From c8b91e24a11f7e2fc6b7b11abd167e3b11f9e923 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 27 Sep 2024 16:30:37 -0700
Subject: [PATCH 02/23] Testing

---
 .github/workflows/test_mobile_job.yml         | 30 ++++++-------
 .../run_on_aws_devicefarm.py                  | 44 ++++++++++++++-----
 2 files changed, 48 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/test_mobile_job.yml b/.github/workflows/test_mobile_job.yml
index 1c14c422e1..5638d4bf88 100644
--- a/.github/workflows/test_mobile_job.yml
+++ b/.github/workflows/test_mobile_job.yml
@@ -9,21 +9,21 @@ on:
   workflow_dispatch:
 
 jobs:
-  test-ios-job:
-    permissions:
-      id-token: write
-      contents: read
-    uses: ./.github/workflows/mobile_job.yml
-    with:
-      device-type: ios
-      # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS
-      runner: ubuntu-latest
-      # There values are prepared beforehand for the test
-      project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0
-      device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:b531574a-fb82-40ae-b687-8f0b81341ae0/da5d902d-45db-477b-ae0a-766e06ef3845
-      ios-ipa-archive: https://ossci-assets.s3.amazonaws.com/DeviceFarm.ipa
-      ios-xctestrun-zip: https://ossci-assets.s3.amazonaws.com/MobileNetClassifierTest_MobileNetClassifierTest_iphoneos17.4-arm64.xctestrun.zip
-      test-spec: https://ossci-assets.s3.amazonaws.com/default-ios-device-farm-appium-test-spec.yml
+#  test-ios-job:
+#    permissions:
+#      id-token: write
+#      contents: read
+#    uses: ./.github/workflows/mobile_job.yml
+#    with:
+#      device-type: ios
+#      # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS
+#      runner: ubuntu-latest
+#      # There values are prepared beforehand for the test
+#      project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0
+#      device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:b531574a-fb82-40ae-b687-8f0b81341ae0/da5d902d-45db-477b-ae0a-766e06ef3845
+#      ios-ipa-archive: https://ossci-assets.s3.amazonaws.com/DeviceFarm.ipa
+#      ios-xctestrun-zip: https://ossci-assets.s3.amazonaws.com/MobileNetClassifierTest_MobileNetClassifierTest_iphoneos17.4-arm64.xctestrun.zip
+#      test-spec: https://ossci-assets.s3.amazonaws.com/default-ios-device-farm-appium-test-spec.yml
 
   test-android-llama2-job:
     permissions:
diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index 6ea1cd1220..8e8e4cb517 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -1,4 +1,4 @@
-#, !/usr/bin/env python3
+# , !/usr/bin/env python3
 
 import datetime
 import logging
@@ -8,10 +8,10 @@
 import string
 import sys
 import time
-from argparse import Action, ArgumentParser
+from argparse import Action, ArgumentParser, Namespace
 from enum import Enum
 from logging import info
-from typing import Any, Dict, List, Optional, Pattern
+from typing import Any, Dict, List, Optional, Pattern, Sequence, Union
 from warnings import warn
 
 import boto3
@@ -43,7 +43,13 @@ class ReportType(Enum):
 
 
 class ValidateArchive(Action):
-    def __call__(self, parser, namespace, values, option_string=None):
+    def __call__(
+        self,
+        parser: ArgumentParser,
+        namespace: Namespace,
+        values: Any,
+        option_string: Optional[str] = None,
+    ) -> None:
         if values.startswith(AWS_ARN_PREFIX) or (
             os.path.isfile(values) and values.endswith(".zip")
         ):
@@ -54,7 +60,13 @@ def __call__(self, parser, namespace, values, option_string=None):
 
 
 class ValidateExtraDataArchive(Action):
-    def __call__(self, parser, namespace, values, option_string=None):
+    def __call__(
+        self,
+        parser: ArgumentParser,
+        namespace: Namespace,
+        values: Any,
+        option_string: Optional[str] = None,
+    ) -> None:
         # This parameter is optional and can accept an empty string, or it can be
         # an existing ARN, or a local zip archive to be uploaded to AWS
         if (
@@ -71,7 +83,13 @@ def __call__(self, parser, namespace, values, option_string=None):
 
 
 class ValidateApp(Action):
-    def __call__(self, parser, namespace, values, option_string=None):
+    def __call__(
+        self,
+        parser: ArgumentParser,
+        namespace: Namespace,
+        values: Any,
+        option_string: Optional[str] = None,
+    ) -> None:
         # This can be a local file or an existing app that has previously been uploaded
         # to AWS
         if values.startswith(AWS_ARN_PREFIX) or (
@@ -87,7 +105,13 @@ def __call__(self, parser, namespace, values, option_string=None):
 
 
 class ValidateTestSpec(Action):
-    def __call__(self, parser, namespace, values, option_string=None):
+    def __call__(
+        self,
+        parser: ArgumentParser,
+        namespace: Namespace,
+        values: Any,
+        option_string: Optional[str] = None,
+    ) -> None:
         if values.startswith(AWS_ARN_PREFIX) or (
             os.path.isfile(values)
             and (values.endswith(".yml") or values.endswith(".yaml"))
@@ -251,7 +275,7 @@ def upload_file_to_s3(
 def print_testspec(
     file_name: str,
     indent: int = 0,
-):
+) -> None:
     """
     The test spec output from AWS Device Farm is the main output of the test job.
     """
@@ -297,9 +321,7 @@ def print_test_artifacts(
 
             # Additional step to print the test output
             if filetype == "TESTSPEC_OUTPUT":
-                print_testspec(
-                    local_filename, indent + 2
-                )
+                print_testspec(local_filename, indent + 2)
 
     return gathered_artifacts
 

From 9d38d52cb962353fb2d2dbaed916a717fde71c0c Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 27 Sep 2024 17:26:13 -0700
Subject: [PATCH 03/23] Save the artifacts output

---
 .../run_on_aws_devicefarm.py                  | 39 ++++++++++++++-----
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index 8e8e4cb517..9ee5376654 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -1,17 +1,17 @@
-# , !/usr/bin/env python3
+#!/usr/bin/env python3
 
+import json
 import datetime
 import logging
 import os
 import random
-import re
 import string
 import sys
 import time
 from argparse import Action, ArgumentParser, Namespace
 from enum import Enum
 from logging import info
-from typing import Any, Dict, List, Optional, Pattern, Sequence, Union
+from typing import Any, Dict, List, Optional
 from warnings import warn
 
 import boto3
@@ -38,6 +38,8 @@ class ReportType(Enum):
 
 
 DEVICE_FARM_BUCKET = "gha-artifacts"
+DYNAMODB_BENCHMARK_DB = "benchmark"
+DYNAMODB_BENCHMARK_TABLE = "oss_ci_benchmark_v2"
 
 logging.basicConfig(level=logging.INFO)
 
@@ -272,6 +274,14 @@ def upload_file_to_s3(
     )
 
 
+def set_output(name: str, val: Any) -> None:
+    if os.getenv("GITHUB_OUTPUT"):
+        with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env:
+            print(f"{name}={val}", file=env)
+    else:
+        print(f"::set-output name={name}::{val}")
+
+
 def print_testspec(
     file_name: str,
     indent: int = 0,
@@ -279,14 +289,19 @@ def print_testspec(
     """
     The test spec output from AWS Device Farm is the main output of the test job.
     """
-    info("::group::Test output")
+    print("::group::Test output")
     with open(file_name) as f:
         info(f.read())
-    info("::endgroup::")
+    print("::endgroup::")
 
 
 def print_test_artifacts(
-    client: Any, test_arn: str, workflow_id: str, workflow_attempt: int, indent: int = 0
+    client: Any,
+    test_arn: str,
+    workflow_id: str,
+    workflow_attempt: int,
+    report_name: str,
+    indent: int = 0,
 ) -> List[Dict[str, str]]:
     """
     Return all artifacts from this specific test. There are 3 types of artifacts
@@ -313,10 +328,16 @@ def print_test_artifacts(
                 s3_key,
             )
 
+            s3_url = f"https://{DEVICE_FARM_BUCKET}.s3.amazonaws.com/{s3_key}"
+            artifact["s3_url"] = s3_url
+
             info(
                 f"{' ' * indent}Saving {artifact_type} {filename}.{extension} ({filetype}) "
-                + f"at https://{DEVICE_FARM_BUCKET}.s3.amazonaws.com/{s3_key}"
+                + f"at {s3_url}"
             )
+
+            # Some more metadata to identify where the artifact comes from
+            artifact["report_name"] = report_name
             gathered_artifacts.append(artifact)
 
             # Additional step to print the test output
@@ -364,7 +385,7 @@ def print_report(
         next_rtype = ReportType.TEST
     elif rtype == ReportType.TEST:
         return print_test_artifacts(
-            client, arn, workflow_id, workflow_attempt, indent + 2
+            client, arn, workflow_id, workflow_attempt, name, indent + 2
         )
 
     artifacts = []
@@ -577,7 +598,7 @@ def main() -> None:
         artifacts = print_report(
             client, r.get("run"), ReportType.RUN, workflow_id, workflow_attempt
         )
-        print(artifacts)
+        set_output("artifacts", json.dumps(artifacts))
 
     if not is_success(result):
         sys.exit(1)

From ee2af2e38da52fb56bd6d303d1790bcf75cd7e26 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 27 Sep 2024 21:26:56 -0700
Subject: [PATCH 04/23] Clean up before review

---
 .github/workflows/mobile_job.yml              | 17 +++++++----
 .github/workflows/test_mobile_job.yml         | 30 +++++++++----------
 .../run_on_aws_devicefarm.py                  |  7 +++--
 3 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 382ceaa9bf..9c51d8fb43 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -92,14 +92,23 @@ on:
         type: string
         default: ''
 
+    outputs:
+      artifacts:
+        description: |
+          The list of artifacts from AWS in JSON format returned to the caller
+        type: string
+        value: ${{ jobs.mobile.outputs.artifacts }}
+
 jobs:
-  job:
+  mobile:
     name: ${{ inputs.job-name }} (${{ inputs.device-type }})
     runs-on: ${{ inputs.runner }}
     timeout-minutes: ${{ inputs.timeout }}
     permissions:
       id-token: write
       contents: read
+    outputs:
+      artifacts: ${{ inputs.device-type == 'ios' && steps.ios-test.outputs.artifacts || input.device-types == 'android' && steps.android-test.outputs.artifacts || '[]' }}
     steps:
       - name: Clean workspace
         run: |
@@ -255,6 +264,7 @@ jobs:
           echo "extra-data-output=${EXTRA_DATA_OUTPUT}" >> "${GITHUB_OUTPUT}"
 
       - name: Run iOS tests on devices
+        id: ios-test
         if: ${{ inputs.device-type == 'ios' }}
         shell: bash
         working-directory: test-infra/tools/device-farm-runner
@@ -273,7 +283,6 @@ jobs:
         run: |
           set -ex
 
-          echo "::group::Running iOS tests"
           ${CONDA_RUN} python run_on_aws_devicefarm.py \
             --project-arn "${PROJECT_ARN}" \
             --device-pool-arn "${DEVICE_POOL_ARN}" \
@@ -284,9 +293,9 @@ jobs:
             --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
             --workflow-id "${RUN_ID}" \
             --workflow-attempt "${RUN_ATTEMPT}"
-          echo "::endgroup::"
 
       - name: Run Android tests on devices
+        id: android-test
         if: ${{ inputs.device-type == 'android' }}
         shell: bash
         working-directory: test-infra/tools/device-farm-runner
@@ -305,7 +314,6 @@ jobs:
         run: |
           set -ex
 
-          echo "::group::Running Android tests"
           ${CONDA_RUN} python run_on_aws_devicefarm.py \
             --project-arn "${PROJECT_ARN}" \
             --device-pool-arn "${DEVICE_POOL_ARN}" \
@@ -316,4 +324,3 @@ jobs:
             --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
             --workflow-id "${RUN_ID}" \
             --workflow-attempt "${RUN_ATTEMPT}"
-          echo "::endgroup::"
diff --git a/.github/workflows/test_mobile_job.yml b/.github/workflows/test_mobile_job.yml
index 5638d4bf88..1c14c422e1 100644
--- a/.github/workflows/test_mobile_job.yml
+++ b/.github/workflows/test_mobile_job.yml
@@ -9,21 +9,21 @@ on:
   workflow_dispatch:
 
 jobs:
-#  test-ios-job:
-#    permissions:
-#      id-token: write
-#      contents: read
-#    uses: ./.github/workflows/mobile_job.yml
-#    with:
-#      device-type: ios
-#      # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS
-#      runner: ubuntu-latest
-#      # There values are prepared beforehand for the test
-#      project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0
-#      device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:b531574a-fb82-40ae-b687-8f0b81341ae0/da5d902d-45db-477b-ae0a-766e06ef3845
-#      ios-ipa-archive: https://ossci-assets.s3.amazonaws.com/DeviceFarm.ipa
-#      ios-xctestrun-zip: https://ossci-assets.s3.amazonaws.com/MobileNetClassifierTest_MobileNetClassifierTest_iphoneos17.4-arm64.xctestrun.zip
-#      test-spec: https://ossci-assets.s3.amazonaws.com/default-ios-device-farm-appium-test-spec.yml
+  test-ios-job:
+    permissions:
+      id-token: write
+      contents: read
+    uses: ./.github/workflows/mobile_job.yml
+    with:
+      device-type: ios
+      # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS
+      runner: ubuntu-latest
+      # There values are prepared beforehand for the test
+      project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0
+      device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:b531574a-fb82-40ae-b687-8f0b81341ae0/da5d902d-45db-477b-ae0a-766e06ef3845
+      ios-ipa-archive: https://ossci-assets.s3.amazonaws.com/DeviceFarm.ipa
+      ios-xctestrun-zip: https://ossci-assets.s3.amazonaws.com/MobileNetClassifierTest_MobileNetClassifierTest_iphoneos17.4-arm64.xctestrun.zip
+      test-spec: https://ossci-assets.s3.amazonaws.com/default-ios-device-farm-appium-test-spec.yml
 
   test-android-llama2-job:
     permissions:
diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index 9ee5376654..bba9d60ff5 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -283,15 +283,16 @@ def set_output(name: str, val: Any) -> None:
 
 
 def print_testspec(
+    report_name: str,
     file_name: str,
     indent: int = 0,
 ) -> None:
     """
     The test spec output from AWS Device Farm is the main output of the test job.
     """
-    print("::group::Test output")
+    print(f"::group::{report_name} test output")
     with open(file_name) as f:
-        info(f.read())
+        print(f.read())
     print("::endgroup::")
 
 
@@ -342,7 +343,7 @@ def print_test_artifacts(
 
             # Additional step to print the test output
             if filetype == "TESTSPEC_OUTPUT":
-                print_testspec(local_filename, indent + 2)
+                print_testspec(report_name, local_filename, indent + 2)
 
     return gathered_artifacts
 

From 1ab6ac8233fa71691e805a7272e002e949581518 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 27 Sep 2024 21:29:09 -0700
Subject: [PATCH 05/23] Fix lint

---
 .github/workflows/mobile_job.yml                  | 3 +--
 tools/device-farm-runner/run_on_aws_devicefarm.py | 2 --
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 9c51d8fb43..ccaaf08ecb 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -96,7 +96,6 @@ on:
       artifacts:
         description: |
           The list of artifacts from AWS in JSON format returned to the caller
-        type: string
         value: ${{ jobs.mobile.outputs.artifacts }}
 
 jobs:
@@ -108,7 +107,7 @@ jobs:
       id-token: write
       contents: read
     outputs:
-      artifacts: ${{ inputs.device-type == 'ios' && steps.ios-test.outputs.artifacts || input.device-types == 'android' && steps.android-test.outputs.artifacts || '[]' }}
+      artifacts: ${{ inputs.device-type == 'ios' && steps.ios-test.outputs.artifacts || inputs.device-type == 'android' && steps.android-test.outputs.artifacts || '[]' }}
     steps:
       - name: Clean workspace
         run: |
diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index bba9d60ff5..7bccb54549 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -38,8 +38,6 @@ class ReportType(Enum):
 
 
 DEVICE_FARM_BUCKET = "gha-artifacts"
-DYNAMODB_BENCHMARK_DB = "benchmark"
-DYNAMODB_BENCHMARK_TABLE = "oss_ci_benchmark_v2"
 
 logging.basicConfig(level=logging.INFO)
 

From 3ec32322fe26e8aa66234a11befbef7a3adf055a Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 27 Sep 2024 21:47:07 -0700
Subject: [PATCH 06/23] Minor tweak

---
 .../run_on_aws_devicefarm.py                  | 36 +++++++++++--------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index 7bccb54549..79b817d9a5 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -281,7 +281,7 @@ def set_output(name: str, val: Any) -> None:
 
 
 def print_testspec(
-    report_name: str,
+    report_name: Optional[str],
     file_name: str,
     indent: int = 0,
 ) -> None:
@@ -299,7 +299,7 @@ def print_test_artifacts(
     test_arn: str,
     workflow_id: str,
     workflow_attempt: int,
-    report_name: str,
+    report_name: Optional[str],
     indent: int = 0,
 ) -> List[Dict[str, str]]:
     """
@@ -349,7 +349,8 @@ def print_test_artifacts(
 def print_report(
     client: Any,
     report: Dict[str, Any],
-    rtype: ReportType,
+    report_name: Optional[str],
+    report_type: ReportType,
     workflow_id: str,
     workflow_attempt: int,
     indent: int = 0,
@@ -363,37 +364,42 @@ def print_report(
         return []
 
     name = report["name"]
+    # Keep the top-level report name as the name of the whole test report, this
+    # is used to connect all artifacts from one report together
+    if not report_name:
+        report_name = name
     result = report["result"]
 
     extra_msg = ""
-    if rtype == ReportType.SUITE or is_success(result):
+    if report_type == ReportType.SUITE or is_success(result):
         counters = report["counters"]
         extra_msg = f"with stats {counters}"
 
     info(f"{' ' * indent}{name} {result} {extra_msg}")
 
     arn = report["arn"]
-    if rtype == ReportType.RUN:
+    if report_type == ReportType.RUN:
         more_reports = client.list_jobs(arn=arn)
-        next_rtype = ReportType.JOB
-    elif rtype == ReportType.JOB:
+        next_report_type = ReportType.JOB
+    elif report_type == ReportType.JOB:
         more_reports = client.list_suites(arn=arn)
-        next_rtype = ReportType.SUITE
-    elif rtype == ReportType.SUITE:
+        next_report_type = ReportType.SUITE
+    elif report_type == ReportType.SUITE:
         more_reports = client.list_tests(arn=arn)
-        next_rtype = ReportType.TEST
-    elif rtype == ReportType.TEST:
+        next_report_type = ReportType.TEST
+    elif report_type == ReportType.TEST:
         return print_test_artifacts(
-            client, arn, workflow_id, workflow_attempt, name, indent + 2
+            client, arn, workflow_id, workflow_attempt, report_name, indent + 2
         )
 
     artifacts = []
-    for more_report in more_reports.get(f"{next_rtype.value}s", []):
+    for more_report in more_reports.get(f"{next_report_type.value}s", []):
         artifacts.extend(
             print_report(
                 client,
                 more_report,
-                next_rtype,
+                report_name,
+                next_report_type,
                 workflow_id,
                 workflow_attempt,
                 indent + 2,
@@ -595,7 +601,7 @@ def main() -> None:
         sys.exit(1)
     finally:
         artifacts = print_report(
-            client, r.get("run"), ReportType.RUN, workflow_id, workflow_attempt
+            client, r.get("run"), None, ReportType.RUN, workflow_id, workflow_attempt
         )
         set_output("artifacts", json.dumps(artifacts))
 

From 9e6cd94621188b935fc06f212377cfa321b394e1 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 13:39:02 -0700
Subject: [PATCH 07/23] Upload artifacts to S3

---
 .github/workflows/mobile_job.yml              | 32 +++++++++++++++++--
 .../run_on_aws_devicefarm.py                  | 18 ++++++++---
 2 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index ccaaf08ecb..d253aa1484 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -279,6 +279,7 @@ jobs:
           DEVICE_TYPE: ${{ inputs.device-type }}
           RUN_ID: ${{ github.run_id }}
           RUN_ATTEMPT: ${{ github.run_attempt }}
+          JOB_ID: ${{ github.job }}
         run: |
           set -ex
 
@@ -291,7 +292,20 @@ jobs:
             --test-spec "${TEST_SPEC}" \
             --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
             --workflow-id "${RUN_ID}" \
-            --workflow-attempt "${RUN_ATTEMPT}"
+            --workflow-attempt "${RUN_ATTEMPT}" \
+            --output "ios-artifacts-${JOB_ID}.json"
+
+      - name: Upload iOS artifacts to S3
+        uses: seemethere/upload-artifact-s3@v5
+        if: ${{ inputs.device-type == 'ios' }}
+        with:
+          name: ios-artifacts
+          retention-days: 14
+          s3-bucket: gha-artifacts
+          s3-prefix: |
+            ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts
+          path: |
+            test-infra/tools/device-farm-runner/ios-artifacts-${{ github.job }}.json
 
       - name: Run Android tests on devices
         id: android-test
@@ -310,6 +324,7 @@ jobs:
           DEVICE_TYPE: ${{ inputs.device-type }}
           RUN_ID: ${{ github.run_id }}
           RUN_ATTEMPT: ${{ github.run_attempt }}
+          JOB_ID: ${{ github.job }}
         run: |
           set -ex
 
@@ -322,4 +337,17 @@ jobs:
             --test-spec "${TEST_SPEC}" \
             --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
             --workflow-id "${RUN_ID}" \
-            --workflow-attempt "${RUN_ATTEMPT}"
+            --workflow-attempt "${RUN_ATTEMPT}" \
+            --output "android-artifacts-${JOB_ID}.json"
+
+      - name: Upload Android artifacts to S3
+        uses: seemethere/upload-artifact-s3@v5
+        if: ${{ inputs.device-type == 'android' }}
+        with:
+          name: android-artifacts
+          retention-days: 14
+          s3-bucket: gha-artifacts
+          s3-prefix: |
+            ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts
+          path: |
+            test-infra/tools/device-farm-runner/android-artifacts-${{ github.job }}.json
diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index 79b817d9a5..d1f9ab936c 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -190,6 +190,11 @@ def parse_args() -> Any:
         default=0,
         help="the workflow run attempt",
     )
+    parser.add_argument(
+        "--output",
+        type=str,
+        help="an optional file to write the list of artifacts from AWS in JSON format",
+    )
 
     return parser.parse_args()
 
@@ -272,12 +277,17 @@ def upload_file_to_s3(
     )
 
 
-def set_output(name: str, val: Any) -> None:
+def set_output(val: Any, gh_var_name: str, filename: Optional[str]) -> None:
     if os.getenv("GITHUB_OUTPUT"):
         with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env:
-            print(f"{name}={val}", file=env)
+            print(f"{gh_var_name}={val}", file=env)
     else:
-        print(f"::set-output name={name}::{val}")
+        print(f"::set-output name={gh_var_name}::{val}")
+
+    # Also write the value to file if it exists
+    if filename:
+        with open(filename, "w") as f:
+            print(val, file=f)
 
 
 def print_testspec(
@@ -603,7 +613,7 @@ def main() -> None:
         artifacts = print_report(
             client, r.get("run"), None, ReportType.RUN, workflow_id, workflow_attempt
         )
-        set_output("artifacts", json.dumps(artifacts))
+        set_output(json.dumps(artifacts), "artifacts", args.output)
 
     if not is_success(result):
         sys.exit(1)

From 842b20471c812a89ea709f152ae5c6e868ef7522 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 13:47:34 -0700
Subject: [PATCH 08/23] Run on linux.2xlarge

---
 .github/workflows/test_mobile_job.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_mobile_job.yml b/.github/workflows/test_mobile_job.yml
index 1c14c422e1..f35ebda420 100644
--- a/.github/workflows/test_mobile_job.yml
+++ b/.github/workflows/test_mobile_job.yml
@@ -17,7 +17,7 @@ jobs:
     with:
       device-type: ios
       # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS
-      runner: ubuntu-latest
+      runner: linux.2xlarge
       # There values are prepared beforehand for the test
       project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0
       device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:b531574a-fb82-40ae-b687-8f0b81341ae0/da5d902d-45db-477b-ae0a-766e06ef3845
@@ -32,7 +32,7 @@ jobs:
     uses: ./.github/workflows/mobile_job.yml
     with:
       device-type: android
-      runner: ubuntu-latest
+      runner: linux.2xlarge
       timeout: 120
       # There values are prepared beforehand for the test
       project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0

From 5a7debc6e895c7bfe0257173b8651b22e3bc0e1a Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:00:00 -0700
Subject: [PATCH 09/23] Set the correct path

---
 .github/workflows/mobile_job.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index d253aa1484..a94fa3bc07 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -303,7 +303,7 @@ jobs:
           retention-days: 14
           s3-bucket: gha-artifacts
           s3-prefix: |
-            ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts
+            device_farm/${{ github.run_id }}/${{ github.run_attempt }}/artifacts
           path: |
             test-infra/tools/device-farm-runner/ios-artifacts-${{ github.job }}.json
 
@@ -348,6 +348,6 @@ jobs:
           retention-days: 14
           s3-bucket: gha-artifacts
           s3-prefix: |
-            ${{ env.REPOSITORY }}/${{ github.run_id }}/artifacts
+            device_farm/${{ github.run_id }}/${{ github.run_attempt }}/artifacts
           path: |
             test-infra/tools/device-farm-runner/android-artifacts-${{ github.job }}.json

From 800681c4697fc2ce396fe4dd70a2fc0048767eaa Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:00:32 -0700
Subject: [PATCH 10/23] Use GHA role

---
 .github/workflows/test_mobile_job.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_mobile_job.yml b/.github/workflows/test_mobile_job.yml
index f35ebda420..1c14c422e1 100644
--- a/.github/workflows/test_mobile_job.yml
+++ b/.github/workflows/test_mobile_job.yml
@@ -17,7 +17,7 @@ jobs:
     with:
       device-type: ios
       # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS
-      runner: linux.2xlarge
+      runner: ubuntu-latest
       # There values are prepared beforehand for the test
       project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0
       device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:b531574a-fb82-40ae-b687-8f0b81341ae0/da5d902d-45db-477b-ae0a-766e06ef3845
@@ -32,7 +32,7 @@ jobs:
     uses: ./.github/workflows/mobile_job.yml
     with:
       device-type: android
-      runner: linux.2xlarge
+      runner: ubuntu-latest
       timeout: 120
       # There values are prepared beforehand for the test
       project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0

From 882426e75089bbcd9086ea1bad683e765d194dad Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:39:53 -0700
Subject: [PATCH 11/23] Use actual job id

---
 .github/workflows/mobile_job.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index a94fa3bc07..8b5dd1a76c 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -262,6 +262,13 @@ jobs:
 
           echo "extra-data-output=${EXTRA_DATA_OUTPUT}" >> "${GITHUB_OUTPUT}"
 
+      - name: Get workflow job id
+        id: get-job-id
+        uses: ./.github/actions/get-workflow-job-id
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Run iOS tests on devices
         id: ios-test
         if: ${{ inputs.device-type == 'ios' }}
@@ -279,7 +286,7 @@ jobs:
           DEVICE_TYPE: ${{ inputs.device-type }}
           RUN_ID: ${{ github.run_id }}
           RUN_ATTEMPT: ${{ github.run_attempt }}
-          JOB_ID: ${{ github.job }}
+          JOB_ID: ${{ steps.get-job-id.outputs.job-id }}
         run: |
           set -ex
 
@@ -324,7 +331,7 @@ jobs:
           DEVICE_TYPE: ${{ inputs.device-type }}
           RUN_ID: ${{ github.run_id }}
           RUN_ATTEMPT: ${{ github.run_attempt }}
-          JOB_ID: ${{ github.job }}
+          JOB_ID: ${{ steps.get-job-id.outputs.job-id }}
         run: |
           set -ex
 

From 1747534be1a48edd7764e4e7c8f6f22d0f548d51 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:42:20 -0700
Subject: [PATCH 12/23] Another job id usage

---
 .github/workflows/mobile_job.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 8b5dd1a76c..4d014b8e29 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -312,7 +312,7 @@ jobs:
           s3-prefix: |
             device_farm/${{ github.run_id }}/${{ github.run_attempt }}/artifacts
           path: |
-            test-infra/tools/device-farm-runner/ios-artifacts-${{ github.job }}.json
+            test-infra/tools/device-farm-runner/ios-artifacts-${{ steps.get-job-id.outputs.job-id }}.json
 
       - name: Run Android tests on devices
         id: android-test
@@ -357,4 +357,4 @@ jobs:
           s3-prefix: |
             device_farm/${{ github.run_id }}/${{ github.run_attempt }}/artifacts
           path: |
-            test-infra/tools/device-farm-runner/android-artifacts-${{ github.job }}.json
+            test-infra/tools/device-farm-runner/android-artifacts-${{ steps.get-job-id.outputs.job-id }}.json

From 8ce9959dbeca10d77d63bb777cea06adaafbebcc Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:46:57 -0700
Subject: [PATCH 13/23] Use test-infra path

---
 .github/workflows/mobile_job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 4d014b8e29..0e34bbe31e 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -264,7 +264,7 @@ jobs:
 
       - name: Get workflow job id
         id: get-job-id
-        uses: ./.github/actions/get-workflow-job-id
+        uses: test-infra/.github/actions/get-workflow-job-id
         if: always()
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

From 93ea9f40aedcea3f6d81395906140dfd6d8deec9 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:47:55 -0700
Subject: [PATCH 14/23] Ugh, GH syntax

---
 .github/workflows/mobile_job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 0e34bbe31e..037bbcd528 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -264,7 +264,7 @@ jobs:
 
       - name: Get workflow job id
         id: get-job-id
-        uses: test-infra/.github/actions/get-workflow-job-id
+        uses: ./test-infra/.github/actions/get-workflow-job-id
         if: always()
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

From c1eb47e1bb68efc09818464596006a837e5bb233 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:49:21 -0700
Subject: [PATCH 15/23] Is GH having an outage atm?

---
 .github/workflows/mobile_job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 037bbcd528..5366f9959a 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -120,7 +120,7 @@ jobs:
         uses: aws-actions/configure-aws-credentials@v3
         with:
           role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_mobile_job
-          # This could go up to 18000, the max duration enforced by the server side
+          # The max duration enforced by the server side
           role-duration-seconds: 18000
           aws-region: us-east-1
 

From 0dd252e4cfd936c0db4c27734921b9776d051fc6 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:57:50 -0700
Subject: [PATCH 16/23] Does this work with working-directory?

---
 .github/workflows/mobile_job.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 5366f9959a..5a2e4db949 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -264,7 +264,8 @@ jobs:
 
       - name: Get workflow job id
         id: get-job-id
-        uses: ./test-infra/.github/actions/get-workflow-job-id
+        working-directory: test-infra
+        uses: .github/actions/get-workflow-job-id
         if: always()
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

From 9cd670c907f3888a7acae52a95f5e0960be21174 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 14:59:20 -0700
Subject: [PATCH 17/23] Wrong syntax again

---
 .github/workflows/mobile_job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 5a2e4db949..bbf87c563c 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -265,7 +265,7 @@ jobs:
       - name: Get workflow job id
         id: get-job-id
         working-directory: test-infra
-        uses: .github/actions/get-workflow-job-id
+        uses: ./.github/actions/get-workflow-job-id
         if: always()
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

From 0464814e753dbe289fb88ae5cc8c2782b8445639 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 15:03:54 -0700
Subject: [PATCH 18/23] Debug get_workflow_job_id script path

---
 .github/actions/get-workflow-job-id/action.yml | 4 ++++
 .github/workflows/mobile_job.yml               | 3 +--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/actions/get-workflow-job-id/action.yml b/.github/actions/get-workflow-job-id/action.yml
index b910b33fe0..910152e027 100644
--- a/.github/actions/get-workflow-job-id/action.yml
+++ b/.github/actions/get-workflow-job-id/action.yml
@@ -25,6 +25,10 @@ runs:
       id: get-job-id
       run: |
         set -eux
+
+        # DEBUG
+        ls -R
+
         python3 .github/scripts/get_workflow_job_id.py "${GITHUB_RUN_ID}" "${RUNNER_NAME}"
       env:
         GITHUB_TOKEN: ${{ inputs.github-token }}
diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index bbf87c563c..5366f9959a 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -264,8 +264,7 @@ jobs:
 
       - name: Get workflow job id
         id: get-job-id
-        working-directory: test-infra
-        uses: ./.github/actions/get-workflow-job-id
+        uses: ./test-infra/.github/actions/get-workflow-job-id
         if: always()
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

From eb6787389607f4eae54e8bf7671b1c235c5cee46 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 15:30:32 -0700
Subject: [PATCH 19/23] Set get-workflow-job-id working directory

---
 .github/actions/get-workflow-job-id/action.yml | 9 ++++-----
 .github/workflows/mobile_job.yml               | 1 +
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/actions/get-workflow-job-id/action.yml b/.github/actions/get-workflow-job-id/action.yml
index 910152e027..ea01ad2337 100644
--- a/.github/actions/get-workflow-job-id/action.yml
+++ b/.github/actions/get-workflow-job-id/action.yml
@@ -3,6 +3,9 @@ name: Get workflow job id
 description: Get the ID of the workflow job that is currently running.
 
 inputs:
+  working-directory:
+    description: The working directory to use
+    default: ''
   github-token:
     description: GITHUB_TOKEN
     required: true
@@ -19,16 +22,12 @@ runs:
   using: composite
   steps:
     - name: Get job id and name or fail
-      # timeout-minutes is unsupported for composite workflows, see https://github.com/actions/runner/issues/1979
-      # timeout-minutes: 10
       shell: bash
       id: get-job-id
+      working-directory: ${{ inputs.working-directory }}
       run: |
         set -eux
 
-        # DEBUG
-        ls -R
-
         python3 .github/scripts/get_workflow_job_id.py "${GITHUB_RUN_ID}" "${RUNNER_NAME}"
       env:
         GITHUB_TOKEN: ${{ inputs.github-token }}
diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 5366f9959a..e3093f9755 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -267,6 +267,7 @@ jobs:
         uses: ./test-infra/.github/actions/get-workflow-job-id
         if: always()
         with:
+          working-directory: test-infra
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run iOS tests on devices

From e2ba055810de05ded13f0c4ef87bb624a36db52e Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 16:46:54 -0700
Subject: [PATCH 20/23] Also attempt to upload to GH

---
 .github/workflows/mobile_job.yml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index e3093f9755..39712e5540 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -303,11 +303,18 @@ jobs:
             --workflow-attempt "${RUN_ATTEMPT}" \
             --output "ios-artifacts-${JOB_ID}.json"
 
+      - name: Upload iOS artifacts to GitHub
+        uses: actions/upload-artifact@v4
+        if: ${{ inputs.device-type == 'ios' }}
+        with:
+          name: ios-artifacts
+          path: |
+            test-infra/tools/device-farm-runner/ios-artifacts-${{ steps.get-job-id.outputs.job-id }}.json
+
       - name: Upload iOS artifacts to S3
         uses: seemethere/upload-artifact-s3@v5
         if: ${{ inputs.device-type == 'ios' }}
         with:
-          name: ios-artifacts
           retention-days: 14
           s3-bucket: gha-artifacts
           s3-prefix: |
@@ -348,6 +355,14 @@ jobs:
             --workflow-attempt "${RUN_ATTEMPT}" \
             --output "android-artifacts-${JOB_ID}.json"
 
+      - name: Upload iOS artifacts to GitHub
+        uses: actions/upload-artifact@v4
+        if: ${{ inputs.device-type == 'android' }}
+        with:
+          name: android-artifacts
+          path: |
+            test-infra/tools/device-farm-runner/android-artifacts-${{ steps.get-job-id.outputs.job-id }}.json
+
       - name: Upload Android artifacts to S3
         uses: seemethere/upload-artifact-s3@v5
         if: ${{ inputs.device-type == 'android' }}

From eb470d34c2fda175fe4c263eb11130c026545268 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 17:11:36 -0700
Subject: [PATCH 21/23] Use job name instead of test run name as the report
 name

---
 .../run_on_aws_devicefarm.py                  | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index d1f9ab936c..9ca2a7b255 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -291,14 +291,14 @@ def set_output(val: Any, gh_var_name: str, filename: Optional[str]) -> None:
 
 
 def print_testspec(
-    report_name: Optional[str],
+    job_name: Optional[str],
     file_name: str,
     indent: int = 0,
 ) -> None:
     """
     The test spec output from AWS Device Farm is the main output of the test job.
     """
-    print(f"::group::{report_name} test output")
+    print(f"::group::{job_name} test output")
     with open(file_name) as f:
         print(f.read())
     print("::endgroup::")
@@ -309,7 +309,7 @@ def print_test_artifacts(
     test_arn: str,
     workflow_id: str,
     workflow_attempt: int,
-    report_name: Optional[str],
+    job_name: Optional[str],
     indent: int = 0,
 ) -> List[Dict[str, str]]:
     """
@@ -346,12 +346,12 @@ def print_test_artifacts(
             )
 
             # Some more metadata to identify where the artifact comes from
-            artifact["report_name"] = report_name
+            artifact["job_name"] = job_name
             gathered_artifacts.append(artifact)
 
             # Additional step to print the test output
             if filetype == "TESTSPEC_OUTPUT":
-                print_testspec(report_name, local_filename, indent + 2)
+                print_testspec(job_name, local_filename, indent + 2)
 
     return gathered_artifacts
 
@@ -359,8 +359,8 @@ def print_test_artifacts(
 def print_report(
     client: Any,
     report: Dict[str, Any],
-    report_name: Optional[str],
     report_type: ReportType,
+    job_name: Optional[str],
     workflow_id: str,
     workflow_attempt: int,
     indent: int = 0,
@@ -374,10 +374,10 @@ def print_report(
         return []
 
     name = report["name"]
-    # Keep the top-level report name as the name of the whole test report, this
-    # is used to connect all artifacts from one report together
-    if not report_name:
-        report_name = name
+    # Keep the top-level job name as the name of the whole report, this
+    # is used to connect all artifacts from one job together
+    if report_type == ReportType.JOB:
+        job_name = name
     result = report["result"]
 
     extra_msg = ""
@@ -399,7 +399,7 @@ def print_report(
         next_report_type = ReportType.TEST
     elif report_type == ReportType.TEST:
         return print_test_artifacts(
-            client, arn, workflow_id, workflow_attempt, report_name, indent + 2
+            client, arn, workflow_id, workflow_attempt, job_name, indent + 2
         )
 
     artifacts = []
@@ -408,8 +408,8 @@ def print_report(
             print_report(
                 client,
                 more_report,
-                report_name,
                 next_report_type,
+                job_name,
                 workflow_id,
                 workflow_attempt,
                 indent + 2,
@@ -611,7 +611,7 @@ def main() -> None:
         sys.exit(1)
     finally:
         artifacts = print_report(
-            client, r.get("run"), None, ReportType.RUN, workflow_id, workflow_attempt
+            client, r.get("run"), ReportType.RUN, None, workflow_id, workflow_attempt
         )
         set_output(json.dumps(artifacts), "artifacts", args.output)
 

From 45a7a5acbd3b2b4c58aef2cab2e0e71dcce6119d Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 17:24:00 -0700
Subject: [PATCH 22/23] Also include app type

---
 .../run_on_aws_devicefarm.py                  | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/tools/device-farm-runner/run_on_aws_devicefarm.py b/tools/device-farm-runner/run_on_aws_devicefarm.py
index 9ca2a7b255..4509df170b 100755
--- a/tools/device-farm-runner/run_on_aws_devicefarm.py
+++ b/tools/device-farm-runner/run_on_aws_devicefarm.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-import json
 import datetime
+import json
 import logging
 import os
 import random
@@ -306,6 +306,7 @@ def print_testspec(
 
 def print_test_artifacts(
     client: Any,
+    app_type: str,
     test_arn: str,
     workflow_id: str,
     workflow_attempt: int,
@@ -346,6 +347,7 @@ def print_test_artifacts(
             )
 
             # Some more metadata to identify where the artifact comes from
+            artifact["app_type"] = app_type
             artifact["job_name"] = job_name
             gathered_artifacts.append(artifact)
 
@@ -358,6 +360,7 @@ def print_test_artifacts(
 
 def print_report(
     client: Any,
+    app_type: str,
     report: Dict[str, Any],
     report_type: ReportType,
     job_name: Optional[str],
@@ -399,7 +402,7 @@ def print_report(
         next_report_type = ReportType.TEST
     elif report_type == ReportType.TEST:
         return print_test_artifacts(
-            client, arn, workflow_id, workflow_attempt, job_name, indent + 2
+            client, app_type, arn, workflow_id, workflow_attempt, job_name, indent + 2
         )
 
     artifacts = []
@@ -407,6 +410,7 @@ def print_report(
         artifacts.extend(
             print_report(
                 client,
+                app_type,
                 more_report,
                 next_report_type,
                 job_name,
@@ -542,6 +546,7 @@ def main() -> None:
         + f"{datetime.date.today().isoformat()}-{''.join(random.sample(string.ascii_letters, 8))}"
     )
 
+    app_type = ""
     if args.app.startswith(AWS_ARN_PREFIX):
         appfile_arn = args.app
         info(f"Use the existing app: {appfile_arn}")
@@ -559,11 +564,13 @@ def main() -> None:
         info(f"Uploaded app: {appfile_arn}")
 
     if args.ios_xctestrun:
+        app_type = "IOS_APP"
         test_to_run = generate_ios_xctestrun(
             client, project_arn, unique_prefix, args.ios_xctestrun, args.test_spec
         )
 
     if args.android_instrumentation_test:
+        app_type = "ANDROID_APP"
         test_to_run = generate_android_instrumentation_test(
             client,
             project_arn,
@@ -611,7 +618,13 @@ def main() -> None:
         sys.exit(1)
     finally:
         artifacts = print_report(
-            client, r.get("run"), ReportType.RUN, None, workflow_id, workflow_attempt
+            client,
+            app_type,
+            r.get("run"),
+            ReportType.RUN,
+            None,
+            workflow_id,
+            workflow_attempt,
         )
         set_output(json.dumps(artifacts), "artifacts", args.output)
 

From 84c7bb46ec8608ac11be225a0f45e5bdf2424b82 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Wed, 2 Oct 2024 17:32:21 -0700
Subject: [PATCH 23/23] GH artifacts doesn't work, need to use S3 then

https://github.com/pytorch/executorch/actions/runs/11152662006/job/31001501683#step:17:26
---
 .github/workflows/mobile_job.yml | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/.github/workflows/mobile_job.yml b/.github/workflows/mobile_job.yml
index 39712e5540..f545c77b02 100644
--- a/.github/workflows/mobile_job.yml
+++ b/.github/workflows/mobile_job.yml
@@ -303,14 +303,6 @@ jobs:
             --workflow-attempt "${RUN_ATTEMPT}" \
             --output "ios-artifacts-${JOB_ID}.json"
 
-      - name: Upload iOS artifacts to GitHub
-        uses: actions/upload-artifact@v4
-        if: ${{ inputs.device-type == 'ios' }}
-        with:
-          name: ios-artifacts
-          path: |
-            test-infra/tools/device-farm-runner/ios-artifacts-${{ steps.get-job-id.outputs.job-id }}.json
-
       - name: Upload iOS artifacts to S3
         uses: seemethere/upload-artifact-s3@v5
         if: ${{ inputs.device-type == 'ios' }}
@@ -355,19 +347,10 @@ jobs:
             --workflow-attempt "${RUN_ATTEMPT}" \
             --output "android-artifacts-${JOB_ID}.json"
 
-      - name: Upload iOS artifacts to GitHub
-        uses: actions/upload-artifact@v4
-        if: ${{ inputs.device-type == 'android' }}
-        with:
-          name: android-artifacts
-          path: |
-            test-infra/tools/device-farm-runner/android-artifacts-${{ steps.get-job-id.outputs.job-id }}.json
-
       - name: Upload Android artifacts to S3
         uses: seemethere/upload-artifact-s3@v5
         if: ${{ inputs.device-type == 'android' }}
         with:
-          name: android-artifacts
           retention-days: 14
           s3-bucket: gha-artifacts
           s3-prefix: |