diff --git a/.ci/scripts/gather_benchmark_configs.py b/.ci/scripts/gather_benchmark_configs.py index b2126f84e78..9a4723d7e56 100755 --- a/.ci/scripts/gather_benchmark_configs.py +++ b/.ci/scripts/gather_benchmark_configs.py @@ -23,6 +23,7 @@ "samsung_galaxy_s22": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa", "samsung_galaxy_s24": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98f8788c-2e25-4a3c-8bb2-0d1e8897c0db", "google_pixel_8_pro": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d65096ab-900b-4521-be8b-a3619b69236a", + "google_pixel_3_private_rooted": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98d23ca8-ea9e-4fb7-b725-d402017b198d", } # Predefined benchmark configurations diff --git a/.github/workflows/android-perf-private-device-experiment.yml b/.github/workflows/android-perf-private-device-experiment.yml new file mode 100644 index 00000000000..ffbd1e064b1 --- /dev/null +++ b/.github/workflows/android-perf-private-device-experiment.yml @@ -0,0 +1,62 @@ +name: android-perf (private devices) + +on: + schedule: + - cron: 0 0,4,8,12,16,20 * * * + pull_request: + paths: + - .github/workflows/android-perf-private-device-experiment.yml + push: + branches: + - main + paths: + - .github/workflows/android-perf-private-device-experiment.yml + # Note: GitHub has an upper limit of 10 inputs + workflow_dispatch: + inputs: + models: + description: Models to be benchmarked + required: false + type: string + default: mv3,meta-llama/Llama-3.2-1B-Instruct-SpinQuant_INT4_EO8,meta-llama/Llama-3.2-1B-Instruct-QLORA_INT4_EO8 + devices: + description: Target devices to run benchmark + required: false + type: string + default: google_pixel_3_private_rooted + benchmark_configs: + description: The list of configs used the benchmark + required: false + type: string + workflow_call: + inputs: + models: + description: Models to be benchmarked + required: false + type: string + default: mv3,meta-llama/Llama-3.2-1B-Instruct-SpinQuant_INT4_EO8,meta-llama/Llama-3.2-1B-Instruct-QLORA_INT4_EO8 + devices: + description: Target devices to run benchmark + required: false + type: string + default: google_pixel_3_private_rooted + benchmark_configs: + description: The list of configs used the benchmark + required: false + type: string + +concurrency: + group: android-perf-private-devices-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} + cancel-in-progress: true + +jobs: + android: + uses: ./.github/workflows/android-perf.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + models: ${{ inputs.models }} + devices: google_pixel_3_private_rooted + benchmark_configs: ${{ inputs.benchmark_configs }}