diff --git a/.ci/scripts/gather_benchmark_configs.py b/.ci/scripts/gather_benchmark_configs.py index 9a4723d7e56..27877d19cae 100755 --- a/.ci/scripts/gather_benchmark_configs.py +++ b/.ci/scripts/gather_benchmark_configs.py @@ -24,6 +24,7 @@ "samsung_galaxy_s24": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98f8788c-2e25-4a3c-8bb2-0d1e8897c0db", "google_pixel_8_pro": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d65096ab-900b-4521-be8b-a3619b69236a", "google_pixel_3_private_rooted": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98d23ca8-ea9e-4fb7-b725-d402017b198d", + "apple_iphone_15_private": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/55929353-2f28-4ee5-bdff-d1a95f58cb28", } # Predefined benchmark configurations diff --git a/.github/workflows/apple-perf-private-device-experiment.yml b/.github/workflows/apple-perf-private-device-experiment.yml new file mode 100644 index 00000000000..2e9e444bada --- /dev/null +++ b/.github/workflows/apple-perf-private-device-experiment.yml @@ -0,0 +1,64 @@ +name: apple-perf (private devices) + +on: + # TODO (huydhn): Disable the schedule run until we land the change to add device pool and device name + # to separate between public and private iOS devices + # schedule: + # - cron: 0 0,4,8,12,16,20 * * * + pull_request: + paths: + - .github/workflows/apple-perf-private-device-experiment.yml + # push: + # branches: + # - main + # paths: + # - .github/workflows/apple-perf-private-device-experiment.yml + # Note: GitHub has an upper limit of 10 inputs + workflow_dispatch: + inputs: + models: + description: Models to be benchmarked + required: false + type: string + default: mv3,meta-llama/Llama-3.2-1B-Instruct-SpinQuant_INT4_EO8,meta-llama/Llama-3.2-1B-Instruct-QLORA_INT4_EO8 + devices: + description: Target devices to run benchmark + required: false + type: string + default: apple_iphone_15_private + benchmark_configs: + description: The list of configs used the benchmark + required: false + type: string + workflow_call: + inputs: + models: + description: Models to be benchmarked + required: false + type: string + default: mv3,meta-llama/Llama-3.2-1B-Instruct-SpinQuant_INT4_EO8,meta-llama/Llama-3.2-1B-Instruct-QLORA_INT4_EO8 + devices: + description: Target devices to run benchmark + required: false + type: string + default: apple_iphone_15_private + benchmark_configs: + description: The list of configs used the benchmark + required: false + type: string + +concurrency: + group: apple-perf-private-devices-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} + cancel-in-progress: true + +jobs: + apple: + uses: ./.github/workflows/apple-perf.yml + secrets: inherit + permissions: + id-token: write + contents: read + with: + models: ${{ inputs.models || 'mv3,meta-llama/Llama-3.2-1B-Instruct-SpinQuant_INT4_EO8,meta-llama/Llama-3.2-1B-Instruct-QLORA_INT4_EO8' }} + devices: apple_iphone_15_private + benchmark_configs: ${{ inputs.benchmark_configs }}