Skip to content

Commit 8be6a6b

Browse files
jackzhxngjackzhxng
committed
Merge branch 'main' into jz/cpp-no-op-tokenizer
2 parents 6cc40bb + 072403b commit 8be6a6b

File tree

133 files changed

+4014
-1104
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

133 files changed

+4014
-1104
lines changed

‎.ci/scripts/check_c10_sync.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -eux
9+
ls pytorch/.git || git clone https://github.com/pytorch/pytorch.git
10+
pytorch_pin="$(< .ci/docker/ci_commit_pins/pytorch.txt)"
11+
pushd pytorch
12+
git checkout "$pytorch_pin"
13+
popd
14+
"$(dirname "${BASH_SOURCE[0]}")"/compare_dirs.sh runtime/core/portable_type/c10/c10 pytorch/c10

‎.ci/scripts/compare_dirs.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -eux
9+
10+
# Check if dir1's files are also found in dir2 with the same
11+
# contents. Exempt files named BUCK, CMakeLists.txt, TARGETS, or
12+
# targets.bzl.
13+
14+
if [ $# -ne 2 ]; then
15+
echo "Usage: $0 dir1 dir2" >&2
16+
exit 1
17+
fi
18+
dir1="$1"
19+
dir2="$2"
20+
21+
if [ ! -d "$dir1" ] || [ ! -d "$dir2" ]; then
22+
echo "Error: Both directories must exist" >&2
23+
exit 1
24+
fi
25+
26+
exit_status=0
27+
while IFS= read -r -d '' file; do
28+
base=$(basename "$file")
29+
case "$base" in
30+
"BUCK"|"CMakeLists.txt"|"TARGETS"|"targets.bzl")
31+
continue
32+
;;
33+
esac
34+
# Construct the corresponding path in the second directory
35+
file2="$dir2/${file#$dir1/}"
36+
# Check if the corresponding file exists in the second directory
37+
if [ ! -f "$file2" ]; then
38+
echo "Error: File '$file' found in '$dir1' but not found in '$dir2'" >&2
39+
exit 1
40+
fi
41+
# Compare the contents of the two files using diff
42+
set +ex
43+
differences=$(diff -u -p "$file" "$file2")
44+
set -e # leave x off
45+
# If there are any differences, print an error message and exit with failure status
46+
if [ -n "$differences" ]; then
47+
echo "Error: Mismatch detected in file '$file':" >&2
48+
echo "$differences" >&2
49+
exit_status=1
50+
fi
51+
set -x
52+
done < <(find "$dir1" -type f -print0)
53+
54+
exit $exit_status

‎.ci/scripts/gather_benchmark_configs.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"apple_iphone_15": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3b5acd2e-92e2-4778-b651-7726bafe129d",
2222
"apple_iphone_15+ios_18": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/12c8b15c-8d03-4e07-950d-0a627e7595b4",
2323
"samsung_galaxy_s22": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa",
24+
"samsung_galaxy_s22_private": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/ea6b049d-1508-4233-9a56-5d9eacbe1078",
2425
"samsung_galaxy_s24": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98f8788c-2e25-4a3c-8bb2-0d1e8897c0db",
2526
"google_pixel_8_pro": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d65096ab-900b-4521-be8b-a3619b69236a",
2627
"google_pixel_3_private_rooted": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98d23ca8-ea9e-4fb7-b725-d402017b198d",

‎.github/scripts/extract_benchmark_results.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,10 @@ def transform(
349349
# Overwrite the device name here with the job name as it has more information about
350350
# the device, i.e. Samsung Galaxy S22 5G instead of just Samsung
351351
for r in benchmark_results:
352-
r["deviceInfo"]["device"] = job_name
352+
is_private_device = job_report.get("is_private_instance", False)
353+
r["deviceInfo"]["device"] = (
354+
f"{job_name} (private)" if is_private_device else job_name
355+
)
353356

354357
# From https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database
355358
return [
@@ -363,6 +366,7 @@ def transform(
363366
"benchmark_config": json.dumps(benchmark_config),
364367
"job_conclusion": "SUCCESS",
365368
"job_arn": job_report.get("arn", ""),
369+
"instance_arn": job_report.get("instance_arn", ""),
366370
},
367371
},
368372
"model": {

‎.github/workflows/android-perf-private-device-experiment.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ on:
2323
description: Target devices to run benchmark
2424
required: false
2525
type: string
26-
default: google_pixel_3_private_rooted
26+
default: samsung_galaxy_s22_private
2727
benchmark_configs:
2828
description: The list of configs used the benchmark
2929
required: false
@@ -39,7 +39,7 @@ on:
3939
description: Target devices to run benchmark
4040
required: false
4141
type: string
42-
default: google_pixel_3_private_rooted
42+
default: samsung_galaxy_s22_private
4343
benchmark_configs:
4444
description: The list of configs used the benchmark
4545
required: false
@@ -58,5 +58,5 @@ jobs:
5858
contents: read
5959
with:
6060
models: ${{ inputs.models || 'mv3,meta-llama/Llama-3.2-1B-Instruct-SpinQuant_INT4_EO8,meta-llama/Llama-3.2-1B-Instruct-QLORA_INT4_EO8' }}
61-
devices: google_pixel_3_private_rooted
61+
devices: samsung_galaxy_s22_private
6262
benchmark_configs: ${{ inputs.benchmark_configs }}

‎.github/workflows/apple-perf-private-device-experiment.yml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,16 @@
11
name: apple-perf (private devices)
22

33
on:
4-
# TODO (huydhn): Disable the schedule run until we land the change to add device pool and device name
5-
# to separate between public and private iOS devices
6-
# schedule:
7-
# - cron: 0 0,4,8,12,16,20 * * *
4+
schedule:
5+
- cron: 0 0,4,8,12,16,20 * * *
86
pull_request:
97
paths:
108
- .github/workflows/apple-perf-private-device-experiment.yml
11-
# push:
12-
# branches:
13-
# - main
14-
# paths:
15-
# - .github/workflows/apple-perf-private-device-experiment.yml
9+
push:
10+
branches:
11+
- main
12+
paths:
13+
- .github/workflows/apple-perf-private-device-experiment.yml
1614
# Note: GitHub has an upper limit of 10 inputs
1715
workflow_dispatch:
1816
inputs:

‎.github/workflows/check-c10-sync.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: check-c10-sync
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- .ci/docker/ci_commit_pins/pytorch.txt
7+
- .ci/scripts/compare_dirs.sh
8+
- runtime/core/portable_type/c10/**
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
check-c10-sync:
16+
permissions:
17+
id-token: write
18+
contents: read
19+
name: check-c10-sync
20+
runs-on: ubuntu-latest
21+
steps:
22+
- uses: actions/checkout@v3
23+
with:
24+
fetch-depth: 0
25+
- name: Clone PyTorch
26+
run: |
27+
.ci/scripts/check_c10_sync.sh

‎.github/workflows/doc-build.yml‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,6 @@ on:
1414
- cron: '0 0 * * *'
1515

1616
jobs:
17-
check-urls:
18-
runs-on: ubuntu-latest
19-
steps:
20-
- uses: actions/checkout@v3
21-
- name: Check URLs
22-
run: bash ./scripts/check_urls.sh
23-
24-
check-xrefs:
25-
runs-on: ubuntu-latest
26-
steps:
27-
- uses: actions/checkout@v3
28-
- name: Check Links
29-
run: bash ./scripts/check_xrefs.sh
30-
3117
build:
3218
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
3319
permissions:

‎.github/workflows/lint.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,30 @@ jobs:
6464
6565
exit $RC
6666
67+
lint-urls:
68+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
69+
with:
70+
runner: linux.2xlarge
71+
docker-image: executorch-ubuntu-22.04-linter
72+
submodules: 'none'
73+
fetch-depth: 0
74+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
75+
timeout: 90
76+
script: |
77+
./scripts/lint_urls.sh
78+
79+
lint-xrefs:
80+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
81+
with:
82+
runner: linux.2xlarge
83+
docker-image: executorch-ubuntu-22.04-linter
84+
submodules: 'none'
85+
fetch-depth: 0
86+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
87+
timeout: 90
88+
script: |
89+
./scripts/lint_xrefs.sh
90+
6791
android-java-format:
6892
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
6993
permissions:

‎.github/workflows/pull.yml‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -481,37 +481,38 @@ jobs:
481481
build-tool: buck2
482482
docker-image: executorch-ubuntu-22.04-clang12
483483

484-
unittest-arm:
484+
unittest-arm-backend-with-no-fvp:
485+
name: unittest-arm-backend-with-no-fvp
485486
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
486487
permissions:
487488
id-token: write
488489
contents: read
490+
strategy:
491+
matrix:
492+
include:
493+
- test_arm_baremetal: test_pytest_ops
494+
- test_arm_baremetal: test_pytest_models
495+
fail-fast: false
489496
with:
490497
runner: linux.2xlarge
491498
docker-image: executorch-ubuntu-22.04-arm-sdk
492499
submodules: 'recursive'
493500
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
494501
timeout: 90
495502
script: |
496-
set -eux
497-
498503
# The generic Linux job chooses to use base env, not the one setup by the image
499504
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
500505
conda activate "${CONDA_ENV}"
501506
502-
BUILD_TOOL="cmake"
503-
504-
# Setup MacOS dependencies as there is no Docker support on MacOS atm
505-
PYTHON_EXECUTABLE=python \
506-
CMAKE_ARGS="-DEXECUTORCH_BUILD_PYBIND=ON" \
507-
EXECUTORCH_BUILD_ARM_BAREMETAL=ON \
508-
.ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
507+
source .ci/scripts/utils.sh
508+
install_executorch "--use-pt-pinned-commit"
509509
510-
# Install Arm dependencies
511510
.ci/scripts/setup-arm-baremetal-tools.sh
512511
513-
# Run pytest without simulator
514-
backends/arm/test/test_arm_baremetal.sh test_pytest
512+
ARM_TEST=${{ matrix.test_arm_baremetal }}
513+
514+
# Test test_arm_baremetal.sh with test
515+
backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
515516
516517
test-llama-runner-qnn-linux:
517518
name: test-llama-runner-qnn-linux

0 commit comments

Comments
 (0)