Skip to content
This repository was archived by the owner on Sep 10, 2025. It is now read-only.

Commit 5e21fff

Browse files
Jack-KhuuJack-Khuu
authored
Merge branch 'main' into patch-35
2 parents 053058d + 9686c79 commit 5e21fff

File tree

7 files changed

+208
-33
lines changed

7 files changed

+208
-33
lines changed

‎.github/workflows/run-readme-pr-macos.yml‎

Lines changed: 111 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ jobs:
3333
sysctl machdep.cpu.core_count
3434
echo "::endgroup::"
3535
36-
.ci/scripts/run-docs readme
36+
echo "using workaround for #1416 and #1315 by setting torchchat device explicitly"
37+
TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs readme
3738
3839
echo "::group::Completion"
3940
echo "tests complete"
@@ -68,7 +69,8 @@ jobs:
6869
sysctl machdep.cpu.core_count
6970
echo "::endgroup::"
7071
71-
.ci/scripts/run-docs quantization
72+
echo "using workaround for #1416 and #1315 by setting torchchat device explicitly"
73+
TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs quantization
7274
7375
echo "::group::Completion"
7476
echo "tests complete"
@@ -103,7 +105,8 @@ jobs:
103105
sysctl machdep.cpu.core_count
104106
echo "::endgroup::"
105107
106-
.ci/scripts/run-docs gguf
108+
echo "using workaround for #1416 and #1315 by setting torchchat device explicitly"
109+
TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs gguf
107110
108111
echo "::group::Completion"
109112
echo "tests complete"
@@ -137,9 +140,113 @@ jobs:
137140
sysctl machdep.cpu.core_count
138141
echo "::endgroup::"
139142
140-
.ci/scripts/run-docs advanced
143+
echo "using workaround for #1416 and #1315 by setting torchchat device explicitly"
144+
TORCHCHAT_DEVICE=cpu .ci/scripts/run-docs advanced
141145
142146
echo "::group::Completion"
143147
echo "tests complete"
144148
echo "*******************************************"
145149
echo "::endgroup::"
150+
151+
test-eval-macos:
152+
runs-on: macos-14-xlarge
153+
steps:
154+
- name: Checkout code
155+
uses: actions/checkout@v2
156+
- uses: actions/setup-python@v4
157+
with:
158+
python-version: '3.10.11'
159+
- name: Setup Xcode
160+
if: runner.os == 'macOS'
161+
uses: maxim-lobanov/setup-xcode@v1
162+
with:
163+
xcode-version: '15.3'
164+
- name: Run script
165+
run: |
166+
set -x
167+
# NS: Remove previous installation of torch first
168+
# as this script does not isntall anything into conda env but rather as system dep
169+
pip3 uninstall -y torch || true
170+
set -eou pipefail
171+
172+
echo "::group::Print machine info"
173+
uname -a
174+
sysctl machdep.cpu.brand_string
175+
sysctl machdep.cpu.core_count
176+
echo "::endgroup::"
177+
178+
.ci/scripts/run-docs evaluation
179+
180+
echo "::group::Completion"
181+
echo "tests complete"
182+
echo "*******************************************"
183+
echo "::endgroup::"
184+
185+
test-multimodal-macos:
186+
runs-on: macos-14-xlarge
187+
steps:
188+
- name: Checkout code
189+
uses: actions/checkout@v2
190+
- uses: actions/setup-python@v4
191+
with:
192+
python-version: '3.10.11'
193+
- name: Setup Xcode
194+
if: runner.os == 'macOS'
195+
uses: maxim-lobanov/setup-xcode@v1
196+
with:
197+
xcode-version: '15.3'
198+
- name: Run script
199+
run: |
200+
set -x
201+
# NS: Remove previous installation of torch first
202+
# as this script does not isntall anything into conda env but rather as system dep
203+
pip3 uninstall -y torch || true
204+
set -eou pipefail
205+
206+
echo "::group::Print machine info"
207+
uname -a
208+
sysctl machdep.cpu.brand_string
209+
sysctl machdep.cpu.core_count
210+
echo "::endgroup::"
211+
212+
.ci/scripts/run-docs multimodal
213+
214+
echo "::group::Completion"
215+
echo "tests complete"
216+
echo "*******************************************"
217+
echo "::endgroup::"
218+
219+
test-native-macos:
220+
runs-on: macos-14-xlarge
221+
steps:
222+
- name: Checkout code
223+
uses: actions/checkout@v2
224+
- uses: actions/setup-python@v4
225+
with:
226+
python-version: '3.10.11'
227+
- name: Setup Xcode
228+
if: runner.os == 'macOS'
229+
uses: maxim-lobanov/setup-xcode@v1
230+
with:
231+
xcode-version: '15.3'
232+
- name: Run script
233+
run: |
234+
set -x
235+
# NS: Remove previous installation of torch first
236+
# as this script does not isntall anything into conda env but rather as system dep
237+
pip3 uninstall -y torch || true
238+
set -eou pipefail
239+
240+
echo "::group::Print machine info"
241+
uname -a
242+
sysctl machdep.cpu.brand_string
243+
sysctl machdep.cpu.core_count
244+
echo "::endgroup::"
245+
246+
.ci/scripts/run-docs native
247+
248+
echo "::group::Completion"
249+
echo "tests complete"
250+
echo "*******************************************"
251+
echo "::endgroup::"
252+

‎.github/workflows/run-readme-pr-mps.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,3 +113,84 @@ jobs:
113113
echo "tests complete"
114114
echo "*******************************************"
115115
echo "::endgroup::"
116+
117+
test-evaluation-mps-macos:
118+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
119+
with:
120+
runner: macos-m1-14 # needs MPS, was macos-m1-stable
121+
script: |
122+
set -x
123+
conda create -y -n test-evaluation-mps-macos python=3.10.11
124+
conda activate test-evaluation-mps-macos
125+
# NS: Remove previous installation of torch first
126+
# as this script does not isntall anything into conda env
127+
# but rather system dep
128+
pip3 uninstall -y torch || true
129+
set -eou pipefail
130+
131+
echo "::group::Print machine info"
132+
uname -a
133+
sysctl machdep.cpu.brand_string
134+
sysctl machdep.cpu.core_count
135+
echo "::endgroup::"
136+
137+
.ci/scripts/run-docs evaluation
138+
139+
echo "::group::Completion"
140+
echo "tests complete"
141+
echo "*******************************************"
142+
echo "::endgroup::"
143+
144+
test-multimodal-mps-macos:
145+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
146+
with:
147+
runner: macos-m1-14 # needs MPS, was macos-m1-stable
148+
script: |
149+
set -x
150+
conda create -y -n test-multimodal-mps-macos python=3.10.11
151+
conda activate test-multimodal-mps-macos
152+
# NS: Remove previous installation of torch first
153+
# as this script does not isntall anything into conda env
154+
# but rather system dep
155+
pip3 uninstall -y torch || true
156+
set -eou pipefail
157+
158+
echo "::group::Print machine info"
159+
uname -a
160+
sysctl machdep.cpu.brand_string
161+
sysctl machdep.cpu.core_count
162+
echo "::endgroup::"
163+
164+
.ci/scripts/run-docs multimodal
165+
166+
echo "::group::Completion"
167+
echo "tests complete"
168+
echo "*******************************************"
169+
echo "::endgroup::"
170+
171+
test-native-mps-macos:
172+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
173+
with:
174+
runner: macos-m1-14 # needs MPS, was macos-m1-stable
175+
script: |
176+
set -x
177+
conda create -y -n test-native-mps-macos python=3.10.11
178+
conda activate test-native-mps-macos
179+
# NS: Remove previous installation of torch first
180+
# as this script does not isntall anything into conda env
181+
# but rather system dep
182+
pip3 uninstall -y torch || true
183+
set -eou pipefail
184+
185+
echo "::group::Print machine info"
186+
uname -a
187+
sysctl machdep.cpu.brand_string
188+
sysctl machdep.cpu.core_count
189+
echo "::endgroup::"
190+
191+
.ci/scripts/run-docs native
192+
193+
echo "::group::Completion"
194+
echo "tests complete"
195+
echo "*******************************************"
196+
echo "::endgroup::"

‎install/install_requirements.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,13 @@ echo "Using pip executable: $PIP_EXECUTABLE"
5151
# NOTE: If a newly-fetched version of the executorch repo changes the value of
5252
# PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary
5353
# package versions.
54-
PYTORCH_NIGHTLY_VERSION=dev20250119
54+
PYTORCH_NIGHTLY_VERSION=dev20250124
5555

5656
# Nightly version for torchvision
57-
VISION_NIGHTLY_VERSION=dev20250119
57+
VISION_NIGHTLY_VERSION=dev20250124
5858

5959
# Nightly version for torchtune
60-
TUNE_NIGHTLY_VERSION=dev20250119
60+
TUNE_NIGHTLY_VERSION=dev20250124
6161

6262
# The pip repository that hosts nightly torch packages. cpu by default.
6363
# If cuda is available, based on presence of nvidia-smi, install the pytorch nightly

‎install/requirements.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@ jinja2
1616
# Miscellaneous
1717
snakeviz
1818
sentencepiece
19-
# numpy version range required by GGUF util
20-
numpy >= 1.17, < 2.0
19+
numpy >= 1.17
2120
blobfile
2221
tomli >= 1.1.0 ; python_version < "3.11"
2322
openai

‎torchchat/cli/builder.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -589,9 +589,8 @@ def do_nothing(max_batch_size, max_seq_length):
589589
# attributes will NOT be seen on by AOTI-compiled forward
590590
# function, e.g. calling model.setup_cache will NOT touch
591591
# AOTI compiled and maintained model buffers such as kv_cache.
592-
from torch._inductor.package import load_package
593592

594-
aoti_compiled_model = load_package(
593+
aoti_compiled_model = torch._inductor.aoti_load_package(
595594
str(builder_args.aoti_package_path.absolute())
596595
)
597596

‎torchchat/export.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -75,17 +75,20 @@ def export_for_server(
7575
if not package:
7676
options = {"aot_inductor.output_path": output_path}
7777

78-
path = torch._export.aot_compile(
78+
ep = torch.export.export(
7979
model,
8080
example_inputs,
8181
dynamic_shapes=dynamic_shapes,
82-
options=options,
8382
)
8483

8584
if package:
86-
from torch._inductor.package import package_aoti
87-
88-
path = package_aoti(output_path, path)
85+
path = torch._inductor.aoti_compile_and_package(
86+
ep, package_path=output_path, inductor_configs=options
87+
)
88+
else:
89+
path = torch._inductor.aot_compile(
90+
ep.module(), example_inputs, options=options
91+
)
8992

9093
print(f"The generated packaged model can be found at: {path}")
9194
return path

‎torchchat/model.py‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -657,7 +657,7 @@ def __init__(self, config: TransformerArgs) -> None:
657657
self.layers[str(layer_id)] = TransformerBlock(config)
658658

659659
if config.stage_idx == config.n_stages - 1:
660-
self.norm = RMSNorm(config.dim, eps=config.norm_eps)
660+
self.norm = nn.RMSNorm(config.dim, eps=config.norm_eps)
661661
self.output = nn.Linear(config.dim, config.vocab_size, bias=False)
662662
if config.tie_word_embeddings:
663663
self.output.weight = self.tok_embeddings.weight
@@ -751,8 +751,8 @@ def __init__(self, config: TransformerArgs) -> None:
751751
super().__init__()
752752
self.attention = Attention(config)
753753
self.feed_forward = FeedForward(config)
754-
self.ffn_norm = RMSNorm(config.dim, config.norm_eps)
755-
self.attention_norm = RMSNorm(config.dim, config.norm_eps)
754+
self.ffn_norm = nn.RMSNorm(config.dim, config.norm_eps)
755+
self.attention_norm = nn.RMSNorm(config.dim, config.norm_eps)
756756
# None for llama architecture, set for granite architectures
757757
self.residual_multiplier = (
758758
config.residual_multiplier
@@ -928,20 +928,6 @@ def forward(self, x: Tensor) -> Tensor:
928928
return self.w2(F.silu(self.w1(x)) * self.w3(x))
929929

930930

931-
class RMSNorm(nn.Module):
932-
def __init__(self, dim: int, eps: float = 1e-5):
933-
super().__init__()
934-
self.eps = eps
935-
self.weight = nn.Parameter(torch.ones(dim))
936-
937-
def _norm(self, x):
938-
return x * torch.rsqrt(torch.mean(x * x, dim=-1, keepdim=True) + self.eps)
939-
940-
def forward(self, x: Tensor) -> Tensor:
941-
output = self._norm(x.float()).type_as(x)
942-
return output * self.weight
943-
944-
945931
def apply_scaling(freqs: torch.Tensor, rope_scaling: Dict[str, Any]):
946932
# Check for the presence of the required keys
947933
required_keys = {

0 commit comments

Comments
 (0)