Skip to content

Commit 40f4ea2

Browse files
winglianwinglian
authored
replace references to random 68m model w 135m smollm2 (axolotl-ai-cloud#2570) [skip ci]
* replace references to random 68m model w 135m smollm2 * use AutoTokenizer for smollm2
1 parent f1df73a commit 40f4ea2

13 files changed

+95
-87
lines changed

‎tests/conftest.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,9 @@ def download_smollm2_135m_model():
7979

8080

8181
@pytest.fixture(scope="session", autouse=True)
82-
def download_llama_68m_random_model():
82+
def download_smollm2_135m_gptq_model():
8383
# download the model
84-
snapshot_download_w_retry("JackFram/llama-68m", repo_type="model")
84+
snapshot_download_w_retry("lilmeaty/SmolLM2-135M-Instruct-GPTQ", repo_type="model")
8585

8686

8787
@pytest.fixture(scope="session", autouse=True)

‎tests/e2e/patched/test_4d_multipack_llama.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def test_sdp_lora_packing(self, temp_dir):
2828
# pylint: disable=duplicate-code
2929
cfg = DictDefault(
3030
{
31-
"base_model": "JackFram/llama-68m",
31+
"base_model": "HuggingFaceTB/SmolLM2-135M",
3232
"flash_attention": False,
3333
"sdp_attention": True,
3434
"sample_packing": True,
@@ -41,6 +41,9 @@ def test_sdp_lora_packing(self, temp_dir):
4141
"lora_target_linear": True,
4242
"sequence_len": 1024,
4343
"val_set_size": 0.02,
44+
"special_tokens": {
45+
"pad_token": "<|endoftext|>",
46+
},
4447
"datasets": [
4548
{
4649
"path": "mhenrichsen/alpaca_2k_test",
@@ -73,7 +76,7 @@ def test_torch_lora_packing(self, temp_dir):
7376
# pylint: disable=duplicate-code
7477
cfg = DictDefault(
7578
{
76-
"base_model": "JackFram/llama-68m",
79+
"base_model": "HuggingFaceTB/SmolLM2-135M",
7780
"flash_attention": False,
7881
"sdp_attention": False,
7982
"sample_packing": True,
@@ -86,6 +89,9 @@ def test_torch_lora_packing(self, temp_dir):
8689
"lora_dropout": 0.05,
8790
"lora_target_linear": True,
8891
"val_set_size": 0.02,
92+
"special_tokens": {
93+
"pad_token": "<|endoftext|>",
94+
},
8995
"datasets": [
9096
{
9197
"path": "mhenrichsen/alpaca_2k_test",

‎tests/e2e/patched/test_fused_llama.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def test_fft_packing(self, temp_dir):
3232
# pylint: disable=duplicate-code
3333
cfg = DictDefault(
3434
{
35-
"base_model": "JackFram/llama-68m",
35+
"base_model": "HuggingFaceTB/SmolLM2-135M",
3636
"flash_attention": True,
3737
"pad_to_sequence_len": True,
3838
"flash_attn_fuse_qkv": True,
@@ -41,9 +41,7 @@ def test_fft_packing(self, temp_dir):
4141
"sequence_len": 1024,
4242
"val_set_size": 0.02,
4343
"special_tokens": {
44-
"unk_token": "<unk>",
45-
"bos_token": "<s>",
46-
"eos_token": "</s>",
44+
"pad_token": "<|endoftext|>",
4745
},
4846
"datasets": [
4947
{

‎tests/e2e/patched/test_llama_s2_attention.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ def test_lora_s2_attn(self, temp_dir):
3131
# pylint: disable=duplicate-code
3232
cfg = DictDefault(
3333
{
34-
"base_model": "JackFram/llama-68m",
35-
"tokenizer_type": "LlamaTokenizer",
34+
"base_model": "HuggingFaceTB/SmolLM2-135M",
35+
"tokenizer_type": "AutoTokenizer",
3636
"sequence_len": 16384,
3737
"sample_packing": False,
3838
"flash_attention": True,
@@ -44,7 +44,9 @@ def test_lora_s2_attn(self, temp_dir):
4444
"lora_dropout": 0.05,
4545
"lora_target_linear": True,
4646
"val_set_size": 0.02,
47-
"special_tokens": {},
47+
"special_tokens": {
48+
"pad_token": "<|endoftext|>",
49+
},
4850
"datasets": [
4951
{
5052
"path": "Yukang/LongAlpaca-12k",
@@ -78,14 +80,16 @@ def test_fft_s2_attn(self, temp_dir):
7880
# pylint: disable=duplicate-code
7981
cfg = DictDefault(
8082
{
81-
"base_model": "JackFram/llama-68m",
82-
"tokenizer_type": "LlamaTokenizer",
83+
"base_model": "HuggingFaceTB/SmolLM2-135M",
84+
"tokenizer_type": "AutoTokenizer",
8385
"sequence_len": 16384,
8486
"sample_packing": False,
8587
"flash_attention": True,
8688
"s2_attention": True,
8789
"val_set_size": 0.02,
88-
"special_tokens": {},
90+
"special_tokens": {
91+
"pad_token": "<|endoftext|>",
92+
},
8993
"datasets": [
9094
{
9195
"path": "Yukang/LongAlpaca-12k",

‎tests/e2e/patched/test_lora_llama_multipack.py

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ def test_lora_packing(self, temp_dir):
3131
# pylint: disable=duplicate-code
3232
cfg = DictDefault(
3333
{
34-
"base_model": "JackFram/llama-68m",
35-
"tokenizer_type": "LlamaTokenizer",
34+
"base_model": "HuggingFaceTB/SmolLM2-135M",
35+
"tokenizer_type": "AutoTokenizer",
3636
"sequence_len": 1024,
3737
"sample_packing": True,
3838
"flash_attention": True,
@@ -44,9 +44,7 @@ def test_lora_packing(self, temp_dir):
4444
"lora_target_linear": True,
4545
"val_set_size": 0.2,
4646
"special_tokens": {
47-
"unk_token": "<unk>",
48-
"bos_token": "<s>",
49-
"eos_token": "</s>",
47+
"pad_token": "<|endoftext|>",
5048
},
5149
"datasets": [
5250
{
@@ -84,9 +82,9 @@ def test_lora_gptq_packed(self, temp_dir):
8482
# pylint: disable=duplicate-code
8583
cfg = DictDefault(
8684
{
87-
"base_model": "TheBlokeAI/jackfram_llama-68m-GPTQ",
85+
"base_model": "lilmeaty/SmolLM2-135M-Instruct-GPTQ",
8886
"model_type": "AutoModelForCausalLM",
89-
"tokenizer_type": "LlamaTokenizer",
87+
"tokenizer_type": "AutoTokenizer",
9088
"sequence_len": 1024,
9189
"sample_packing": True,
9290
"flash_attention": True,
@@ -100,9 +98,7 @@ def test_lora_gptq_packed(self, temp_dir):
10098
"lora_target_linear": True,
10199
"val_set_size": 0.02,
102100
"special_tokens": {
103-
"unk_token": "<unk>",
104-
"bos_token": "<s>",
105-
"eos_token": "</s>",
101+
"pad_token": "<|endoftext|>",
106102
},
107103
"datasets": [
108104
{

‎tests/e2e/test_dpo.py

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,18 @@ def test_dpo_lora(self, temp_dir):
3131
# pylint: disable=duplicate-code
3232
cfg = DictDefault(
3333
{
34-
"base_model": "JackFram/llama-68m",
35-
"tokenizer_type": "LlamaTokenizer",
34+
"base_model": "HuggingFaceTB/SmolLM2-135M",
35+
"tokenizer_type": "AutoTokenizer",
3636
"sequence_len": 1024,
3737
"load_in_8bit": True,
3838
"adapter": "lora",
3939
"lora_r": 64,
4040
"lora_alpha": 32,
4141
"lora_dropout": 0.1,
4242
"lora_target_linear": True,
43-
"special_tokens": {},
43+
"special_tokens": {
44+
"pad_token": "<|endoftext|>",
45+
},
4446
"rl": "dpo",
4547
"datasets": [
4648
{
@@ -77,16 +79,18 @@ def test_dpo_nll_lora(self, temp_dir):
7779
# pylint: disable=duplicate-code
7880
cfg = DictDefault(
7981
{
80-
"base_model": "JackFram/llama-68m",
81-
"tokenizer_type": "LlamaTokenizer",
82+
"base_model": "HuggingFaceTB/SmolLM2-135M",
83+
"tokenizer_type": "AutoTokenizer",
8284
"sequence_len": 1024,
8385
"load_in_8bit": True,
8486
"adapter": "lora",
8587
"lora_r": 64,
8688
"lora_alpha": 32,
8789
"lora_dropout": 0.1,
8890
"lora_target_linear": True,
89-
"special_tokens": {},
91+
"special_tokens": {
92+
"pad_token": "<|endoftext|>",
93+
},
9094
"rl": "dpo",
9195
"rpo_alpha": 0.5,
9296
"datasets": [
@@ -124,16 +128,18 @@ def test_dpo_use_weighting(self, temp_dir):
124128
# pylint: disable=duplicate-code
125129
cfg = DictDefault(
126130
{
127-
"base_model": "JackFram/llama-68m",
128-
"tokenizer_type": "LlamaTokenizer",
131+
"base_model": "HuggingFaceTB/SmolLM2-135M",
132+
"tokenizer_type": "AutoTokenizer",
129133
"sequence_len": 1024,
130134
"load_in_8bit": True,
131135
"adapter": "lora",
132136
"lora_r": 64,
133137
"lora_alpha": 32,
134138
"lora_dropout": 0.1,
135139
"lora_target_linear": True,
136-
"special_tokens": {},
140+
"special_tokens": {
141+
"pad_token": "<|endoftext|>",
142+
},
137143
"rl": "dpo",
138144
"dpo_use_weighting": True,
139145
"datasets": [
@@ -172,16 +178,18 @@ def test_kto_pair_lora(self, temp_dir):
172178
# pylint: disable=duplicate-code
173179
cfg = DictDefault(
174180
{
175-
"base_model": "JackFram/llama-68m",
176-
"tokenizer_type": "LlamaTokenizer",
181+
"base_model": "HuggingFaceTB/SmolLM2-135M",
182+
"tokenizer_type": "AutoTokenizer",
177183
"sequence_len": 1024,
178184
"load_in_8bit": True,
179185
"adapter": "lora",
180186
"lora_r": 64,
181187
"lora_alpha": 32,
182188
"lora_dropout": 0.1,
183189
"lora_target_linear": True,
184-
"special_tokens": {},
190+
"special_tokens": {
191+
"pad_token": "<|endoftext|>",
192+
},
185193
"rl": "kto_pair",
186194
"datasets": [
187195
{
@@ -218,16 +226,18 @@ def test_ipo_lora(self, temp_dir):
218226
# pylint: disable=duplicate-code
219227
cfg = DictDefault(
220228
{
221-
"base_model": "JackFram/llama-68m",
222-
"tokenizer_type": "LlamaTokenizer",
229+
"base_model": "HuggingFaceTB/SmolLM2-135M",
230+
"tokenizer_type": "AutoTokenizer",
223231
"sequence_len": 1024,
224232
"load_in_8bit": True,
225233
"adapter": "lora",
226234
"lora_r": 64,
227235
"lora_alpha": 32,
228236
"lora_dropout": 0.1,
229237
"lora_target_linear": True,
230-
"special_tokens": {},
238+
"special_tokens": {
239+
"pad_token": "<|endoftext|>",
240+
},
231241
"rl": "ipo",
232242
"datasets": [
233243
{
@@ -264,16 +274,18 @@ def test_orpo_lora(self, temp_dir):
264274
# pylint: disable=duplicate-code
265275
cfg = DictDefault(
266276
{
267-
"base_model": "JackFram/llama-68m",
268-
"tokenizer_type": "LlamaTokenizer",
277+
"base_model": "HuggingFaceTB/SmolLM2-135M",
278+
"tokenizer_type": "AutoTokenizer",
269279
"sequence_len": 1024,
270280
"load_in_8bit": True,
271281
"adapter": "lora",
272282
"lora_r": 64,
273283
"lora_alpha": 32,
274284
"lora_dropout": 0.1,
275285
"lora_target_linear": True,
276-
"special_tokens": {},
286+
"special_tokens": {
287+
"pad_token": "<|endoftext|>",
288+
},
277289
"rl": "orpo",
278290
"orpo_alpha": 0.1,
279291
"remove_unused_columns": False,
@@ -314,7 +326,7 @@ def test_kto_lora(self, temp_dir):
314326
# pylint: disable=duplicate-code
315327
cfg = DictDefault(
316328
{
317-
"base_model": "JackFram/llama-68m",
329+
"base_model": "HuggingFaceTB/SmolLM2-135M",
318330
"tokenizer_type": "LlamaTokenizer",
319331
"sequence_len": 1024,
320332
"load_in_8bit": True,
@@ -323,7 +335,9 @@ def test_kto_lora(self, temp_dir):
323335
"lora_alpha": 32,
324336
"lora_dropout": 0.1,
325337
"lora_target_linear": True,
326-
"special_tokens": {},
338+
"special_tokens": {
339+
"pad_token": "<|endoftext|>",
340+
},
327341
"rl": "kto",
328342
"rl_beta": 0.5,
329343
"kto_desirable_weight": 1.0,

‎tests/e2e/test_llama.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,13 @@ def test_fft_trust_remote_code(self, temp_dir):
2626
# pylint: disable=duplicate-code
2727
cfg = DictDefault(
2828
{
29-
"base_model": "JackFram/llama-68m",
30-
"tokenizer_type": "LlamaTokenizer",
29+
"base_model": "HuggingFaceTB/SmolLM2-135M",
30+
"tokenizer_type": "AutoTokenizer",
3131
"trust_remote_code": True,
3232
"sequence_len": 512,
3333
"val_set_size": 0.02,
3434
"special_tokens": {
35-
"unk_token": "<unk>",
36-
"bos_token": "<s>",
37-
"eos_token": "</s>",
35+
"pad_token": "<|endoftext|>",
3836
},
3937
"datasets": [
4038
{

‎tests/e2e/test_load_model.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ def setup_method(self):
2626
# load config
2727
self.cfg = DictDefault(
2828
{
29-
"base_model": "JackFram/llama-68m",
30-
"tokenizer_type": "LlamaTokenizer",
31-
"tokenizer_config": "JackFram/llama-68m",
29+
"base_model": "HuggingFaceTB/SmolLM2-135M",
30+
"tokenizer_type": "AutoTokenizer",
31+
"tokenizer_config": "HuggingFaceTB/SmolLM2-135M",
3232
"sequence_len": 1024,
3333
"load_in_8bit": False,
3434
"adapter": "lora",
@@ -38,9 +38,7 @@ def setup_method(self):
3838
"lora_target_linear": True,
3939
"val_set_size": 0.02,
4040
"special_tokens": {
41-
"unk_token": "<unk>",
42-
"bos_token": "<s>",
43-
"eos_token": "</s>",
41+
"pad_token": "<|endoftext|>",
4442
},
4543
"datasets": [
4644
{

‎tests/e2e/test_lora_llama.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ def test_lora(self, temp_dir):
2828
# pylint: disable=duplicate-code
2929
cfg = DictDefault(
3030
{
31-
"base_model": "JackFram/llama-68m",
32-
"tokenizer_type": "LlamaTokenizer",
31+
"base_model": "HuggingFaceTB/SmolLM2-135M",
32+
"tokenizer_type": "AutoTokenizer",
3333
"sequence_len": 1024,
3434
"load_in_8bit": True,
3535
"adapter": "lora",
@@ -39,9 +39,7 @@ def test_lora(self, temp_dir):
3939
"lora_target_linear": True,
4040
"val_set_size": 0.02,
4141
"special_tokens": {
42-
"unk_token": "<unk>",
43-
"bos_token": "<s>",
44-
"eos_token": "</s>",
42+
"pad_token": "<|endoftext|>",
4543
},
4644
"datasets": [
4745
{

0 commit comments

Comments
 (0)