Skip to content

Commit 241bbd6

Browse files
ysjprojectsshijie.yupre-commit-ci[bot]
authored
phi-4 reasoning models (#2047)
Co-authored-by: shijie.yu <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 1602e96 commit 241bbd6

File tree

5 files changed

+110
-2
lines changed

5 files changed

+110
-2
lines changed

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,9 @@ Every model is written from scratch to maximize performance and remove layers of
139139
| Phi 3 | 3.8B | Microsoft Research | [Abdin et al. 2024](https://arxiv.org/abs/2404.14219) |
140140
| Phi 4 | 14B | Microsoft Research | [Abdin et al. 2024](https://arxiv.org/abs/2412.08905) |
141141
| Phi 4 Mini Instruct | 3.8B | Microsoft Research | [Microsoft 2025](https://arxiv.org/abs/2503.01743) |
142+
| Phi 4 Mini Reasoning | 3.8B | Microsoft Research | [Xu, Peng et al. 2025](https://arxiv.org/abs/2504.21233) |
143+
| Phi 4 Reasoning | 3.8B | Microsoft Research | [Abdin et al. 2025](https://arxiv.org/abs/2504.21318) |
144+
| Phi 4 Reasoning Plus | 3.8B | Microsoft Research | [Abdin et al. 2025](https://arxiv.org/abs/2504.21318) |
142145
| Platypus | 7B, 13B, 70B | Lee et al. | [Lee, Hunter, and Ruiz 2023](https://arxiv.org/abs/2308.07317) |
143146
| Pythia | {14,31,70,160,410}M, {1,1.4,2.8,6.9,12}B | EleutherAI | [Biderman et al. 2023](https://arxiv.org/abs/2304.01373) |
144147
| Qwen2.5 | 0.5B, 1.5B, 3B, 7B, 14B, 32B, 72B | Alibaba Group | [Qwen Team 2024](https://qwenlm.github.io/blog/qwen2.5/) |

‎litgpt/config.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,6 +1744,44 @@ def norm_class(self) -> Type:
17441744
mlp_class_name="LLaMAMLP",
17451745
parallel_residual=False,
17461746
),
1747+
# https://huggingface.co/microsoft/Phi-4-reasoning/blob/main/config.json
1748+
dict(
1749+
name="Phi-4-reasoning",
1750+
hf_config=dict(org="microsoft", name="Phi-4-reasoning"),
1751+
vocab_size=100352,
1752+
padded_vocab_size=100352,
1753+
block_size=32768,
1754+
n_embd=5120,
1755+
n_layer=40,
1756+
n_head=40,
1757+
n_query_groups=10,
1758+
rotary_percentage=1.0,
1759+
bias=False,
1760+
norm_class_name="RMSNorm",
1761+
intermediate_size=17920,
1762+
rope_base=500000,
1763+
mlp_class_name="LLaMAMLP",
1764+
parallel_residual=False,
1765+
),
1766+
# https://huggingface.co/microsoft/Phi-4-reasoning-plus/blob/main/config.json
1767+
dict(
1768+
name="Phi-4-reasoning-plus",
1769+
hf_config=dict(org="microsoft", name="Phi-4-reasoning-plus"),
1770+
vocab_size=100352,
1771+
padded_vocab_size=100352,
1772+
block_size=32768,
1773+
n_embd=5120,
1774+
n_layer=40,
1775+
n_head=40,
1776+
n_query_groups=10,
1777+
rotary_percentage=1.0,
1778+
bias=False,
1779+
norm_class_name="RMSNorm",
1780+
intermediate_size=17920,
1781+
rope_base=500000,
1782+
mlp_class_name="LLaMAMLP",
1783+
parallel_residual=False,
1784+
),
17471785
# https://huggingface.co/microsoft/Phi-4-mini-instruct/blob/main/config.json
17481786
dict(
17491787
name="Phi-4-mini-instruct",
@@ -1763,6 +1801,25 @@ def norm_class(self) -> Type:
17631801
parallel_residual=False,
17641802
sliding_window_size=262145,
17651803
),
1804+
# https://huggingface.co/microsoft/Phi-4-mini-reasoning/blob/main/config.json
1805+
dict(
1806+
name="Phi-4-mini-reasoning",
1807+
hf_config=dict(org="microsoft", name="Phi-4-mini-reasoning"),
1808+
vocab_size=200019,
1809+
padded_vocab_size=200064,
1810+
block_size=131072,
1811+
n_embd=3072,
1812+
n_layer=32,
1813+
n_head=24,
1814+
n_query_groups=8,
1815+
rotary_percentage=0.75,
1816+
bias=False,
1817+
norm_class_name="RMSNorm",
1818+
intermediate_size=8192,
1819+
mlp_class_name="LLaMAMLP",
1820+
parallel_residual=False,
1821+
sliding_window_size=262145,
1822+
),
17661823
]
17671824
configs.extend(phi)
17681825

‎litgpt/prompts.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,35 @@ def apply(self, prompt: str, *, sys_prompt: Optional[str] = None, **kwargs: str)
325325

326326
class Phi4(PromptStyle):
327327
def apply(self, prompt: str, *, sys_prompt: Optional[str] = None, **kwargs: str) -> str:
328-
return f"<|im_start|>user<|im_sep|>{prompt}<|im_end|><|im_start|>assistant<|im_sep|>"
328+
res = ""
329+
if sys_prompt:
330+
res += f"<|im_start|>system<|im_sep|>{sys_prompt}<|im_end|>"
331+
res += f"<|im_start|>user<|im_sep|>{prompt}<|im_end|><|im_start|>assistant<|im_sep|>"
332+
return res
333+
334+
335+
class Phi4Reasoning(PromptStyle):
336+
def apply(self, prompt: str, *, sys_prompt: Optional[str] = None, **kwargs: str) -> str:
337+
sys_prompt = (
338+
sys_prompt
339+
or "You are Phi, a language model trained by Microsoft to help users. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracing, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: <think> {Thought section} </think> {Solution section}. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion. Now, try to solve the following question through the above guidelines:"
340+
)
341+
return f"<|im_start>system<|im_sep|>{sys_prompt}<|im_end|><|im_start|>user<|im_sep|>{prompt}<|im_end|><|im_start|>assistant<|im_sep|>"
342+
343+
344+
class Phi4Mini(PromptStyle):
345+
def apply(self, prompt: str, *, sys_prompt: Optional[str] = None, **kwargs: str) -> str:
346+
res = ""
347+
if sys_prompt:
348+
res += f"<|system|>{sys_prompt}<|end|>"
349+
res += f"<|user|>{prompt}<|end|><|assistant|>"
350+
return res
351+
352+
353+
class Phi4MiniReasoning(PromptStyle):
354+
def apply(self, prompt: str, *, sys_prompt: Optional[str] = None, **kwargs: str) -> str:
355+
sys_prompt = sys_prompt or "Your name is Phi, an AI math expert developed by Microsoft."
356+
return f"<|system|>{sys_prompt}<|end|><|user|>{prompt}<|end|><|assistant|>"
329357

330358

331359
class TinyLlama(PromptStyle):
@@ -409,6 +437,9 @@ def __init__(self):
409437
"phi-2": Phi2,
410438
"phi-3": Phi3,
411439
"phi-4": Phi4,
440+
"phi-4-reasoning": Phi4Reasoning,
441+
"phi-4-mini": Phi4Mini,
442+
"phi-4-mini-reasoning": Phi4MiniReasoning,
412443
"tinyllama": TinyLlama,
413444
"gemma": Gemma,
414445
"llama3": Llama3,
@@ -455,6 +486,12 @@ def model_name_to_prompt_style(model_name: str) -> PromptStyle:
455486
return Phi2()
456487
if re.search("Phi-3", model_name):
457488
return Phi3()
489+
if re.search("Phi-4-reasoning", model_name):
490+
return Phi4Reasoning()
491+
if re.search("Phi-4-mini-reasoning", model_name):
492+
return Phi4MiniReasoning()
493+
if re.search("Phi-4-mini", model_name):
494+
return Phi4Mini()
458495
if re.search("phi-4", model_name):
459496
return Phi4()
460497
if re.search(r"tiny-llama.*chat", model_name):

‎tests/test_model.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,15 @@ def test_against_hf_phi(model_name, device, dtype):
337337
@torch.inference_mode()
338338
@pytest.mark.parametrize(
339339
"model_name",
340-
("Phi-3-mini-4k-instruct", "Phi-3-mini-128k-instruct", "Phi-3.5-mini-instruct", "phi-4", "Phi-4-mini-instruct"),
340+
(
341+
"Phi-3-mini-4k-instruct",
342+
"Phi-3-mini-128k-instruct",
343+
"Phi-3.5-mini-instruct",
344+
"phi-4",
345+
"Phi-4-mini-instruct",
346+
"Phi-4-reasoning",
347+
"Phi-4-mini-reasoning",
348+
),
341349
)
342350
@pytest.mark.parametrize(
343351
("device", "dtype"),

‎tutorials/download_model_weights.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ LitGPT supports a variety of LLM architectures with publicly available weights.
3737
| Phi 3 & 3.5 | 3.8B | Microsoft Research | [Abdin et al. 2024](https://arxiv.org/abs/2404.14219)
3838
| Phi 4 | 14B | Microsoft Research | [Abdin et al. 2024](https://arxiv.org/abs/2412.08905) |
3939
| Phi 4 Mini Instruct | 3.8B | Microsoft Research | [Microsoft 2025](https://arxiv.org/abs/2503.01743) |
40+
| Phi 4 Mini Reasoning | 3.8B | Microsoft Research | [Xu, Peng et al. 2025](https://arxiv.org/abs/2504.21233) |
41+
| Phi 4 Reasoning | 3.8B | Microsoft Research | [Abdin et al. 2025](https://arxiv.org/abs/2504.21318) |
42+
| Phi 4 Reasoning Plus | 3.8B | Microsoft Research | [Abdin et al. 2025](https://arxiv.org/abs/2504.21318) |
4043
| Platypus | 7B, 13B, 70B | Lee et al. | [Lee, Hunter, and Ruiz 2023](https://arxiv.org/abs/2308.07317) |
4144
| Pythia | {14,31,70,160,410}M, {1,1.4,2.8,6.9,12}B | EleutherAI | [Biderman et al. 2023](https://arxiv.org/abs/2304.01373) |
4245
| Qwen2.5 | 0.5B, 1.5B, 3B, 7B, 14B, 32B, 72B | Alibaba Group | [Qwen Team 2024](https://qwenlm.github.io/blog/qwen2.5/) |

0 commit comments

Comments
 (0)