Skip to content
This repository was archived by the owner on Sep 10, 2025. It is now read-only.

Flag misnomer: GeneratorArgs.is_torchtune_model #1274

Merged
merged 1 commit into from
Oct 7, 2024
Merged

Flag misnomer: GeneratorArgs.is_torchtune_model #1274

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions torchchat/generate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ class GeneratorArgs:
speculate_k: int = 5
sequential_prefill: bool = False
max_autotune: bool = False
# (Misnomer) See Issue: https://github.com/pytorch/torchchat/issues/1273
is_torchtune_model: bool = False

def __post_init__(self):
Expand Down Expand Up @@ -958,16 +959,19 @@ def chat(
if get_system_prompt == "y" or get_system_prompt == "Y":
self.system_prompt = input("What is your system prompt? \n")

# elif not generator_args.is_torchtune_model:
# max_seq_length = min(
# encoded.size(0) + generator_args.max_new_tokens,
# (
# text_transformer_args.block_size
# if text_transformer_args is not None
# else 2048
# ),
# max_seq_length,
# )
# `is_torchtune_model` is a misnomer since it doesn't capture all
# torchtune models (i.e. Flamingo)
# See Issue: https://github.com/pytorch/torchchat/issues/1273
elif not generator_args.is_torchtune_model and self.model.config.model_type != ModelType.Flamingo:
max_seq_length = min(
encoded.size(0) + generator_args.max_new_tokens,
(
text_transformer_args.block_size
if text_transformer_args is not None
else 2048
Copy link
Contributor

@byjlw byjlw Oct 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

discrete numbers and strings should be constants or at the very least have comments explaining why that number.

),
max_seq_length,
)

max_seq_length = (
max_seq_length + self.speculative_builder_args.speculate_k + 1
Expand Down
Loading