Skip to content

Fix reasoning model output #83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Fix reasoning model output #83

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
56e98e1
Fix context stream
Alex-Welsh Feb 27, 2025
cadec0f
Add tab title and text copy buttons
Alex-Welsh Feb 28, 2025
335b7d1
Fix reasoning model generation output
Alex-Welsh Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 25 additions & 11 deletions web-apps/chat/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ class PossibleSystemPromptException(Exception):
def inference(latest_message, history):
# Allow mutating global variable
global BACKEND_INITIALISED
log.debug("Inference request received with history: %s", history)

try:
context = []
Expand All @@ -81,26 +82,32 @@ def inference(latest_message, history):
else:
if role != "assistant":
log.warn(f"Message role {role} converted to 'assistant'")
context.append(AIMessage(content=(content or "")))
context.append(AIMessage(content=(content or "")))
context.append(HumanMessage(content=latest_message))

log.debug("Chat context: %s", context)


response = ""
thinking = False

for chunk in llm.stream(context):
# If this is our first successful response from the backend
# then update the status variable to allow future error messages
# to be more informative
if not BACKEND_INITIALISED and len(response) > 0:
BACKEND_INITIALISED = True

# NOTE(sd109): For some reason the '>' character breaks the UI
# so we need to escape it here.
# response += chunk.content.replace('>', '\>')
# UPDATE(sd109): Above bug seems to have been fixed as of gradio 4.15.0
# but keeping this note here incase we enounter it again
response += chunk.content
# The "think" tags mark the chatbot's reasoning. Remove the content
# and replace with "Thinking..." until the closing tag is found.
content = chunk.content
if '<think>' in content or thinking:
thinking = True
response = "Thinking..."
if '</think>' in content:
thinking = False
response = ""
else:
response += content
yield response

# Handle any API errors here. See OpenAI Python client for possible error responses
Expand Down Expand Up @@ -163,13 +170,20 @@ def inference_wrapper(*args):
fill_height=True,
theme=theme,
css=settings.css_overrides,
js=settings.custom_javascript
js=settings.custom_javascript,
title=settings.page_title,
) as demo:
gr.ChatInterface(
gr.Markdown('# ' + settings.page_title)
gr.ChatInterface(
inference_wrapper,
type="messages",
title=settings.page_title,
analytics_enabled=False,
chatbot=gr.Chatbot(
show_copy_button=True,
height="75vh",
resizable=True,
sanitize_html=True,
),
)


Expand Down