From dc1ac6edef792e6b1dd26889b60813aa8e9caa0a Mon Sep 17 00:00:00 2001 From: Hudson Cooper Date: Thu, 21 Aug 2025 17:16:07 -0700 Subject: [PATCH] stub out gpt_oss notebook --- notebooks/tutorials/gpt_oss.ipynb | 437 ++++++++++++++++++++++++++++++ 1 file changed, 437 insertions(+) create mode 100644 notebooks/tutorials/gpt_oss.ipynb diff --git a/notebooks/tutorials/gpt_oss.ipynb b/notebooks/tutorials/gpt_oss.ipynb new file mode 100644 index 000000000..89dbc0ccf --- /dev/null +++ b/notebooks/tutorials/gpt_oss.ipynb @@ -0,0 +1,437 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a71069d3", + "metadata": {}, + "source": [ + "# Using Guidance With the new OpenAI Harmony Response Format" + ] + }, + { + "cell_type": "markdown", + "id": "6b177583", + "metadata": {}, + "source": [ + "First, we define a guidance grammar in accordance with the cookbook https://cookbook.openai.com/articles/openai-harmony" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "fbce1652", + "metadata": {}, + "outputs": [], + "source": [ + "from guidance import gen, special_token\n", + "\n", + "def analysis():\n", + " return (\n", + " special_token(\"<|start|>\")\n", + " + \"assistant\"\n", + " + special_token(\"<|channel|>\")\n", + " + \"analysis\"\n", + " + special_token(\"<|message|>\")\n", + " # We could add a more specific grammar here instead of `gen` if we want control over the analysis / reasoning channel\n", + " + gen()\n", + " + special_token(\"<|end|>\")\n", + " )\n", + "\n", + "def commentary():\n", + " return (\n", + " special_token(\"<|start|>\")\n", + " + \"assistant\"\n", + " + special_token(\"<|channel|>\")\n", + " + \"commentary\"\n", + " + special_token(\"<|message|>\")\n", + " # We could add a more specific grammar here instead of `gen` if we want control over the commentary channel\n", + " + gen()\n", + " + special_token(\"<|end|>\")\n", + " )\n", + "\n", + "def final():\n", + " return (\n", + " special_token(\"<|start|>\")\n", + " + \"assistant\"\n", + " + special_token(\"<|channel|>\")\n", + " + \"final\"\n", + " + special_token(\"<|message|>\")\n", + " # We could add a more specific grammar here instead of `gen` if we want control over the final channel\n", + " + gen()\n", + " + special_token(\"<|return|>\")\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "7403fd6d", + "metadata": {}, + "source": [ + "Adding in support for tool calling (just JSON for now):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f275f7b", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Any\n", + "from guidance import json\n", + "\n", + "# https://cookbook.openai.com/articles/openai-harmony#receiving-tool-calls\n", + "def constrained_tool_call(name: str, schema: dict[str, Any]):\n", + " return (\n", + " special_token(\"<|start|>\")\n", + " + \"assistant\"\n", + " + special_token(\"<|channel|>\")\n", + " + f\"commentary to={name} \"\n", + " + special_token(\"<|constrain|>\")\n", + " + \"json\"\n", + " + special_token(\"<|message|>\")\n", + " # Constrain the message format to match the JSON schema -- we could instead use other custom formats here\n", + " + json(schema=schema)\n", + " + special_token(\"<|call|>\")\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "1b88582c", + "metadata": {}, + "source": [ + "Define a \"top-level\" guidance grammar that we can parametrize with tools" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4b309237", + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from typing import Callable\n", + "from guidance import optional, select\n", + "\n", + "@dataclass\n", + "class Tool:\n", + " name: str\n", + " schema: dict[str, Any]\n", + " description: str\n", + " callable: Callable\n", + "\n", + "def harmony_response_format(tools: list[Tool]):\n", + " return (\n", + " # Always include reasoning\n", + " analysis()\n", + " # Optional commentary\n", + " + optional(commentary())\n", + " # Either final output or constrained tool call\n", + " + select([final(), *(constrained_tool_call(tool.name, tool.schema) for tool in tools)])\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "e9ea6b9e", + "metadata": {}, + "source": [ + "Set up some tools" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94628a45", + "metadata": {}, + "outputs": [], + "source": [ + "def get_weather(location: str) -> dict:\n", + " # Simulated weather data\n", + " weather_data = {\n", + " \"location\": location,\n", + " \"temperature\": \"22°C\",\n", + " \"condition\": \"Sunny\",\n", + " }\n", + " return weather_data\n", + "\n", + "weather_tool = Tool(\n", + " name=\"get_weather\",\n", + " schema={\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\"type\": \"string\"},\n", + " },\n", + " \"required\": [\"location\"],\n", + " \"additionalProperties\": False,\n", + " },\n", + " description=\"Fetch the weather information for a specific location.\",\n", + " callable=get_weather\n", + ")\n", + "\n", + "tools = [weather_tool]" + ] + }, + { + "cell_type": "markdown", + "id": "5c943cbf", + "metadata": {}, + "source": [ + "Prepare our grammar into [llguidance/lark format](https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "98929b9b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "%llguidance {}\n", + "\n", + "start: <|start|> \"assistant\" <|channel|> \"analysis\" <|message|> GEN <|end|> optional select\n", + "GEN: /(?s:.*)/\n", + "optional: (<|start|> \"assistant\" <|channel|> \"commentary\" <|message|> GEN <|end|>)?\n", + "\n", + "select: <|start|> \"assistant\" <|channel|> \"final\" <|message|> GEN <|return|>\n", + " | <|start|> \"assistant\" <|channel|> \"commentary to=get_weather \" <|constrain|> \"json\" <|message|> json <|call|>\n", + "\n", + "json: %json {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": [\n", + " \"location\"\n", + " ],\n", + " \"x-guidance\": {\n", + " \"whitespace_flexible\": false,\n", + " \"whitespace_pattern\": null,\n", + " \"item_separator\": \", \",\n", + " \"key_separator\": \": \",\n", + " \"coerce_one_of\": true,\n", + " \"lenient\": false\n", + " }\n", + "}\n", + "\n", + "\n" + ] + } + ], + "source": [ + "guidance_grammar = harmony_response_format(tools)\n", + "llguidance_grammar = guidance_grammar.ll_grammar()\n", + "\n", + "# Let's take a look\n", + "print(llguidance_grammar)" + ] + }, + { + "cell_type": "markdown", + "id": "17556033", + "metadata": {}, + "source": [ + "Set up the conversation using harmony" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "3be2f3c0", + "metadata": {}, + "outputs": [], + "source": [ + "from openai_harmony import (\n", + " Conversation,\n", + " DeveloperContent,\n", + " Message,\n", + " ReasoningEffort,\n", + " Role,\n", + " SystemContent,\n", + " load_harmony_encoding,\n", + " HarmonyEncodingName\n", + ")\n", + "\n", + "enc = load_harmony_encoding(\n", + " HarmonyEncodingName.HARMONY_GPT_OSS\n", + ")\n", + "\n", + "convo = Conversation.from_messages(\n", + " [\n", + " Message.from_role_and_content(\n", + " Role.SYSTEM,\n", + " SystemContent.new()\n", + " .with_reasoning_effort(ReasoningEffort.LOW)\n", + " ),\n", + " Message.from_role_and_content(\n", + " Role.DEVELOPER,\n", + " DeveloperContent.new()\n", + " .with_instructions(\"You are a helpful assistant.\")\n", + " .with_function_tools([\n", + " {\"name\": tool.name, \"description\": tool.description, \"parameters\": tool.schema}\n", + " for tool in tools\n", + " ])\n", + " ),\n", + " Message.from_role_and_content(\n", + " Role.USER,\n", + " \"What's the weather like in Paris?\"\n", + " ),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2c0b29c5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\n", + "Knowledge cutoff: 2024-06\n", + "\n", + "Reasoning: low\n", + "\n", + "# Valid channels: analysis, commentary, final. Channel must be included for every message.\n", + "Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions\n", + "\n", + "You are a helpful assistant.\n", + "\n", + "# Tools\n", + "\n", + "## functions\n", + "\n", + "namespace functions {\n", + "\n", + "// Fetch the weather information for a specific location.\n", + "type get_weather = (_: {\n", + "location: string,\n", + "}) => any;\n", + "\n", + "} // namespace functions<|end|><|start|>user<|message|>What's the weather like in Paris?<|end|>\n" + ] + } + ], + "source": [ + "# Prompt tokens for completion\n", + "# NOTE: harmony docs suggest using `enc.render_conversation_for_completion`, which includes the next turn role in the prompt.\n", + "# We don't need to do this, because we include this in the completion grammar itself!\n", + "prompt = enc.render_conversation(\n", + " conversation=convo,\n", + ")\n", + "\n", + "# Let's take a look\n", + "prompt_text = enc.decode(prompt)\n", + "print(prompt_text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1601c3c", + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(\n", + " base_url=\"URL_TO_YOUR_DEPLOYMENT\"\n", + ")\n", + "\n", + "response = client.completions.create(\n", + " model=\"gpt-oss-120b\",\n", + " prompt=prompt,\n", + " max_tokens=1024,\n", + " logprobs=True,\n", + " # We're using sglang, so we need to provide the llguidance grammar like so\n", + " # -- please check the documentation of your favorite inference engine\n", + " extra_body={\"ebnf\": llguidance_grammar},\n", + ")\n", + "\n", + "# Note that we need to get tokens from the logprobs because they're currently \n", + "# improperly formatted in the response's messages\n", + "# TODO: deal with lower-level utf-8 issues\n", + "tokens = enc.encode(text=\"\".join(response.choices[0].logprobs.tokens), allowed_special=\"all\")\n", + "messages = enc.parse_messages_from_completion_tokens(tokens=tokens, role=Role.ASSISTANT)\n", + "\n", + "# Extend the conversation with our new messages\n", + "convo.messages += messages" + ] + }, + { + "cell_type": "markdown", + "id": "b9b700a1", + "metadata": {}, + "source": [ + "# TODO" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "58675dcc", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'messages' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[12], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Check if last message was a tool call. If so, call it!\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m last_message \u001b[38;5;241m=\u001b[39m \u001b[43mmessages\u001b[49m[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m last_message\u001b[38;5;241m.\u001b[39mrecipient \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m last_message\u001b[38;5;241m.\u001b[39mrecipient\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfunctions.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mNameError\u001b[0m: name 'messages' is not defined" + ] + } + ], + "source": [ + "# Check if last message was a tool call. If so, call it!\n", + "last_message = messages[-1]\n", + "if last_message.recipient is not None:\n", + " assert last_message.recipient.startswith(\"functions.\")\n", + " tool_name = last_message.recipient[len(\"functions.\"):]\n", + " tool = next((t for t in tools if t.name == tool_name), None)\n", + " assert tool is not None, f\"Tool {tool_name} not found in tools list.\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f44b5d2a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "guidance-ai (3.10.15)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}