Chad/0.1.0/final edits (#13)

* add upgrade cta to studio header, fix bug with llm_log failure in SDK, add reasoning token support

* bump sdk version from 0.0.5 to 0.0.6

* fix updated_at search path (thanks supabase), consolidated all streaming prompt accumulation, added resoningTokens to streaming response, adjusted sdk reference docs, added manifesto to docs

* add reasoning to ExecuteNonStreamingResult reference

Author: chad-syntax

__tests__/lib/template-utils.test.ts __tests__/utils/template-utils.test.ts__tests__/lib/template-utils.test.ts renamed to __tests__/utils/template-utils.test.ts

@@ -1,4 +1,4 @@
 {
   "title": "Core Concepts",
-  "pages": ["architecture", "self-hosting"]
+  "pages": ["manifesto", "architecture", "self-hosting"]
 }

content/docs/core-concepts/manifesto.mdx

@@ -20,7 +20,7 @@ Agentsmith is an open-source platform that brings the rigor of software developm
       </CardHeader>
     </Card>
   </a>
-  <a href="/docs/core-concepts/architecture" className="h-full no-underline group">
+  <a href="/docs/core-concepts/manifesto" className="h-full no-underline group">
     <Card className="h-full transition-shadow duration-200 group-hover:shadow-lg">
       <CardHeader>
         <Box className="h-8 w-8 mb-2 text-primary" />

content/docs/core-concepts/meta.json

@@ -24,6 +24,29 @@ console.log(''); // Newline at the end
 await client.shutdown();
 ```
 
+You can also access the reasoning tokens and tool calls as they arrive.
+
+```ts
+const { reasoningTokens, toolCalls } = await helloWorldPrompt.execute(
+  { firstName: 'Jane', lastName: 'Doe' },
+  { config: { stream: true } },
+);
+
+console.log('Streaming reasoning:');
+for await (const reasoningToken of reasoningTokens) {
+  process.stdout.write(reasoningToken);
+}
+
+console.log(''); // Newline at the end
+
+console.log('Streaming tool calls:');
+for await (const toolCall of toolCalls) {
+  console.log(toolCall);
+}
+
+await client.shutdown();
+```
+
 ### Overriding Model Configuration
 
 You can override the default model configuration set in the Studio on a per-call basis. This is useful for A/B testing different models, adjusting parameters for specific users, or dynamically changing behavior.

content/docs/index.mdx

@@ -165,6 +165,11 @@ Executes the prompt with the given variables. The `execute` method returns an ob
         'The AI response content, equivalent to `completion.choices[0].message.content`.',
       type: 'string | null',
     },
+    reasoning: {
+      description:
+        'The AI response reasoning, equivalent to `completion.choices[0].message.reasoning`.',
+      type: 'string | null',
+    },
     completion: {
       description: 'The full OpenRouter completion object.',
       type: 'OpenrouterNonStreamingResponse',
@@ -196,6 +201,10 @@ Executes the prompt with the given variables. The `execute` method returns an ob
       description: 'An async generator that yields content tokens as strings.',
       type: 'AsyncGenerator<string | undefined, void, unknown>',
     },
+    reasoningTokens: {
+      description: 'An async generator that yields reasoning tokens as strings.',
+      type: 'AsyncGenerator<string | undefined, void, unknown>',
+    },
     toolCalls: {
       description: 'An async generator that yields tool calls as they arrive.',
       type: 'AsyncGenerator<ToolCall, void, unknown>',

content/docs/sdk/advanced-usage.mdx

@@ -7,9 +7,10 @@ import { CompletionConfig } from '@/lib/openrouter';
 import { AgentsmithServices } from '@/lib/AgentsmithServices';
 import { validateGlobalContext, validateVariables } from '@/utils/template-utils';
 import merge from 'lodash.merge';
-import { streamToIterator } from '@/utils/stream-to-iterator';
+import { OpenrouterStreamEvent, streamToIterator } from '@/utils/stream-to-iterator';
 import { LLMLogsService } from '@/lib/LLMLogsService';
 import { mergeIncludedVariables } from '@/utils/merge-included-variables';
+import { accumulateStreamToCompletion } from '@/utils/accumulate-stream';
 
 export const maxDuration = 320; // 5m20s minute function timeout
 
@@ -146,31 +147,10 @@ export async function POST(
       const [streamForClient, streamForLogging] = response.stream.tee();
 
       const logStreamedCompletion = async () => {
-        let fullCompletion: any = {};
-        let content = '';
-
         try {
-          const stream = streamToIterator(streamForLogging);
-          for await (const event of stream) {
-            if (event.type === 'logUuid') {
-              response.logUuid = event.data.logUuid;
-            } else {
-              const chunk = event.data;
-              // usage chunk contains null stop values we don't want to merge
-              if (chunk.usage) {
-                fullCompletion.usage = merge(fullCompletion.usage, chunk.usage);
-              } else if (chunk.choices) {
-                content += chunk.choices[0].delta.content ?? '';
-                fullCompletion = merge(fullCompletion, chunk);
-              }
-            }
-          }
-
-          // rewrite delta to message
-          if (fullCompletion.choices?.[0]) {
-            delete fullCompletion.choices[0].delta;
-            fullCompletion.choices[0].message = { role: 'assistant', content };
-          }
+          const fullCompletion = await accumulateStreamToCompletion(
+            streamToIterator<OpenrouterStreamEvent>(streamForLogging),
+          );
 
           await agentsmith.services.llmLogs.updateLogWithCompletion(
             response.logUuid,
@@ -179,7 +159,7 @@ export async function POST(
         } catch (error) {
           agentsmith.logger.error(error, 'Error logging streamed completion');
           await agentsmith.services.llmLogs.updateLogWithCompletion(response.logUuid, {
-            error: 'Failed to log stream',
+            error: `Failed to log stream: ${error instanceof Error ? error.message : 'Unknown error'}`,
           });
         }
       };

content/docs/sdk/reference.mdx

@@ -26,11 +26,13 @@ export async function POST(request: Request) {
 
   let jwt: string | null = null;
   let expiresAt: number | null = null;
+  let organizationUuid: string | null = null;
 
   try {
     const exchangeResult = await exchangeApiKeyForJwt(apiKey);
     jwt = exchangeResult.jwt;
     expiresAt = exchangeResult.expiresAt;
+    organizationUuid = exchangeResult.organizationUuid;
   } catch (error) {
     logger.error(error, 'Failed to exchange API key for JWT');
     return NextResponse.json(
@@ -44,6 +46,7 @@ export async function POST(request: Request) {
   const response: SdkExchangeResponse = {
     jwt,
     expiresAt,
+    organizationUuid,
   };
 
   return NextResponse.json(response);

src/app/(api)/api/[apiVersion]/promptVersion/[promptVersionUuid]/execute/route.tsx

@@ -3,7 +3,7 @@
 import { useState } from 'react';
 import Link from 'next/link';
 import { useApp } from '@/providers/app';
-import { NonStreamingChoice } from '@/lib/openrouter';
+import { NonStreamingChoice, OpenrouterNonStreamingResponse } from '@/lib/openrouter';
 import { connectOpenrouter } from '@/app/actions/openrouter';
 import { routes } from '@/utils/routes';
 import { Button } from '@/components/ui/button';
@@ -17,8 +17,7 @@ import {
 } from '@/components/ui/dialog';
 import { Alert, AlertDescription } from '@/components/ui/alert';
 import { Card, CardContent } from '@/components/ui/card';
-import merge from 'lodash.merge';
-import { streamToIterator } from '@/utils/stream-to-iterator';
+import { OpenrouterStreamEvent, StreamEvent, streamToIterator } from '@/utils/stream-to-iterator';
 import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
 import { MarkdownEditor } from '../editors/markdown-editor';
 import { MarkdownRenderer } from '../markdown-renderer';
@@ -29,6 +28,7 @@ import { VariableInput } from '../variable-input';
 import { JsonEditor } from '../editors/json-editor';
 import { Loader2 } from 'lucide-react';
 import { toast } from 'sonner';
+import { accumulateStreamToCompletion } from '@/utils/accumulate-stream';
 
 const TABS = {
   content: 'content',
@@ -58,8 +58,12 @@ export const PromptTestModal = () => {
   } = state;
 
   const [isRunning, setIsRunning] = useState(false);
-  const [testResult, setTestResult] = useState<string | null>(null);
-  const [fullResult, setFullResult] = useState<any | null>(null);
+  const [completionContent, setCompletionContent] = useState<string | null>(null);
+  const [reasoningContent, setReasoningContent] = useState<string | null>(null);
+  const [fullResult, setFullResult] = useState<{
+    completion?: OpenrouterNonStreamingResponse;
+    logUuid?: string;
+  } | null>(null);
   const [testError, setTestError] = useState<string | null>(null);
   const [selectedTab, setSelectedTab] = useState<Tab>(TABS.content);
 
@@ -70,7 +74,7 @@ export const PromptTestModal = () => {
 
   const handleTestPrompt = async () => {
     setIsRunning(true);
-    setTestResult(null);
+    setCompletionContent(null);
     setFullResult(null);
     setTestError(null);
 
@@ -106,38 +110,34 @@ export const PromptTestModal = () => {
       }
 
       if (editorConfig.stream && response.body) {
-        let fullResult: any = {};
-        let content = '';
-
         try {
-          const stream = streamToIterator(response.body);
-          for await (const event of stream) {
+          const [streamA, streamB] = response.body.tee();
+
+          const iterator = streamToIterator<StreamEvent>(streamA);
+
+          for await (const event of iterator) {
             if (event.type === 'logUuid') {
               if (event.data.logUuid) {
-                fullResult.logUuid = event.data.logUuid;
+                setFullResult((prev) => ({ ...prev, logUuid: event.data.logUuid }));
               }
-            } else {
-              const chunk = event.data;
-              // usage chunk contains null stop values we don't want to merge
-              if (chunk.usage) {
-                fullResult.completion.usage = merge(fullResult.completion.usage, chunk.usage);
-              } else if (chunk.choices) {
-                content += chunk.choices[0].delta.content ?? '';
-                setTestResult(content);
-              }
-              fullResult.completion = merge(fullResult.completion, chunk);
+            }
+            if (event.type === 'message' && event.data.choices?.[0]?.delta?.content) {
+              setCompletionContent(
+                (prev) => (prev ?? '') + (event.data.choices[0].delta.content ?? ''),
+              );
+            }
+            if (event.type === 'message' && event.data.choices?.[0]?.delta?.reasoning) {
+              setReasoningContent(
+                (prev) => (prev ?? '') + (event.data.choices[0].delta.reasoning ?? ''),
+              );
             }
           }
 
-          if (fullResult.completion.choices?.[0]) {
-            delete fullResult.completion.choices[0].delta;
-            fullResult.completion.choices[0].message = {
-              role: 'assistant',
-              content,
-            };
-          }
+          const completion = await accumulateStreamToCompletion(
+            streamToIterator<OpenrouterStreamEvent>(streamB),
+          );
 
-          setFullResult(fullResult);
+          setFullResult((prev) => ({ ...prev, completion }));
         } catch (error) {
           console.error('Error testing prompt:', error);
           setTestError(error instanceof Error ? error.message : 'Unknown error occurred');
@@ -150,10 +150,17 @@ export const PromptTestModal = () => {
 
       const data = await response.json();
 
-      const result =
-        (data.completion.choices[0] as NonStreamingChoice).message.content || 'No response content';
+      const choice = data.completion.choices[0] as NonStreamingChoice;
+
+      const result = choice.message.content || 'No response content';
+      const reasoning = choice.message.reasoning;
+
+      setCompletionContent(result);
+
+      if (reasoning) {
+        setReasoningContent(reasoning);
+      }
 
-      setTestResult(result);
       setFullResult(data);
     } catch (error) {
       console.error('Error testing prompt:', error);
@@ -166,7 +173,8 @@ export const PromptTestModal = () => {
 
   const resetTest = () => {
     setInputVariables({});
-    setTestResult(null);
+    setCompletionContent(null);
+    setReasoningContent(null);
     setFullResult(null);
     setTestError(null);
   };
@@ -219,8 +227,8 @@ export const PromptTestModal = () => {
 
   let isContentJson = false;
   try {
-    JSON.parse(testResult as string);
-    isContentJson = true;
+    const parsed = JSON.parse(completionContent as string);
+    if (parsed !== null && parsed !== undefined) isContentJson = true;
   } catch (error) {
     //
   }
@@ -271,7 +279,7 @@ export const PromptTestModal = () => {
               >
                 {isRunning ? 'Running...' : 'Run Test'}
               </Button>
-              {testResult && (
+              {completionContent && (
                 <Button onClick={resetTest} variant="outline">
                   Reset
                 </Button>
@@ -294,7 +302,7 @@ export const PromptTestModal = () => {
               )}
             </div>
 
-            {testResult ? (
+            {completionContent || reasoningContent ? (
               <Tabs
                 value={selectedTab}
                 onValueChange={(value) => setSelectedTab(value as Tab)}
@@ -316,29 +324,53 @@ export const PromptTestModal = () => {
                   />
                 </TabsContent>
                 <TabsContent value={TABS.content} className="flex-1 overflow-auto">
+                  {reasoningContent && (
+                    <>
+                      <h3 className="font-medium text-lg">Reasoning</h3>
+                      <MarkdownEditor
+                        className="mb-4"
+                        value={reasoningContent}
+                        readOnly
+                        minHeight="100%"
+                      />
+                    </>
+                  )}
+                  {completionContent && reasoningContent && (
+                    <h3 className="font-medium text-lg">Completion</h3>
+                  )}
                   {isContentJson ? (
                     <JsonEditor
-                      value={JSON.parse(testResult as string)}
+                      value={JSON.parse(completionContent as string)}
                       readOnly
                       minHeight="100%"
                     />
+                  ) : completionContent ? (
+                    <MarkdownEditor value={completionContent} readOnly minHeight="100%" />
                   ) : (
-                    <MarkdownEditor value={testResult} readOnly minHeight="100%" />
+                    <div className="flex items-center justify-center h-full">
+                      <Loader2 className="w-4 h-4 animate-spin" />
+                    </div>
                   )}
                 </TabsContent>
                 <TabsContent value={TABS.markdown} className="flex-1 overflow-auto">
                   <div className="p-4 border rounded-md">
-                    <MarkdownRenderer>{testResult}</MarkdownRenderer>
+                    {completionContent ? (
+                      <MarkdownRenderer>{completionContent}</MarkdownRenderer>
+                    ) : (
+                      <div className="flex items-center justify-center h-full">
+                        <Loader2 className="w-4 h-4 animate-spin" />
+                      </div>
+                    )}
                   </div>
                 </TabsContent>
                 <TabsContent value={TABS.response} className="flex-1 overflow-auto">
                   {isRunning ? (
                     <div className="flex items-center justify-center h-full">
                       <Loader2 className="w-4 h-4 animate-spin" />
                     </div>
-                  ) : (
+                  ) : fullResult ? (
                     <JsonEditor value={fullResult} readOnly minHeight="100%" />
-                  )}
+                  ) : null}
                 </TabsContent>
               </Tabs>
             ) : (

src/app/(api)/api/[apiVersion]/sdk-exchange/route.ts

@@ -145,12 +145,17 @@ export const SignInForm = ({ className, requiredTos = true, ...props }: SignInFo
                       <Link href={routes.marketing.terms} className="underline">
                         Terms of Service
                       </Link>
+                      and the{' '}
+                      <Link href={routes.marketing.privacy} className="underline">
+                        Privacy Policy
+                      </Link>
+                      .
                     </span>
                   </Label>
                 </div>
                 {mustAgreeTos && (
                   <p className="text-sm text-destructive mt-2 text-center">
-                    You must agree to the Terms of Service.
+                    You must agree to the Terms of Service and Privacy Policy.
                   </p>
                 )}
               </div>