Run sanity/integration tests in CI

Author: kzu

.github/workflows/build.yml

@@ -72,6 +72,9 @@ jobs:
         run: dotnet build -m:1 -bl:build.binlog
 
       - name: 🧪 test
+        env:
+          CI_XAI_API_KEY: ${{ secrets.CI_XAI_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
         shell: pwsh
         run: dnx --yes retest -- --no-build
 

.github/workflows/publish.yml

@@ -34,6 +34,9 @@ jobs:
         run: dotnet build -m:1 -bl:build.binlog
 
       - name: 🧪 test
+        env:
+          CI_XAI_API_KEY: ${{ secrets.CI_XAI_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
         shell: pwsh
         run: dnx --yes retest -- --no-build
 

src/xAI.Tests/SanityChecks.cs

@@ -1,17 +1,20 @@
 using System.Text.Json;
+using Devlooped.Extensions.AI;
+using Microsoft.Extensions.AI;
 using Microsoft.Extensions.DependencyInjection;
 using xAI.Protocol;
 using Xunit.Abstractions;
+using ChatConversation = Devlooped.Extensions.AI.Chat;
 
 namespace xAI.Tests;
 
 public class SanityChecks(ITestOutputHelper output)
 {
-    [SecretsFact("XAI_API_KEY")]
+    [SecretsFact("CI_XAI_API_KEY")]
     public async Task ListModelsAsync()
     {
         var services = new ServiceCollection()
-            .AddxAIProtocol(Environment.GetEnvironmentVariable("XAI_API_KEY")!)
+            .AddxAIProtocol(Environment.GetEnvironmentVariable("CI_XAI_API_KEY")!)
             .BuildServiceProvider();
 
         var client = services.GetRequiredService<Models.ModelsClient>();
@@ -24,14 +27,14 @@ public async Task ListModelsAsync()
             output.WriteLine(model.Name);
     }
 
-    [SecretsFact("XAI_API_KEY")]
+    [SecretsFact("CI_XAI_API_KEY")]
     public async Task ExecuteLocalFunctionWithWebSearch()
     {
         var services = new ServiceCollection()
-            .AddxAIProtocol(Environment.GetEnvironmentVariable("XAI_API_KEY")!)
+            .AddxAIProtocol(Environment.GetEnvironmentVariable("CI_XAI_API_KEY")!)
             .BuildServiceProvider();
 
-        var client = services.GetRequiredService<Chat.ChatClient>();
+        var client = services.GetRequiredService<xAI.Protocol.Chat.ChatClient>();
 
         // Define a local function to get the current date
         var getDateFunction = new Function
@@ -132,4 +135,206 @@ public async Task ExecuteLocalFunctionWithWebSearch()
         Assert.NotNull(finalOutput.Message.Content);
         Assert.NotEmpty(finalOutput.Message.Content);
     }
+
+    /// <summary>
+    /// Comprehensive integration test (non-streaming) that exercises all major features:
+    /// - Client-side tool invocation (AIFunctionFactory)
+    /// - Hosted web search tool
+    /// - Hosted code interpreter tool
+    /// - Hosted MCP server tool (GitHub)
+    /// - Citations and annotations
+    /// </summary>
+    [SecretsFact("CI_XAI_API_KEY", "GITHUB_TOKEN")]
+    public async Task IntegrationTest()
+    {
+        var (grok, options, getDateCalls) = SetupIntegrationTest();
+
+        var response = await grok.GetResponseAsync(CreateIntegrationChat(), options);
+
+        AssertIntegrationTest(response, getDateCalls);
+    }
+
+    [SecretsFact("CI_XAI_API_KEY", "GITHUB_TOKEN")]
+    public async Task IntegrationTestStreaming()
+    {
+        var (grok, options, getDateCalls) = SetupIntegrationTest();
+
+        var updates = await grok.GetStreamingResponseAsync(CreateIntegrationChat(), options).ToListAsync();
+        var response = updates.ToChatResponse();
+
+        AssertIntegrationTest(response, getDateCalls);
+    }
+
+    static ChatConversation CreateIntegrationChat() => new()
+    {
+        { "system", "You are a helpful assistant that uses all available tools to answer questions accurately." },
+        { "user",
+            """
+            Please answer the following questions using the appropriate tools:
+            1. What is today's date? (use get_date tool)
+            2. What is the current price of Tesla (TSLA) stock? (use Yahoo news web search)
+            3. Calculate the compound interest for $5,000 at 4% annually for 5 years (use code interpreter)
+            4. What is the latest release version of the devlooped/GrokClient repository? (use GitHub MCP tool)
+            
+            Respond with a JSON object in this exact format:
+            {
+              "today": "[date from get_date in YYYY-MM-DD format]",
+              "tesla_price": [numeric price from web search],
+              "compound_interest": [numeric result from code interpreter],
+              "latest_release": "[version string from GitHub]"
+            }
+            """
+        }
+    };
+
+    static (IChatClient grok, GrokChatOptions options, Func<int> getDateCalls) SetupIntegrationTest()
+    {
+        var getDateCalls = 0;
+        var grok = new GrokClient(Environment.GetEnvironmentVariable("CI_XAI_API_KEY")!)
+            .AsIChatClient("grok-4-fast")
+            .AsBuilder()
+            .UseFunctionInvocation()
+            .Build();
+
+        var options = new GrokChatOptions
+        {
+            Include =
+            [
+                IncludeOption.InlineCitations,
+                IncludeOption.WebSearchCallOutput,
+                IncludeOption.CodeExecutionCallOutput,
+                IncludeOption.McpCallOutput
+            ],
+            Tools =
+            [
+                // Client-side tool
+                AIFunctionFactory.Create(() =>
+                {
+                    getDateCalls++;
+                    return DateTime.Now.ToString("yyyy-MM-dd");
+                }, "get_date", "Gets the current date in YYYY-MM-DD format"),
+
+                // Hosted web search tool
+                new HostedWebSearchTool(),
+
+                // Hosted code interpreter tool
+                new HostedCodeInterpreterTool(),
+
+                // Hosted MCP server tool (GitHub)
+                new HostedMcpServerTool("GitHub", "https://api.githubcopilot.com/mcp/")
+                {
+                    AuthorizationToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN")!,
+                    AllowedTools = ["list_releases", "get_release_by_tag"],
+                }
+            ]
+        };
+
+        return (grok, options, () => getDateCalls);
+    }
+
+    void AssertIntegrationTest(ChatResponse response, Func<int> getDateCalls)
+    {
+        // Verify response basics
+        Assert.NotNull(response);
+        Assert.NotNull(response.ModelId);
+        Assert.NotEmpty(response.Messages);
+
+        // Verify client-side tool was invoked
+        Assert.True(getDateCalls() >= 1);
+
+        // Verify web search tool was used
+        var webSearchCalls = response.Messages
+            .SelectMany(x => x.Contents.Select(c => c.RawRepresentation as xAI.Protocol.ToolCall))
+            .Where(x => x?.Type == xAI.Protocol.ToolCallType.WebSearchTool)
+            .ToList();
+        Assert.NotEmpty(webSearchCalls);
+
+        // Verify code interpreter tool was used
+        var codeInterpreterCalls = response.Messages
+            .SelectMany(x => x.Contents)
+            .OfType<CodeInterpreterToolCallContent>()
+            .ToList();
+        Assert.NotEmpty(codeInterpreterCalls);
+
+        // Verify code interpreter output was included
+        var codeInterpreterResults = response.Messages
+            .SelectMany(x => x.Contents)
+            .OfType<CodeInterpreterToolResultContent>()
+            .ToList();
+        Assert.NotEmpty(codeInterpreterResults);
+
+        // Verify MCP tool was used
+        var mcpCalls = response.Messages
+            .SelectMany(x => x.Contents)
+            .OfType<McpServerToolCallContent>()
+            .ToList();
+        Assert.NotEmpty(mcpCalls);
+
+        // Verify MCP output was included
+        var mcpResults = response.Messages
+            .SelectMany(x => x.Contents)
+            .OfType<McpServerToolResultContent>()
+            .ToList();
+        Assert.NotEmpty(mcpResults);
+
+        // Verify citations from web search
+        var citations = response.Messages
+            .SelectMany(x => x.Contents)
+            .SelectMany(x => x.Annotations?.OfType<CitationAnnotation>() ?? [])
+            .Where(x => x.Url is not null)
+            .Select(x => x.Url!)
+            .ToList();
+        Assert.NotEmpty(citations);
+
+        // Parse and validate the JSON response
+        var responseText = response.Messages.Last().Text;
+        Assert.NotNull(responseText);
+
+        output.WriteLine("Response text:");
+        output.WriteLine(responseText);
+
+        // Extract JSON from response (may be wrapped in markdown code blocks)
+        var jsonStart = responseText.IndexOf('{');
+        var jsonEnd = responseText.LastIndexOf('}');
+        if (jsonStart >= 0 && jsonEnd > jsonStart)
+        {
+            var json = responseText.Substring(jsonStart, jsonEnd - jsonStart + 1);
+            var result = JsonSerializer.Deserialize<IntegrationTestResponse>(json, new JsonSerializerOptions(JsonSerializerDefaults.Web)
+            {
+                PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower
+            });
+
+            Assert.NotNull(result);
+
+            // Verify date is today
+            Assert.Equal(DateTime.Today.ToString("yyyy-MM-dd"), result.Today);
+
+            // Verify Tesla price is reasonable (greater than $100)
+            Assert.True(result.TeslaPrice > 100, $"Tesla price {result.TeslaPrice} should be > 100");
+
+            // Verify compound interest calculation is approximately correct
+            // Formula: P(1 + r)^t - P = 5000 * (1.04)^5 - 5000 ≈ $1,083.26
+            Assert.True(result.CompoundInterest > 1000 && result.CompoundInterest < 1200,
+                $"Compound interest {result.CompoundInterest} should be between 1000 and 1200");
+
+            // Verify latest release contains version pattern
+            Assert.NotNull(result.LatestRelease);
+            Assert.Contains(".", result.LatestRelease);
+
+            output.WriteLine($"Parsed response: Today={result.Today}, TeslaPrice={result.TeslaPrice}, CompoundInterest={result.CompoundInterest}, LatestRelease={result.LatestRelease}");
+        }
+
+        output.WriteLine($"Web search citations: {citations.Count}");
+        foreach (var url in citations.Take(5))
+            output.WriteLine($"  - {url}");
+
+        output.WriteLine($"Code interpreter calls: {codeInterpreterCalls.Count}");
+        output.WriteLine($"MCP calls: {mcpCalls.Count}");
+    }
+
+    record IntegrationTestResponse(
+        string Today,
+        decimal TeslaPrice,
+        decimal CompoundInterest,
+        string LatestRelease);
 }