[Firebase AI] Add support for URL Context (#1340)

* [Firebase AI] Add support for URL Context

* Update URLContext.cs

* Update UIHandlerAutomated.cs

Author: a-maurice

docs/readme.md

@@ -109,6 +109,10 @@ Support
 
 Release Notes
 -------------
+### Upcoming
+-   Changes
+    - Firebase AI: Add support for Gemini's URL context tool.
+
 ### 13.3.0
 -   Changes
     - Firebase AI: Add support for enabling the model to use Code Execution.

firebaseai/src/Candidate.cs

@@ -101,16 +101,22 @@ public IReadOnlyList<SafetyRating> SafetyRatings {
   /// Grounding metadata for the response, if any.
   /// </summary>
   public GroundingMetadata? GroundingMetadata { get; }
+  
+  /// <summary>
+  /// Metadata related to the `URLContext` tool.
+  /// </summary>
+  public UrlContextMetadata? UrlContextMetadata { get; }
 
   // Hidden constructor, users don't need to make this.
   private Candidate(ModelContent content, List<SafetyRating> safetyRatings,
       FinishReason? finishReason, CitationMetadata? citationMetadata,
-      GroundingMetadata? groundingMetadata) {
+      GroundingMetadata? groundingMetadata, UrlContextMetadata? urlContextMetadata) {
     Content = content;
     _safetyRatings = safetyRatings ?? new List<SafetyRating>();
     FinishReason = finishReason;
     CitationMetadata = citationMetadata;
     GroundingMetadata = groundingMetadata;
+    UrlContextMetadata = urlContextMetadata;
   }
 
   private static FinishReason ParseFinishReason(string str) {
@@ -141,7 +147,9 @@ internal static Candidate FromJson(Dictionary<string, object> jsonDict,
       jsonDict.ParseNullableObject("citationMetadata",
           (d) => Firebase.AI.CitationMetadata.FromJson(d, backend)),
       jsonDict.ParseNullableObject("groundingMetadata",
-          Firebase.AI.GroundingMetadata.FromJson));
+          Firebase.AI.GroundingMetadata.FromJson),
+      jsonDict.ParseNullableObject("urlContextMetadata",
+          Firebase.AI.UrlContextMetadata.FromJson));
   }
 }
 

firebaseai/src/FunctionCalling.cs

@@ -102,6 +102,7 @@ public readonly struct Tool {
   private List<FunctionDeclaration> FunctionDeclarations { get; }
   private GoogleSearch? GoogleSearch { get; }
   private CodeExecution? CodeExecution { get; }
+  private UrlContext? UrlContext { get; }
 
   /// <summary>
   /// Creates a tool that allows the model to perform function calling.
@@ -112,6 +113,7 @@ public Tool(params FunctionDeclaration[] functionDeclarations) {
     FunctionDeclarations = new List<FunctionDeclaration>(functionDeclarations);
     GoogleSearch = null;
     CodeExecution = null;
+    UrlContext = null;
   }
   /// <summary>
   /// Creates a tool that allows the model to perform function calling.
@@ -122,6 +124,7 @@ public Tool(IEnumerable<FunctionDeclaration> functionDeclarations) {
     FunctionDeclarations = new List<FunctionDeclaration>(functionDeclarations);
     GoogleSearch = null;
     CodeExecution = null;
+    UrlContext = null;
   }
 
   /// <summary>
@@ -133,6 +136,7 @@ public Tool(GoogleSearch googleSearch) {
     FunctionDeclarations = null;
     GoogleSearch = googleSearch;
     CodeExecution = null;
+    UrlContext = null;
   }
 
   /// <summary>
@@ -144,6 +148,20 @@ public Tool(CodeExecution codeExecution) {
     FunctionDeclarations = null;
     GoogleSearch = null;
     CodeExecution = codeExecution;
+    UrlContext = null;
+  }
+  
+  /// <summary>
+  /// Creates a tool that allows you to provide additional context to the models in the form of
+  /// public web URLs.
+  /// </summary>
+  /// <param name="urlContext">An empty `UrlContext` object. The presence of this object
+  ///     in the list of tools enables the model to use Url Contexts.</param>
+  public Tool(UrlContext urlContext) {
+    FunctionDeclarations = null;
+    GoogleSearch = null;
+    CodeExecution = null;
+    UrlContext = urlContext;
   }
 
   /// <summary>
@@ -161,6 +179,9 @@ internal Dictionary<string, object> ToJson() {
     if (CodeExecution.HasValue) {
       json["codeExecution"] = new Dictionary<string, object>();
     }
+    if (UrlContext.HasValue) {
+      json["urlContext"] = new Dictionary<string, object>();
+    }
     return json;
   }
 }

firebaseai/src/GenerateContentResponse.cs

@@ -471,33 +471,57 @@ public readonly struct UsageMetadata {
   /// </summary>
   public int ThoughtsTokenCount { get; }
   /// <summary>
+  /// The number of tokens used by any enabled tools.
+  /// </summary>
+  public int ToolUsePromptTokenCount { get; }
+  /// <summary>
   /// The total number of tokens in both the request and response.
   /// </summary>
   public int TotalTokenCount { get; }
 
   private readonly IReadOnlyList<ModalityTokenCount> _promptTokensDetails;
+  /// <summary>
+  /// The breakdown, by modality, of how many tokens are consumed by the prompt.
+  /// </summary>
   public IReadOnlyList<ModalityTokenCount> PromptTokensDetails {
     get {
       return _promptTokensDetails ?? new List<ModalityTokenCount>();
     }
   }
 
   private readonly IReadOnlyList<ModalityTokenCount> _candidatesTokensDetails;
+  /// <summary>
+  /// The breakdown, by modality, of how many tokens are consumed by the candidates.
+  /// </summary>
   public IReadOnlyList<ModalityTokenCount> CandidatesTokensDetails {
     get {
       return _candidatesTokensDetails ?? new List<ModalityTokenCount>();
     }
   }
 
+  private readonly IReadOnlyList<ModalityTokenCount> _toolUsePromptTokensDetails;
+  /// <summary>
+  /// The breakdown, by modality, of how many tokens were consumed by the tools used to process
+  /// the request.
+  /// </summary>
+  public IReadOnlyList<ModalityTokenCount> ToolUsePromptTokensDetails {
+    get {
+      return _toolUsePromptTokensDetails ?? new List<ModalityTokenCount>();
+    }
+  }
+
   // Hidden constructor, users don't need to make this.
-  private UsageMetadata(int promptTC, int candidatesTC, int thoughtsTC, int totalTC,
-                        List<ModalityTokenCount> promptDetails, List<ModalityTokenCount> candidateDetails) {
+  private UsageMetadata(int promptTC, int candidatesTC, int thoughtsTC, int toolUseTC, int totalTC,
+                        List<ModalityTokenCount> promptDetails, List<ModalityTokenCount> candidateDetails,
+                        List<ModalityTokenCount> toolUseDetails) {
     PromptTokenCount = promptTC;
     CandidatesTokenCount = candidatesTC;
     ThoughtsTokenCount = thoughtsTC;
+    ToolUsePromptTokenCount = toolUseTC;
     TotalTokenCount = totalTC;
     _promptTokensDetails = promptDetails;
     _candidatesTokensDetails = candidateDetails;
+    _toolUsePromptTokensDetails = toolUseDetails;
   }
 
   /// <summary>
@@ -509,9 +533,11 @@ internal static UsageMetadata FromJson(Dictionary<string, object> jsonDict) {
       jsonDict.ParseValue<int>("promptTokenCount"),
       jsonDict.ParseValue<int>("candidatesTokenCount"),
       jsonDict.ParseValue<int>("thoughtsTokenCount"),
+      jsonDict.ParseValue<int>("toolUsePromptTokenCount"),
       jsonDict.ParseValue<int>("totalTokenCount"),
       jsonDict.ParseObjectList("promptTokensDetails", ModalityTokenCount.FromJson),
-      jsonDict.ParseObjectList("candidatesTokensDetails", ModalityTokenCount.FromJson));
+      jsonDict.ParseObjectList("candidatesTokensDetails", ModalityTokenCount.FromJson),
+      jsonDict.ParseObjectList("toolUsePromptTokensDetails", ModalityTokenCount.FromJson));
   }
 }
 

firebaseai/src/URLContext.cs

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using Firebase.AI.Internal;
+
+namespace Firebase.AI {
+
+  /// <summary>
+  /// A tool that allows you to provide additional context to the models in the form of
+  /// public web URLs.
+  ///
+  /// By including URLs in your request, the Gemini model will access the content from those pages
+  /// to inform and enhance its response.
+  /// </summary>
+  public readonly struct UrlContext { }
+
+  /// <summary>
+  /// Metadata for a single URL retrieved by the UrlContext tool.
+  /// </summary>
+  public readonly struct UrlMetadata {
+    /// <summary>
+    /// Status of the URL retrieval.
+    /// </summary>
+    public enum UrlRetrievalStatus {
+      /// <summary>
+      /// Unspecified retrieval status
+      /// </summary>
+      Unspecified = 0,
+      /// <summary>
+      /// The URL retrieval was successful.
+      /// </summary>
+      Success,
+      /// <summary>
+      /// The URL retrieval failed.
+      /// </summary>
+      Error,
+      /// <summary>
+      /// The URL retrieval failed because the content is behind a paywall.
+      /// </summary>
+      Paywall,
+      /// <summary>
+      /// The URL retrieval failed because the content is unsafe.
+      /// </summary>
+      Unsafe,
+    }
+
+    /// <summary>
+    /// The retrieved URL.
+    /// </summary>
+    public System.Uri Url { get; }
+    /// <summary>
+    /// The status of the URL retrieval.
+    /// </summary>
+    public UrlRetrievalStatus RetrievalStatus { get; }
+
+    private UrlMetadata(string urlString, UrlRetrievalStatus? retrievalStatus) {
+      if (string.IsNullOrEmpty(urlString)) {
+        Url = null;
+      }
+      else {
+        Url = new Uri(urlString);
+      }
+      RetrievalStatus = retrievalStatus ?? UrlRetrievalStatus.Unspecified;
+    }
+
+    private static UrlRetrievalStatus ParseUrlRetrievalStatus(string str) {
+      return str switch {
+        "URL_RETRIEVAL_STATUS_SUCCESS" => UrlRetrievalStatus.Success,
+        "URL_RETRIEVAL_STATUS_ERROR" => UrlRetrievalStatus.Error,
+        "URL_RETRIEVAL_STATUS_PAYWALL" => UrlRetrievalStatus.Paywall,
+        "URL_RETRIEVAL_STATUS_UNSAFE" => UrlRetrievalStatus.Unsafe,
+        _ => UrlRetrievalStatus.Unspecified,
+      };
+    }
+
+    /// <summary>
+    /// Intended for internal use only.
+    /// This method is used for deserializing JSON responses and should not be called directly.
+    /// </summary>
+    internal static UrlMetadata FromJson(Dictionary<string, object> jsonDict) {
+      return new UrlMetadata(
+        jsonDict.ParseValue<string>("retrievedUrl"),
+        jsonDict.ParseNullableEnum("urlRetrievalStatus", ParseUrlRetrievalStatus)
+      );
+    }
+  }
+
+  /// <summary>
+  /// Metadata related to the UrlContext tool.
+  /// </summary>
+  public readonly struct UrlContextMetadata {
+    private readonly IReadOnlyList<UrlMetadata> _urlMetadata;
+    /// <summary>
+    /// List of URL metadata used to provide context to the Gemini model.
+    /// </summary>
+    public IReadOnlyList<UrlMetadata> UrlMetadata {
+      get {
+        return _urlMetadata ?? new List<UrlMetadata>();
+      }
+    }
+
+    private UrlContextMetadata(List<UrlMetadata> urlMetadata) {
+      _urlMetadata = urlMetadata;
+    }
+
+    /// <summary>
+    /// Intended for internal use only.
+    /// This method is used for deserializing JSON responses and should not be called directly.
+    /// </summary>
+    internal static UrlContextMetadata FromJson(Dictionary<string, object> jsonDict) {
+      return new UrlContextMetadata(
+        jsonDict.ParseObjectList("urlMetadata", Firebase.AI.UrlMetadata.FromJson)
+      );
+    }
+  }
+
+}