diff --git a/examples/tool/webfetch/geminifetch/README.md b/examples/tool/webfetch/geminifetch/README.md new file mode 100644 index 000000000..ea5c0c13e --- /dev/null +++ b/examples/tool/webfetch/geminifetch/README.md @@ -0,0 +1,114 @@ +# Gemini Web Fetch Tool Example + +This example demonstrates how to use the Gemini web fetch tool with an AI agent for interactive conversations. The tool leverages Gemini's URL Context feature for server-side web fetching and intelligent content analysis. + + +## Running the Example + +### Using environment variables: + +```bash +export OPENAI_API_KEY="your-openai-api-key" +export GEMINI_API_KEY="your-gemini-api-key" +go run main.go +``` +### Example Session + +``` + trpc-agent-go % ./dpskv3.sh +πŸš€ Gemini Web Fetch Chat Demo +Model: deepseek-v3-local-II +Gemini Fetch Model: gemini-2.5-flash +Type 'exit' to end the conversation +Available tools: gemini_web_fetch +================================================== +βœ… Gemini web fetch chat ready! Session: gemini-web-fetch-session-1763990940 + +πŸ’‘ Try asking questions like: + - Summarize https://example.com + - Compare https://site1.com and https://site2.com + - What's the main content of https://news.ycombinator.com + - Analyze the article at https://blog.example.com/post + - Extract key points from https://ai.google.dev/gemini-api/docs/url-context + +ℹ️ Note: URLs are automatically detected and fetched by Gemini's server + +πŸ‘€ You: compare https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-fetch-tool#how-to-use-web-fetch and https://ai.google.dev/gemini-api/docs/url-context +πŸ€– Assistant: 🌐 Gemini web fetch initiated: + β€’ gemini_web_fetch (ID: chatcmpl-tool-309371dd1ac24ee5a76f6aa7a6ebe8b7) + Prompt: {"prompt": "Compare the web fetch tool documentation for Claude AI (https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-fetch-tool#how-to-use-web-fetch) and Google's Gemini API (https://ai.google.dev/gemini-api/docs/url-context). Highlight the key differences and similarities in their functionality, usage, and capabilities."} + +πŸ”„ Gemini fetching and analyzing content... +βœ… Fetch result (ID: chatcmpl-tool-309371dd1ac24ee5a76f6aa7a6ebe8b7): {"content":"Both Claude AI's \"Web fetch tool\" and Google's Gemini API's \"URL context\" feature allow their respective models to access and process content from specified URLs to enhance their respo... + +πŸ€– Assistant: Here’s a detailed comparison of Claude AI's **Web Fetch Tool** and Google's **Gemini API URL Context** feature: + +### **Similarities:** +1. **Core Functionality**: Both tools allow AI models to fetch and process content from URLs for tasks like summarization, analysis, and data extraction. +2. **PDF Support**: Both can retrieve and process PDF documents. +3. **Token Consumption**: Content fetched from URLs counts toward input token limits and affects pricing. +4. **Error Handling**: Both provide mechanisms to handle inaccessible or unprocessable URLs. +5. **Integration with Search**: Both can be combined with their respective search tools (Claude's Web Search and Gemini's Grounding with Google Search) for broader information gathering. + +--- + +### **Key Differences:** + +#### **1. Usage and Activation:** +- **Claude**: + - Requires enabling a beta header (`web-fetch-2025-09-10`) in API requests. + - Explicitly defined in the `tools` array with a `type` and `name`. +- **Gemini**: + - Activated by including `{"url_context": {}}` in the `tools` configuration. + - URLs are included directly in the prompt's `contents`. + +#### **2. Content Retrieval Mechanism:** +- **Claude**: + - Fetches full text content from URLs and extracts text from PDFs. + - Does not support dynamically rendered JavaScript websites. +- **Gemini**: + - Uses a two-step process: first checks an internal cache, then fetches live if needed. + - Balances speed, cost, and access to fresh data. + +#### **3. Security and Control:** +- **Claude**: + - Emphasizes data exfiltration risks; recommends trusted environments. + - Offers fine-grained control with `max_uses`, `allowed_domains`, and `blocked_domains`. +- **Gemini**: + - Content undergoes safety checks. + - Documentation lacks detailed controls like Claude's domain restrictions. + +#### **4. Supported Content Types:** +- **Claude**: + - Supports web pages and PDFs; no JavaScript-rendered sites. +- **Gemini**: + - Supports text-based formats (HTML, JSON, etc.), images (PNG, JPEG), and PDFs. + - Excludes paywalled content, YouTube videos, and large files (>34MB per URL). + +#### **5. Citations:** +- **Claude**: + - Citations are optional and must be enabled (`"citations": {"enabled": true}`). +- **Gemini**: + - Provides `url_context_metadata` for verification but lacks explicit in-text citations. + +#### **6. Model Support:** +- **Claude**: + - Available on specific versions of Sonnet, Haiku, and Opus models. +- **Gemini**: + - Supports models like `gemini-2.5-pro` and `gemini-2.5-flash`. + +#### **7. Additional Features:** +- **Claude**: + - Offers prompt caching, streaming, and batch request integration. +- **Gemini**: + - Provides `url_context_metadata` and `usage_metadata` for debugging and token tracking. + +--- + +### **Summary:** +- **Claude** excels in security controls and fine-grained URL management but has stricter limitations on content types. +- **Gemini** offers broader content support and a hybrid retrieval system but lacks some of Claude's granular security features. + +Choose Claude for stricter control over sensitive data and Gemini for versatility in handling diverse content types. +``` + diff --git a/examples/tool/webfetch/geminifetch/go.mod b/examples/tool/webfetch/geminifetch/go.mod new file mode 100644 index 000000000..1581097a1 --- /dev/null +++ b/examples/tool/webfetch/geminifetch/go.mod @@ -0,0 +1,59 @@ +module trpc.group/trpc-go/trpc-agent-go/examples/tool/webfetch/geminifetch + +go 1.24.1 + +replace ( + trpc.group/trpc-go/trpc-agent-go => ../../../.. + trpc.group/trpc-go/trpc-agent-go/tool/webfetch/geminifetch => ../../../../tool/webfetch/geminifetch +) + +require ( + trpc.group/trpc-go/trpc-agent-go v0.2.0 + trpc.group/trpc-go/trpc-agent-go/tool/webfetch/geminifetch v0.0.0-00010101000000-000000000000 +) + +require ( + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/openai/openai-go v1.12.0 // indirect + github.com/panjf2000/ants/v2 v2.10.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/sdk v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genai v1.36.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.2 // indirect + google.golang.org/protobuf v1.34.2 // indirect + trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a // indirect +) diff --git a/examples/tool/webfetch/geminifetch/go.sum b/examples/tool/webfetch/geminifetch/go.sum new file mode 100644 index 000000000..162968f26 --- /dev/null +++ b/examples/tool/webfetch/geminifetch/go.sum @@ -0,0 +1,192 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U= +cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0= +github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y= +github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= +github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 h1:dIIDULZJpgdiHz5tXrTgKIMLkus6jEFa7x5SOKcyR7E= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 h1:nSiV3s7wiCam610XcLbYOmMfJxB9gO4uK3Xgv5gmTgg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0/go.mod h1:hKn/e/Nmd19/x1gvIHwtOwVWM+VhuITSWip3JUDghj0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 h1:JAv0Jwtl01UFiyWZEMiJZBiTlv5A50zNs8lsthXqIio= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0/go.mod h1:QNKLmUEAq2QUbPQUfvw4fmv0bgbK7UlOSFCnXyfvSNc= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY= +go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genai v1.36.0 h1:sJCIjqTAmwrtAIaemtTiKkg2TO1RxnYEusTmEQ3nGxM= +google.golang.org/genai v1.36.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a h1:dOon6HF2sPRFnhCLEiAeKPc21JHL2eX7UBWjIR8PLaY= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a/go.mod h1:Gtytau9Uoc3oPo/dpHvKit+tQn9Qlk5XFG1RiZTGqfk= diff --git a/examples/tool/webfetch/geminifetch/main.go b/examples/tool/webfetch/geminifetch/main.go new file mode 100644 index 000000000..2caa30c15 --- /dev/null +++ b/examples/tool/webfetch/geminifetch/main.go @@ -0,0 +1,278 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package main demonstrates interactive chat using Gemini web fetch tool. +// The tool uses Gemini's URL Context feature for server-side web fetching and analysis. +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "log" + "os" + "strings" + "time" + + "trpc.group/trpc-go/trpc-agent-go/agent/llmagent" + "trpc.group/trpc-go/trpc-agent-go/event" + "trpc.group/trpc-go/trpc-agent-go/model" + "trpc.group/trpc-go/trpc-agent-go/model/openai" + "trpc.group/trpc-go/trpc-agent-go/runner" + "trpc.group/trpc-go/trpc-agent-go/tool" + "trpc.group/trpc-go/trpc-agent-go/tool/webfetch/geminifetch" +) + +func main() { + // Parse command line flags. + modelName := flag.String("model", "deepseek-chat", "Name of the model to use") + geminiModel := flag.String("gemini-model", "gemini-2.5-flash", "Gemini model for web fetching") + flag.Parse() + + fmt.Printf("πŸš€ Gemini Web Fetch Chat Demo\n") + fmt.Printf("Model: %s\n", *modelName) + fmt.Printf("Gemini Fetch Model: %s\n", *geminiModel) + fmt.Printf("Type 'exit' to end the conversation\n") + fmt.Printf("Available tools: gemini_web_fetch\n") + fmt.Println(strings.Repeat("=", 50)) + + // Create and run the chat. + chat := &geminiWebFetchChat{ + modelName: *modelName, + geminiModel: *geminiModel, + } + + if err := chat.run(); err != nil { + log.Fatalf("Chat failed: %v", err) + } +} + +// geminiWebFetchChat manages the conversation with Gemini web fetch capability. +type geminiWebFetchChat struct { + modelName string + geminiModel string + runner runner.Runner + userID string + sessionID string +} + +// run starts the interactive chat session. +func (c *geminiWebFetchChat) run() error { + ctx := context.Background() + + // Setup the runner. + if err := c.setup(ctx); err != nil { + return fmt.Errorf("setup failed: %w", err) + } + + // Start interactive chat. + return c.startChat(ctx) +} + +// setup creates the runner with LLM agent and Gemini web fetch tool. +func (c *geminiWebFetchChat) setup(ctx context.Context) error { + // Create OpenAI model. + modelInstance := openai.New(c.modelName) + + // Create Gemini web fetch tool. + fetchTool, err := geminifetch.NewTool(c.geminiModel) + if err != nil { + return fmt.Errorf("failed to create gemini fetch tool: %w", err) + } + + // Create LLM agent with Gemini web fetch tool. + genConfig := model.GenerationConfig{ + MaxTokens: intPtr(2000), + Temperature: floatPtr(0.7), + Stream: true, // Enable streaming + } + + agentName := "gemini-web-fetch-assistant" + llmAgent := llmagent.New( + agentName, + llmagent.WithModel(modelInstance), + llmagent.WithDescription("A helpful AI assistant with Gemini's server-side web fetching capability."), + llmagent.WithInstruction("Use the gemini_web_fetch tool to retrieve and analyze web content. "+ + "Simply include URLs naturally in your prompt and Gemini will automatically fetch and analyze them on the server side. "+ + "You can include up to 20 URLs in a single request. "+ + "The tool leverages Gemini's URL Context feature for intelligent content extraction and analysis. "+ + "When analyzing web content, provide clear summaries and extract key information relevant to the user's question."), + llmagent.WithGenerationConfig(genConfig), + llmagent.WithTools([]tool.Tool{fetchTool}), + ) + + // Create runner. + appName := "gemini-web-fetch-chat" + c.runner = runner.NewRunner( + appName, + llmAgent, + ) + + // Setup identifiers. + c.userID = "user" + c.sessionID = fmt.Sprintf("gemini-web-fetch-session-%d", time.Now().Unix()) + + fmt.Printf("βœ… Gemini web fetch chat ready! Session: %s\n\n", c.sessionID) + + return nil +} + +// startChat runs the interactive conversation loop. +func (c *geminiWebFetchChat) startChat(ctx context.Context) error { + scanner := bufio.NewScanner(os.Stdin) + + // Print welcome message with examples. + fmt.Println("πŸ’‘ Try asking questions like:") + fmt.Println(" - Summarize https://example.com") + fmt.Println(" - Compare https://site1.com and https://site2.com") + fmt.Println(" - What's the main content of https://news.ycombinator.com") + fmt.Println(" - Analyze the article at https://blog.example.com/post") + fmt.Println(" - Extract key points from https://ai.google.dev/gemini-api/docs/url-context") + fmt.Println() + fmt.Println("ℹ️ Note: URLs are automatically detected and fetched by Gemini's server") + fmt.Println() + + for { + fmt.Print("πŸ‘€ You: ") + if !scanner.Scan() { + break + } + + userInput := strings.TrimSpace(scanner.Text()) + if userInput == "" { + continue + } + + // Handle exit command. + if strings.ToLower(userInput) == "exit" { + fmt.Println("πŸ‘‹ Goodbye!") + return nil + } + + // Process the user message. + if err := c.processMessage(ctx, userInput); err != nil { + fmt.Printf("❌ Error: %v\n", err) + } + + fmt.Println() // Add spacing between turns + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("input scanner error: %w", err) + } + + return nil +} + +// processMessage handles a single message exchange. +func (c *geminiWebFetchChat) processMessage(ctx context.Context, userMessage string) error { + message := model.NewUserMessage(userMessage) + + // Run the agent through the runner. + eventChan, err := c.runner.Run(ctx, c.userID, c.sessionID, message) + if err != nil { + return fmt.Errorf("failed to run agent: %w", err) + } + + // Process streaming response. + return c.processStreamingResponse(eventChan) +} + +// processStreamingResponse handles the streaming response with Gemini web fetch tool visualization. +func (c *geminiWebFetchChat) processStreamingResponse(eventChan <-chan *event.Event) error { + fmt.Print("πŸ€– Assistant: ") + + var ( + fullContent string + toolCallsDetected bool + assistantStarted bool + ) + + for event := range eventChan { + + // Handle errors. + if event.Error != nil { + fmt.Printf("\n❌ Error: %s\n", event.Error.Message) + continue + } + + // Detect and display tool calls. + if len(event.Response.Choices) > 0 && len(event.Response.Choices[0].Message.ToolCalls) > 0 { + toolCallsDetected = true + if assistantStarted { + fmt.Printf("\n") + } + fmt.Printf("🌐 Gemini web fetch initiated:\n") + for _, toolCall := range event.Response.Choices[0].Message.ToolCalls { + fmt.Printf(" β€’ %s (ID: %s)\n", toolCall.Function.Name, toolCall.ID) + if len(toolCall.Function.Arguments) > 0 { + fmt.Printf(" Prompt: %s\n", string(toolCall.Function.Arguments)) + } + } + fmt.Printf("\nπŸ”„ Gemini fetching and analyzing content...\n") + } + + // Detect tool responses. + if event.Response != nil && len(event.Response.Choices) > 0 { + hasToolResponse := false + for _, choice := range event.Response.Choices { + if choice.Message.Role == model.RoleTool && choice.Message.ToolID != "" { + // Truncate long tool responses for display + content := strings.TrimSpace(choice.Message.Content) + if len(content) > 200 { + content = content[:200] + "..." + } + fmt.Printf("βœ… Fetch result (ID: %s): %s\n", + choice.Message.ToolID, + content) + hasToolResponse = true + } + } + if hasToolResponse { + continue + } + } + + // Process streaming content. + if len(event.Response.Choices) > 0 { + choice := event.Response.Choices[0] + + // Handle streaming delta content. + if choice.Delta.Content != "" { + if !assistantStarted { + if toolCallsDetected { + fmt.Printf("\nπŸ€– Assistant: ") + } + assistantStarted = true + } + fmt.Print(choice.Delta.Content) + fullContent += choice.Delta.Content + } + } + + // Check if this is the final event. + if event.IsFinalResponse() { + fmt.Printf("\n") + break + } + } + + return nil +} + +// intPtr returns a pointer to the given int. +func intPtr(i int) *int { + return &i +} + +// floatPtr returns a pointer to the given float64. +func floatPtr(f float64) *float64 { + return &f +} diff --git a/examples/tool/webfetch/httpfetch/README.md b/examples/tool/webfetch/httpfetch/README.md new file mode 100644 index 000000000..14c84194f --- /dev/null +++ b/examples/tool/webfetch/httpfetch/README.md @@ -0,0 +1,92 @@ +# HTTP Web Fetch Tool Example + +This example demonstrates how to use the HTTP web fetch tool with an AI agent for interactive conversations. The tool enables fetching and extracting content from web pages, converting HTML to markdown for better readability, and supporting various text formats. + +## Running the Example + +### Using environment variables: + +```bash +export OPENAI_API_KEY="your-api-key-here" +export OPENAI_BASE_URL="https://api.openai.com/v1" # Optional +go run main.go +``` + +### Using custom model: + +```bash +export OPENAI_API_KEY="your-api-key-here" +go run main.go -model gpt-4o-mini +``` + +## Example Session + +``` +amdahliu@AMDAHLIU-MC2 trpc-agent-go % ./dpskv3.sh +πŸš€ HTTP Web Fetch Chat Demo +Model: deepseek-v3-local-II +Type 'exit' to end the conversation +Available tools: web_fetch +================================================== +βœ… Web fetch chat ready! Session: web-fetch-session-1763989894 + +πŸ’‘ Try asking questions like: + - Summarize the content from https://example.com + - Fetch and compare https://site1.com and https://site2.com + - What's on the homepage of https://news.ycombinator.com + - Extract the main points from https://blog.example.com/article + - Get the API documentation from https://api.example.com/docs + +ℹ️ Note: The tool supports HTML, JSON, XML, and plain text formats + +πŸ‘€ You: Summarize the content from https://ai.google.dev/gemini-api/docs/text-generation +πŸ€– Assistant: 🌐 Web fetch initiated: + β€’ web_fetch (ID: chatcmpl-tool-2f80eb6504fc43b0adb62f36f21ee339) + Args: {"urls":["https://ai.google.dev/gemini-api/docs/text-generation"]} + +πŸ”„ Fetching web content... +βœ… Fetch result (ID: chatcmpl-tool-2f80eb6504fc43b0adb62f36f21ee339): {"results":[{"retrieved_url":"https://ai.google.dev/gemini-api/docs/text-generation","status_code":200,"content_type":"text/html","content":"[Skip to main content](#main-content)\n\n[![Gemini API](htt... + +πŸ€– Assistant: The page provides a comprehensive guide on using the Gemini API for text generation. Here's a summary of the key points: + +### **Text Generation with Gemini API** +1. **Basic Text Generation**: + - The API can generate text from various inputs (text, images, video, audio). + - Example code snippets are provided for Python, JavaScript, Go, Java, REST, and Apps Script. + +2. **Thinking with Gemini 2.5**: + - Models like Gemini 2.5 Flash and Pro have "thinking" enabled by default for enhanced quality. + - Thinking can be disabled by setting the `thinking_budget` to zero. + +3. **System Instructions**: + - You can guide the model's behavior using system instructions (e.g., "You are a cat. Your name is Neko."). + - Examples are provided for multiple programming languages. + +4. **Multimodal Inputs**: + - The API supports combining text with media files (e.g., images). + - Code examples demonstrate how to process multimodal inputs. + +5. **Streaming Responses**: + - For real-time interactions, streaming allows incremental responses. + - Examples are provided for streaming in Python, JavaScript, Go, Java, REST, and Apps Script. + +6. **Multi-Turn Conversations (Chat)**: + - The SDKs support chat functionality to maintain conversation history. + - Examples show how to implement multi-turn conversations and streaming chats. + +7. **Supported Models**: + - All Gemini models support text generation. Details about models and their capabilities are available on the [Models](https://ai.google.dev/gemini-api/docs/models) page. + +8. **Best Practices**: + - **Prompting Tips**: Use zero-shot or few-shot prompts for tailored outputs. + - **Structured Output**: The API can generate structured outputs like JSON. + +### **Next Steps** +- Try the [Gemini API getting started Colab](https://colab.research.google.com/github/google-gemini/cookbook/blob/main/quickstarts/Get_started.ipynb). +- Explore multimodal capabilities (image, video, audio, document understanding). + +For more details, refer to the [official documentation](https://ai.google.dev/gemini-api/docs/text-generation). + +πŸ‘€ You: exit +πŸ‘‹ Goodbye! +``` diff --git a/examples/tool/webfetch/httpfetch/go.mod b/examples/tool/webfetch/httpfetch/go.mod new file mode 100644 index 000000000..0ec515cac --- /dev/null +++ b/examples/tool/webfetch/httpfetch/go.mod @@ -0,0 +1,50 @@ +module trpc.group/trpc-go/trpc-agent-go/examples/tool/webfetch/httpfetch + +go 1.24.1 + +replace ( + trpc.group/trpc-go/trpc-agent-go => ../../../.. + trpc.group/trpc-go/trpc-agent-go/tool/webfetch/httpfetch => ../../../../tool/webfetch/httpfetch +) + +require ( + trpc.group/trpc-go/trpc-agent-go v0.2.0 + trpc.group/trpc-go/trpc-agent-go/tool/webfetch/httpfetch v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/JohannesKaufmann/dom v0.2.0 // indirect + github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/openai/openai-go v1.12.0 // indirect + github.com/panjf2000/ants/v2 v2.10.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/sdk v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a // indirect +) diff --git a/examples/tool/webfetch/httpfetch/go.sum b/examples/tool/webfetch/httpfetch/go.sum new file mode 100644 index 000000000..71c72ecd3 --- /dev/null +++ b/examples/tool/webfetch/httpfetch/go.sum @@ -0,0 +1,103 @@ +github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= +github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 h1:C0/TerKdQX9Y9pbYi1EsLr5LDNANsqunyI/btpyfCg8= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0/go.mod h1:OLaKh+giepO8j7teevrNwiy/fwf8LXgoc9g7rwaE1jk= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0= +github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y= +github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= +github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5E= +github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 h1:dIIDULZJpgdiHz5tXrTgKIMLkus6jEFa7x5SOKcyR7E= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0 h1:nSiV3s7wiCam610XcLbYOmMfJxB9gO4uK3Xgv5gmTgg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.29.0/go.mod h1:hKn/e/Nmd19/x1gvIHwtOwVWM+VhuITSWip3JUDghj0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 h1:JAv0Jwtl01UFiyWZEMiJZBiTlv5A50zNs8lsthXqIio= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0/go.mod h1:QNKLmUEAq2QUbPQUfvw4fmv0bgbK7UlOSFCnXyfvSNc= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY= +go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd h1:BBOTEWLuuEGQy9n1y9MhVJ9Qt0BDu21X8qZs71/uPZo= +google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a h1:dOon6HF2sPRFnhCLEiAeKPc21JHL2eX7UBWjIR8PLaY= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a/go.mod h1:Gtytau9Uoc3oPo/dpHvKit+tQn9Qlk5XFG1RiZTGqfk= diff --git a/examples/tool/webfetch/httpfetch/main.go b/examples/tool/webfetch/httpfetch/main.go new file mode 100644 index 000000000..e142dc14a --- /dev/null +++ b/examples/tool/webfetch/httpfetch/main.go @@ -0,0 +1,273 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package main demonstrates interactive chat using HTTP web fetch tool. +// The tool provides the ability to fetch and extract content from web pages. +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "log" + "os" + "strings" + "time" + + "trpc.group/trpc-go/trpc-agent-go/agent/llmagent" + "trpc.group/trpc-go/trpc-agent-go/event" + "trpc.group/trpc-go/trpc-agent-go/model" + "trpc.group/trpc-go/trpc-agent-go/model/openai" + "trpc.group/trpc-go/trpc-agent-go/runner" + "trpc.group/trpc-go/trpc-agent-go/tool" + "trpc.group/trpc-go/trpc-agent-go/tool/webfetch/httpfetch" +) + +func main() { + // Parse command line flags. + modelName := flag.String("model", "deepseek-chat", "Name of the model to use") + flag.Parse() + + fmt.Printf("πŸš€ HTTP Web Fetch Chat Demo\n") + fmt.Printf("Model: %s\n", *modelName) + fmt.Printf("Type 'exit' to end the conversation\n") + fmt.Printf("Available tools: web_fetch\n") + fmt.Println(strings.Repeat("=", 50)) + + // Create and run the chat. + chat := &webFetchChat{ + modelName: *modelName, + } + + if err := chat.run(); err != nil { + log.Fatalf("Chat failed: %v", err) + } +} + +// webFetchChat manages the conversation with web fetch capability. +type webFetchChat struct { + modelName string + runner runner.Runner + userID string + sessionID string +} + +// run starts the interactive chat session. +func (c *webFetchChat) run() error { + ctx := context.Background() + + // Setup the runner. + if err := c.setup(ctx); err != nil { + return fmt.Errorf("setup failed: %w", err) + } + + // Start interactive chat. + return c.startChat(ctx) +} + +// setup creates the runner with LLM agent and web fetch tool. +func (c *webFetchChat) setup(ctx context.Context) error { + // Create OpenAI model. + modelInstance := openai.New(c.modelName) + + // Create HTTP web fetch tool. + fetchTool := httpfetch.NewTool( + httpfetch.WithMaxContentLength(50000), // Limit single URL content to 50KB + httpfetch.WithMaxTotalContentLength(150000), // Limit total content to 150KB + ) + + // Create LLM agent with web fetch tool. + genConfig := model.GenerationConfig{ + MaxTokens: intPtr(2000), + Temperature: floatPtr(0.7), + Stream: true, // Enable streaming + } + + agentName := "web-fetch-assistant" + llmAgent := llmagent.New( + agentName, + llmagent.WithModel(modelInstance), + llmagent.WithDescription("A helpful AI assistant with the ability to fetch and analyze web content."), + llmagent.WithInstruction("Use the web_fetch tool to retrieve and extract content from web pages. "+ + "You can fetch multiple URLs at once (up to 20). "+ + "The tool converts HTML to markdown for better readability and supports various text formats including JSON, XML, plain text, etc. "+ + "When analyzing web content, provide clear summaries and extract key information relevant to the user's question."), + llmagent.WithGenerationConfig(genConfig), + llmagent.WithTools([]tool.Tool{fetchTool}), + ) + + // Create runner. + appName := "web-fetch-chat" + c.runner = runner.NewRunner( + appName, + llmAgent, + ) + + // Setup identifiers. + c.userID = "user" + c.sessionID = fmt.Sprintf("web-fetch-session-%d", time.Now().Unix()) + + fmt.Printf("βœ… Web fetch chat ready! Session: %s\n\n", c.sessionID) + + return nil +} + +// startChat runs the interactive conversation loop. +func (c *webFetchChat) startChat(ctx context.Context) error { + scanner := bufio.NewScanner(os.Stdin) + + // Print welcome message with examples. + fmt.Println("πŸ’‘ Try asking questions like:") + fmt.Println(" - Summarize the content from https://example.com") + fmt.Println(" - Fetch and compare https://site1.com and https://site2.com") + fmt.Println(" - What's on the homepage of https://news.ycombinator.com") + fmt.Println(" - Extract the main points from https://blog.example.com/article") + fmt.Println(" - Get the API documentation from https://api.example.com/docs") + fmt.Println() + fmt.Println("ℹ️ Note: The tool supports HTML, JSON, XML, and plain text formats") + fmt.Println() + + for { + fmt.Print("πŸ‘€ You: ") + if !scanner.Scan() { + break + } + + userInput := strings.TrimSpace(scanner.Text()) + if userInput == "" { + continue + } + + // Handle exit command. + if strings.ToLower(userInput) == "exit" { + fmt.Println("πŸ‘‹ Goodbye!") + return nil + } + + // Process the user message. + if err := c.processMessage(ctx, userInput); err != nil { + fmt.Printf("❌ Error: %v\n", err) + } + + fmt.Println() // Add spacing between turns + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("input scanner error: %w", err) + } + + return nil +} + +// processMessage handles a single message exchange. +func (c *webFetchChat) processMessage(ctx context.Context, userMessage string) error { + message := model.NewUserMessage(userMessage) + + // Run the agent through the runner. + eventChan, err := c.runner.Run(ctx, c.userID, c.sessionID, message) + if err != nil { + return fmt.Errorf("failed to run agent: %w", err) + } + + // Process streaming response. + return c.processStreamingResponse(eventChan) +} + +// processStreamingResponse handles the streaming response with web fetch tool visualization. +func (c *webFetchChat) processStreamingResponse(eventChan <-chan *event.Event) error { + fmt.Print("πŸ€– Assistant: ") + + var ( + fullContent string + toolCallsDetected bool + assistantStarted bool + ) + + for event := range eventChan { + + // Handle errors. + if event.Error != nil { + fmt.Printf("\n❌ Error: %s\n", event.Error.Message) + continue + } + + // Detect and display tool calls. + if len(event.Response.Choices) > 0 && len(event.Response.Choices[0].Message.ToolCalls) > 0 { + toolCallsDetected = true + if assistantStarted { + fmt.Printf("\n") + } + fmt.Printf("🌐 Web fetch initiated:\n") + for _, toolCall := range event.Response.Choices[0].Message.ToolCalls { + fmt.Printf(" β€’ %s (ID: %s)\n", toolCall.Function.Name, toolCall.ID) + if len(toolCall.Function.Arguments) > 0 { + fmt.Printf(" Args: %s\n", string(toolCall.Function.Arguments)) + } + } + fmt.Printf("\nπŸ”„ Fetching web content...\n") + } + + // Detect tool responses. + if event.Response != nil && len(event.Response.Choices) > 0 { + hasToolResponse := false + for _, choice := range event.Response.Choices { + if choice.Message.Role == model.RoleTool && choice.Message.ToolID != "" { + // Truncate long tool responses for display + content := strings.TrimSpace(choice.Message.Content) + if len(content) > 200 { + content = content[:200] + "..." + } + fmt.Printf("βœ… Fetch result (ID: %s): %s\n", + choice.Message.ToolID, + content) + hasToolResponse = true + } + } + if hasToolResponse { + continue + } + } + + // Process streaming content. + if len(event.Response.Choices) > 0 { + choice := event.Response.Choices[0] + + // Handle streaming delta content. + if choice.Delta.Content != "" { + if !assistantStarted { + if toolCallsDetected { + fmt.Printf("\nπŸ€– Assistant: ") + } + assistantStarted = true + } + fmt.Print(choice.Delta.Content) + fullContent += choice.Delta.Content + } + } + + // Check if this is the final event. + if event.IsFinalResponse() { + fmt.Printf("\n") + break + } + } + + return nil +} + +// intPtr returns a pointer to the given int. +func intPtr(i int) *int { + return &i +} + +// floatPtr returns a pointer to the given float64. +func floatPtr(f float64) *float64 { + return &f +} diff --git a/tool/webfetch/README.md b/tool/webfetch/README.md new file mode 100644 index 000000000..3fbe40c75 --- /dev/null +++ b/tool/webfetch/README.md @@ -0,0 +1,29 @@ +# WebFetch Tools + +This directory contains web fetching implementations for the trpc-agent-go framework. + +## Implementations + +### Client-Side Fetch (Current HTTP Implementation) +Your tool directly fetches content and returns it to the agent. + +**How it works:** +- The agent framework makes HTTP requests on behalf of the LLM +- Content is fetched, processed, and returned as tool results +- Provides full control over fetching, parsing, and filtering + +**Implementation:** `httpfetch/` + +### LLM Server-Side Fetch (Claude/Gemini Style) +The LLM provider handles fetching; you just configure and enable the tool. + +**How it works:** +- The LLM provider's infrastructure fetches web content +- You only provide configuration (domain filters, limits, etc.) +- Content fetching happens on the provider's side, reducing latency + +**Implementations:** +- `geminifetch/` +- `claudefetch/` - Reserved for future use + + diff --git a/tool/webfetch/claudefetch/fetch.go b/tool/webfetch/claudefetch/fetch.go new file mode 100644 index 000000000..475cfc19b --- /dev/null +++ b/tool/webfetch/claudefetch/fetch.go @@ -0,0 +1,11 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package claudefetch provides a Cladude webfetch tool. +package claudefetch diff --git a/tool/webfetch/claudefetch/go.mod b/tool/webfetch/claudefetch/go.mod new file mode 100644 index 000000000..8738ef279 --- /dev/null +++ b/tool/webfetch/claudefetch/go.mod @@ -0,0 +1,5 @@ +module trpc.group/trpc-go/trpc-agent-go/tool/webfetch/claudefetch + +go 1.21.0 + +replace trpc.group/trpc-go/trpc-agent-go => ../../../ diff --git a/tool/webfetch/geminifetch/fetch.go b/tool/webfetch/geminifetch/fetch.go new file mode 100644 index 000000000..7c018aabf --- /dev/null +++ b/tool/webfetch/geminifetch/fetch.go @@ -0,0 +1,194 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package geminifetch provides a Gemini webfetch tool. +package geminifetch + +import ( + "context" + "fmt" + "os" + + "google.golang.org/genai" + "trpc.group/trpc-go/trpc-agent-go/tool" + "trpc.group/trpc-go/trpc-agent-go/tool/function" +) + +// Option configures the GeminiFetch tool. +type Option func(*config) + +// modelCaller is the interface for the model caller. for testing purposes, we can inject a stub model caller. +type modelCaller interface { + GenerateContent(ctx context.Context, model string, contents []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) +} + +type config struct { + apiKey string + model string + client *genai.Client + modelCaller modelCaller +} + +// WithAPIKey sets the Google AI API key. +// If not provided, it will use the GEMINI_API_KEY environment variable. +func WithAPIKey(apiKey string) Option { + return func(cfg *config) { + cfg.apiKey = apiKey + } +} + +// WithClient sets a custom Gemini client. +func WithClient(client *genai.Client) Option { + return func(cfg *config) { + cfg.client = client + } +} + +// fetchRequest is the input for the tool. +type fetchRequest struct { + Prompt string `json:"prompt" jsonschema:"description=Prompt that includes URLs to fetch and instructions for processing. URLs will be automatically detected and fetched by Gemini."` +} + +// fetchResponse is the output. +type fetchResponse struct { + Content string `json:"content"` + URLContextMetadata *urlContextMetadata `json:"url_context_metadata,omitempty"` +} + +type urlContextMetadata struct { + URLMetadata []urlMetadata `json:"url_metadata"` +} + +type urlMetadata struct { + RetrievedURL string `json:"retrieved_url"` + URLRetrievalStatus string `json:"url_retrieval_status"` +} + +// NewTool creates the Gemini web-fetch tool. +// This tool uses Gemini's URL Context feature to fetch and process web content. +// modelName: the Gemini model-id to use. +func NewTool(modelName string, opts ...Option) (tool.CallableTool, error) { + if modelName == "" { + return nil, fmt.Errorf("model name is required") + } + cfg := &config{ + apiKey: os.Getenv("GEMINI_API_KEY"), + model: modelName, + } + for _, opt := range opts { + opt(cfg) + } + + return function.NewFunctionTool( + newGeminiFetchTool(cfg).fetch, + function.WithName("gemini_web_fetch"), + function.WithDescription("Fetches and analyzes web content using Gemini's URL Context feature. "+ + "Simply include URLs in your prompt and Gemini will automatically fetch and analyze them. "+ + "Supports up to 20 URLs per request. "+ + "Example: 'Summarize https://example.com/article and compare it with https://example.com/another'"), + ), nil +} + +type geminiFetchTool struct { + cfg *config +} + +func newGeminiFetchTool(cfg *config) *geminiFetchTool { + return &geminiFetchTool{ + cfg: cfg, + } +} + +func (t *geminiFetchTool) fetch(ctx context.Context, req fetchRequest) (fetchResponse, error) { + if req.Prompt == "" { + return fetchResponse{}, nil + } + + // Resolve the model caller so tests can inject a stub without hitting the API. + modelCaller := t.cfg.modelCaller + client := t.cfg.client + if modelCaller == nil { + if client == nil { + var err error + client, err = genai.NewClient(ctx, &genai.ClientConfig{ + APIKey: t.cfg.apiKey, + Backend: genai.BackendGeminiAPI, + }) + if err != nil { + return fetchResponse{}, fmt.Errorf("failed to create Gemini client: %w", err) + } + // Note: Client doesn't have Close method in this version + } + if client == nil || client.Models == nil { + return fetchResponse{}, fmt.Errorf("gemini client is missing the Models service") + } + modelCaller = client.Models + } + if modelCaller == nil { + return fetchResponse{}, fmt.Errorf("gemini model caller is not available") + } + + // Build content parts with the user's prompt + // Gemini will automatically detect URLs in the prompt and fetch them + contents := []*genai.Content{ + { + Parts: []*genai.Part{ + {Text: req.Prompt}, + }, + }, + } + + // Configure with URL context tool + // This enables Gemini to automatically fetch URLs mentioned in the prompt + config := &genai.GenerateContentConfig{ + Tools: []*genai.Tool{ + { + URLContext: &genai.URLContext{}, + }, + }, + } + + // Generate content + resp, err := modelCaller.GenerateContent(ctx, t.cfg.model, contents, config) + if err != nil { + return fetchResponse{}, fmt.Errorf("failed to generate content: %w", err) + } + + // Extract content from response + var content string + if len(resp.Candidates) > 0 && resp.Candidates[0].Content != nil { + for _, part := range resp.Candidates[0].Content.Parts { + if part.Text != "" { + content += part.Text + } + } + } + + // Extract URL metadata if available + var urlCtxMetadata *urlContextMetadata + if len(resp.Candidates) > 0 && resp.Candidates[0].URLContextMetadata != nil { + var urlMetadataList []urlMetadata + for _, urlMeta := range resp.Candidates[0].URLContextMetadata.URLMetadata { + urlMetadataList = append(urlMetadataList, urlMetadata{ + RetrievedURL: urlMeta.RetrievedURL, + URLRetrievalStatus: string(urlMeta.URLRetrievalStatus), + }) + } + if len(urlMetadataList) > 0 { + urlCtxMetadata = &urlContextMetadata{ + URLMetadata: urlMetadataList, + } + } + } + + return fetchResponse{ + Content: content, + URLContextMetadata: urlCtxMetadata, + }, nil +} diff --git a/tool/webfetch/geminifetch/fetch_real_test.go b/tool/webfetch/geminifetch/fetch_real_test.go new file mode 100644 index 000000000..665a330da --- /dev/null +++ b/tool/webfetch/geminifetch/fetch_real_test.go @@ -0,0 +1,57 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package geminifetch_test + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" + "trpc.group/trpc-go/trpc-agent-go/tool/webfetch/geminifetch" +) + +func TestGeminiFetch_Real(t *testing.T) { + // Skip if no API key + if os.Getenv("GEMINI_API_KEY") == "" { + t.Skip("GEMINI_API_KEY not set, skipping real API test") + } + + tool, err := geminifetch.NewTool("gemini-2.5-flash") + require.NoError(t, err) + + // Test with a real URL embedded in the prompt + args := `{"prompt": "Summarize the key features described at https://ai.google.dev/gemini-api/docs/url-context"}` + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + // Print the result for inspection + t.Logf("Result: %+v", res) +} + +func TestGeminiFetch_MultipleURLs(t *testing.T) { + // Skip if no API key + if os.Getenv("GEMINI_API_KEY") == "" { + t.Skip("GEMINI_API_KEY not set, skipping real API test") + } + + tool, err := geminifetch.NewTool("gemini-2.5-flash") + require.NoError(t, err) + + // Test with multiple URLs in the prompt + args := `{"prompt": "Compare the ingredients and cooking times from the recipes at https://www.foodnetwork.com/recipes/ina-garten/perfect-roast-chicken-recipe-1940592 and https://www.allrecipes.com/recipe/21151/simple-whole-roast-chicken/"}` + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + // Print the result for inspection + t.Logf("Result: %+v", res) +} diff --git a/tool/webfetch/geminifetch/fetch_test.go b/tool/webfetch/geminifetch/fetch_test.go new file mode 100644 index 000000000..f2d6806d2 --- /dev/null +++ b/tool/webfetch/geminifetch/fetch_test.go @@ -0,0 +1,205 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package geminifetch + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/genai" +) + +func TestNewToolRequiresModelName(t *testing.T) { + tool, err := NewTool("") + require.Error(t, err) + assert.Nil(t, tool) +} + +func TestNewToolCreatesCallableTool(t *testing.T) { + tool, err := NewTool("gemini-2.5-flash") + require.NoError(t, err) + assert.NotNil(t, tool) +} + +func TestOptionHelpers(t *testing.T) { + cfg := &config{} + client := &genai.Client{} + + WithAPIKey("secret")(cfg) + require.Equal(t, "secret", cfg.apiKey) + + WithClient(client)(cfg) + assert.Equal(t, client, cfg.client) +} + +func TestFetchReturnsEmptyWhenPromptMissing(t *testing.T) { + stub := &stubModelCaller{} + tool := newGeminiFetchTool(&config{ + model: "gemini-test", + modelCaller: stub, + }) + + resp, err := tool.fetch(context.Background(), fetchRequest{}) + require.NoError(t, err) + assert.Empty(t, resp.Content) + assert.Nil(t, resp.URLContextMetadata) + assert.False(t, stub.called) +} + +func TestFetchUsesInjectedModelCaller(t *testing.T) { + stub := &stubModelCaller{ + resp: responseWithParts([]string{"Hello ", "World"}, nil), + } + tool := newGeminiFetchTool(&config{ + model: "gemini-test", + modelCaller: stub, + }) + + req := fetchRequest{Prompt: "Summarize https://example.com"} + resp, err := tool.fetch(context.Background(), req) + require.NoError(t, err) + + assert.Equal(t, "Hello World", resp.Content) + require.True(t, stub.called) + require.Len(t, stub.capturedContents, 1) + require.Len(t, stub.capturedContents[0].Parts, 1) + assert.Equal(t, req.Prompt, stub.capturedContents[0].Parts[0].Text) +} + +func TestFetchPropagatesGenerationErrors(t *testing.T) { + stub := &stubModelCaller{ + err: errors.New("boom"), + } + tool := newGeminiFetchTool(&config{ + model: "gemini-test", + modelCaller: stub, + }) + + _, err := tool.fetch(context.Background(), fetchRequest{Prompt: "prompt"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to generate content") +} + +func TestFetchExtractsMetadata(t *testing.T) { + stub := &stubModelCaller{ + resp: responseWithParts( + []string{"Summary"}, + []*genai.URLMetadata{ + newURLMetadata("https://example.com/a", "SUCCESS"), + newURLMetadata("https://example.com/b", "FAILED"), + }, + ), + } + tool := newGeminiFetchTool(&config{ + model: "gemini-test", + modelCaller: stub, + }) + + resp, err := tool.fetch(context.Background(), fetchRequest{Prompt: "prompt"}) + require.NoError(t, err) + + require.NotNil(t, resp.URLContextMetadata) + require.Len(t, resp.URLContextMetadata.URLMetadata, 2) + assert.Equal(t, "https://example.com/a", resp.URLContextMetadata.URLMetadata[0].RetrievedURL) + assert.Equal(t, "SUCCESS", resp.URLContextMetadata.URLMetadata[0].URLRetrievalStatus) +} + +func TestFetchHandlesMissingCandidatesGracefully(t *testing.T) { + stub := &stubModelCaller{ + resp: &genai.GenerateContentResponse{ + Candidates: []*genai.Candidate{ + {Content: nil}, + }, + }, + } + tool := newGeminiFetchTool(&config{ + model: "gemini-test", + modelCaller: stub, + }) + + resp, err := tool.fetch(context.Background(), fetchRequest{Prompt: "prompt"}) + require.NoError(t, err) + + assert.Empty(t, resp.Content) + assert.Nil(t, resp.URLContextMetadata) +} + +func TestFetchFailsWhenClientHasNoModelsService(t *testing.T) { + tool := newGeminiFetchTool(&config{ + model: "gemini-test", + client: &genai.Client{}, + }) + + _, err := tool.fetch(context.Background(), fetchRequest{Prompt: "prompt"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing the Models service") +} + +func TestFetchClientCreationError(t *testing.T) { + t.Setenv("GEMINI_API_KEY", "") + tool := newGeminiFetchTool(&config{ + model: "gemini-test", + }) + + _, err := tool.fetch(context.Background(), fetchRequest{Prompt: "prompt"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create Gemini client") +} + +type stubModelCaller struct { + resp *genai.GenerateContentResponse + err error + called bool + capturedContents []*genai.Content +} + +func (s *stubModelCaller) GenerateContent(ctx context.Context, model string, contents []*genai.Content, config *genai.GenerateContentConfig) (*genai.GenerateContentResponse, error) { + s.called = true + s.capturedContents = append([]*genai.Content(nil), contents...) + if s.err != nil { + return nil, s.err + } + if s.resp != nil { + return s.resp, nil + } + return &genai.GenerateContentResponse{}, nil +} + +func responseWithParts(parts []string, urlMetadata []*genai.URLMetadata) *genai.GenerateContentResponse { + genaiParts := make([]*genai.Part, len(parts)) + for i, text := range parts { + genaiParts[i] = &genai.Part{Text: text} + } + + candidate := &genai.Candidate{ + Content: &genai.Content{ + Parts: genaiParts, + }, + } + if len(urlMetadata) > 0 { + candidate.URLContextMetadata = &genai.URLContextMetadata{ + URLMetadata: urlMetadata, + } + } + + return &genai.GenerateContentResponse{ + Candidates: []*genai.Candidate{candidate}, + } +} + +func newURLMetadata(url, status string) *genai.URLMetadata { + return &genai.URLMetadata{ + RetrievedURL: url, + URLRetrievalStatus: genai.URLRetrievalStatus(status), + } +} diff --git a/tool/webfetch/geminifetch/go.mod b/tool/webfetch/geminifetch/go.mod new file mode 100644 index 000000000..da60596ef --- /dev/null +++ b/tool/webfetch/geminifetch/go.mod @@ -0,0 +1,38 @@ +module trpc.group/trpc-go/trpc-agent-go/tool/webfetch/geminifetch + +go 1.24 + +toolchain go1.24.4 + +replace trpc.group/trpc-go/trpc-agent-go => ../../../ + +require ( + github.com/stretchr/testify v1.11.1 + google.golang.org/genai v1.36.0 + trpc.group/trpc-go/trpc-agent-go v0.0.0-00010101000000-000000000000 +) + +require ( + cloud.google.com/go v0.116.0 // indirect + cloud.google.com/go/auth v0.9.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.2 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a // indirect +) diff --git a/tool/webfetch/geminifetch/go.sum b/tool/webfetch/geminifetch/go.sum new file mode 100644 index 000000000..f75a8b24b --- /dev/null +++ b/tool/webfetch/geminifetch/go.sum @@ -0,0 +1,140 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U= +cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genai v1.36.0 h1:sJCIjqTAmwrtAIaemtTiKkg2TO1RxnYEusTmEQ3nGxM= +google.golang.org/genai v1.36.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a h1:dOon6HF2sPRFnhCLEiAeKPc21JHL2eX7UBWjIR8PLaY= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a/go.mod h1:Gtytau9Uoc3oPo/dpHvKit+tQn9Qlk5XFG1RiZTGqfk= diff --git a/tool/webfetch/httpfetch/fetch.go b/tool/webfetch/httpfetch/fetch.go new file mode 100644 index 000000000..72fb76aa6 --- /dev/null +++ b/tool/webfetch/httpfetch/fetch.go @@ -0,0 +1,346 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package httpfetch provides the HTTP webfetch tool. +package httpfetch + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + "github.com/JohannesKaufmann/html-to-markdown/v2/converter" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark" + "trpc.group/trpc-go/trpc-agent-go/tool" + "trpc.group/trpc-go/trpc-agent-go/tool/function" + "trpc.group/trpc-go/trpc-agent-go/tool/webfetch/internal/urlfilter" +) + +const ( + defaultTimeout = 30 * time.Second + maxURLs = 20 +) + +// Option configures the WebFetch tool. +type Option func(*config) + +type config struct { + httpClient *http.Client + maxContentLength int + maxTotalContentLength int + allowedDomains []string + blockedDomains []string +} + +// WithHTTPClient sets the HTTP client. +func WithHTTPClient(c *http.Client) Option { + return func(cfg *config) { + cfg.httpClient = c + } +} + +// WithMaxContentLength sets the maximum content length for a single URL. +// 0 means unlimited. +func WithMaxContentLength(limit int) Option { + return func(cfg *config) { + cfg.maxContentLength = limit + } +} + +// WithMaxTotalContentLength sets the maximum total content length for all URLs. +// 0 means unlimited. +func WithMaxTotalContentLength(limit int) Option { + return func(cfg *config) { + cfg.maxTotalContentLength = limit + } +} + +// WithAllowedDomains sets the list of allowed domains or URL patterns. +// If provided, only URLs matching one of these patterns (host and optional path prefix) will be allowed. +// Examples: "example.com" (allows all paths), "example.com/docs" (allows /docs/...). +func WithAllowedDomains(domains []string) Option { + return func(cfg *config) { + cfg.allowedDomains = domains + } +} + +// WithBlockedDomains sets the list of blocked domains or URL patterns. +// URLs matching one of these patterns will be blocked. +func WithBlockedDomains(domains []string) Option { + return func(cfg *config) { + cfg.blockedDomains = domains + } +} + +// fetchRequest is the input for the tool. +type fetchRequest struct { + URLS []string `json:"urls" jsonschema:"description=The list of URLs to fetch content from"` +} + +// fetchResponse is the output. +type fetchResponse struct { + Results []resultItem `json:"results"` + Summary string `json:"summary"` +} + +type resultItem struct { + RetrievedURL string `json:"retrieved_url"` + StatusCode int `json:"status_code,omitempty"` + ContentType string `json:"content_type,omitempty"` + Content string `json:"content,omitempty"` + Error string `json:"error,omitempty"` +} + +// NewTool creates the web-fetch tool. +func NewTool(opts ...Option) tool.CallableTool { + cfg := &config{ + httpClient: &http.Client{Timeout: defaultTimeout}, + } + for _, opt := range opts { + opt(cfg) + } + + t := &webFetchTool{ + client: cfg.httpClient, + maxContentLength: cfg.maxContentLength, + maxTotalContentLength: cfg.maxTotalContentLength, + } + + // Register urlValidators + // 1. Blocked domains + for _, blocked := range cfg.blockedDomains { + t.urlValidators = append(t.urlValidators, urlfilter.URLValidator{ + Filter: urlfilter.NewBlockPatternFilter(blocked), + ErrMsg: fmt.Sprintf("URL matches blocked pattern: %s", blocked), + }) + } + + // 2. Allowed domains + if len(cfg.allowedDomains) > 0 { + t.urlValidators = append(t.urlValidators, urlfilter.URLValidator{ + Filter: urlfilter.NewAllowPatternsFilter(cfg.allowedDomains), + ErrMsg: "URL does not match any allowed pattern", + }) + } + + return function.NewFunctionTool( + t.fetch, + function.WithName("web_fetch"), + function.WithDescription("Fetches and extracts text content from a list of URLs. "+ + "Supports up to 20 URLs. Useful for summarizing, comparing, or extracting information from web pages."), + ) +} + +type webFetchTool struct { + client *http.Client + maxContentLength int + maxTotalContentLength int + urlValidators []urlfilter.URLValidator +} + +func (t *webFetchTool) fetch(ctx context.Context, req fetchRequest) (fetchResponse, error) { + if len(req.URLS) == 0 { + return fetchResponse{Summary: "No URLs provided"}, nil + } + + // Deduplicate URLs + uniqueURLs := make(map[string]struct{}) + var targetURLs []string + for _, u := range req.URLS { + u = strings.TrimSpace(u) + if u == "" { + continue + } + if _, exists := uniqueURLs[u]; !exists { + uniqueURLs[u] = struct{}{} + targetURLs = append(targetURLs, u) + } + } + + if len(targetURLs) > maxURLs { + targetURLs = targetURLs[:maxURLs] + } + + var wg sync.WaitGroup + results := make([]resultItem, len(targetURLs)) + + for i, u := range targetURLs { + wg.Add(1) + go func(index int, urlStr string) { + defer wg.Done() + item := t.fetchOne(ctx, urlStr) + results[index] = item + }(i, u) + } + wg.Wait() + + // Apply total length limit + if t.maxTotalContentLength > 0 { + currentTotal := 0 + for i := range results { + if results[i].Error != "" { + continue + } + contentLen := len(results[i].Content) + if currentTotal >= t.maxTotalContentLength { + results[i].Content = "" // Or maybe a note like "[Truncated due to total limit]" + results[i].Error = "Content truncated due to total length limit" + } else if currentTotal+contentLen > t.maxTotalContentLength { + allowed := t.maxTotalContentLength - currentTotal + results[i].Content = truncateString(results[i].Content, allowed) + currentTotal += len(results[i].Content) + } else { + currentTotal += contentLen + } + } + } + + return fetchResponse{ + Results: results, + Summary: fmt.Sprintf("Fetched %d URLs", len(targetURLs)), + }, nil +} + +func (t *webFetchTool) fetchOne(ctx context.Context, urlStr string) resultItem { + item := resultItem{ + RetrievedURL: urlStr, + } + + if err := urlfilter.CheckURL(t.urlValidators, urlStr); err != nil { + item.Error = err.Error() + return item + } + + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return item + } + req.Header.Set("User-Agent", "trpc-agent-go/web-fetch") + + resp, err := t.client.Do(req) + if err != nil { + item.Error = err.Error() + return item + } + defer resp.Body.Close() + + contentType := resp.Header.Get("Content-Type") + // Parse media type (ignore parameters like charset) + item.ContentType = strings.Split(contentType, ";")[0] + item.ContentType = strings.TrimSpace(item.ContentType) + item.StatusCode = resp.StatusCode + + if item.StatusCode < 200 || item.StatusCode >= 300 { + item.Error = fmt.Sprintf("HTTP status %d", item.StatusCode) + return item + } + + var content string + var processErr error + + if item.ContentType == "text/html" { + content, processErr = convertHTMLToMarkdown(resp.Body) + } else if isSupportedTextType(item.ContentType) { + content, processErr = readBodyAsString(resp.Body) + } else { + item.Error = fmt.Sprintf("unsupported content type: %s", item.ContentType) + return item + } + + if processErr != nil { + item.Error = processErr.Error() + return item + } + + // Apply per-URL limit + if t.maxContentLength > 0 && len(content) > t.maxContentLength { + content = truncateString(content, t.maxContentLength) + } + + item.Content = content + return item +} + +// truncateString truncates a string to n bytes, ensuring valid UTF-8. +func truncateString(s string, n int) string { + if len(s) <= n { + return s + } + // If we cut exactly at n, check if it's a valid boundary. + // Simple approach: convert to runes if we cared about rune count, but "length" usually implies bytes/storage. + // However, chopping bytes can split characters. + // Safe approach: iterate runes until byte count exceeds n. + + if n <= 0 { + return "" + } + + var sb strings.Builder + currentLen := 0 + for _, r := range s { + rLen := len(string(r)) + if currentLen+rLen > n { + break + } + sb.WriteRune(r) + currentLen += rLen + } + return sb.String() +} + +func isSupportedTextType(mediaType string) bool { + switch mediaType { + case "application/json", + "text/plain", + "text/xml", + "text/css", + "text/javascript", + "text/csv", + "text/rtf": + return true + default: + return false + } +} + +// readBodyAsString reads the entire content of an io.Reader into a string. +func readBodyAsString(r io.Reader) (string, error) { + buf := new(strings.Builder) + _, err := io.Copy(buf, r) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + return buf.String(), nil +} + +func convertHTMLToMarkdown(r io.Reader) (string, error) { + conv := converter.NewConverter( + converter.WithPlugins( + base.NewBasePlugin(), + commonmark.NewCommonmarkPlugin(), + ), + ) + + bodyBytes, err := io.ReadAll(r) + if err != nil { + return "", err + } + + markdown, err := conv.ConvertString(string(bodyBytes)) + if err != nil { + return "", err + } + + return markdown, nil +} diff --git a/tool/webfetch/httpfetch/fetch_real_test.go b/tool/webfetch/httpfetch/fetch_real_test.go new file mode 100644 index 000000000..5ae3a2779 --- /dev/null +++ b/tool/webfetch/httpfetch/fetch_real_test.go @@ -0,0 +1,46 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package httpfetch_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "trpc.group/trpc-go/trpc-agent-go/tool/webfetch/httpfetch" +) + +func TestWebFetch_1(t *testing.T) { + wft := httpfetch.NewTool() + + // The Call method expects JSON input with a "urls" field + args := `{"urls": ["https://geminicli.com/docs/tools/web-fetch/"]}` + + res, err := wft.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + // Print the result for inspection + t.Logf("Result: %+v", res) +} + +func TestWebFetch_2(t *testing.T) { + + wft := httpfetch.NewTool() + + // The Call method expects JSON input with a "urls" field + args := `{"urls": ["https://ai.google.dev/gemini-api/docs/url-context"]}` + + res, err := wft.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + // Print the result for inspection + t.Logf("Result: %+v", res) + +} diff --git a/tool/webfetch/httpfetch/fetch_test.go b/tool/webfetch/httpfetch/fetch_test.go new file mode 100644 index 000000000..0e130bbce --- /dev/null +++ b/tool/webfetch/httpfetch/fetch_test.go @@ -0,0 +1,684 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package httpfetch + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWebFetch(t *testing.T) { + // Mock server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/page1" { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `

Hello

World

`) + } else if r.URL.Path == "/page2" { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, `
Foo Bar
`) + } else { + w.WriteHeader(404) + } + })) + defer ts.Close() + + tool := NewTool() + + // We call the tool via the interface + args := fmt.Sprintf(`{"urls": ["%s/page1", "%s/page2"]}`, ts.URL, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp, ok := res.(fetchResponse) + require.True(t, ok, "Response should be of type fetchResponse") + + assert.Len(t, resp.Results, 2) + + // Order isn't guaranteed due to concurrency, so we check existence. + foundPage1 := false + foundPage2 := false + + for _, r := range resp.Results { + if r.RetrievedURL == ts.URL+"/page1" { + assert.Contains(t, r.Content, "# Hello") + assert.Contains(t, r.Content, "World") + assert.Equal(t, http.StatusOK, r.StatusCode) + assert.Equal(t, "text/html", r.ContentType) + foundPage1 = true + } + if r.RetrievedURL == ts.URL+"/page2" { + assert.Contains(t, r.Content, "Foo Bar") + assert.Equal(t, http.StatusOK, r.StatusCode) + assert.Equal(t, "text/html", r.ContentType) + foundPage2 = true + } + } + + assert.True(t, foundPage1, "Should have found page1") + assert.True(t, foundPage2, "Should have found page2") + + // Test 404 case + args404 := fmt.Sprintf(`{"urls": ["%s/nonexistent"]}`, ts.URL) + + res404, err404 := tool.Call(context.Background(), []byte(args404)) + require.NoError(t, err404) + + resp404, ok404 := res404.(fetchResponse) + require.True(t, ok404, "Response should be of type fetchResponse") + assert.Len(t, resp404.Results, 1) + assert.Equal(t, ts.URL+"/nonexistent", resp404.Results[0].RetrievedURL) + assert.Equal(t, http.StatusNotFound, resp404.Results[0].StatusCode) + assert.Equal(t, "", resp404.Results[0].ContentType) // 404 response might not have a content type + assert.Contains(t, resp404.Results[0].Error, "HTTP status 404") +} + +func TestWebFetch_NoURLs(t *testing.T) { + tool := NewTool() + res, err := tool.Call(context.Background(), []byte(`{"urls": []}`)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Empty(t, resp.Results) + assert.Equal(t, "No URLs provided", resp.Summary) +} + +func TestWebFetch_PlainText(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") // With params to test cleaning + fmt.Fprint(w, "This is plain text content.") + })) + defer ts.Close() + + tool := NewTool() + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp, ok := res.(fetchResponse) + require.True(t, ok, "Response should be of type fetchResponse") + assert.Len(t, resp.Results, 1) + assert.Equal(t, ts.URL, resp.Results[0].RetrievedURL) + assert.Equal(t, http.StatusOK, resp.Results[0].StatusCode) + assert.Equal(t, "text/plain", resp.Results[0].ContentType) + assert.Equal(t, "This is plain text content.", resp.Results[0].Content) +} + +func TestWebFetch_JSON(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"key": "value", "number": 123}`) + })) + defer ts.Close() + + tool := NewTool() + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp, ok := res.(fetchResponse) + require.True(t, ok, "Response should be of type fetchResponse") + assert.Len(t, resp.Results, 1) + assert.Equal(t, ts.URL, resp.Results[0].RetrievedURL) + assert.Equal(t, http.StatusOK, resp.Results[0].StatusCode) + assert.Equal(t, "application/json", resp.Results[0].ContentType) + assert.Equal(t, `{"key": "value", "number": 123}`, resp.Results[0].Content) +} + +func TestWebFetch_UnsupportedType(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + fmt.Fprint(w, `binary data`) + })) + defer ts.Close() + + tool := NewTool() + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp, ok := res.(fetchResponse) + require.True(t, ok, "Response should be of type fetchResponse") + assert.Len(t, resp.Results, 1) + assert.Equal(t, ts.URL, resp.Results[0].RetrievedURL) + assert.Equal(t, http.StatusOK, resp.Results[0].StatusCode) + assert.Equal(t, "application/octet-stream", resp.Results[0].ContentType) + assert.Empty(t, resp.Results[0].Content) + assert.Contains(t, resp.Results[0].Error, "unsupported content type: application/octet-stream") +} + +func TestWebFetch_PerUrlLimit(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "1234567890") + })) + defer ts.Close() + + // Limit to 5 bytes + tool := NewTool(WithMaxContentLength(5)) + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp, ok := res.(fetchResponse) + require.True(t, ok) + assert.Len(t, resp.Results, 1) + assert.Equal(t, "12345", resp.Results[0].Content) + assert.Equal(t, "text/plain", resp.Results[0].ContentType) +} + +func TestWebFetch_TotalLimit(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "12345") + })) + defer ts.Close() + + // Total limit 7. Fetch two URLs (5 bytes each). + // 1st: 5 bytes. Total 5. (OK) + // 2nd: 5 bytes. Total 10 > 7. Truncate 2nd to 2 bytes (7-5). + // Note: Order depends on concurrency, but results array is ordered by input. + // The implementation applies total limit strictly on result array order. + + tool := NewTool(WithMaxTotalContentLength(7)) + args := fmt.Sprintf(`{"urls": ["%s/1", "%s/2"]}`, ts.URL, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp, ok := res.(fetchResponse) + require.True(t, ok) + assert.Len(t, resp.Results, 2) + + // The implementation iterates results in order. + // Result 0: "12345" (len 5) + // Result 1: "12" (len 2) -> Total 7 + + assert.Equal(t, "12345", resp.Results[0].Content) + assert.Equal(t, "text/plain", resp.Results[0].ContentType) + assert.Equal(t, "12", resp.Results[1].Content) + assert.Equal(t, "text/plain", resp.Results[1].ContentType) +} + +func TestWebFetch_TruncateUTF8(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + // "δ½ ε₯½" is 6 bytes (3 per rune). + fmt.Fprint(w, "δ½ ε₯½") + })) + defer ts.Close() + + // Limit to 4 bytes. + // "δ½ " is 3 bytes. "ε₯½" is 3 bytes. + // 4 bytes is not enough for "δ½ ε₯½". + // truncateString iterates runes. + // Rune 1 'δ½ ': len 3. Current 3 <= 4. Keep. + // Rune 2 'ε₯½': len 3. Current 3+3=6 > 4. Stop. + // Result should be "δ½ " (3 bytes). + + tool := NewTool(WithMaxContentLength(4)) + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp, ok := res.(fetchResponse) + require.True(t, ok) + assert.Equal(t, "δ½ ", resp.Results[0].Content) + assert.Equal(t, "text/plain", resp.Results[0].ContentType) +} + +func TestConvertHTMLToMarkdown(t *testing.T) { + htmlContent := ` + + + Test Page + + + +

Header

+

Subheader

+ +

Paragraph text.

+ +

Check this link.

+

Bold and Italic

+ + + ` + // Mock reader + result, err := convertHTMLToMarkdown(strings.NewReader(htmlContent)) + require.NoError(t, err) + + // Debug output if test fails + t.Logf("Converted Markdown:%s", result) + + // Check expected markdown content + expectedParts := []string{ + "# Header", + "## Subheader", + "Paragraph text.", + "- Item 1", + "- Item 2", + "Check [this link](http://example.com).", + "**Bold**", + "*Italic*", + } + + for _, part := range expectedParts { + assert.Contains(t, result, part) + } + + assert.NotContains(t, result, "console.log") + assert.NotContains(t, result, "color: red") +} + +func TestWebFetch_WithHTTPClient(t *testing.T) { + client := &http.Client{Timeout: defaultTimeout} + tool := NewTool(WithHTTPClient(client)) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "OK") + })) + defer ts.Close() + + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.Equal(t, "OK", resp.Results[0].Content) +} + +// failReader is an io.Reader that returns an error on Read. +type failReader struct{} + +func (f *failReader) Read(p []byte) (n int, err error) { + return 0, errors.New("simulated read error") +} + +func TestReadBodyAsString_Error(t *testing.T) { + _, err := readBodyAsString(&failReader{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to read response body") +} + +func TestConvertHTMLToMarkdown_ReadError(t *testing.T) { + _, err := convertHTMLToMarkdown(&failReader{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "simulated read error") +} + +func TestTruncateString_EdgeCases(t *testing.T) { + assert.Equal(t, "", truncateString("hello", 0)) + assert.Equal(t, "", truncateString("hello", -1)) + assert.Equal(t, "h", truncateString("hello", 1)) + // Test UTF-8 splitting + // "δ½ ε₯½" -> bytes: [e4 bd a0 e5 a5 bd] + // limit 4. "δ½ " is 3 bytes. "ε₯½" is 3 bytes. 3 <= 4. Result "δ½ " + assert.Equal(t, "δ½ ", truncateString("δ½ ε₯½", 4)) + // limit 2. "δ½ " is 3 bytes. 0+3 > 2. Result "" + assert.Equal(t, "", truncateString("δ½ ε₯½", 2)) +} + +func TestWebFetch_InvalidURL(t *testing.T) { + // Test http.NewRequestWithContext error + // URL with control character should fail parsing/request creation + tool := NewTool() + // A URL with a space is invalid for NewRequest + args := `{"urls": ["http://example.com/ foo"]}` + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.NotEmpty(t, resp.Results[0].Error) +} + +func TestWebFetch_ClientDoError(t *testing.T) { + // Simulate a client error (e.g., connection refused) + // We can use a closed server URL or invalid port + tool := NewTool() + args := `{"urls": ["http://127.0.0.1:0"]}` // Invalid port 0 usually fails immediately or connection refused + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.NotEmpty(t, resp.Results[0].Error) +} + +func TestFetch_TotalLimitExact(t *testing.T) { + // Test exact limit match + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "abc") + })) + defer ts.Close() + + tool := NewTool(WithMaxTotalContentLength(3)) + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + resp := res.(fetchResponse) + assert.Equal(t, "abc", resp.Results[0].Content) +} + +func TestFetch_TotalLimitExceeded(t *testing.T) { + // Test limit exceeded where next item is skipped entirely? + // Code: + // if currentTotal >= t.maxTotalContentLength { results[i].Content = ""; Error = "..." } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "abc") + })) + defer ts.Close() + + // Limit 2. Fetch "abc". Should be truncated to "ab". Next should be skipped. + tool := NewTool(WithMaxTotalContentLength(2)) + args := fmt.Sprintf(`{"urls": ["%s/1", "%s/2"]}`, ts.URL, ts.URL) + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + resp := res.(fetchResponse) + + assert.Equal(t, "ab", resp.Results[0].Content) + + // The second one should be marked as truncated/skipped + // In loop: currentTotal becomes 2. + // Next iter: currentTotal (2) >= limit (2). + assert.Empty(t, resp.Results[1].Content) + assert.Contains(t, resp.Results[1].Error, "truncated due to total length limit") +} + +// ============================================================================ +// URL Filtering Tests +// ============================================================================ + +func TestWebFetch_WithAllowedDomains(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "Allowed content") + })) + defer ts.Close() + + // Only allow localhost + tool := NewTool(WithAllowedDomains([]string{"127.0.0.1"})) + + // This should be allowed + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.Equal(t, "Allowed content", resp.Results[0].Content) + assert.Empty(t, resp.Results[0].Error) +} + +func TestWebFetch_WithAllowedDomains_Blocked(t *testing.T) { + // Only allow example.com + tool := NewTool(WithAllowedDomains([]string{"example.com"})) + + // Try to fetch from google.com (should be blocked) + args := `{"urls": ["https://google.com"]}` + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.NotEmpty(t, resp.Results[0].Error) + assert.Contains(t, resp.Results[0].Error, "does not match any allowed pattern") +} + +func TestWebFetch_WithBlockedDomains(t *testing.T) { + // Block malicious.com + tool := NewTool(WithBlockedDomains([]string{"malicious.com"})) + + // Try to fetch from blocked domain + args := `{"urls": ["https://malicious.com/page"]}` + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.NotEmpty(t, resp.Results[0].Error) + assert.Contains(t, resp.Results[0].Error, "matches blocked pattern") +} + +func TestWebFetch_WithBlockedDomains_Allowed(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "Not blocked") + })) + defer ts.Close() + + // Block example.com but allow localhost + tool := NewTool(WithBlockedDomains([]string{"example.com"})) + + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.Equal(t, "Not blocked", resp.Results[0].Content) + assert.Empty(t, resp.Results[0].Error) +} + +func TestWebFetch_CombinedFilters(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "Success") + })) + defer ts.Close() + + // Allow 127.0.0.1 and block nothing specific + tool := NewTool( + WithAllowedDomains([]string{"127.0.0.1"}), + WithBlockedDomains([]string{"evil.com"}), + ) + + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + assert.Equal(t, "Success", resp.Results[0].Content) +} + +// ============================================================================ +// Additional Edge Case Tests +// ============================================================================ + +func TestWebFetch_DuplicateURLs(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "content") + })) + defer ts.Close() + + // Submit same URL twice - should be deduplicated + tool := NewTool() + args := fmt.Sprintf(`{"urls": ["%s", "%s", "%s"]}`, ts.URL, ts.URL, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + // Should only fetch once due to deduplication + assert.Len(t, resp.Results, 1) + assert.Equal(t, "content", resp.Results[0].Content) + assert.Contains(t, resp.Summary, "Fetched 1 URLs") +} + +func TestWebFetch_EmptyURLs(t *testing.T) { + // Test with empty strings in URL list + tool := NewTool() + args := `{"urls": ["", " ", ""]}` + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Empty(t, resp.Results) + // After trimming and deduplication, empty URLs result in "Fetched 0 URLs" + assert.Equal(t, "Fetched 0 URLs", resp.Summary) +} + +func TestWebFetch_MixedEmptyAndValid(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "valid") + })) + defer ts.Close() + + // Mix of empty and valid URLs + tool := NewTool() + args := fmt.Sprintf(`{"urls": ["", "%s", " ", "%s", ""]}`, ts.URL, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + // Should only fetch the one unique valid URL + assert.Len(t, resp.Results, 1) + assert.Equal(t, "valid", resp.Results[0].Content) +} + +func TestWebFetch_MaxURLsLimit(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "ok") + })) + defer ts.Close() + + // Generate 25 unique URLs (more than maxURLs=20) + urls := make([]string, 25) + for i := 0; i < 25; i++ { + urls[i] = fmt.Sprintf("%s/page%d", ts.URL, i) + } + + tool := NewTool() + args := fmt.Sprintf(`{"urls": ["%s"]}`, strings.Join(urls, `","`)) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + // Should only fetch first 20 due to limit + assert.Len(t, resp.Results, 20) + assert.Contains(t, resp.Summary, "Fetched 20 URLs") +} + +func TestWebFetch_SupportedTextTypes(t *testing.T) { + tests := []struct { + name string + contentType string + content string + supported bool + }{ + {"application/json", "application/json", `{"test": "json"}`, true}, + {"text/plain", "text/plain", "plain text", true}, + {"text/xml", "text/xml", "data", true}, + {"text/css", "text/css", "body { color: red; }", true}, + {"text/javascript", "text/javascript", "console.log('test');", true}, + {"text/csv", "text/csv", "a,b,c\n1,2,3", true}, + {"text/rtf", "text/rtf", "{\\rtf1 test}", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", tt.contentType) + fmt.Fprint(w, tt.content) + })) + defer ts.Close() + + tool := NewTool() + args := fmt.Sprintf(`{"urls": ["%s"]}`, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 1) + + if tt.supported { + assert.Equal(t, tt.content, resp.Results[0].Content) + assert.Empty(t, resp.Results[0].Error) + } else { + assert.NotEmpty(t, resp.Results[0].Error) + } + }) + } +} + +func TestIsSupportedTextType(t *testing.T) { + // Direct function test + assert.True(t, isSupportedTextType("application/json")) + assert.True(t, isSupportedTextType("text/plain")) + assert.True(t, isSupportedTextType("text/xml")) + assert.True(t, isSupportedTextType("text/css")) + assert.True(t, isSupportedTextType("text/javascript")) + assert.True(t, isSupportedTextType("text/csv")) + assert.True(t, isSupportedTextType("text/rtf")) + + assert.False(t, isSupportedTextType("application/pdf")) + assert.False(t, isSupportedTextType("image/png")) + assert.False(t, isSupportedTextType("video/mp4")) + assert.False(t, isSupportedTextType("")) +} + +func TestWebFetch_TotalLimitWithError(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "error") { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, "12345") + })) + defer ts.Close() + + // Total limit 10. First URL errors, second succeeds + tool := NewTool(WithMaxTotalContentLength(10)) + args := fmt.Sprintf(`{"urls": ["%s/error", "%s/ok"]}`, ts.URL, ts.URL) + + res, err := tool.Call(context.Background(), []byte(args)) + require.NoError(t, err) + + resp := res.(fetchResponse) + assert.Len(t, resp.Results, 2) + + // First should have error (not counted toward total) + assert.NotEmpty(t, resp.Results[0].Error) + + // Second should succeed with full content + assert.Equal(t, "12345", resp.Results[1].Content) +} + +func TestTruncateString_ExactLength(t *testing.T) { + // Test when string length equals limit + assert.Equal(t, "hello", truncateString("hello", 5)) + assert.Equal(t, "hello", truncateString("hello", 10)) +} diff --git a/tool/webfetch/httpfetch/go.mod b/tool/webfetch/httpfetch/go.mod new file mode 100644 index 000000000..7e15f82d6 --- /dev/null +++ b/tool/webfetch/httpfetch/go.mod @@ -0,0 +1,22 @@ +module trpc.group/trpc-go/trpc-agent-go/tool/webfetch/httpfetch + +go 1.23.0 + +replace trpc.group/trpc-go/trpc-agent-go => ../../../ + +require ( + github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 + github.com/stretchr/testify v1.10.0 + trpc.group/trpc-go/trpc-agent-go v0.0.0 +) + +require ( + github.com/JohannesKaufmann/dom v0.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a // indirect +) diff --git a/tool/webfetch/httpfetch/go.sum b/tool/webfetch/httpfetch/go.sum new file mode 100644 index 000000000..c77b9831b --- /dev/null +++ b/tool/webfetch/httpfetch/go.sum @@ -0,0 +1,30 @@ +github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= +github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 h1:C0/TerKdQX9Y9pbYi1EsLr5LDNANsqunyI/btpyfCg8= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0/go.mod h1:OLaKh+giepO8j7teevrNwiy/fwf8LXgoc9g7rwaE1jk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5E= +github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a h1:dOon6HF2sPRFnhCLEiAeKPc21JHL2eX7UBWjIR8PLaY= +trpc.group/trpc-go/trpc-a2a-go v0.2.5-0.20251023030722-7f02b57fd14a/go.mod h1:Gtytau9Uoc3oPo/dpHvKit+tQn9Qlk5XFG1RiZTGqfk= diff --git a/tool/webfetch/internal/urlfilter/filter.go b/tool/webfetch/internal/urlfilter/filter.go new file mode 100644 index 000000000..4dd253cc5 --- /dev/null +++ b/tool/webfetch/internal/urlfilter/filter.go @@ -0,0 +1,134 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package urlfilter provides URL filtering functions. +package urlfilter + +import ( + "fmt" + "net/url" + "strings" +) + +// urlFilter is a function that determines if a URL should be allowed. +// It returns true if the URL is allowed, false otherwise. +type urlFilter func(url string) bool + +// URLValidator combines a filter with an error message. +type URLValidator struct { + Filter urlFilter + ErrMsg string +} + +// CheckURL checks the URL against the configured validators. +func CheckURL(validators []URLValidator, urlStr string) error { + for _, v := range validators { + if !v.Filter(urlStr) { + return fmt.Errorf("%s", v.ErrMsg) + } + } + return nil +} + +// NewBlockPatternFilter creates a filter that blocks URLs matching the pattern. +func NewBlockPatternFilter(pattern string) urlFilter { + return func(urlStr string) bool { + u, err := url.Parse(urlStr) + if err != nil { + // Fail-safe: treat unparsable URLs as blocked + return false + } + return !matchPattern(u, pattern) + } +} + +// NewAllowPatternsFilter creates a filter that allows URLs matching any of the patterns. +func NewAllowPatternsFilter(patterns []string) urlFilter { + return func(urlStr string) bool { + u, err := url.Parse(urlStr) + if err != nil { + return false + } + for _, p := range patterns { + if matchPattern(u, p) { + return true + } + } + return false + } +} + +// matchPattern checks if the URL matches the given pattern (host + path prefix). +func matchPattern(u *url.URL, pattern string) bool { + // Split pattern into host and path + // Pattern is expected to be like "example.com" or "example.com/foo" + var patternHost, patternPath string + if idx := strings.Index(pattern, "/"); idx != -1 { + patternHost = pattern[:idx] + patternPath = pattern[idx:] + } else { + patternHost = pattern + patternPath = "" + } + + // 1. Host match (case-insensitive) + if !matchHost(u.Hostname(), patternHost) { + return false + } + + // 2. Path match + if patternPath == "" { + return true + } + + // Normalize URL path + uPath := u.Path + if uPath == "" { + uPath = "/" + } + // Ensure absolute path comparison if pattern starts with / (which it does from split) + if !strings.HasPrefix(uPath, "/") { + uPath = "/" + uPath + } + + if !strings.HasPrefix(uPath, patternPath) { + return false + } + + // Boundary check to avoid "/doc" matching "/docserver" + // Match if: + // - lengths are equal (exact match) + // - pattern ends with '/' (explicit directory match) + // - next char in uPath is '/' (sub-path match) + if len(uPath) == len(patternPath) { + return true + } + if strings.HasSuffix(patternPath, "/") { + return true + } + if uPath[len(patternPath)] == '/' { + return true + } + + return false +} + +// matchHost checks if hostname matches target domain (exact or suffix). +// e.g., matchHost("www.example.com", "example.com") -> true +func matchHost(hostname, target string) bool { + hostname = strings.ToLower(hostname) + target = strings.ToLower(target) + if hostname == target { + return true + } + if strings.HasSuffix(hostname, "."+target) { + return true + } + return false +} diff --git a/tool/webfetch/internal/urlfilter/filter_test.go b/tool/webfetch/internal/urlfilter/filter_test.go new file mode 100644 index 000000000..6c514664a --- /dev/null +++ b/tool/webfetch/internal/urlfilter/filter_test.go @@ -0,0 +1,195 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2025 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package urlfilter + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWebFetch_DomainFiltering(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "OK") + })) + defer ts.Close() + + // ts.URL looks like http://127.0.0.1:xxxxx + // We'll use 127.0.0.1 for filtering tests. + + t.Run("AllowedDomains", func(t *testing.T) { + validators := []URLValidator{ + { + Filter: NewAllowPatternsFilter([]string{"127.0.0.1"}), + ErrMsg: "URL does not match any allowed pattern", + }, + } + err := CheckURL(validators, ts.URL) + assert.NoError(t, err) + }) + + t.Run("AllowedDomains_Blocked", func(t *testing.T) { + validators := []URLValidator{ + { + Filter: NewAllowPatternsFilter([]string{"example.com"}), + ErrMsg: "URL does not match any allowed pattern", + }, + } + err := CheckURL(validators, ts.URL) + assert.Contains(t, err.Error(), "does not match any allowed pattern") + }) + + t.Run("BlockedDomains", func(t *testing.T) { + validators := []URLValidator{ + { + Filter: NewBlockPatternFilter("127.0.0.1"), + ErrMsg: "URL matches blocked pattern: 127.0.0.1", + }, + } + err := CheckURL(validators, ts.URL) + assert.Contains(t, err.Error(), "matches blocked pattern") + }) + + t.Run("AllowedDomains_Subpath", func(t *testing.T) { + validators := []URLValidator{ + { + Filter: NewAllowPatternsFilter([]string{"127.0.0.1/docs"}), + ErrMsg: "URL does not match any allowed pattern", + }, + } + + // Allowed path + errOK := CheckURL(validators, ts.URL+"/docs/page1") + assert.NoError(t, errOK, "Should allow /docs/page1") + + // Blocked path + errBlock := CheckURL(validators, ts.URL+"/admin") + assert.Contains(t, errBlock.Error(), "not match any allowed pattern", "Should block /admin") + }) + + t.Run("BlockedDomains_Subpath", func(t *testing.T) { + validators := []URLValidator{ + { + Filter: NewBlockPatternFilter("127.0.0.1/private"), + ErrMsg: "URL matches blocked pattern: 127.0.0.1/private", + }, + } + + // Allowed path (not blocked) + errOK := CheckURL(validators, ts.URL+"/public") + assert.NoError(t, errOK, "Should allow /public") + + // Blocked path + errBlock := CheckURL(validators, ts.URL+"/private/secret") + assert.Contains(t, errBlock.Error(), "matches blocked pattern", "Should block /private/secret") + }) + + t.Run("CustomURLFilter", func(t *testing.T) { + // Filter that only allows paths containing "secure" + filter := func(u string) bool { + return strings.Contains(u, "secure") + } + validators := []URLValidator{ + { + Filter: filter, + ErrMsg: "URL rejected by custom filter", + }, + } + + // Allowed path + errOK := CheckURL(validators, ts.URL+"/secure/page") + assert.NoError(t, errOK, "Should allow /secure/page") + + // Blocked path + errBlock := CheckURL(validators, ts.URL+"/unsafe/page") + assert.Contains(t, errBlock.Error(), "rejected by custom filter", "Should block /unsafe/page") + }) +} + +func TestMatchHost(t *testing.T) { + tests := []struct { + host string + target string + want bool + }{ + {"example.com", "example.com", true}, + {"www.example.com", "example.com", true}, + {"sub.www.example.com", "example.com", true}, + {"example.com", "google.com", false}, + {"notexample.com", "example.com", false}, // Suffix but not dot separator + {"example.com.evil.com", "example.com", false}, + } + + for _, tt := range tests { + got := matchHost(tt.host, tt.target) + assert.Equal(t, tt.want, got, "matchHost(%q, %q)", tt.host, tt.target) + } +} + +func TestMatchPattern(t *testing.T) { + // Helper to create URL + parse := func(s string) *url.URL { + u, _ := url.Parse(s) + return u + } + + tests := []struct { + urlStr string + pattern string + want bool + }{ + // Host only + {"http://example.com", "example.com", true}, + {"http://www.example.com", "example.com", true}, + {"http://google.com", "example.com", false}, + + // Host + Path + {"http://example.com/docs", "example.com/docs", true}, + {"http://example.com/docs/api", "example.com/docs", true}, + {"http://example.com/other", "example.com/docs", false}, + {"http://example.com/docserver", "example.com/docs", false}, // boundary check + {"http://example.com", "example.com/docs", false}, // path too short + + // Subdomains + {"http://www.example.com/docs", "example.com/docs", true}, + + // Trailing slash in pattern + {"http://example.com/docs/", "example.com/docs/", true}, + {"http://example.com/docs", "example.com/docs/", false}, // url path missing slash + {"http://example.com/docs/api", "example.com/docs/", true}, + + // Trailing slash in URL + {"http://example.com/docs/", "example.com/docs", true}, + } + + for _, tt := range tests { + u := parse(tt.urlStr) + got := matchPattern(u, tt.pattern) + assert.Equal(t, tt.want, got, "matchPattern(%q, %q)", tt.urlStr, tt.pattern) + } +} + +// Tests moved from webfetch_coverage_test.go that relate to urlfilter logic +func TestNewBlockPatternFilter_InvalidURL(t *testing.T) { + filter := NewBlockPatternFilter("example.com") + // Pass an invalid URL string that url.Parse fails on. + // Control character in URL path + assert.False(t, filter("http://example.com/\x00")) +} + +func TestNewAllowPatternsFilter_InvalidURL(t *testing.T) { + filter := NewAllowPatternsFilter([]string{"example.com"}) + assert.False(t, filter("http://example.com/\x00")) +}