anthropic calls can now be cached (by default)

Author: rainu

frontend/src/components/Consumption.vue

@@ -1,7 +1,7 @@
 <template>
 	<div class="text-center">
 		<v-chip prepend-icon="mdi-import" class="mx-2" variant="outlined" label v-if="inputToken !== undefined">
-			{{(inputToken - cachedToken).toLocaleString()}}
+			{{inputToken.toLocaleString()}}
 		</v-chip>
 		<v-chip prepend-icon="mdi-cached" class="mx-2" variant="outlined" label v-if="inputToken !== undefined && cachedToken">
 			{{cachedToken.toLocaleString()}}

frontend/wailsjs/go/models.ts

@@ -1688,10 +1688,27 @@ export namespace http {
 
 export namespace llm {
 
+	export class AnthropicCache {
+	    SystemMessage: boolean;
+	    Tools: boolean;
+	    Chat: boolean;
+	
+	    static createFrom(source: any = {}) {
+	        return new AnthropicCache(source);
+	    }
+	
+	    constructor(source: any = {}) {
+	        if ('string' === typeof source) source = JSON.parse(source);
+	        this.SystemMessage = source["SystemMessage"];
+	        this.Tools = source["Tools"];
+	        this.Chat = source["Chat"];
+	    }
+	}
 	export class AnthropicConfig {
 	    Token: common.Secret;
 	    BaseUrl: string;
 	    Model: string;
+	    Cache?: AnthropicCache;
 
 	    static createFrom(source: any = {}) {
 	        return new AnthropicConfig(source);
@@ -1702,6 +1719,7 @@ export namespace llm {
 	        this.Token = this.convertValues(source["Token"], common.Secret);
 	        this.BaseUrl = source["BaseUrl"];
 	        this.Model = source["Model"];
+	        this.Cache = this.convertValues(source["Cache"], AnthropicCache);
 	    }
 
 		convertValues(a: any, classs: any, asMap: boolean = false): any {

go.mod

@@ -72,4 +72,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/tmc/langchaingo => github.com/rainu/langchaingo v0.0.0-20250529115311-2c0e8fe0b340
+replace github.com/tmc/langchaingo => github.com/rainu/langchaingo v0.0.0-20250529142520-123929cc4bd7

go.sum

@@ -99,8 +99,8 @@ github.com/rainu/go-command-chain v0.4.0 h1:qgrNbNsqkTfJHdwGzVuGPPK+p+XSnGAhAT/8
 github.com/rainu/go-command-chain v0.4.0/go.mod h1:RvLsDKnTGD9XoUY7nmBz73ayffI0bFCDH/EVJPRgfks=
 github.com/rainu/go-yacl v0.2.1 h1:BZdwonr/JA8RiE/8xptE9RKT0+COwyfTmyB7kiANGPw=
 github.com/rainu/go-yacl v0.2.1/go.mod h1:cZwUkCDYE1w6xlTUi6vCqdV1O3iLvM/govQdUn6I9NU=
-github.com/rainu/langchaingo v0.0.0-20250529115311-2c0e8fe0b340 h1:bPPPcONFkLvC2dq9e/fQ4z4t4R3/OF4qbMIlmHkLcXc=
-github.com/rainu/langchaingo v0.0.0-20250529115311-2c0e8fe0b340/go.mod h1:5TXP7bKcWjN05g9e2+MpqUXprf5jwI62Q2xGtAKLno8=
+github.com/rainu/langchaingo v0.0.0-20250529142520-123929cc4bd7 h1:sQyN78q6HJVyoCwfGhxIqkCGB41ds6Xd7QIORrrtEbM=
+github.com/rainu/langchaingo v0.0.0-20250529142520-123929cc4bd7/go.mod h1:5TXP7bKcWjN05g9e2+MpqUXprf5jwI62Q2xGtAKLno8=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=

internal/config/model/llm/anthropic.go

@@ -5,27 +5,34 @@ import (
 	"github.com/rainu/ask-mai/internal/config/model/common"
 	iAnthropic "github.com/rainu/ask-mai/internal/llms/anthropic"
 	llmCommon "github.com/rainu/ask-mai/internal/llms/common"
+	"github.com/rainu/go-yacl"
 	"github.com/tmc/langchaingo/llms/anthropic"
 )
 
 type AnthropicConfig struct {
 	Token   common.Secret `yaml:"api-key,omitempty" usage:"API Key"`
 	BaseUrl string        `yaml:"base-url,omitempty" usage:"BaseUrl"`
 	Model   string        `yaml:"model,omitempty" usage:"Model"`
+
+	Cache AnthropicCache `yaml:"disable-cache,omitempty" usage:"disable "`
 }
 
-func (c *AnthropicConfig) AsOptions() (opts []anthropic.Option) {
-	if v := c.Token.GetOrPanicWithDefaultTimeout(); v != nil {
-		opts = append(opts, anthropic.WithToken(string(v)))
+type AnthropicCache struct {
+	SystemMessage *bool `yaml:"system-message,omitempty" usage:"system message cache"`
+	Tools         *bool `yaml:"tools,omitempty" usage:"tools cache"`
+	Chat          *bool `yaml:"chat,omitempty" usage:"chat cache"`
+}
+
+func (c *AnthropicCache) SetDefaults() {
+	if c.SystemMessage == nil {
+		c.SystemMessage = yacl.P(false)
 	}
-	if c.BaseUrl != "" {
-		opts = append(opts, anthropic.WithBaseURL(c.BaseUrl))
+	if c.Tools == nil {
+		c.Tools = yacl.P(false)
 	}
-	if c.Model != "" {
-		opts = append(opts, anthropic.WithModel(c.Model))
+	if c.Chat == nil {
+		c.Chat = yacl.P(false)
 	}
-
-	return
 }
 
 func (c *AnthropicConfig) SetDefaults() {
@@ -45,6 +52,29 @@ func (c *AnthropicConfig) Validate() error {
 	return nil
 }
 
+func (c *AnthropicConfig) AsOptions() (opts []anthropic.Option) {
+	if v := c.Token.GetOrPanicWithDefaultTimeout(); v != nil {
+		opts = append(opts, anthropic.WithToken(string(v)))
+	}
+	if c.BaseUrl != "" {
+		opts = append(opts, anthropic.WithBaseURL(c.BaseUrl))
+	}
+	if c.Model != "" {
+		opts = append(opts, anthropic.WithModel(c.Model))
+	}
+	if c.Cache.SystemMessage != nil && !*c.Cache.SystemMessage {
+		opts = append(opts, anthropic.WithCacheSystemMessage())
+	}
+	if c.Cache.Tools != nil && !*c.Cache.Tools {
+		opts = append(opts, anthropic.WithCacheTools())
+	}
+	if c.Cache.Chat != nil && !*c.Cache.Chat {
+		opts = append(opts, anthropic.WithCacheChat())
+	}
+
+	return
+}
+
 func (c *AnthropicConfig) BuildLLM() (llmCommon.Model, error) {
 	return iAnthropic.New(c.AsOptions())
 }

internal/llms/anthropic/consumption.go

@@ -6,8 +6,10 @@ import (
 )
 
 const (
-	tokenKeyInput  = "InputTokens"
-	tokenKeyOutput = "OutputTokens"
+	tokenKeyInput          = "InputTokens"
+	tokenKeyOutput         = "OutputTokens"
+	tokenKeyCachedCreation = "CacheCreationInputTokens"
+	tokenKeyCachedRead     = "CacheReadInputTokens"
 )
 
 func (a *Anthropic) ConsumptionOf(resp *llms.ContentResponse) common.Consumption {
@@ -23,6 +25,12 @@ func (a *Anthropic) ConsumptionOf(resp *llms.ContentResponse) common.Consumption
 		if t, ok := choice.GenerationInfo[tokenKeyOutput]; ok {
 			result.output += uint64(t.(int))
 		}
+		if t, ok := choice.GenerationInfo[tokenKeyCachedCreation]; ok {
+			result.cachedCreation += uint64(t.(int))
+		}
+		if t, ok := choice.GenerationInfo[tokenKeyCachedRead]; ok {
+			result.cachedRead += uint64(t.(int))
+		}
 	}
 
 	return result
@@ -31,15 +39,23 @@ func (a *Anthropic) ConsumptionOf(resp *llms.ContentResponse) common.Consumption
 type consumption struct {
 	input  uint64
 	output uint64
+
+	cachedCreation uint64
+	cachedRead     uint64
 }
 
 func (t *consumption) Summary() common.ConsumptionSummary {
-	return common.NewSimpleConsumption(t.input, t.output)
+	base := common.NewCachedConsumption(t.input, t.cachedRead, t.output)
+	base["CacheCreation"] = t.cachedCreation
+
+	return base
 }
 
 func (t *consumption) Add(add common.Consumption) {
 	if token, ok := add.(*consumption); ok {
 		t.input += token.input
 		t.output += token.output
+		t.cachedCreation += token.cachedCreation
+		t.cachedRead += token.cachedRead
 	}
 }

internal/llms/openai/consumption.go

@@ -27,6 +27,7 @@ func (o *OpenAI) ConsumptionOf(resp *llms.ContentResponse) common.Consumption {
 		}
 		if t, ok := choice.GenerationInfo[tokenKeyCached]; ok {
 			result.cached += uint64(t.(int))
+			result.input -= uint64(t.(int)) // cached tokens are already part of the input
 		}
 		if t, ok := choice.GenerationInfo[tokenKeyReasoning]; ok {
 			result.reason += uint64(t.(int))

internal/llms/openai/consumption_test.go

@@ -30,14 +30,16 @@ func TestOpenAI_ConsumptionOf(t *testing.T) {
 					{
 						GenerationInfo: map[string]interface{}{
 							tokenKeyInput:      42,
+							tokenKeyCached:     40,
 							tokenKeyCompletion: 123,
 							tokenKeyReasoning:  7,
 						},
 					},
 				},
 			},
 			expectedTokens: consumption{
-				input:  42,
+				input:  2,
+				cached: 40,
 				output: 123,
 				reason: 7,
 			},