diff --git a/genai/count_tokens/count_tokens_examples_test.go b/genai/count_tokens/count_tokens_examples_test.go index 22da59959a..8775f36685 100644 --- a/genai/count_tokens/count_tokens_examples_test.go +++ b/genai/count_tokens/count_tokens_examples_test.go @@ -81,4 +81,30 @@ func TestTextGeneration(t *testing.T) { t.Error("expected non-empty output, got empty") } }) + + t.Run("count local tokens with text input", func(t *testing.T) { + buf.Reset() + err := countTokenLocalWithTxt(buf) + if err != nil { + t.Fatalf("countTokenLocalWithTxt failed: %v", err) + } + + output := buf.String() + if output == "" { + t.Error("expected non-empty output, got empty") + } + }) + + t.Run("compute local tokens with text input", func(t *testing.T) { + buf.Reset() + err := countTokenLocalComputeWithTxt(buf) + if err != nil { + t.Fatalf("countTokenLocalComputeWithTxt failed: %v", err) + } + + output := buf.String() + if output == "" { + t.Error("expected non-empty output, got empty") + } + }) } diff --git a/genai/count_tokens/counttoken_localtokenizer_compute_with_txt.go b/genai/count_tokens/counttoken_localtokenizer_compute_with_txt.go new file mode 100644 index 0000000000..2dbb62aa47 --- /dev/null +++ b/genai/count_tokens/counttoken_localtokenizer_compute_with_txt.go @@ -0,0 +1,69 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package count_tokens shows examples of counting tokens using the GenAI SDK. +package count_tokens + +// [START googlegenaisdk_counttoken_localtokenizer_compute_with_txt] +import ( + "fmt" + "io" + "strings" + + "google.golang.org/genai" + "google.golang.org/genai/tokenizer" +) + +// countTokenLocalComputeWithTxt shows how to compute tokens using the local tokenizer with text input. +func countTokenLocalComputeWithTxt(w io.Writer) error { + modelName := "gemini-2.5-flash" + client, err := tokenizer.NewLocalTokenizer(modelName) + if err != nil { + return fmt.Errorf("failed to create local tokenizer: %w", err) + } + + contents := []*genai.Content{ + {Parts: []*genai.Part{ + {Text: "What's the longest word in the English language?"}, + }}, + } + + resp, err := client.ComputeTokens(contents) + if err != nil { + return fmt.Errorf("failed to compute tokens: %w", err) + } + + for _, tokenInfo := range resp.TokensInfo { + fmt.Fprintf(w, "Role: %s\n", tokenInfo.Role) + fmt.Fprintf(w, "Token IDs: %v\n", tokenInfo.TokenIDs) + fmt.Fprintf(w, "Tokens: [%s]\n", formatTokens(tokenInfo.Tokens)) + } + + // Example response: + // Role: user + // Token IDs: [3689 236789 236751 506 27801 3658 528 506 5422 5192 236881] + // Tokens: [[What, ', s, ▁the, ▁longest, ▁word, ▁in, ▁the, ▁English, ▁language, ?]] ... + + return nil +} + +func formatTokens(tokens [][]byte) string { + parts := make([]string, len(tokens)) + for i, t := range tokens { + parts[i] = fmt.Sprintf("%s", t) + } + return fmt.Sprintf("[%s]", strings.Join(parts, ", ")) +} + +// [END googlegenaisdk_counttoken_localtokenizer_compute_with_txt] diff --git a/genai/count_tokens/counttoken_localtokenizer_with_txt.go b/genai/count_tokens/counttoken_localtokenizer_with_txt.go new file mode 100644 index 0000000000..bcb78e3899 --- /dev/null +++ b/genai/count_tokens/counttoken_localtokenizer_with_txt.go @@ -0,0 +1,54 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package count_tokens shows examples of counting tokens using the GenAI SDK. +package count_tokens + +// [START googlegenaisdk_counttoken_localtokenizer_with_txt] +import ( + "fmt" + "io" + + "google.golang.org/genai" + "google.golang.org/genai/tokenizer" +) + +// countTokenLocalWithTxt shows how to count tokens using the local tokenizer with text input. +func countTokenLocalWithTxt(w io.Writer) error { + modelName := "gemini-2.5-flash" + client, err := tokenizer.NewLocalTokenizer(modelName) + if err != nil { + return fmt.Errorf("failed to create local tokenizer: %w", err) + } + + contents := []*genai.Content{ + {Parts: []*genai.Part{ + {Text: "What's the highest mountain in Africa?"}, + }}, + } + + resp, err := client.CountTokens(contents, nil) + if err != nil { + return fmt.Errorf("failed to count tokens: %w", err) + } + + fmt.Fprintf(w, "Total: %d\n", resp.TotalTokens) + + // Example response: + // Total: 9 + + return nil +} + +// [END googlegenaisdk_counttoken_localtokenizer_with_txt] diff --git a/genai/go.mod b/genai/go.mod index 033bf8cca1..1ef99fc648 100644 --- a/genai/go.mod +++ b/genai/go.mod @@ -5,7 +5,7 @@ go 1.24.0 require ( github.com/GoogleCloudPlatform/golang-samples v0.0.0-20250201051611-5fb145d1e974 golang.org/x/oauth2 v0.25.0 - google.golang.org/genai v1.17.0 + google.golang.org/genai v1.35.0 ) require ( @@ -43,11 +43,11 @@ require ( go.opentelemetry.io/otel/sdk v1.34.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect go.opentelemetry.io/otel/trace v1.34.0 // indirect - golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.34.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.9.0 // indirect google.golang.org/api v0.217.0 // indirect google.golang.org/genproto v0.0.0-20250115164207-1a7da9e5054f // indirect diff --git a/genai/go.sum b/genai/go.sum index 263bb22be2..e12ee22414 100644 --- a/genai/go.sum +++ b/genai/go.sum @@ -97,24 +97,24 @@ go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= google.golang.org/api v0.217.0 h1:GYrUtD289o4zl1AhiTZL0jvQGa2RDLyC+kX1N/lfGOU= google.golang.org/api v0.217.0/go.mod h1:qMc2E8cBAbQlRypBTBWHklNJlaZZJBwDv81B1Iu8oSI= -google.golang.org/genai v1.17.0 h1:lXYSnWShPYjxTouxRj0zF8RsNmSF+SKo7SQ7dM35NlI= -google.golang.org/genai v1.17.0/go.mod h1:QPj5NGJw+3wEOHg+PrsWwJKvG6UC84ex5FR7qAYsN/M= +google.golang.org/genai v1.35.0 h1:Jo6g25CzVqFzGrX5mhWyBgQqXAUzxcx5jeK7U74zv9c= +google.golang.org/genai v1.35.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= google.golang.org/genproto v0.0.0-20250115164207-1a7da9e5054f h1:387Y+JbxF52bmesc8kq1NyYIp33dnxCw6eiA7JMsTmw= google.golang.org/genproto v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:0joYwWwLQh18AOj8zMYeZLjzuqcYTU3/nC5JdCvC3JI= google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA=