adding count token samples

Author: Guiners

genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js

@@ -0,0 +1,55 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or written agreement, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the License for the License.
+// limitations under the License.
+
+'use strict';
+
+// [START googlegenaisdk_counttoken_localtokenizer_compute_with_txt]
+const {GoogleGenAI} = require('@google/genai');
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
+
+async function counttokenLocalTokenizerCompute() {
+  const client = new GoogleGenAI({
+    vertexai: true,
+    project: GOOGLE_CLOUD_PROJECT,
+    location: GOOGLE_CLOUD_LOCATION,
+    httpOptions: {apiVersion: 'v1'},
+  });
+
+  const response = await client.models.computeTokens({
+    model: 'gemini-2.5-flash',
+    contents: "What's the longest word in the English language?",
+  });
+
+  console.log(response.tokensInfo);
+
+  // Example output:
+  // {
+  //   tokensInfo: [
+  //     {
+  //       role: 'user',
+  //       tokenIds: [...],
+  //       tokens: [...],
+  //     }
+  //   ]
+  // }
+
+  return response.tokensInfo;
+}
+// [END googlegenaisdk_counttoken_localtokenizer_compute_with_txt]
+
+module.exports = {
+  counttokenLocalTokenizerCompute,
+};

genai/count-tokens/counttoken-localtokenizer-with-txt.js

@@ -0,0 +1,48 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+// [START googlegenaisdk_counttoken_localtokenizer_with_txt]
+
+const {GoogleGenAI} = require('@google/genai');
+
+const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT;
+const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global';
+
+async function counttokenLocalTokenizer() {
+  const client = new GoogleGenAI({
+    vertexai: true,
+    project: GOOGLE_CLOUD_PROJECT,
+    location: GOOGLE_CLOUD_LOCATION,
+    httpOptions: {apiVersion: 'v1'},
+  });
+
+  const response = await client.models.countTokens({
+    model: 'gemini-2.5-flash',
+    contents: "What's the highest mountain in Africa?",
+  });
+
+  console.log(response.totalTokens);
+
+  // Example output:
+  //   { totalTokens: 10 }
+
+  return response.totalTokens;
+}
+// [END googlegenaisdk_counttoken_localtokenizer_with_txt]
+
+module.exports = {
+  counttokenLocalTokenizer,
+};

genai/test/counttoken-localtokenizer-compute-with-txt.test.js

@@ -0,0 +1,32 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+
+const projectId = process.env.CAIP_PROJECT_ID;
+const sample = require('../count-tokens/counttoken-localtokenizer-compute-with-txt.js');
+const {delay} = require('./util');
+
+describe('counttoken-localtokenizer-compute-with-txt', () => {
+  it('should return totalTokens from text prompt', async function () {
+    this.timeout(18000);
+    this.retries(4);
+    await delay(this.test);
+    const output = await sample.counttokenLocalTokenizerCompute(projectId);
+    assert(output.length > 0);
+  });
+});

genai/test/counttoken-localtokenizer-with-txt.test.js

@@ -0,0 +1,32 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {assert} = require('chai');
+const {describe, it} = require('mocha');
+
+const projectId = process.env.CAIP_PROJECT_ID;
+const sample = require('../count-tokens/counttoken-localtokenizer-with-txt.js');
+const {delay} = require('./util');
+
+describe('counttoken-localtokenizer-with-txt', () => {
+  it('should return totalTokens from text prompt', async function () {
+    this.timeout(18000);
+    this.retries(4);
+    await delay(this.test);
+    const output = await sample.counttokenLocalTokenizer(projectId);
+    assert(output > 0);
+  });
+});