refactor: (GenAI) Reorganized Token Count Samples (Group C) (GoogleCloudPlatform#12616)

* Created new Token Count folder and new Sample with test.

* Rename file to resolve conflicts

* Changed model

Author: Thoughtseize1

generative_ai/token_count/api_example.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def count_token_service() -> int:
+    # [START generativeaionvertexai_token_count_sample_with_genai]
+    import vertexai
+    from vertexai.generative_models import GenerativeModel
+
+    # TODO(developer): Update and un-comment below line
+    # PROJECT_ID = "your-project-id"
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    # using Vertex AI Model as tokenzier
+    model = GenerativeModel("gemini-1.5-flash")
+
+    prompt = "hello world"
+    response = model.count_tokens(prompt)
+    print(f"Prompt Token Count: {response.total_tokens}")
+    print(f"Prompt Character Count: {response.total_billable_characters}")
+
+    prompt = ["hello world", "what's the weather today"]
+    response = model.count_tokens(prompt)
+    print(f"Prompt Token Count: {response.total_tokens}")
+    print(f"Prompt Character Count: {response.total_billable_characters}")
+    # Example response:
+    # Prompt Token Count: 2
+    # Prompt Character Count: 10
+    # Prompt Token Count: 8
+    # Prompt Character Count: 31
+
+    # [END generativeaionvertexai_token_count_sample_with_genai]
+    return response.total_tokens
+
+
+if __name__ == "__main__":
+    count_token_service()

generative_ai/token_count/local_sdk_example.py

@@ -0,0 +1,39 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def count_token_locally() -> int:
+    # [START generativeaionvertexai_token_count_sample_with_local_sdk]
+    from vertexai.preview.tokenization import get_tokenizer_for_model
+
+    # Using local tokenzier
+    tokenizer = get_tokenizer_for_model("gemini-1.5-flash")
+
+    prompt = "hello world"
+    response = tokenizer.count_tokens(prompt)
+    print(f"Prompt Token Count: {response.total_tokens}")
+
+    prompt = ["hello world", "what's the weather today"]
+    response = tokenizer.count_tokens(prompt)
+    print(f"Prompt Token Count: {response.total_tokens}")
+    # Example response:
+    # Prompt Token Count: 2
+    # Prompt Token Count: 8
+
+    # [END generativeaionvertexai_token_count_sample_with_local_sdk]
+    return response.total_tokens
+
+
+if __name__ == "__main__":
+    count_token_locally()

generative_ai/token_count/multimodal_token_count_example.py

@@ -0,0 +1,65 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from vertexai.generative_models import GenerationResponse
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def count_tokens_multimodal() -> GenerationResponse:
+    # [START generativeaionvertexai_gemini_token_count_multimodal]
+    import vertexai
+    from vertexai.generative_models import GenerativeModel, Part
+
+    # TODO(developer): Update and un-comment below line
+    # PROJECT_ID = "your-project-id"
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    model = GenerativeModel("gemini-1.5-flash-002")
+
+    contents = [
+        Part.from_uri(
+            "gs://cloud-samples-data/generative-ai/video/pixel8.mp4",
+            mime_type="video/mp4",
+        ),
+        "Provide a description of the video.",
+    ]
+
+    # Prompt tokens count
+    response = model.count_tokens(contents)
+    print(f"Prompt Token Count: {response.total_tokens}")
+    print(f"Prompt Character Count: {response.total_billable_characters}")
+
+    # Send text to Gemini
+    response = model.generate_content(contents)
+    usage_metadata = response.usage_metadata
+
+    # Response tokens count
+    print(f"Prompt Token Count: {usage_metadata.prompt_token_count}")
+    print(f"Candidates Token Count: {usage_metadata.candidates_token_count}")
+    print(f"Total Token Count: {usage_metadata.total_token_count}")
+    # Example response:
+    # Prompt Token Count: 16822
+    # Prompt Character Count: 30
+    # Prompt Token Count: 16822
+    # Candidates Token Count: 71
+    # Total Token Count: 16893
+
+    # [END generativeaionvertexai_gemini_token_count_multimodal]
+    return response
+
+
+if __name__ == "__main__":
+    count_tokens_multimodal()

generative_ai/token_count/simple_example.py

@@ -0,0 +1,58 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from vertexai.generative_models import GenerationResponse
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def count_tokens() -> GenerationResponse:
+    # [START generativeaionvertexai_gemini_token_count]
+    import vertexai
+    from vertexai.generative_models import GenerativeModel
+
+    # TODO(developer): Update and un-comment below line
+    # PROJECT_ID = "your-project-id"
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    model = GenerativeModel("gemini-1.5-flash-002")
+
+    prompt = "Why is the sky blue?"
+    # Prompt tokens count
+    response = model.count_tokens(prompt)
+    print(f"Prompt Token Count: {response.total_tokens}")
+    print(f"Prompt Character Count: {response.total_billable_characters}")
+
+    # Send text to Gemini
+    response = model.generate_content(prompt)
+
+    # Response tokens count
+    usage_metadata = response.usage_metadata
+    print(f"Prompt Token Count: {usage_metadata.prompt_token_count}")
+    print(f"Candidates Token Count: {usage_metadata.candidates_token_count}")
+    print(f"Total Token Count: {usage_metadata.total_token_count}")
+    # Example response:
+    # Prompt Token Count: 6
+    # Prompt Character Count: 16
+    # Prompt Token Count: 6
+    # Candidates Token Count: 315
+    # Total Token Count: 321
+
+    # [END generativeaionvertexai_gemini_token_count]
+    return response
+
+
+if __name__ == "__main__":
+    count_tokens()

generative_ai/token_count/token_count_test.py

@@ -0,0 +1,33 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import api_example
+import local_sdk_example
+import multimodal_token_count_example
+import simple_example
+
+
+def test_count_token() -> None:
+    assert local_sdk_example.count_token_locally()
+    assert api_example.count_token_service()
+
+
+def test_gemini_count_token_example() -> None:
+    response = simple_example.count_tokens()
+    assert response
+    assert response.usage_metadata
+
+    response = multimodal_token_count_example.count_tokens_multimodal()
+    assert response
+    assert response.usage_metadata