From 2c772727e2bbc900955e8098ffe5dd081f9f62ce Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 13:24:16 +0200 Subject: [PATCH 1/8] Initial working samples --- generative_ai/gemma2/gemma2_predict_gpu.py | 79 +++++++++++++++++++++ generative_ai/gemma2/gemma2_predict_tpu.py | 80 ++++++++++++++++++++++ generative_ai/gemma2/requirements.txt | 2 + 3 files changed, 161 insertions(+) create mode 100644 generative_ai/gemma2/gemma2_predict_gpu.py create mode 100644 generative_ai/gemma2/gemma2_predict_tpu.py create mode 100644 generative_ai/gemma2/requirements.txt diff --git a/generative_ai/gemma2/gemma2_predict_gpu.py b/generative_ai/gemma2/gemma2_predict_gpu.py new file mode 100644 index 00000000000..5aeaa47a1fa --- /dev/null +++ b/generative_ai/gemma2/gemma2_predict_gpu.py @@ -0,0 +1,79 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHcontent WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") + + +def gemma2_predict_gpu(ENDPOINT_REGION: str, ENDPOINT_ID: str) -> str: + # [START generativeaionvertexai_gemma2_predict_gpu] + """ + Sample to run interference on a Gemma2 model deployed to a Vertex AI endpoint with GPU accellerators. + """ + + from google.cloud import aiplatform + from google.protobuf import json_format + from google.protobuf.struct_pb2 import Value + + # TODO(developer): Update & uncomment lines below + # PROJECT_ID = "your-project-id" + # ENDPOINT_REGION = "your-vertex-endpoint-region" + # ENDPOINT_ID = "your-vertex-endpoint-id" + + # Default configuration + config = {"max_tokens": 1024, "temperature": 0.9, "top_p": 1.0, "top_k": 1} + + # Prompt used in the prediction + prompt = "Why is the sky blue?" + + # Encapsulate the prompt in a correct format for GPUs + # Example format: [{'inputs': 'Why is the sky blue?', 'parameters': {'temperature': 0.9}}] + input = {"inputs": prompt, "parameters": config} + + # Convert input message to a list of GAPIC instances for model input + instances = [json_format.ParseDict(input, Value())] + + # Create a client + api_endpoint = f"{ENDPOINT_REGION}-aiplatform.googleapis.com" + client = aiplatform.gapic.PredictionServiceClient( + client_options={"api_endpoint": api_endpoint} + ) + + # Call the Gemma2 endpoint + gemma2_end_point = ( + f"projects/{PROJECT_ID}/locations/{ENDPOINT_REGION}/endpoints/{ENDPOINT_ID}" + ) + response = client.predict( + endpoint=gemma2_end_point, + instances=instances, + ) + text_responses = response.predictions + print(text_responses[0]) + + # [END generativeaionvertexai_gemma2_predict_gpu] + return text_responses[0] + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print( + "Usage: python gemma2_predict_gpu.py " + ) + sys.exit(1) + + ENDPOINT_REGION = sys.argv[1] + ENDPOINT_ID = sys.argv[2] + gemma2_predict_gpu(ENDPOINT_REGION, ENDPOINT_ID) diff --git a/generative_ai/gemma2/gemma2_predict_tpu.py b/generative_ai/gemma2/gemma2_predict_tpu.py new file mode 100644 index 00000000000..c3eb616b36b --- /dev/null +++ b/generative_ai/gemma2/gemma2_predict_tpu.py @@ -0,0 +1,80 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHcontent WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") + + +def gemma2_predict_tpu(ENDPOINT_REGION: str, ENDPOINT_ID: str) -> str: + # [START generativeaionvertexai_gemma2_predict_tpu] + """ + Sample to run interference on a Gemma2 model deployed to a Vertex AI endpoint with TPU accellerators. + """ + + from google.cloud import aiplatform + from google.protobuf import json_format + from google.protobuf.struct_pb2 import Value + + # TODO(developer): Update & uncomment lines below + # PROJECT_ID = "your-project-id" + # ENDPOINT_REGION = "your-vertex-endpoint-region" + # ENDPOINT_ID = "your-vertex-endpoint-id" + + # Default configuration + config = {"max_tokens": 1024, "temperature": 0.9, "top_p": 1.0, "top_k": 1} + + # Prompt used in the prediction + prompt = "Why is the sky blue?" + + # Encapsulate the prompt in a correct format for TPUs + # Example format: [{'prompt': 'Why is the sky blue?', 'temperature': 0.9}] + input = {"prompt": prompt} + input.update(config) + + # Convert input message to a list of GAPIC instances for model input + instances = [json_format.ParseDict(input, Value())] + + # Create a client + api_endpoint = f"{ENDPOINT_REGION}-aiplatform.googleapis.com" + client = aiplatform.gapic.PredictionServiceClient( + client_options={"api_endpoint": api_endpoint} + ) + + # Call the Gemma2 endpoint + gemma2_end_point = ( + f"projects/{PROJECT_ID}/locations/{ENDPOINT_REGION}/endpoints/{ENDPOINT_ID}" + ) + response = client.predict( + endpoint=gemma2_end_point, + instances=instances, + ) + text_responses = response.predictions + print(text_responses[0]) + + # [END generativeaionvertexai_gemma2_predict_tpu] + return text_responses[0] + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print( + "Usage: python gemma2_predict_tpu.py " + ) + sys.exit(1) + + ENDPOINT_REGION = sys.argv[1] + ENDPOINT_ID = sys.argv[2] + gemma2_predict_tpu(ENDPOINT_REGION, ENDPOINT_ID) diff --git a/generative_ai/gemma2/requirements.txt b/generative_ai/gemma2/requirements.txt new file mode 100644 index 00000000000..1b733fb6d58 --- /dev/null +++ b/generative_ai/gemma2/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-aiplatform[all]==1.64.0 +protobuf==5.28.1 \ No newline at end of file From 7dc6503cd213898c22423cf1b8021041e9cdbe87 Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 15:52:49 +0200 Subject: [PATCH 2/8] Tests ready with mocks --- generative_ai/gemma2/gemma2_test.py | 97 +++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 generative_ai/gemma2/gemma2_test.py diff --git a/generative_ai/gemma2/gemma2_test.py b/generative_ai/gemma2/gemma2_test.py new file mode 100644 index 00000000000..1ef9f99acba --- /dev/null +++ b/generative_ai/gemma2/gemma2_test.py @@ -0,0 +1,97 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHcontent WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest import mock +from unittest.mock import MagicMock + +from google.cloud.aiplatform_v1.types import prediction_service +from google.protobuf.struct_pb2 import Value + +from gemma2_predict_gpu import gemma2_predict_gpu +from gemma2_predict_tpu import gemma2_predict_tpu + +# Global variables +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +GPU_ENDPOINT_REGION = "us-east1" +GPU_ENDPOINT_ID = "123456789" # Mock ID used to check if GPU was called + +TPU_ENDPOINT_REGION = "us-west1" +TPU_ENDPOINT_ID = "987654321" # Mock ID used to check if TPU was called + +# MOCKED RESPONSE +MODEL_RESPONSES = """ +The sky appears blue due to a phenomenon called **Rayleigh scattering**. + +**Here's how it works:** + +1. **Sunlight:** Sunlight is composed of all the colors of the rainbow. + +2. **Earth's Atmosphere:** When sunlight enters the Earth's atmosphere, it collides with tiny particles like nitrogen and oxygen molecules. + +3. **Scattering:** These particles scatter the sunlight in all directions. However, blue light (which has a shorter wavelength) is scattered more effectively than other colors. + +4. **Our Perception:** As a result, we see a blue sky because the scattered blue light reaches our eyes from all directions. + +**Why not other colors?** + +* **Violet light** has an even shorter wavelength than blue and is scattered even more. However, our eyes are less sensitive to violet light, so we perceive the sky as blue. +* **Longer wavelengths** like red, orange, and yellow are scattered less and travel more directly through the atmosphere. This is why we see these colors during sunrise and sunset, when sunlight has to travel through more of the atmosphere. +""" + + +# Mocked function - we check if proper format was used depending on selected architecture +def mock_predict(endpoint, instances): + gpu_endpoint = f"projects/{PROJECT_ID}/locations/{GPU_ENDPOINT_REGION}/endpoints/{GPU_ENDPOINT_ID}" + tpu_endpoint = f"projects/{PROJECT_ID}/locations/{TPU_ENDPOINT_REGION}/endpoints/{TPU_ENDPOINT_ID}" + instance_fields = instances[0].struct_value.fields + + if endpoint == gpu_endpoint: + assert "string_value" in instance_fields["inputs"] + assert "struct_value" in instance_fields["parameters"] + parameters = instance_fields["parameters"].struct_value.fields + assert "number_value" in parameters["max_tokens"] + assert "number_value" in parameters["temperature"] + assert "number_value" in parameters["top_p"] + assert "number_value" in parameters["top_k"] + elif endpoint == tpu_endpoint: + assert "string_value" in instance_fields["prompt"] + assert "number_value" in instance_fields["max_tokens"] + assert "number_value" in instance_fields["temperature"] + assert "number_value" in instance_fields["top_p"] + assert "number_value" in instance_fields["top_k"] + else: + assert False + + response = prediction_service.PredictResponse() + response.predictions.append(Value(string_value=MODEL_RESPONSES)) + return response + + +@mock.patch("google.cloud.aiplatform.gapic.PredictionServiceClient") +def test_gemma2_predict_gpu(mock_client: MagicMock) -> None: + mock_client_instance = mock_client.return_value + mock_client_instance.predict = mock_predict + + response = gemma2_predict_gpu(GPU_ENDPOINT_REGION, GPU_ENDPOINT_ID) + assert "Rayleigh scattering" in response + + +@mock.patch("google.cloud.aiplatform.gapic.PredictionServiceClient") +def test_gemma2_predict_tpu(mock_client: MagicMock) -> None: + mock_client_instance = mock_client.return_value + mock_client_instance.predict = mock_predict + + response = gemma2_predict_tpu(TPU_ENDPOINT_REGION, TPU_ENDPOINT_ID) + assert "Rayleigh scattering" in response From 7bc964c400cfd2a5798470246b68cf24cd34ad68 Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 15:54:19 +0200 Subject: [PATCH 3/8] Updated CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3b9e40d0bfa..5815c3c67eb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,6 +29,7 @@ /cdn/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /compute/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /dns/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers +/generative_ai/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /iam/cloud-client/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /kms/**/** @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /media_cdn/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers From e0776aebc352780b84a05dd7a168d81a0b6fe156 Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 21:32:49 +0200 Subject: [PATCH 4/8] Fix lint and tests --- generative_ai/gemma2/gemma2_test.py | 8 +++-- generative_ai/gemma2/noxfile_config.py | 42 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 generative_ai/gemma2/noxfile_config.py diff --git a/generative_ai/gemma2/gemma2_test.py b/generative_ai/gemma2/gemma2_test.py index 1ef9f99acba..72b898d64d8 100644 --- a/generative_ai/gemma2/gemma2_test.py +++ b/generative_ai/gemma2/gemma2_test.py @@ -13,11 +13,12 @@ # limitations under the License. import os +from typing import MutableSequence, Optional from unittest import mock from unittest.mock import MagicMock from google.cloud.aiplatform_v1.types import prediction_service -from google.protobuf.struct_pb2 import Value +from google.protobuf.struct_pb2 import struct_pb2, Value from gemma2_predict_gpu import gemma2_predict_gpu from gemma2_predict_tpu import gemma2_predict_tpu @@ -52,7 +53,10 @@ # Mocked function - we check if proper format was used depending on selected architecture -def mock_predict(endpoint, instances): +def mock_predict( + endpoint: Optional[str] = None, + instances: Optional[MutableSequence[struct_pb2.Value]] = None, +) -> prediction_service.PredictResponse: gpu_endpoint = f"projects/{PROJECT_ID}/locations/{GPU_ENDPOINT_REGION}/endpoints/{GPU_ENDPOINT_ID}" tpu_endpoint = f"projects/{PROJECT_ID}/locations/{TPU_ENDPOINT_REGION}/endpoints/{TPU_ENDPOINT_ID}" instance_fields = instances[0].struct_value.fields diff --git a/generative_ai/gemma2/noxfile_config.py b/generative_ai/gemma2/noxfile_config.py new file mode 100644 index 00000000000..3c9d52c6e2f --- /dev/null +++ b/generative_ai/gemma2/noxfile_config.py @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default TEST_CONFIG_OVERRIDE for python repos. + +# You can copy this file into your directory, then it will be imported from +# the noxfile.py. + +# The source of truth: +# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py + +TEST_CONFIG_OVERRIDE = { + # You can opt out from the test for specific Python versions. + "ignored_versions": ["3.11"], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": True, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} From 89f1e8cafe6b8c696a555c3097c85ec5cf54aa04 Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 21:54:48 +0200 Subject: [PATCH 5/8] Fix lint and tests --- generative_ai/gemma2/noxfile_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generative_ai/gemma2/noxfile_config.py b/generative_ai/gemma2/noxfile_config.py index 3c9d52c6e2f..494cf15318d 100644 --- a/generative_ai/gemma2/noxfile_config.py +++ b/generative_ai/gemma2/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["3.11"], + "ignored_versions": ["2.7", "3.7", "3.9", "3.10", "3.11"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, From 76e5f50be8965713d52da5e8dd8712185e5f4f86 Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 22:00:50 +0200 Subject: [PATCH 6/8] Moving samples to fix testing issues when in subfolder of generative-ai --- .github/CODEOWNERS | 1 + {generative_ai/gemma2 => gemma2}/gemma2_predict_gpu.py | 0 {generative_ai/gemma2 => gemma2}/gemma2_predict_tpu.py | 0 {generative_ai/gemma2 => gemma2}/gemma2_test.py | 3 ++- {generative_ai/gemma2 => gemma2}/noxfile_config.py | 0 {generative_ai/gemma2 => gemma2}/requirements.txt | 0 6 files changed, 3 insertions(+), 1 deletion(-) rename {generative_ai/gemma2 => gemma2}/gemma2_predict_gpu.py (100%) rename {generative_ai/gemma2 => gemma2}/gemma2_predict_tpu.py (100%) rename {generative_ai/gemma2 => gemma2}/gemma2_test.py (97%) rename {generative_ai/gemma2 => gemma2}/noxfile_config.py (100%) rename {generative_ai/gemma2 => gemma2}/requirements.txt (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5815c3c67eb..dbf7a02ea26 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,6 +29,7 @@ /cdn/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /compute/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /dns/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers +/gemma2/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /generative_ai/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /iam/cloud-client/**/* @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers /kms/**/** @GoogleCloudPlatform/dee-infra @GoogleCloudPlatform/python-samples-reviewers @GoogleCloudPlatform/cloud-samples-reviewers diff --git a/generative_ai/gemma2/gemma2_predict_gpu.py b/gemma2/gemma2_predict_gpu.py similarity index 100% rename from generative_ai/gemma2/gemma2_predict_gpu.py rename to gemma2/gemma2_predict_gpu.py diff --git a/generative_ai/gemma2/gemma2_predict_tpu.py b/gemma2/gemma2_predict_tpu.py similarity index 100% rename from generative_ai/gemma2/gemma2_predict_tpu.py rename to gemma2/gemma2_predict_tpu.py diff --git a/generative_ai/gemma2/gemma2_test.py b/gemma2/gemma2_test.py similarity index 97% rename from generative_ai/gemma2/gemma2_test.py rename to gemma2/gemma2_test.py index 72b898d64d8..91c93336727 100644 --- a/generative_ai/gemma2/gemma2_test.py +++ b/gemma2/gemma2_test.py @@ -18,7 +18,8 @@ from unittest.mock import MagicMock from google.cloud.aiplatform_v1.types import prediction_service -from google.protobuf.struct_pb2 import struct_pb2, Value +import google.protobuf.struct_pb2 as struct_pb2 +from google.protobuf.struct_pb2 import Value from gemma2_predict_gpu import gemma2_predict_gpu from gemma2_predict_tpu import gemma2_predict_tpu diff --git a/generative_ai/gemma2/noxfile_config.py b/gemma2/noxfile_config.py similarity index 100% rename from generative_ai/gemma2/noxfile_config.py rename to gemma2/noxfile_config.py diff --git a/generative_ai/gemma2/requirements.txt b/gemma2/requirements.txt similarity index 100% rename from generative_ai/gemma2/requirements.txt rename to gemma2/requirements.txt From 1d53d3c603a625851e82b62d727f782ce71bfae2 Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 22:08:19 +0200 Subject: [PATCH 7/8] Add missing pytest --- gemma2/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 gemma2/requirements-test.txt diff --git a/gemma2/requirements-test.txt b/gemma2/requirements-test.txt new file mode 100644 index 00000000000..40543aababf --- /dev/null +++ b/gemma2/requirements-test.txt @@ -0,0 +1 @@ +pytest==8.3.3 From 206a67db56961518750b8bf6e0bd6f10a659f088 Mon Sep 17 00:00:00 2001 From: rsamborski Date: Tue, 17 Sep 2024 22:13:37 +0200 Subject: [PATCH 8/8] Trailling newline --- gemma2/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemma2/requirements.txt b/gemma2/requirements.txt index 1b733fb6d58..cd3ab556b42 100644 --- a/gemma2/requirements.txt +++ b/gemma2/requirements.txt @@ -1,2 +1,2 @@ google-cloud-aiplatform[all]==1.64.0 -protobuf==5.28.1 \ No newline at end of file +protobuf==5.28.1