feat(generativeai): New Samples for Model Tuning folder (#12605)

Author: Thoughtseize1

generative_ai/model_tuning/create_evaluation_task_example.py

@@ -0,0 +1,92 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from vertexai.preview.evaluation import EvalResult
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def create_evaluation_task() -> EvalResult:
+    # [START generativeaionvertexai_create_evaluation_task]
+    import pandas as pd
+
+    import vertexai
+    from vertexai.preview.evaluation import EvalTask, MetricPromptTemplateExamples
+
+    # TODO(developer): Update and un-comment below line
+    # PROJECT_ID = "your-project-id"
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    eval_dataset = pd.DataFrame(
+        {
+            "instruction": [
+                "Summarize the text in one sentence.",
+                "Summarize the text such that a five-year-old can understand.",
+            ],
+            "context": [
+                """As part of a comprehensive initiative to tackle urban congestion and foster
+                sustainable urban living, a major city has revealed ambitious plans for an
+                extensive overhaul of its public transportation system. The project aims not
+                only to improve the efficiency and reliability of public transit but also to
+                reduce the city\'s carbon footprint and promote eco-friendly commuting options.
+                City officials anticipate that this strategic investment will enhance
+                accessibility for residents and visitors alike, ushering in a new era of
+                efficient, environmentally conscious urban transportation.""",
+                """A team of archaeologists has unearthed ancient artifacts shedding light on a
+                previously unknown civilization. The findings challenge existing historical
+                narratives and provide valuable insights into human history.""",
+            ],
+            "response": [
+                "A major city is revamping its public transportation system to fight congestion, reduce emissions, and make getting around greener and easier.",
+                "Some people who dig for old things found some very special tools and objects that tell us about people who lived a long, long time ago! What they found is like a new puzzle piece that helps us understand how people used to live.",
+            ],
+        }
+    )
+
+    eval_task = EvalTask(
+        dataset=eval_dataset,
+        metrics=[
+            MetricPromptTemplateExamples.Pointwise.SUMMARIZATION_QUALITY,
+            MetricPromptTemplateExamples.Pointwise.GROUNDEDNESS,
+            MetricPromptTemplateExamples.Pointwise.VERBOSITY,
+            MetricPromptTemplateExamples.Pointwise.INSTRUCTION_FOLLOWING,
+        ],
+    )
+
+    prompt_template = (
+        "Instruction: {instruction}. Article: {context}. Summary: {response}"
+    )
+    result = eval_task.evaluate(prompt_template=prompt_template)
+
+    print("Summary Metrics:\n")
+
+    for key, value in result.summary_metrics.items():
+        print(f"{key}: \t{value}")
+
+    print("\n\nMetrics Table:\n")
+    print(result.metrics_table)
+    # Example response:
+    # Summary Metrics:
+    # row_count:      2
+    # summarization_quality/mean:     3.5
+    # summarization_quality/std:      2.1213203435596424
+    # ...
+
+    # [END generativeaionvertexai_create_evaluation_task]
+    return result
+
+
+if __name__ == "__main__":
+    create_evaluation_task()

generative_ai/model_tuning/create_evaluation_task_example_test.py

@@ -0,0 +1,20 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import create_evaluation_task_example
+
+
+def test_create_evaluation_task() -> None:
+    response = create_evaluation_task_example.create_evaluation_task()
+    assert response

generative_ai/model_tuning/distillation_example.py

@@ -0,0 +1,70 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START generativeaionvertexai_sdk_distillation]
+from __future__ import annotations
+
+import os
+
+from typing import Optional
+
+import vertexai
+from vertexai.preview.language_models import TextGenerationModel, TuningEvaluationSpec
+
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def distill_model(
+    dataset: str,
+    source_model: str,
+    evaluation_dataset: Optional[str] = None,
+) -> None:
+    """Distill a new model using a teacher model and a dataset.
+    Args:
+        dataset (str): GCS URI of the JSONL file containing the training data.
+            E.g., "gs://[BUCKET]/[FILENAME].jsonl".
+        source_model (str): Name of the teacher model to distill from.
+            E.g., "text-unicorn@001".
+        evaluation_dataset (Optional[str]): GCS URI of the JSONL file containing the evaluation data.
+    """
+    # TODO developer - override these parameters as needed:
+    vertexai.init(project=PROJECT_ID, location="us-central1")
+
+    # Create a tuning evaluation specification with the evaluation dataset
+    eval_spec = TuningEvaluationSpec(evaluation_data=evaluation_dataset)
+
+    # Load the student model from a pre-trained model
+    student_model = TextGenerationModel.from_pretrained("text-bison@002")
+
+    # Start the distillation job using the teacher model and dataset
+    distillation_job = student_model.distill_from(
+        teacher_model=source_model,
+        dataset=dataset,
+        # Optional:
+        train_steps=300,  # Number of training steps to use when tuning the model.
+        evaluation_spec=eval_spec,
+    )
+
+    return distillation_job
+
+
+# [END generativeaionvertexai_sdk_distillation]
+
+if __name__ == "__main__":
+    distill_model(
+        dataset="your-dataset-uri",
+        source_model="your-source-model",
+        evaluation_dataset="your-evaluation-dataset-uri",
+    )

generative_ai/model_tuning/distillation_example_test.py

@@ -0,0 +1,107 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import uuid
+
+import distillation_example
+
+from google.cloud import aiplatform
+from google.cloud import storage
+
+from google.cloud.aiplatform.compat.types import pipeline_state
+
+import pytest
+
+from vertexai.preview.language_models import TextGenerationModel
+
+_BUCKET = os.environ["CLOUD_STORAGE_BUCKET"]
+
+
+def get_model_display_name(tuned_model: TextGenerationModel) -> str:
+    language_model_tuning_job = tuned_model._job
+    pipeline_job = language_model_tuning_job._job
+    return dict(pipeline_job._gca_resource.runtime_config.parameter_values)[
+        "model_display_name"
+    ]
+
+
+def upload_to_gcs(bucket: str, name: str, data: str) -> None:
+    client = storage.Client()
+    bucket = client.get_bucket(bucket)
+    blob = bucket.blob(name)
+    blob.upload_from_string(data)
+
+
+def download_from_gcs(bucket: str, name: str) -> str:
+    client = storage.Client()
+    bucket = client.get_bucket(bucket)
+    blob = bucket.blob(name)
+    data = blob.download_as_bytes()
+    return "\n".join(data.decode().splitlines()[:10])
+
+
+def delete_from_gcs(bucket: str, name: str) -> None:
+    client = storage.Client()
+    bucket = client.get_bucket(bucket)
+    blob = bucket.blob(name)
+    blob.delete()
+
+
+@pytest.fixture(scope="function")
+def training_data_filename() -> str:
+    temp_filename = f"{uuid.uuid4()}.jsonl"
+    data = download_from_gcs(
+        "cloud-samples-data", "ai-platform/generative_ai/headline_classification.jsonl"
+    )
+    upload_to_gcs(_BUCKET, temp_filename, data)
+    try:
+        yield f"gs://{_BUCKET}/{temp_filename}"
+    finally:
+        delete_from_gcs(_BUCKET, temp_filename)
+
+
+def teardown_model(
+    tuned_model: TextGenerationModel, training_data_filename: str
+) -> None:
+    for tuned_model_name in tuned_model.list_tuned_model_names():
+        model_registry = aiplatform.models.ModelRegistry(model=tuned_model_name)
+        if (
+            training_data_filename
+            in model_registry.get_version_info("1").model_display_name
+        ):
+            display_name = model_registry.get_version_info("1").model_display_name
+            for endpoint in aiplatform.Endpoint.list():
+                for _ in endpoint.list_models():
+                    if endpoint.display_name == display_name:
+                        endpoint.undeploy_all()
+                        endpoint.delete()
+            aiplatform.Model(model_registry.model_resource_name).delete()
+
+
+@pytest.mark.skip("Blocked on b/277959219")
+def test_distill_model(training_data_filename: str) -> None:
+    """Takes approx. 60 minutes."""
+    student_model = distillation_example.distill_model(
+        dataset=training_data_filename,
+        teacher_model="text-unicorn@001",
+        evaluation_dataset=training_data_filename,
+    )
+    try:
+        assert (
+            student_model._job.state
+            == pipeline_state.PipelineState.PIPELINE_STATE_SUCCEEDED
+        )
+    finally:
+        teardown_model(student_model, training_data_filename)

generative_ai/model_tuning/evaluate_model_example.py

@@ -0,0 +1,65 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START generativeaionvertexai_evaluate_model]
+import os
+
+from google.auth import default
+
+import vertexai
+from vertexai.preview.language_models import (
+    EvaluationTextClassificationSpec,
+    TextGenerationModel,
+)
+
+PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+
+
+def evaluate_model() -> object:
+    """Evaluate the performance of a generative AI model."""
+
+    # Set credentials for the pipeline components used in the evaluation task
+    credentials, _ = default(scopes=["https://www.googleapis.com/auth/cloud-platform"])
+
+    vertexai.init(project=PROJECT_ID, location="us-central1", credentials=credentials)
+
+    # Create a reference to a generative AI model
+    model = TextGenerationModel.from_pretrained("text-bison@002")
+
+    # Define the evaluation specification for a text classification task
+    task_spec = EvaluationTextClassificationSpec(
+        ground_truth_data=[
+            "gs://cloud-samples-data/ai-platform/generative_ai/llm_classification_bp_input_prompts_with_ground_truth.jsonl"
+        ],
+        class_names=["nature", "news", "sports", "health", "startups"],
+        target_column_name="ground_truth",
+    )
+
+    # Evaluate the model
+    eval_metrics = model.evaluate(task_spec=task_spec)
+    print(eval_metrics)
+    # Example response:
+    # ...
+    # PipelineJob run completed.
+    # Resource name: projects/123456789/locations/us-central1/pipelineJobs/evaluation-llm-classification-...
+    # EvaluationClassificationMetric(label_name=None, auPrc=0.53833705, auRoc=0.8...
+
+    return eval_metrics
+
+
+# [END generativeaionvertexai_evaluate_model]
+
+
+if __name__ == "__main__":
+    evaluate_model()

generative_ai/model_tuning/evaluate_model_example_test.py

@@ -0,0 +1,32 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import backoff
+
+import evaluate_model_example
+
+from google.api_core.exceptions import ResourceExhausted
+
+import pytest
+
+
+@pytest.mark.skip(
+    reason="Model is giving 404 Not found error."
+    "Need to investigate. Created an issue tracker is at "
+    "python-docs-samples/issues/11264"
+)
+@backoff.on_exception(backoff.expo, ResourceExhausted, max_time=10)
+def test_evaluate_model() -> None:
+    eval_metrics = evaluate_model_example.evaluate_model()
+    assert hasattr(eval_metrics, "auRoc")