From 7783d5ad15fb041c04e134b31e904a2ec3bcad11 Mon Sep 17 00:00:00 2001 From: Joanna Grycz Date: Thu, 19 Sep 2024 08:22:53 +0200 Subject: [PATCH 1/3] feat: gemma2 models samples --- generative-ai/snippets/gemma2PredictGpu.js | 73 +++++++++++++++++++ .../snippets/test/gemma2Prediction.test.js | 57 +++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 generative-ai/snippets/gemma2PredictGpu.js create mode 100644 generative-ai/snippets/test/gemma2Prediction.test.js diff --git a/generative-ai/snippets/gemma2PredictGpu.js b/generative-ai/snippets/gemma2PredictGpu.js new file mode 100644 index 0000000000..c811f22f3c --- /dev/null +++ b/generative-ai/snippets/gemma2PredictGpu.js @@ -0,0 +1,73 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START generativeaionvertexai_gemma2_predict_gpu] +// Imports the Google Cloud Prediction Service Client library +const { + // TODO(developer): Uncomment PredictionServiceClient before running the sample. + // PredictionServiceClient, + helpers, +} = require('@google-cloud/aiplatform'); + +async function gemma2PredictGpu(predictionServiceClient) { + /** + * TODO(developer): Update these variables before running the sample. + */ + const projectId = 'your-project-id'; + const endpointRegion = 'your-vertex-endpoint-region'; + const endpointId = 'your-vertex-endpoint-id'; + + // Default configuration + const config = {maxOutputTokens: 1024, temperature: 0.9, topP: 1.0, topK: 1}; + // Prompt used in the prediction + const prompt = 'Why is the sky blue?'; + + // Encapsulate the prompt in a correct format for GPUs + // Example format: [{inputs: 'Why is the sky blue?', parameters: {temperature: 0.9}}] + const input = { + inputs: prompt, + parameters: config, + }; + + // Convert input message to a list of GAPIC instances for model input + const instances = [helpers.toValue(input)]; + + // TODO(developer): Uncomment apiEndpoint and predictionServiceClient before running the sample. + // const apiEndpoint = `${endpointRegion}-aiplatform.googleapis.com`; + + // Create a client + // predictionServiceClient = new PredictionServiceClient({apiEndpoint}); + + // Call the Gemma2 endpoint + const gemma2Endpoint = `projects/${projectId}/locations/${endpointRegion}/endpoints/${endpointId}`; + + const [response] = await predictionServiceClient.predict({ + endpoint: gemma2Endpoint, + instances, + }); + + const predictions = response.predictions; + const text = predictions[0].stringValue; + + console.log('Predictions:', text); + // [END generativeaionvertexai_gemma2_predict_gpu] + return text; +} + +module.exports = gemma2PredictGpu; + +gemma2PredictGpu(...process.argv.slice(2)).catch(err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/generative-ai/snippets/test/gemma2Prediction.test.js b/generative-ai/snippets/test/gemma2Prediction.test.js new file mode 100644 index 0000000000..151a19ad3f --- /dev/null +++ b/generative-ai/snippets/test/gemma2Prediction.test.js @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {expect} = require('chai'); +const {afterEach, describe, it} = require('mocha'); +const sinon = require('sinon'); +const gemma2PredictGpu = require('../gemma2PredictGpu.js'); + +const text = `The sky appears blue due to a phenomenon called **Rayleigh scattering**. +**Here's how it works:** +1. **Sunlight:** Sunlight is composed of all the colors of the rainbow. +2. **Earth's Atmosphere:** When sunlight enters the Earth's atmosphere, it collides with tiny particles like nitrogen and oxygen molecules. +3. **Scattering:** These particles scatter the sunlight in all directions. However, blue light (which has a shorter wavelength) is scattered more effectively than other colors. +4. **Our Perception:** As a result, we see a blue sky because the scattered blue light reaches our eyes from all directions. +**Why not other colors?** +* **Violet light** has an even shorter wavelength than blue and is scattered even more. However, our eyes are less sensitive to violet light, so we perceive the sky as blue. +* **Longer wavelengths** like red, orange, and yellow are scattered less and travel more directly through the atmosphere. This is why we see these colors during sunrise and sunset, when sunlight has to travel through more of the atmosphere. +`; + +describe('Gemma2 predictions', async () => { + const predictionServiceClientMock = { + predict: sinon.stub().resolves([ + { + predictions: [ + { + stringValue: text, + }, + ], + }, + ]), + }; + + afterEach(() => { + sinon.restore(); + }); + + it('should run interference with GPU', async () => { + const output = await gemma2PredictGpu(predictionServiceClientMock); + + expect(output).include('Rayleigh scattering'); + }); +}); From e5cc8c6c78006df24361fec43fa1568215a895fd Mon Sep 17 00:00:00 2001 From: Joanna Grycz Date: Mon, 23 Sep 2024 15:24:54 +0200 Subject: [PATCH 2/3] feat: add generativeaionvertexai_gemma2_predict_tpu --- generative-ai/snippets/gemma2PredictGpu.js | 24 +++--- generative-ai/snippets/gemma2PredictTpu.js | 77 +++++++++++++++++++ .../snippets/test/gemma2Prediction.test.js | 40 +++++++--- 3 files changed, 121 insertions(+), 20 deletions(-) create mode 100644 generative-ai/snippets/gemma2PredictTpu.js diff --git a/generative-ai/snippets/gemma2PredictGpu.js b/generative-ai/snippets/gemma2PredictGpu.js index c811f22f3c..47fce36788 100644 --- a/generative-ai/snippets/gemma2PredictGpu.js +++ b/generative-ai/snippets/gemma2PredictGpu.js @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -// [START generativeaionvertexai_gemma2_predict_gpu] -// Imports the Google Cloud Prediction Service Client library -const { - // TODO(developer): Uncomment PredictionServiceClient before running the sample. - // PredictionServiceClient, - helpers, -} = require('@google-cloud/aiplatform'); +'use strict'; async function gemma2PredictGpu(predictionServiceClient) { + // [START generativeaionvertexai_gemma2_predict_gpu] + // Imports the Google Cloud Prediction Service Client library + const { + // TODO(developer): Uncomment PredictionServiceClient before running the sample. + // PredictionServiceClient, + helpers, + } = require('@google-cloud/aiplatform'); /** * TODO(developer): Update these variables before running the sample. */ @@ -67,7 +68,8 @@ async function gemma2PredictGpu(predictionServiceClient) { module.exports = gemma2PredictGpu; -gemma2PredictGpu(...process.argv.slice(2)).catch(err => { - console.error(err.message); - process.exitCode = 1; -}); +// TODO(developer): Uncomment below lines before running the sample. +// gemma2PredictGpu(...process.argv.slice(2)).catch(err => { +// console.error(err.message); +// process.exitCode = 1; +// }); diff --git a/generative-ai/snippets/gemma2PredictTpu.js b/generative-ai/snippets/gemma2PredictTpu.js new file mode 100644 index 0000000000..c854e97009 --- /dev/null +++ b/generative-ai/snippets/gemma2PredictTpu.js @@ -0,0 +1,77 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +async function gemma2PredictTpu(predictionServiceClient) { + // [START generativeaionvertexai_gemma2_predict_tpu] + // Imports the Google Cloud Prediction Service Client library + const { + // TODO(developer): Uncomment PredictionServiceClient before running the sample. + // PredictionServiceClient, + helpers, + } = require('@google-cloud/aiplatform'); + /** + * TODO(developer): Update these variables before running the sample. + */ + const projectId = 'your-project-id'; + const endpointRegion = 'your-vertex-endpoint-region'; + const endpointId = 'your-vertex-endpoint-id'; + + // Prompt used in the prediction + const prompt = 'Why is the sky blue?'; + + // Encapsulate the prompt in a correct format for TPUs + // Example format: [{prompt: 'Why is the sky blue?', temperature: 0.9}] + const input = { + prompt, + // Parameters for default configuration + maxOutputTokens: 1024, + temperature: 0.9, + topP: 1.0, + topK: 1, + }; + + // Convert input message to a list of GAPIC instances for model input + const instances = [helpers.toValue(input)]; + + // TODO(developer): Uncomment apiEndpoint and predictionServiceClient before running the sample. + // const apiEndpoint = `${endpointRegion}-aiplatform.googleapis.com`; + + // Create a client + // predictionServiceClient = new PredictionServiceClient({apiEndpoint}); + + // Call the Gemma2 endpoint + const gemma2Endpoint = `projects/${projectId}/locations/${endpointRegion}/endpoints/${endpointId}`; + + const [response] = await predictionServiceClient.predict({ + endpoint: gemma2Endpoint, + instances, + }); + + const predictions = response.predictions; + const text = predictions[0].stringValue; + + console.log('Predictions:', text); + // [END generativeaionvertexai_gemma2_predict_tpu] + return text; +} + +module.exports = gemma2PredictTpu; + +// TODO(developer): Uncomment below lines before running the sample. +// gemma2PredictTpu(...process.argv.slice(2)).catch(err => { +// console.error(err.message); +// process.exitCode = 1; +// }); diff --git a/generative-ai/snippets/test/gemma2Prediction.test.js b/generative-ai/snippets/test/gemma2Prediction.test.js index 151a19ad3f..f52b3e6e71 100644 --- a/generative-ai/snippets/test/gemma2Prediction.test.js +++ b/generative-ai/snippets/test/gemma2Prediction.test.js @@ -20,8 +20,9 @@ const {expect} = require('chai'); const {afterEach, describe, it} = require('mocha'); const sinon = require('sinon'); const gemma2PredictGpu = require('../gemma2PredictGpu.js'); +const gemma2PredictTpu = require('../gemma2PredictTpu.js'); -const text = `The sky appears blue due to a phenomenon called **Rayleigh scattering**. +const gpuResponse = `The sky appears blue due to a phenomenon called **Rayleigh scattering**. **Here's how it works:** 1. **Sunlight:** Sunlight is composed of all the colors of the rainbow. 2. **Earth's Atmosphere:** When sunlight enters the Earth's atmosphere, it collides with tiny particles like nitrogen and oxygen molecules. @@ -32,25 +33,46 @@ const text = `The sky appears blue due to a phenomenon called **Rayleigh scatter * **Longer wavelengths** like red, orange, and yellow are scattered less and travel more directly through the atmosphere. This is why we see these colors during sunrise and sunset, when sunlight has to travel through more of the atmosphere. `; +const tpuResponse = + 'The sky appears blue due to a phenomenon called **Rayleigh scattering**.'; + describe('Gemma2 predictions', async () => { const predictionServiceClientMock = { - predict: sinon.stub().resolves([ + predict: sinon.stub().resolves([]), + }; + + afterEach(() => { + sinon.restore(); + }); + + it('should run interference with GPU', async () => { + predictionServiceClientMock.predict.resolves([ { predictions: [ { - stringValue: text, + stringValue: gpuResponse, }, ], }, - ]), - }; + ]); - afterEach(() => { - sinon.restore(); + const output = await gemma2PredictGpu(predictionServiceClientMock); + + expect(output).include('Rayleigh scattering'); }); - it('should run interference with GPU', async () => { - const output = await gemma2PredictGpu(predictionServiceClientMock); + it('should run interference with TPU', async () => { + predictionServiceClientMock.predict.resolves([ + { + predictions: [ + { + stringValue: tpuResponse, + }, + ], + }, + ]); + + const output = await gemma2PredictTpu(predictionServiceClientMock); expect(output).include('Rayleigh scattering'); }); From d5be357b1f32e2deb9d3ba256b321ad9f213e0f2 Mon Sep 17 00:00:00 2001 From: Joanna Grycz Date: Tue, 24 Sep 2024 12:31:51 +0200 Subject: [PATCH 3/3] Add check for request structure --- .../snippets/gemma2PredictGpu.js | 0 .../snippets/gemma2PredictTpu.js | 0 .../snippets/test/gemma2Prediction.test.js | 58 ++++++++++++++++++- 3 files changed, 57 insertions(+), 1 deletion(-) rename {generative-ai => ai-platform}/snippets/gemma2PredictGpu.js (100%) rename {generative-ai => ai-platform}/snippets/gemma2PredictTpu.js (100%) rename {generative-ai => ai-platform}/snippets/test/gemma2Prediction.test.js (63%) diff --git a/generative-ai/snippets/gemma2PredictGpu.js b/ai-platform/snippets/gemma2PredictGpu.js similarity index 100% rename from generative-ai/snippets/gemma2PredictGpu.js rename to ai-platform/snippets/gemma2PredictGpu.js diff --git a/generative-ai/snippets/gemma2PredictTpu.js b/ai-platform/snippets/gemma2PredictTpu.js similarity index 100% rename from generative-ai/snippets/gemma2PredictTpu.js rename to ai-platform/snippets/gemma2PredictTpu.js diff --git a/generative-ai/snippets/test/gemma2Prediction.test.js b/ai-platform/snippets/test/gemma2Prediction.test.js similarity index 63% rename from generative-ai/snippets/test/gemma2Prediction.test.js rename to ai-platform/snippets/test/gemma2Prediction.test.js index f52b3e6e71..16b18d8f59 100644 --- a/generative-ai/snippets/test/gemma2Prediction.test.js +++ b/ai-platform/snippets/test/gemma2Prediction.test.js @@ -37,15 +37,47 @@ const tpuResponse = 'The sky appears blue due to a phenomenon called **Rayleigh scattering**.'; describe('Gemma2 predictions', async () => { + const gemma2Endpoint = + 'projects/your-project-id/locations/your-vertex-endpoint-region/endpoints/your-vertex-endpoint-id'; + const configValues = { + maxOutputTokens: {kind: 'numberValue', numberValue: 1024}, + temperature: {kind: 'numberValue', numberValue: 0.9}, + topP: {kind: 'numberValue', numberValue: 1}, + topK: {kind: 'numberValue', numberValue: 1}, + }; + const prompt = 'Why is the sky blue?'; const predictionServiceClientMock = { predict: sinon.stub().resolves([]), }; afterEach(() => { - sinon.restore(); + sinon.reset(); }); it('should run interference with GPU', async () => { + const expectedGpuRequest = { + endpoint: gemma2Endpoint, + instances: [ + { + kind: 'structValue', + structValue: { + fields: { + inputs: { + kind: 'stringValue', + stringValue: prompt, + }, + parameters: { + kind: 'structValue', + structValue: { + fields: configValues, + }, + }, + }, + }, + }, + ], + }; + predictionServiceClientMock.predict.resolves([ { predictions: [ @@ -59,9 +91,30 @@ describe('Gemma2 predictions', async () => { const output = await gemma2PredictGpu(predictionServiceClientMock); expect(output).include('Rayleigh scattering'); + expect(predictionServiceClientMock.predict.calledOnce).to.be.true; + expect(predictionServiceClientMock.predict.calledWith(expectedGpuRequest)) + .to.be.true; }); it('should run interference with TPU', async () => { + const expectedTpuRequest = { + endpoint: gemma2Endpoint, + instances: [ + { + kind: 'structValue', + structValue: { + fields: { + ...configValues, + prompt: { + kind: 'stringValue', + stringValue: prompt, + }, + }, + }, + }, + ], + }; + predictionServiceClientMock.predict.resolves([ { predictions: [ @@ -75,5 +128,8 @@ describe('Gemma2 predictions', async () => { const output = await gemma2PredictTpu(predictionServiceClientMock); expect(output).include('Rayleigh scattering'); + expect(predictionServiceClientMock.predict.calledOnce).to.be.true; + expect(predictionServiceClientMock.predict.calledWith(expectedTpuRequest)) + .to.be.true; }); });