diff --git a/genkit/package.json b/genkit/package.json index 32872a4a..5f7bab5e 100644 --- a/genkit/package.json +++ b/genkit/package.json @@ -14,16 +14,16 @@ "license": "ISC", "dependencies": { "@compass/backend": "file:compass-generated", - "@genkit-ai/checks": "^1.0.4", - "@genkit-ai/express": "^1.0.4", - "@genkit-ai/firebase": "^1.0.4", - "@genkit-ai/google-cloud": "^1.0.4", - "@genkit-ai/googleai": "^1.0.4", - "@genkit-ai/vertexai": "^1.0.4", + "@genkit-ai/checks": "^1.14.1", + "@genkit-ai/express": "^1.14.1", + "@genkit-ai/firebase": "^1.14.1", + "@genkit-ai/google-cloud": "^1.14.1", + "@genkit-ai/googleai": "^1.14.1", + "@genkit-ai/vertexai": "^1.14.1", "axios": "^1.7.8", "express": "^4.21.1", "firebase": "^11.0.2", - "genkit": "^1.0.4" + "genkit": "^1.14.1" }, "devDependencies": { "typescript": "^5.7.2" diff --git a/genkit/prompts/imageDescription.prompt b/genkit/prompts/imageDescription.prompt index c51d53d8..0874d932 100644 --- a/genkit/prompts/imageDescription.prompt +++ b/genkit/prompts/imageDescription.prompt @@ -1,5 +1,5 @@ --- -model: googleai/gemini-1.5-flash +model: googleai/gemma-3-27b-it config: temperature: 1.0 input: diff --git a/genkit/prompts/itineraryPlanningAgent.prompt b/genkit/prompts/itineraryPlanningAgent.prompt index 6e206c9c..12b02566 100644 --- a/genkit/prompts/itineraryPlanningAgent.prompt +++ b/genkit/prompts/itineraryPlanningAgent.prompt @@ -1,5 +1,5 @@ --- -model: googleai/gemini-1.5-flash +model: googleai/gemma-3-27b-it config: temperature: 1.0 safetySettings: @@ -18,22 +18,6 @@ input: knownFor: string, A description of that place. activities(array, a stringify list of activities that can be found at the specified place): string mealOptions?(array, a stringify list of all the restaurants and supermarkets found at that location): string -output: - schema: - place: string, The place the user is traveling to. - itineraryName: string, a catchy itinerary name that encapsulates the spirit of the trip and includes the place name - startDate: string, the start date of the trip - endDate: string, the end date of the trip - tags(array, relevant tags for the trip): string - itinerary(array): - day: number - date: string - planForDay(array): - activityRef: string, the reference value for the activity this comes from the available activities JSON. If no value is present use a ref value of restaurant. - activityTitle: string, a catchy title for the activity - activityDesc: string, a six word description of the activity - photoUri?: string, set the photo uri value for restaurants and supermarket only. - googleMapsUri?: string, if this is a restaurant include the googleMapsUri --- Generate an itinerary for a tourist planning on traveling to the location specified based in their request. @@ -56,7 +40,22 @@ Limit activity descriptions to 6 words. If no mealOptions are supplied, do not recommend any mealOptions to eat at. -Output must be in JSON format. +Output must be in JSON format and follow the following schema: + +place: string, The place the user is traveling to. +itineraryName: string, a catchy itinerary name that encapsulates the spirit of the trip and includes the place name +startDate: string, the start date of the trip +endDate: string, the end date of the trip +tags(array, relevant tags for the trip): string +itinerary(array): + day: number + date: string + planForDay(array): + activityRef: string, the reference value for the activity this comes from the available activities JSON. If no value is present use a ref value of restaurant. + activityTitle: string, a catchy title for the activity + activityDesc: string, a six word description of the activity + photoUri?: string, set the photo uri value for restaurants and supermarket only. + googleMapsUri?: string, if this is a restaurant include the googleMapsUri {{#if mealOptions}} Find a mealOptions to eat at each day. diff --git a/genkit/prompts/mealsPlanningAgent.prompt b/genkit/prompts/mealsPlanningAgent.prompt index 3877f60e..1da19288 100644 --- a/genkit/prompts/mealsPlanningAgent.prompt +++ b/genkit/prompts/mealsPlanningAgent.prompt @@ -1,5 +1,5 @@ --- -model: googleai/gemini-1.5-pro +model: googleai/gemma-3-27b-it config: temperature: 1.0 safetySettings: diff --git a/genkit/prompts/suggestDestinationsWithContextAgent.prompt b/genkit/prompts/suggestDestinationsWithContextAgent.prompt index 5b030d91..667cd056 100644 --- a/genkit/prompts/suggestDestinationsWithContextAgent.prompt +++ b/genkit/prompts/suggestDestinationsWithContextAgent.prompt @@ -1,5 +1,5 @@ --- -model: googleai/gemini-1.5-flash +model: googleai/gemma-3-27b-it config: temperature: 0.6 input: @@ -15,16 +15,6 @@ input: name: string, name for the selected destination ref: string, the reference id of the selected destination tags(array, tags that are associated with a destination): string -output: - format: json - schema: - destinations(array, a list of 3 suggested destinations): - knownFor: string, a three-sentence description enticing the user to pick this vacation - country: string, country for the selected destination - continent: string, continent for the selected destination - imageUrl: string, imageUrl for the selected destination - name: string, name for the selected destination - ref: string, the reference id of the selected destination. This should be the ref from the metadata context object. --- You are a travel assistant helping the user decide on a vacation destination. @@ -36,4 +26,13 @@ Avoid picking multiple destinations in the same country. Take into account the following preferences from the user: {{description}} Context: -{{context}} \ No newline at end of file +{{context}} + +Output must be in JSON format and follow the following schema: + destinations(array, a list of 3 suggested destinations): + knownFor: string, a three-sentence description enticing the user to pick this vacation + country: string, country for the selected destination + continent: string, continent for the selected destination + imageUrl: string, imageUrl for the selected destination + name: string, name for the selected destination + ref: string, the reference id of the selected destination. This should be the ref from the metadata context object. \ No newline at end of file diff --git a/genkit/prompts/textRefinement.prompt b/genkit/prompts/textRefinement.prompt index 44f293d5..22cdd4cf 100644 --- a/genkit/prompts/textRefinement.prompt +++ b/genkit/prompts/textRefinement.prompt @@ -1,5 +1,5 @@ --- -model: googleai/gemini-1.5-flash +model: googleai/gemma-3-27b-it config: temperature: 1.0 safetySettings: @@ -14,11 +14,6 @@ config: input: schema: request: string, a request from the user on where they want to travel to. -output: - schema: - cost: boolean, Do we have information about the cost? - kids: boolean, Do we have information from the user about whether their children will be attending? - date: boolean, Do have information on when the user wants to start their trip? --- Refine this request by asking the user for more information. If no extra information is needed, @@ -53,4 +48,9 @@ Example Output: Input: {{request}} +Output must be in JSON format and follow the following schema: + cost: boolean, Do we have information about the cost? + kids: boolean, Do we have information from the user about whether their children will be attending? + date: boolean, Do have information on when the user wants to start their trip? + Output: \ No newline at end of file diff --git a/genkit/src/common/gemmaParsingTools.ts b/genkit/src/common/gemmaParsingTools.ts new file mode 100644 index 00000000..f9fd11b1 --- /dev/null +++ b/genkit/src/common/gemmaParsingTools.ts @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ModelResponseData } from "genkit"; +import { GemmaResponse } from "./types"; + +const JSON_HEADER = '```json'; +const JSON_FOOTER = '```'; + +export function parseGemmaResponseText(response: ModelResponseData): string { + const gemmaResponse = response as GemmaResponse; + if (!gemmaResponse.raw || !gemmaResponse.raw.text || typeof gemmaResponse.raw.text !== "function") { + throw new Error('Unexpected Gemma response'); + } + + return gemmaResponse.raw.text(); +} + +// Gemma does not support JSON mode. Therefore, we attempt to parse the text as a JSON object. +// Even if no JSON mode is supported, we can tell Gemma to respect a JSON format, even if returned as text. +export function parseGemmaResponseAsJson(gemmaResponse: ModelResponseData): unknown { + const text = parseGemmaResponseText(gemmaResponse); + const jsonText = text.slice(JSON_HEADER.length, -JSON_FOOTER.length); + return JSON.parse(jsonText); +} diff --git a/genkit/src/common/types.ts b/genkit/src/common/types.ts index a0b55f2b..fde748cf 100644 --- a/genkit/src/common/types.ts +++ b/genkit/src/common/types.ts @@ -94,4 +94,10 @@ export interface ItineraryGeneratorOutput { itinerary: Itinerary[], itineraryImageUrl?: string, placeRef?: string, +} + +export interface GemmaResponse { + raw: { + text: () => string, + } } \ No newline at end of file diff --git a/genkit/src/flows/itineraryGenerator.ts b/genkit/src/flows/itineraryGenerator.ts index cdb115fa..28e416f2 100644 --- a/genkit/src/flows/itineraryGenerator.ts +++ b/genkit/src/flows/itineraryGenerator.ts @@ -37,7 +37,7 @@ export const itineraryGenerator2 = ai.defineFlow( const imageDescriptionPrompt = await ai.prompt('imageDescription'); const result = await imageDescriptionPrompt({ - input: { images: userInputs.images }, + images: userInputs.images, }, { use: [...myMiddleware], }, diff --git a/genkit/src/flows/shared/iteneraryManager.ts b/genkit/src/flows/shared/iteneraryManager.ts index 63a93b1b..67b861ab 100644 --- a/genkit/src/flows/shared/iteneraryManager.ts +++ b/genkit/src/flows/shared/iteneraryManager.ts @@ -21,6 +21,7 @@ import { restaurantFinder } from '../../tools/restaurantFinder'; import { supermarketFinder } from '../../tools/supermarketFinder'; import axios from 'axios'; import { ai, myMiddleware } from '../../config/genkit'; +import { parseGemmaResponseAsJson } from '../../common/gemmaParsingTools'; const MAPS_API_KEY = process.env.MAPS_API_KEY; @@ -52,7 +53,7 @@ const generateItineraryForPlace = async (request: string, location: Destination, },{ use: [...myMiddleware], }); - return itineraries.output as ItineraryGeneratorOutput; + return parseGemmaResponseAsJson(itineraries) as ItineraryGeneratorOutput; } /** diff --git a/genkit/src/flows/textRefinement.ts b/genkit/src/flows/textRefinement.ts index b6faa997..b98ee98e 100644 --- a/genkit/src/flows/textRefinement.ts +++ b/genkit/src/flows/textRefinement.ts @@ -16,6 +16,7 @@ import { ai, myMiddleware } from '../config/genkit'; import { z } from 'genkit'; +import { parseGemmaResponseAsJson } from '../common/gemmaParsingTools' // FINAL STEP // We realize that our initial prompts aren't great, so we want to @@ -40,6 +41,6 @@ export const textRefinement = ai.defineFlow( ...myMiddleware ]} ); - return result.output; + return parseGemmaResponseAsJson(result); }); // [END text_refinement_flow] \ No newline at end of file diff --git a/genkit/src/retrievers/placeRetriever.ts b/genkit/src/retrievers/placeRetriever.ts index be30ea81..7b83ad09 100644 --- a/genkit/src/retrievers/placeRetriever.ts +++ b/genkit/src/retrievers/placeRetriever.ts @@ -38,13 +38,15 @@ export const placeRetriever = ai.defineRetriever( configSchema: QueryOptions, }, async (input, options) => { + const text = input.content.find(item => item.text)?.text; const requestEmbedding = await ai.embed({ embedder: textEmbeddingGecko001, - content: input.text, + content: text!, }); + const embedding = requestEmbedding.find(item => item.embedding)?.embedding; const result = await getNearestPlace( dataConnectInstance, - { placeDescriptionVector: requestEmbedding } + { placeDescriptionVector: embedding! } ); const resultData = result.data;