diff --git a/README.md b/README.md index b2a110b4..8bc63a69 100644 --- a/README.md +++ b/README.md @@ -1,126 +1,333 @@ # SwiftBedrockService -Work in progress, feel free to open issue, but do not use in your projects. +This library is a work in progress, feel free to open an issue, but do not use it in your projects just yet. -## How to add a new model family? +## How to get started with BedrockService -As an example we will add the Llama 3.1 70B Instruct model from the Meta family as an example. +1. Import the BedrockService and BedrockTypes -"meta.llama3-70b-instruct-v1:0" +```swift +import BedrockService +import BedrockTypes +``` + +2. Initialize the BedrockService + +Choose what Region to use, whether to use AWS SSO authentication instead of standard credentials and pass a logger. If no region is passed it will default to `.useast1`, if no logger is provided a default logger with the name `bedrock.service` is created. The log level will be set to the environment variable `SWIFT_BEDROCK_LOG_LEVEL` or default to `.trace`. If `useSSO` is not defined it will default to `false` and use the standard credentials for authentication. + +```swift +let bedrock = try await BedrockService( + region: .uswest1, + logger: logger, + useSSO: true +) +``` -### 1. Add create BedrockModel instance +3. List the available models + +Use the `listModels()` function to test your set-up. This function will return an array of `ModelSummary` objects, each one representing a model supported by Amazon Bedrock. The ModelSummaries that contain a `BedrockModel` object are the models supported by BedrockService. ```swift -extension BedrockModel { - public static let llama3_70b_instruct: BedrockModel = BedrockModel( - id: "meta.llama3-70b-instruct-v1:0", - modality: LlamaText() - ) -} +let models = try await bedrock.listModels() ``` -### 2. Create family-specific request and response struct +## How to generate text using the InvokeModel API + +Choose a BedrockModel that supports text generation, you can verify this using the `hasTextModality` function. when calling the `completeText` function you can provide some inference parameters: -Make sure to create a struct that reflects exactly how the body of the request for an invokeModel call to this family should look. Make sure to add the public initializer with parameters `prompt`, `maxTokens` and `temperature` to comply to the `BedrockBodyCodable` protocol. +- `maxTokens`: The maximum amount of tokens that the model is allowed to return +- `temperature`: Controls the randomness of the model's output +- `topP`: Nucleus sampling, this parameter controls the cumulative probability threshold for token selection +- `topK`: Limits the number of tokens the model considers for each step of text generation to the K most likely ones +- `stopSequences`: An array of strings that will cause the model to stop generating further text when encountered -```json -{ - "prompt": "\(prompt)", - "temperature": 1, - "top_p": 0.9, - "max_tokens": 200, - "stop": ["END"] +The function returns a `TextCompletion` object containg the generated text. + +```swift +let model = .anthropicClaude3Sonnet + +guard model.hasTextModality() else { + print("\(model.name) does not support text completion") } + +let textCompletion = try await bedrock.completeText( + "Write a story about a space adventure", + with: model, + maxTokens: 1000, + temperature: 0.7, + topP: 0.9, + topK: 250, + stopSequences: ["THE END"] +) ``` +Note that the minimum, maximum and default values for each parameter are model specific and defined when the BedrockModel is created. Some parameters might not be supported by certain models. + +## How to generate an image using the InvokeModel API + +Choose a BedrockModel that supports image generation - you can verify this using the `hasImageModality` and the `hasTextToImageModality` function. The `generateImage` function allows you to create images from text descriptions with various optional parameters: + +- `prompt`: Text description of the desired image +- `negativePrompt`: Text describing what to avoid in the generated image +- `nrOfImages`: Number of images to generate +- `cfgScale`: Classifier free guidance scale to control how closely the image follows the prompt +- `seed`: Seed for reproducible image generation +- `quality`: Parameter to control the quality of generated images +- `resolution`: Desired image resolution for the generated images + +The function returns an ImageGenerationOutput object containing an array of generated images in base64 format. + ```swift -public struct LlamaRequestBody: BedrockBodyCodable { - let prompt: String - let max_gen_len: Int - let temperature: Double - let top_p: Double - - public init(prompt: String, maxTokens: Int = 512, temperature: Double = 0.5) { - self.prompt = - "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\(prompt)<|eot_id|><|start_header_id|>assistant<|end_header_id|>" - self.max_gen_len = maxTokens - self.temperature = temperature - self.top_p = 0.9 - } +let model = .nova_canvas + +guard model.hasImageModality(), + model.hasTextToImageModality() else { + print("\(model.name) does not support image generation") } + +let imageGeneration = try await bedrock.generateImage( + "A serene landscape with mountains at sunset", + with: model, + negativePrompt: "dark, stormy, people", + nrOfImages: 3, + cfgScale: 7.0, + seed: 42, + quality: .standard, + resolution: ImageResolution(width: 100, height: 100) +) ``` -Do the same for the response and ensure to add the `getTextCompletion` method to extract the completion from the response body and to comply to the `ContainsTextCompletion` protocol. +Note that the minimum, maximum and default values for each parameter are model specific and defined when the BedrockModel is created. Some parameters might not be supported by certain models. + +## How to generate image variations using the InvokeModel API +Choose a BedrockModel that supports image variations - you can verify this using the `hasImageVariationModality` and the `hasImageVariationModality` function. The `generateImageVariation` function allows you to create variations of an existing image with these parameters: + +- `images`: The base64-encoded source images used to create variations from +- `negativePrompt`: Text describing what to avoid in the generated image +- `similarity`: Controls how similar the variations will be to the source images +- `nrOfImages`: Number of variations to generate +- `cfgScale`: Classifier free guidance scale to control how closely variations follow the original image +- `seed`: Seed for reproducible variation generation +- `quality`: Parameter to control the quality of generated variations +- `resolution`: Desired resolution for the output variations -```json -{ - "generation": "\n\n", - "prompt_token_count": int, - "generation_token_count": int, - "stop_reason" : string +This function returns an `ImageGenerationOutput` object containing an array of generated image variations in base64 format. Each variation will maintain key characteristics of the source images while introducing creative differences. + +```swift +let model = .nova_canvas + +guard model.hasImageVariationModality(), + model.hasImageVariationModality() else { + print("\(model.name) does not support image variations") } + +let imageVariations = try await bedrock.generateImageVariation( + images: [base64EncodedImage], + prompt: "A dog drinking out of this teacup", + with: model, + negativePrompt: "Cats, worms, rain", + similarity: 0.8, + nrOfVariations: 4, + cfgScale: 7.0, + seed: 42, + quality: .standard, + resolution: ImageResolution(width: 100, height: 100) +) ``` +Note that the minimum, maximum and default values for each parameter are model specific and defined when the BedrockModel is created. Some parameters might not be supported by certain models. + +## How to chat using the Converse API + +### Text prompt + ```swift -struct LlamaResponseBody: ContainsTextCompletion { - let generation: String - let prompt_token_count: Int - let generation_token_count: Int - let stop_reason: String - - public func getTextCompletion() throws -> TextCompletion { - TextCompletion(generation) - } +let model = .nova_lite + +guard model.hasConverseModality() else { + print("\(model.name) does not support converse") } + +var (reply, history) = try await bedrock.converse( + with: model, + prompt: "Tell me about rainbows", + history: [], + maxTokens: 512, +) + +print("Assistant: \(reply)") + +(reply, history) = try await bedrock.converse( + with: model, + prompt: "Do you think birds can see them too?", + history: history, + maxTokens: 512, +) + +print("Assistant: \(reply)") ``` -### 3. Add the Modality (TextModality or ImageModality) -For a text generation create a struct conforming to TextModality. Use the request body and response body you created in [the previous step](#2-create-family-specific-request-and-response-struct). +### Vision ```swift -struct LlamaText: TextModality { - func getName() -> String { "Llama Text Generation" } - - func getTextRequestBody(prompt: String, maxTokens: Int, temperature: Double) throws -> BedrockBodyCodable { - LlamaRequestBody(prompt: prompt, maxTokens: maxTokens, temperature: temperature) - } +let model = .nova_lite - func getTextResponseBody(from data: Data) throws -> ContainsTextCompletion { - let decoder = JSONDecoder() - return try decoder.decode(LlamaResponseBody.self, from: data) - } +guard model.hasConverseModality(.vision) else { + print("\(model.name) does not support converse") } + +let (reply, history) = try await bedrock.converse( + with model: model, + prompt: "Can you tell me about this plant?", + imageFormat: .jpeg, + imageBytes: base64EncodedImage, + temperature: 0.8, +) + +print("Assistant: \(reply)") ``` -### 4. Optionally you can create a BedrockModel initializer for your newly implemented models +### Tools + ```swift -extension BedrockModel { - init?(_ id: String) { - switch id { - case "meta.llama3-70b-instruct-v1:0": self = .llama3_70b_instruct - // ... - default: - return nil - } - } +let model = .nova_lite + +// verify that the model supports tool usage +guard model.hasConverseModality(.toolUse) else { + print("\(model.name) does not support converse tools") } -``` +// define the inputschema for your tool +let inputSchema = JSON([ + "type": "object", + "properties": [ + "sign": [ + "type": "string", + "description": "The call sign for the radio station for which you want the most popular song. Example calls signs are WZPZ and WKRP." + ] + ], + "required": [ + "sign" + ] +]) + +// create a Tool object +let tool = Tool(name: "top_song", inputSchema: inputSchema, description: "Get the most popular song played on a radio station.") -## How to add a new model? +// pass a prompt and the tool to converse +var (reply, history) = try await bedrock.converse( + with model: model, + prompt: "What is the most popular song on WZPZ?", + tools: [tool] +) -If you want to add a model that has a request and response structure that is already implemented you can skip a few steps. Simply create a typealias for the Modality that matches the structure and use it to create a BedrockModel instance. +print("Assistant: \(reply)") +// The reply will be similar to this: "I need to use the \"top_song\" tool to find the most popular song on the radio station WZPZ. I will input the call sign \"WZPZ\" into the tool to get the required information." +// The last message in the history will contain the tool use request + +if case .toolUse(let toolUse) = history.last?.content.last { + let id = toolUse.id + let name = toolUse.name + let input = toolUse.input + + // Logic to use the tool here + + let toolResult = ToolResultBlock(id: id, content: [.text("The Best Song Ever")], status: .success) + + // Send the toolResult back to the model + (reply, history) = try await bedrock.converse( + with: model, + history: history, + tools: [tool], + toolResult: toolResult +) +} + +print("Assistant: \(reply)") +// The final reply will be similar to: "The most popular song currently played on WZPZ is \"The Best Song Ever\". If you need more information or have another request, feel free to ask!" +``` + +### Make your own `Message` + +Alternatively use the `converse` function that does not take a `prompt`, `toolResult` or `image` and construct the `Message` yourself. ```swift -typealias ClaudeNewModel = AnthropicText +// Message with prompt +let (reply, history) = try await bedrock.converse( + with model: model, + conversation: [Message("What day of the week is it?")], + maxTokens: 512, + temperature: 1, + stopSequences: ["THE END"], + systemPrompts: ["Today is Wednesday, make sure to mention that."] +) + +// Message with an image and prompt +let (reply, history) = try await bedrock.converse( + with model: model, + conversation: [Message(prompt: "What is in the this teacup?", imageFormat: .jpeg, imageBytes: base64EncodedImage)], +) +// Message with toolResult +let (reply, history) = try await bedrock.converse( + with model: model, + conversation: [Message(toolResult)], + tools: [toolA, toolB] +) +``` + +## How to add a BedrockModel + +### Text + +-- Under Construction -- + +### Image + +-- Under Construction -- + +### Converse + +To add a new model that only needs the ConverseModality, simply use the `StandardConverse` and add the correct [inferece parameters](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html) and [supported converse features](https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html). + +```swift extension BedrockModel { - public static let instant: BedrockModel = BedrockModel( - id: "anthropic.claude-new-model", - modality: ClaudeNewModel() + public static let new_bedrock_model = BedrockModel( + id: "family.model-id-v1:0", + name: "New Model Name", + modality: StandardConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.3), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: nil, defaultValue: nil), + topP: Parameter(.topP, minValue: 0.01, maxValue: 0.99, defaultValue: 0.75), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] + ) ) } ``` -Note that the model will not automatically be included in the BedrockModel initializer that creates an instance from a raw string value. Consider creating a custom initializer that includes your models. +If the model also implements other modalities you might need to create you own `Modality` and make sure it conforms to `ConverseModality` by implementing the `getConverseParameters` and `getConverseFeatures` functions. Note that the `ConverseParameters` can be extracted from `TextGenerationParameters` by using the public initializer. + +```swift +struct ModelFamilyModality: TextModality, ConverseModality { + func getName() -> String { "Model Family Text and Converse Modality" } + + let parameters: TextGenerationParameters + let converseFeatures: [ConverseFeature] + let converseParameters: ConverseParameters + + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration]) { + self.parameters = parameters + self.converseFeatures = features + + // public initializer to extract `ConverseParameters` from `TextGenerationParameters` + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) + } + + // ... +} +``` diff --git a/backend/Package.swift b/backend/Package.swift index fa65c639..81a719cd 100644 --- a/backend/Package.swift +++ b/backend/Package.swift @@ -41,6 +41,10 @@ let package = Package( ), .target( name: "BedrockTypes", + dependencies: [ + .product(name: "AWSBedrockRuntime", package: "aws-sdk-swift"), + .product(name: "Smithy", package: "smithy-swift"), + ], path: "Sources/BedrockTypes" ), .testTarget( diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index 4e807eae..2e6536c6 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -13,10 +13,11 @@ // //===----------------------------------------------------------------------===// -import Hummingbird -import Logging import BedrockService import BedrockTypes +import Hummingbird +import Logging +import Foundation /// Application arguments protocol. We use a protocol so we can call /// `buildApplication` inside Tests as well as in the App executable. @@ -168,18 +169,23 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router Logger { var logger: Logger = Logger(label: name) logger.logLevel = @@ -98,6 +100,11 @@ public struct BedrockService: Sendable { } /// Creates a BedrockClient + /// - Parameters: + /// - region: The AWS region to configure the client for + /// - useSSO: Whether to use SSO authentication + /// - Returns: Configured BedrockClientProtocol instance + /// - Throws: Error if client creation fails static private func createBedrockClient( region: Region, useSSO: Bool = false @@ -114,6 +121,11 @@ public struct BedrockService: Sendable { } /// Creates a BedrockRuntimeClient + /// - Parameters: + /// - region: The AWS region to configure the client for + /// - useSSO: Whether to use SSO authentication + /// - Returns: Configured BedrockRuntimeClientProtocol instance + /// - Throws: Error if client creation fails static private func createBedrockRuntimeClient( region: Region, useSSO: Bool = false @@ -135,7 +147,7 @@ public struct BedrockService: Sendable { /// Lists all available foundation models from Amazon Bedrock /// - Throws: BedrockServiceError.invalidResponse - /// - Returns: An array of ModelInfo objects containing details about each available model. + /// - Returns: An array of ModelSummary objects containing details about each available model public func listModels() async throws -> [ModelSummary] { logger.trace("Fetching foundation models") do { @@ -166,16 +178,21 @@ public struct BedrockService: Sendable { } } - /// Generates a text completion using a specified model. + /// Generates a text completion using a specified model /// - Parameters: - /// - text: the text to be completed - /// - model: the BedrockModel that will be used to generate the completion - /// - maxTokens: the maximum amount of tokens in the completion (must be at least 1) optional, default 300 - /// - temperature: the temperature used to generate the completion (must be a value between 0 and 1) optional, default 0.6 - /// - Throws: BedrockServiceError.invalidMaxTokens if maxTokens is less than 1 - /// BedrockServiceError.invalidTemperature if temperature is not between 0 and 1 - /// BedrockServiceError.invalidPrompt if the prompt is empty - /// BedrockServiceError.invalidResponse if the response body is missing + /// - prompt: The text to be completed + /// - model: The BedrockModel that will be used to generate the completion + /// - maxTokens: The maximum amount of tokens in the completion (optional, default 300) + /// - temperature: The temperature used to generate the completion (optional, default 0.6) + /// - topP: Optional top-p parameter for nucleus sampling + /// - topK: Optional top-k parameter for filtering + /// - stopSequences: Optional array of sequences where generation should stop + /// - Throws: BedrockServiceError.notSupported for parameters or functionalities that are not supported + /// BedrockServiceError.invalidParameter for invalid parameters + /// BedrockServiceError.invalidPrompt for a prompt that is empty or too long + /// BedrockServiceError.invalidStopSequences if too many stop sequences were provided + /// BedrockServiceError.invalidModality for invalid modality from the selected model + /// BedrockServiceError.invalidSDKResponse if the response body is missing /// - Returns: a TextCompletion object containing the generated text from the model public func completeText( _ prompt: String, @@ -276,15 +293,22 @@ public struct BedrockService: Sendable { } } - /// Generates 1 to 5 image(s) from a text prompt using a specific model. + /// Generates 1 to 5 image(s) from a text prompt using a specific model /// - Parameters: - /// - prompt: the prompt describing the image that should be generated - /// - model: the BedrockModel that will be used to generate the image - /// - nrOfImages: the number of images that will be generated (must be a number between 1 and 5) optional, default 3 - /// - Throws: BedrockServiceError.invalidNrOfImages if nrOfImages is not between 1 and 5 - /// BedrockServiceError.invalidPrompt if the prompt is empty - /// BedrockServiceError.invalidResponse if the response body is missing - /// - Returns: a ImageGenerationOutput object containing an array of generated images + /// - prompt: The text prompt describing the image that should be generated + /// - model: The BedrockModel that will be used to generate the image + /// - negativePrompt: Optional text describing what to avoid in the generated image + /// - nrOfImages: Optional number of images to generate (must be between 1 and 5, default 3) + /// - cfgScale: Optional classifier free guidance scale to control prompt adherence + /// - seed: Optional seed for reproducible image generation + /// - quality: Optional parameter to control the quality of generated images + /// - resolution: Optional parameter to specify the desired image resolution + /// - Throws: BedrockServiceError.notSupported for parameters or functionalities that are not supported + /// BedrockServiceError.invalidParameter for invalid parameters + /// BedrockServiceError.invalidPrompt if the prompt is empty or too long + /// BedrockServiceError.invalidModality for invalid modality from the selected model + /// BedrockServiceError.invalidSDKResponse if the response body is missing + /// - Returns: An ImageGenerationOutput object containing an array of generated images public func generateImage( _ prompt: String, with model: BedrockModel, @@ -360,17 +384,24 @@ public struct BedrockService: Sendable { } } - /// Generates 1 to 5 imagevariation(s) from reference images and a text prompt using a specific model. + /// Generates 1 to 5 image variation(s) from reference images and a text prompt using a specific model /// - Parameters: - /// - image: the reference images - /// - prompt: the prompt describing the image that should be generated - /// - model: the BedrockModel that will be used to generate the image - /// - nrOfImages: the number of images that will be generated (must be a number between 1 and 5) optional, default 3 - /// - Throws: BedrockServiceError.invalidNrOfImages if nrOfImages is not between 1 and 5 - /// BedrockServiceError.similarity if similarity is not between 0.2 - 1.0 - /// BedrockServiceError.invalidPrompt if the prompt is empty - /// BedrockServiceError.invalidResponse if the response body is missing - /// - Returns: a ImageGenerationOutput object containing an array of generated images + /// - images: Array of base64 encoded reference images to generate variations from + /// - prompt: The text prompt describing desired modifications to the reference images + /// - model: The BedrockModel that will be used to generate the variations + /// - negativePrompt: Optional text describing what to avoid in the generated variations + /// - similarity: Optional parameter controlling how closely variations should match reference (between 0.2 and 1.0) + /// - nrOfImages: Optional number of variations to generate (must be between 1 and 5, default 3) + /// - cfgScale: Optional classifier free guidance scale to control prompt adherence + /// - seed: Optional seed for reproducible variation generation + /// - quality: Optional parameter to control the quality of generated variations + /// - resolution: Optional parameter to specify the desired image resolution + /// - Throws: BedrockServiceError.notSupported for parameters or functionalities that are not supported + /// BedrockServiceError.invalidParameter for invalid parameters + /// BedrockServiceError.invalidPrompt if the prompt is empty or too long + /// BedrockServiceError.invalidModality for invalid modality from the selected model + /// BedrockServiceError.invalidSDKResponse if the response body is missing + /// - Returns: An ImageGenerationOutput object containing an array of generated image variations public func generateImageVariation( images: [String], prompt: String, @@ -456,52 +487,77 @@ public struct BedrockService: Sendable { } } - /// Use Converse API + /// Converse with a model using the Bedrock Converse API + /// - Parameters: + /// - model: The BedrockModel to converse with + /// - conversation: Array of previous messages in the conversation + /// - maxTokens: Optional maximum number of tokens to generate + /// - temperature: Optional temperature parameter for controlling randomness + /// - topP: Optional top-p parameter for nucleus sampling + /// - stopSequences: Optional array of sequences where generation should stop + /// - systemPrompts: Optional array of system prompts to guide the conversation + /// - tools: Optional array of tools the model can use + /// - Throws: BedrockServiceError.notSupported for parameters or functionalities that are not supported + /// BedrockServiceError.invalidParameter for invalid parameters + /// BedrockServiceError.invalidPrompt if the prompt is empty or too long + /// BedrockServiceError.invalidModality for invalid modality from the selected model + /// BedrockServiceError.invalidSDKResponse if the response body is missing + /// - Returns: A Message containing the model's response public func converse( with model: BedrockModel, - prompt: String, - history: [Message] = [], + conversation: [Message], maxTokens: Int? = nil, temperature: Double? = nil, topP: Double? = nil, - stopSequences: [String]? = nil - ) async throws -> (String, [Message]) { - logger.trace( - "Conversing", - metadata: [ - "model.id": .string(model.id), - "model.modality": .string(model.modality.getName()), - "prompt": .string(prompt), - ] - ) + stopSequences: [String]? = nil, + systemPrompts: [String]? = nil, + tools: [Tool]? = nil + ) async throws -> Message { do { - let modality = try model.getTextModality() // FIXME later: ConverseModality? + let modality: ConverseModality = try model.getConverseModality() try validateConverseParams( modality: modality, - prompt: prompt, - history: history, maxTokens: maxTokens, temperature: temperature, topP: topP, stopSequences: stopSequences ) - var messages = history - messages.append(Message(from: .user, content: [.text(prompt)])) - + logger.trace( + "Creating ConverseRequest", + metadata: [ + "model.name": "\(model.name)", + "model.id": "\(model.id)", + "conversation.count": "\(conversation.count)", + "maxToken": "\(String(describing: maxTokens))", + "temperature": "\(String(describing: temperature))", + "topP": "\(String(describing: topP))", + "stopSequences": "\(String(describing: stopSequences))", + "systemPrompts": "\(String(describing: systemPrompts))", + "tools": "\(String(describing: tools))", + ] + ) let converseRequest = ConverseRequest( model: model, - messages: messages, + messages: conversation, maxTokens: maxTokens, temperature: temperature, topP: topP, - stopSequences: stopSequences + stopSequences: stopSequences, + systemPrompts: systemPrompts, + tools: tools ) - let input = converseRequest.getConverseInput() + + logger.trace("Creating ConverseInput") + let input = try converseRequest.getConverseInput() logger.trace( "Created ConverseInput", - metadata: ["messages.count": "\(messages.count)", "model": "\(model.id)"] + metadata: [ + "input.messages.count": "\(String(describing:input.messages!.count))", + "input.modelId": "\(String(describing:input.modelId!))", + ] ) + let response = try await self.bedrockRuntimeClient.converse(input: input) logger.trace("Received response", metadata: ["response": "\(response)"]) @@ -518,11 +574,113 @@ public struct BedrockService: Sendable { ) } let converseResponse = try ConverseResponse(converseOutput) - messages.append(converseResponse.message) + return converseResponse.message + } catch { + logger.trace("Error while conversing", metadata: ["error": "\(error)"]) + throw error + } + } + + /// Use Converse API without needing to make Messages + /// - Parameters: + /// - model: The BedrockModel to converse with + /// - prompt: Optional text prompt for the conversation + /// - imageFormat: Optional format for image input + /// - imageBytes: Optional base64 encoded image data + /// - history: Optional array of previous messages + /// - maxTokens: Optional maximum number of tokens to generate + /// - temperature: Optional temperature parameter for controlling randomness + /// - topP: Optional top-p parameter for nucleus sampling + /// - stopSequences: Optional array of sequences where generation should stop + /// - systemPrompts: Optional array of system prompts to guide the conversation + /// - tools: Optional array of tools the model can use + /// - toolResult: Optional result from a previous tool invocation + /// - Throws: BedrockServiceError.notSupported for parameters or functionalities that are not supported + /// BedrockServiceError.invalidParameter for invalid parameters + /// BedrockServiceError.invalidPrompt if the prompt is empty or too long + /// BedrockServiceError.invalidModality for invalid modality from the selected model + /// BedrockServiceError.invalidSDKResponse if the response body is missing + /// - Returns: Tuple containing the model's response text and updated message history + public func converse( + with model: BedrockModel, + prompt: String? = nil, + imageFormat: ImageBlock.Format? = nil, + imageBytes: String? = nil, + history: [Message] = [], + maxTokens: Int? = nil, + temperature: Double? = nil, + topP: Double? = nil, + stopSequences: [String]? = nil, + systemPrompts: [String]? = nil, + tools: [Tool]? = nil, + toolResult: ToolResultBlock? = nil + ) async throws -> (String, [Message]) { + logger.trace( + "Conversing", + metadata: [ + "model.id": .string(model.id), + "model.modality": .string(model.modality.getName()), + "prompt": .string(prompt ?? "No prompt"), + ] + ) + do { + var messages = history + let modality: ConverseModality = try model.getConverseModality() + + if tools != nil || toolResult != nil { + guard model.hasConverseModality(.toolUse) else { + throw BedrockServiceError.invalidModality( + model, + modality, + "This model does not support converse tool." + ) + } + } + + if let toolResult { + guard let _: [Tool] = tools else { + throw BedrockServiceError.invalidPrompt("Tool result is defined but tools are not.") + } + guard case .toolUse(_) = messages.last?.content.last else { + throw BedrockServiceError.invalidPrompt("Tool result is defined but last message is not tool use.") + } + messages.append(Message(toolResult)) + } else { + guard let prompt = prompt else { + throw BedrockServiceError.invalidPrompt("Prompt is not defined.") + } + + if let imageFormat, let imageBytes { + guard model.hasConverseModality(.vision) else { + throw BedrockServiceError.invalidModality( + model, + modality, + "This model does not support converse vision." + ) + } + messages.append( + Message(prompt: prompt, imageFormat: imageFormat, imageBytes: imageBytes) + ) + } else { + messages.append(Message(prompt)) + } + } + let message = try await converse( + with: model, + conversation: messages, + maxTokens: maxTokens, + temperature: temperature, + topP: topP, + stopSequences: stopSequences, + systemPrompts: systemPrompts, + tools: tools + ) + messages.append(message) logger.trace( "Received message", - metadata: ["replyMessage": "\(converseResponse.message)", "messages.count": "\(messages.count)"] + metadata: ["replyMessage": "\(message)", "messages.count": "\(messages.count)"] ) + let converseResponse = ConverseResponse(message) return (converseResponse.getReply(), messages) } catch { logger.trace("Error while conversing", metadata: ["error": "\(error)"]) diff --git a/backend/Sources/BedrockService/Converse/ConverseRequest.swift b/backend/Sources/BedrockService/Converse/ConverseRequest.swift index c11f8488..b9a37795 100644 --- a/backend/Sources/BedrockService/Converse/ConverseRequest.swift +++ b/backend/Sources/BedrockService/Converse/ConverseRequest.swift @@ -14,13 +14,15 @@ //===----------------------------------------------------------------------===// @preconcurrency import AWSBedrockRuntime -import Foundation import BedrockTypes +import Foundation public struct ConverseRequest { let model: BedrockModel let messages: [Message] let inferenceConfig: InferenceConfig? + let toolConfig: ToolConfig? + let systemPrompts: [String]? init( model: BedrockModel, @@ -28,7 +30,9 @@ public struct ConverseRequest { maxTokens: Int?, temperature: Double?, topP: Double?, - stopSequences: [String]? + stopSequences: [String]?, + systemPrompts: [String]?, + tools: [Tool]? ) { self.messages = messages self.model = model @@ -38,24 +42,32 @@ public struct ConverseRequest { topP: topP, stopSequences: stopSequences ) - } - - func getConverseInput() -> ConverseInput { - let sdkInferenceConfig: BedrockRuntimeClientTypes.InferenceConfiguration? - if inferenceConfig != nil { - sdkInferenceConfig = inferenceConfig!.getSDKInferenceConfig() + self.systemPrompts = systemPrompts + if tools != nil { + self.toolConfig = ToolConfig(tools: tools!) } else { - sdkInferenceConfig = nil + self.toolConfig = nil } - return ConverseInput( - inferenceConfig: sdkInferenceConfig, - messages: getSDKMessages(), - modelId: model.id + } + + func getConverseInput() throws -> ConverseInput { + ConverseInput( + inferenceConfig: inferenceConfig?.getSDKInferenceConfig(), + messages: try getSDKMessages(), + modelId: model.id, + system: getSDKSystemPrompts(), + toolConfig: try toolConfig?.getSDKToolConfig() ) } - private func getSDKMessages() -> [BedrockRuntimeClientTypes.Message] { - messages.map { $0.getSDKMessage() } + private func getSDKMessages() throws -> [BedrockRuntimeClientTypes.Message] { + try messages.map { try $0.getSDKMessage() } + } + + private func getSDKSystemPrompts() -> [BedrockRuntimeClientTypes.SystemContentBlock]? { + systemPrompts?.map { + BedrockRuntimeClientTypes.SystemContentBlock.text($0) + } } struct InferenceConfig { @@ -85,4 +97,24 @@ public struct ConverseRequest { ) } } + + public struct ToolConfig { + // let toolChoice: ToolChoice? + let tools: [Tool] + + func getSDKToolConfig() throws -> BedrockRuntimeClientTypes.ToolConfiguration { + BedrockRuntimeClientTypes.ToolConfiguration( + tools: try tools.map { .toolspec(try $0.getSDKToolSpecification()) } + ) + } + } } + +// public enum ToolChoice { +// /// (Default). The Model automatically decides if a tool should be called or whether to generate text instead. +// case auto(_) +// /// The model must request at least one tool (no text is generated). +// case any(_) +// /// The Model must request the specified tool. Only supported by Anthropic Claude 3 models. +// case tool(String) +// } diff --git a/backend/Sources/BedrockService/Converse/ConverseResponse.swift b/backend/Sources/BedrockService/Converse/ConverseResponse.swift index 8be13ab0..819e8e70 100644 --- a/backend/Sources/BedrockService/Converse/ConverseResponse.swift +++ b/backend/Sources/BedrockService/Converse/ConverseResponse.swift @@ -20,6 +20,10 @@ import BedrockTypes public struct ConverseResponse { let message: Message + public init(_ message: Message) { + self.message = message + } + public init(_ output: BedrockRuntimeClientTypes.ConverseOutput) throws { guard case .message(let sdkMessage) = output else { throw BedrockServiceError.invalidSDKResponse("Could not extract message from ConverseOutput") diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions.swift deleted file mode 100644 index d52a11c1..00000000 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift +++ /dev/null @@ -1,88 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Foundation Models Playground open source project -// -// Copyright (c) 2025 Amazon.com, Inc. or its affiliates -// and the Swift Foundation Models Playground project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -@preconcurrency import AWSBedrockRuntime -import Foundation -import BedrockTypes - -extension Message { - - init(from sdkMessage: BedrockRuntimeClientTypes.Message) throws { - guard let sdkRole = sdkMessage.role else { - throw BedrockServiceError.decodingError("Could not extract role from BedrockRuntimeClientTypes.Message") - } - guard let sdkContent = sdkMessage.content else { - throw BedrockServiceError.decodingError("Could not extract content from BedrockRuntimeClientTypes.Message") - } - let content: [Content] = try sdkContent.map { try Content(from: $0) } - self = Message(from: try Role(from: sdkRole), content: content) - } - - func getSDKMessage() -> BedrockRuntimeClientTypes.Message { - let contentBlocks: [BedrockRuntimeClientTypes.ContentBlock] = content.map { - content -> BedrockRuntimeClientTypes.ContentBlock in - return content.getSDKContentBlock() - } - return BedrockRuntimeClientTypes.Message( - content: contentBlocks, - role: role.getSDKConversationRole() - ) - } -} - -extension Content { - - init(from sdkContentBlock: BedrockRuntimeClientTypes.ContentBlock) throws { - switch sdkContentBlock { - case .text(let text): - self = .text(text) - case .sdkUnknown(let unknownContentBlock): - throw BedrockServiceError.notImplemented( - "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" - ) - default: - throw BedrockServiceError.notImplemented( - "\(sdkContentBlock.self) is not implemented by this library" - ) - } - } - - func getSDKContentBlock() -> BedrockRuntimeClientTypes.ContentBlock { - switch self { - case .text(let text): - return BedrockRuntimeClientTypes.ContentBlock.text(text) - } - } -} - -extension Role { - init(from sdkConversationRole: BedrockRuntimeClientTypes.ConversationRole) throws { - switch sdkConversationRole { - case .user: self = .user - case .assistant: self = .assistant - case .sdkUnknown(let unknownRole): - throw BedrockServiceError.notImplemented( - "Role \(unknownRole) is not implemented by BedrockRuntimeClientTypes" - ) - } - } - - func getSDKConversationRole() -> BedrockRuntimeClientTypes.ConversationRole { - switch self { - case .user: return .user - case .assistant: return .assistant - } - } -} diff --git a/backend/Sources/BedrockService/InvokeModel/ImageGeneration.swift b/backend/Sources/BedrockService/InvokeModel/ImageGeneration.swift new file mode 100644 index 00000000..e695cfb1 --- /dev/null +++ b/backend/Sources/BedrockService/InvokeModel/ImageGeneration.swift @@ -0,0 +1,105 @@ +// //===----------------------------------------------------------------------===// +// // +// // This source file is part of the Swift Foundation Models Playground open source project +// // +// // Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// // and the Swift Foundation Models Playground project authors +// // Licensed under Apache License v2.0 +// // +// // See LICENSE.txt for license information +// // See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// // +// // SPDX-License-Identifier: Apache-2.0 +// // +// //===----------------------------------------------------------------------===// + +// @preconcurrency import AWSBedrockRuntime +// import BedrockTypes +// import Foundation + +// extension BedrockService { + +// /// Generates 1 to 5 image(s) from a text prompt using a specific model. +// /// - Parameters: +// /// - prompt: the prompt describing the image that should be generated +// /// - model: the BedrockModel that will be used to generate the image +// /// - nrOfImages: the number of images that will be generated (must be a number between 1 and 5) optional, default 3 +// /// - Throws: BedrockServiceError.invalidNrOfImages if nrOfImages is not between 1 and 5 +// /// BedrockServiceError.invalidPrompt if the prompt is empty +// /// BedrockServiceError.invalidResponse if the response body is missing +// /// - Returns: a ImageGenerationOutput object containing an array of generated images +// public func generateImage( +// _ prompt: String, +// with model: BedrockModel, +// negativePrompt: String? = nil, +// nrOfImages: Int? = nil, +// cfgScale: Double? = nil, +// seed: Int? = nil, +// quality: ImageQuality? = nil, +// resolution: ImageResolution? = nil +// ) async throws -> ImageGenerationOutput { +// logger.trace( +// "Generating image(s)", +// metadata: [ +// "model.id": .string(model.id), +// "model.modality": .string(model.modality.getName()), +// "prompt": .string(prompt), +// "negativePrompt": .stringConvertible(negativePrompt ?? "not defined"), +// "nrOfImages": .stringConvertible(nrOfImages ?? "not defined"), +// "cfgScale": .stringConvertible(cfgScale ?? "not defined"), +// "seed": .stringConvertible(seed ?? "not defined"), +// ] +// ) +// do { +// let modality = try model.getImageModality() +// try validateImageGenerationParams( +// modality: modality, +// nrOfImages: nrOfImages, +// cfgScale: cfgScale, +// resolution: resolution, +// seed: seed +// ) +// let textToImageModality = try model.getTextToImageModality() +// try validateTextToImageParams(modality: textToImageModality, prompt: prompt, negativePrompt: negativePrompt) + +// let request: InvokeModelRequest = try InvokeModelRequest.createTextToImageRequest( +// model: model, +// prompt: prompt, +// negativeText: negativePrompt, +// nrOfImages: nrOfImages, +// cfgScale: cfgScale, +// seed: seed, +// quality: quality, +// resolution: resolution +// ) +// let input: InvokeModelInput = try request.getInvokeModelInput() +// logger.trace( +// "Sending request to invokeModel", +// metadata: [ +// "model": .string(model.id), "request": .string(String(describing: input)), +// ] +// ) +// let response = try await self.bedrockRuntimeClient.invokeModel(input: input) +// guard let responseBody = response.body else { +// logger.trace( +// "Invalid response", +// metadata: [ +// "response": .string(String(describing: response)), +// "hasBody": .stringConvertible(response.body != nil), +// ] +// ) +// throw BedrockServiceError.invalidSDKResponse( +// "Something went wrong while extracting body from response." +// ) +// } +// let invokemodelResponse: InvokeModelResponse = try InvokeModelResponse.createImageResponse( +// body: responseBody, +// model: model +// ) +// return try invokemodelResponse.getGeneratedImage() +// } catch { +// logger.trace("Error while generating image", metadata: ["error": "\(error)"]) +// throw error +// } +// } +// } diff --git a/backend/Sources/BedrockService/ParameterValidation.swift b/backend/Sources/BedrockService/ParameterValidation.swift index a097c44a..65259064 100644 --- a/backend/Sources/BedrockService/ParameterValidation.swift +++ b/backend/Sources/BedrockService/ParameterValidation.swift @@ -13,20 +13,29 @@ // //===----------------------------------------------------------------------===// -import Foundation import BedrockTypes +import Foundation extension BedrockService { - /// Validate parameters for a text completion request + /// Validates text completion parameters against the model's capabilities and constraints + /// - Parameters: + /// - modality: The text modality of the model + /// - prompt: The input text prompt to validate + /// - maxTokens: Optional maximum number of tokens to generate + /// - temperature: Optional temperature parameter for controlling randomness + /// - topP: Optional top-p parameter for nucleus sampling + /// - topK: Optional top-k parameter for filtering + /// - stopSequences: Optional array of sequences where generation should stop + /// - Throws: BedrockServiceError for invalid parameters public func validateTextCompletionParams( modality: any TextModality, - prompt: String?, - maxTokens: Int?, - temperature: Double?, - topP: Double?, - topK: Int?, - stopSequences: [String]? + prompt: String? = nil, + maxTokens: Int? = nil, + temperature: Double? = nil, + topP: Double? = nil, + topK: Int? = nil, + stopSequences: [String]? = nil ) throws { let parameters = modality.getParameters() if maxTokens != nil { @@ -49,13 +58,20 @@ extension BedrockService { } } - /// Validate parameters for an image generation request + /// Validates image generation parameters against the model's capabilities and constraints + /// - Parameters: + /// - modality: The image modality of the model + /// - nrOfImages: Optional number of images to generate + /// - cfgScale: Optional classifier free guidance scale + /// - resolution: Optional image resolution settings + /// - seed: Optional seed for reproducible generation + /// - Throws: BedrockServiceError for invalid parameters public func validateImageGenerationParams( modality: any ImageModality, - nrOfImages: Int?, - cfgScale: Double?, - resolution: ImageResolution?, - seed: Int? + nrOfImages: Int? = nil, + cfgScale: Double? = nil, + resolution: ImageResolution? = nil, + seed: Int? = nil ) throws { let parameters = modality.getParameters() if nrOfImages != nil { @@ -72,11 +88,16 @@ extension BedrockService { } } - /// Validate specific parameters for a text to image request + /// Validates parameters for text-to-image generation requests + /// - Parameters: + /// - modality: The text-to-image modality of the model to use + /// - prompt: The input text prompt describing the desired image + /// - negativePrompt: Optional text describing what to avoid in the generated image + /// - Throws: BedrockServiceError if the parameters are invalid or exceed model constraints public func validateTextToImageParams( modality: any TextToImageModality, prompt: String, - negativePrompt: String? + negativePrompt: String? = nil ) throws { let textToImageParameters = modality.getTextToImageParameters() try validatePrompt(prompt, maxPromptTokens: textToImageParameters.prompt.maxSize) @@ -85,13 +106,20 @@ extension BedrockService { } } - /// Validate specific parameters for an image variation request + /// Validates image variation generation parameters + /// - Parameters: + /// - modality: The image variation modality of the model + /// - images: Array of base64 encoded images to use as reference + /// - prompt: Text prompt describing desired variations + /// - similarity: Optional parameter controlling variation similarity + /// - negativePrompt: Optional text describing what to avoid + /// - Throws: BedrockServiceError for invalid parameters public func validateImageVariationParams( modality: any ImageVariationModality, images: [String], - prompt: String?, - similarity: Double?, - negativePrompt: String? + prompt: String? = nil, + similarity: Double? = nil, + negativePrompt: String? = nil ) throws { let imageVariationParameters = modality.getImageVariationParameters() try validateParameterValue(images.count, parameter: imageVariationParameters.images) @@ -106,18 +134,26 @@ extension BedrockService { } } - /// Validate parameters for a converse request + /// Validates conversation parameters for the model + /// - Parameters: + /// - modality: The conversation modality of the model + /// - maxTokens: Optional maximum number of tokens to generate + /// - temperature: Optional temperature parameter for controlling randomness + /// - topP: Optional top-p parameter for nucleus sampling + /// - stopSequences: Optional array of sequences where generation should stop + /// - Throws: BedrockServiceError for invalid parameters public func validateConverseParams( - modality: any TextModality, - prompt: String, - history: [Message], - maxTokens: Int?, - temperature: Double?, - topP: Double?, - stopSequences: [String]? + modality: any ConverseModality, + prompt: String? = nil, + maxTokens: Int? = nil, + temperature: Double? = nil, + topP: Double? = nil, + stopSequences: [String]? = nil ) throws { - let parameters = modality.getParameters() - try validatePrompt(prompt, maxPromptTokens: parameters.prompt.maxSize) + let parameters = modality.getConverseParameters() + if prompt != nil { + try validatePrompt(prompt!, maxPromptTokens: parameters.prompt.maxSize) + } if maxTokens != nil { try validateParameterValue(maxTokens!, parameter: parameters.maxTokens) } @@ -177,7 +213,7 @@ extension BedrockService { metadata: [ "parameter": "\(parameter.name)", "value": "\(value)", "value.min": "\(String(describing: parameter.minValue))", - "value.max": "\(String(describing: parameter.maxValue))" + "value.max": "\(String(describing: parameter.maxValue))", ] ) } @@ -205,6 +241,13 @@ extension BedrockService { ) } } + logger.trace( + "Valid prompt", + metadata: [ + "prompt": "\(prompt)", + "prompt.maxPromptTokens": "\(String(describing: maxPromptTokens))", + ] + ) } /// Validate that not more stopsequences than allowed were given @@ -225,5 +268,12 @@ extension BedrockService { ) } } + logger.trace( + "Valid stop sequences", + metadata: [ + "stopSequences": "\(stopSequences)", + "stopSequences.maxNrOfStopSequences": "\(String(describing: maxNrOfStopSequences))", + ] + ) } } diff --git a/backend/Sources/BedrockTypes/BedrockModel.swift b/backend/Sources/BedrockTypes/BedrockModel.swift index fafea5b3..0d3d778a 100644 --- a/backend/Sources/BedrockTypes/BedrockModel.swift +++ b/backend/Sources/BedrockTypes/BedrockModel.swift @@ -69,13 +69,17 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { self = BedrockModel.titan_text_g1_express case BedrockModel.titan_text_g1_lite.id: self = BedrockModel.titan_text_g1_lite - // nova - case BedrockModel.nova_micro.id: - self = BedrockModel.nova_micro case BedrockModel.titan_image_g1_v2.id: self = BedrockModel.titan_image_g1_v2 case BedrockModel.titan_image_g1_v1.id: self = BedrockModel.titan_image_g1_v1 + // nova + case BedrockModel.nova_micro.id: + self = BedrockModel.nova_micro + case BedrockModel.nova_lite.id: + self = BedrockModel.nova_lite + case BedrockModel.nova_pro.id: + self = BedrockModel.nova_pro case BedrockModel.nova_canvas.id: self = BedrockModel.nova_canvas // deepseek @@ -89,11 +93,23 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { case BedrockModel.llama3_2_1b_instruct.id: self = BedrockModel.llama3_2_1b_instruct case BedrockModel.llama3_2_3b_instruct.id: self = BedrockModel.llama3_2_3b_instruct case BedrockModel.llama3_3_70b_instruct.id: self = BedrockModel.llama3_3_70b_instruct + // mistral + case BedrockModel.mistral_large_2402.id: self = BedrockModel.mistral_large_2402 + case BedrockModel.mistral_small_2402.id: self = BedrockModel.mistral_small_2402 + case BedrockModel.mistral_7B_instruct.id: self = BedrockModel.mistral_7B_instruct + case BedrockModel.mistral_8x7B_instruct.id: self = BedrockModel.mistral_8x7B_instruct + //cohere + // case BedrockModel.cohere_command_R_plus.id: self = BedrockModel.cohere_command_R_plus + // case BedrockModel.cohere_command_R.id: self = BedrockModel.cohere_command_R default: return nil } } + // MARK: Modality checks + + // MARK - Text completion + /// Checks if the model supports text generation /// - Returns: True if the model supports text generation public func hasTextModality() -> Bool { @@ -113,6 +129,8 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return textModality } + // MARK - Image generation + /// Checks if the model supports image generation /// - Returns: True if the model supports image generation public func hasImageModality() -> Bool { @@ -132,6 +150,12 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return imageModality } + /// Checks if the model supports text to image generation + /// - Returns: True if the model supports text to image generation + public func hasTextToImageModality() -> Bool { + modality as? any TextToImageModality != nil + } + /// Checks if the model supports text to image generation and returns TextToImageModality /// - Returns: TextToImageModality if the model supports image modality public func getTextToImageModality() throws -> any TextToImageModality { @@ -145,6 +169,12 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return textToImageModality } + /// Checks if the model supports image variation + /// - Returns: True if the model supports image variation + public func hasImageVariationModality() -> Bool { + modality as? any ImageVariationModality != nil + } + /// Checks if the model supports image variation and returns ImageVariationModality /// - Returns: ImageVariationModality if the model supports image modality public func getImageVariationModality() throws -> any ImageVariationModality { @@ -158,23 +188,44 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return modality } - /// Checks if the model supports text to image generation - /// - Returns: True if the model supports text to image generation - public func hasTextToImageModality() -> Bool { - modality as? any TextToImageModality != nil - } - - /// Checks if the model supports image variation - /// - Returns: True if the model supports image variation - public func hasImageVariationModality() -> Bool { - modality as? any ImageVariationModality != nil - } - /// Checks if the model supports conditioned text to image generation /// - Returns: True if the model supports conditioned text to image generation public func hasConditionedTextToImageModality() -> Bool { modality as? any ConditionedTextToImageModality != nil } + + // MARK - Converse + + /// Checks if the model supports converse + /// - Returns: True if the model supports converse + public func hasConverseModality() -> Bool { + modality as? any ConverseModality != nil + } + + /// Checks if the model supports a specific converse feature + /// - Parameters: + /// - feature: the ConverseFeature that will be checked + /// - Returns: True if the model supports the converse feature + public func hasConverseModality(_ feature: ConverseFeature = .textGeneration) -> Bool { + if let converseModality = modality as? any ConverseModality { + let features = converseModality.getConverseFeatures() + return features.contains(feature) + } + return false + } + + /// Checks if the model supports text generation and returns ConverseModality + /// - Returns: ConverseModality if the model supports text modality + public func getConverseModality() throws -> any ConverseModality { + guard let modality = modality as? any ConverseModality else { + throw BedrockServiceError.invalidModality( + self, + modality, + "Model \(id) does not support text generation" + ) + } + return modality + } } extension BedrockModel: Encodable { diff --git a/backend/Sources/BedrockTypes/BedrockServiceError.swift b/backend/Sources/BedrockTypes/BedrockServiceError.swift index e5c56fef..28ab1d5c 100644 --- a/backend/Sources/BedrockTypes/BedrockServiceError.swift +++ b/backend/Sources/BedrockTypes/BedrockServiceError.swift @@ -18,9 +18,9 @@ import Foundation public enum BedrockServiceError: Error { case invalidParameter(ParameterName, String) case invalidModality(BedrockModel, Modality, String) - // case invalidModel(BedrockModel, String) case invalidPrompt(String) case invalidStopSequences([String], String) + case invalidURI(String) case invalidSDKResponse(String) case invalidSDKResponseBody(Data?) case completionNotFound(String) diff --git a/backend/Sources/BedrockTypes/Converse/Content.swift b/backend/Sources/BedrockTypes/Converse/Content.swift new file mode 100644 index 00000000..ac7000c0 --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/Content.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public enum Content: Codable { + case text(String) + case image(ImageBlock) + case toolUse(ToolUseBlock) + case toolResult(ToolResultBlock) + case document(DocumentBlock) + case video(VideoBlock) + // case reasoningcontent(ReasoningBlock) + + public init(from sdkContentBlock: BedrockRuntimeClientTypes.ContentBlock) throws { + switch sdkContentBlock { + case .text(let text): + self = .text(text) + case .image(let sdkImage): + self = .image(try ImageBlock(from: sdkImage)) + case .document(let sdkDocumentBlock): + self = .document(try DocumentBlock(from: sdkDocumentBlock)) + case .tooluse(let sdkToolUseBlock): + self = .toolUse(try ToolUseBlock(from: sdkToolUseBlock)) + case .toolresult(let sdkToolResultBlock): + self = .toolResult(try ToolResultBlock(from: sdkToolResultBlock)) + case .video(let sdkVideoBlock): + self = .video(try VideoBlock(from: sdkVideoBlock)) + case .sdkUnknown(let unknownContentBlock): + throw BedrockServiceError.notImplemented( + "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" + ) + default: + throw BedrockServiceError.notImplemented( + "\(sdkContentBlock.self) is not implemented by this library" + ) + } + } + + public func getSDKContentBlock() throws -> BedrockRuntimeClientTypes.ContentBlock { + switch self { + case .text(let text): + return BedrockRuntimeClientTypes.ContentBlock.text(text) + case .image(let imageBlock): + return BedrockRuntimeClientTypes.ContentBlock.image(try imageBlock.getSDKImageBlock()) + case .document(let documentBlock): + return BedrockRuntimeClientTypes.ContentBlock.document(try documentBlock.getSDKDocumentBlock()) + case .toolResult(let toolResultBlock): + return BedrockRuntimeClientTypes.ContentBlock.toolresult(try toolResultBlock.getSDKToolResultBlock()) + case .toolUse(let toolUseBlock): + return BedrockRuntimeClientTypes.ContentBlock.tooluse(try toolUseBlock.getSDKToolUseBlock()) + case .video(let videoBlock): + return BedrockRuntimeClientTypes.ContentBlock.video(try videoBlock.getSDKVideoBlock()) + // default: + // print("TODO") + // return BedrockRuntimeClientTypes.ContentBlock.text("TODO") + } + } +} diff --git a/backend/Sources/BedrockTypes/Converse/DocumentBlock.swift b/backend/Sources/BedrockTypes/Converse/DocumentBlock.swift new file mode 100644 index 00000000..8b7b6798 --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/DocumentBlock.swift @@ -0,0 +1,113 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public struct DocumentBlock: Codable { + public let name: String + public let format: Format + public let source: String // 64 encoded + + public init(name: String, format: Format, source: String) { + self.name = name + self.format = format + self.source = source + } + + public init(from sdkDocumentBlock: BedrockRuntimeClientTypes.DocumentBlock) throws { + guard let sdkDocumentSource = sdkDocumentBlock.source else { + throw BedrockServiceError.decodingError( + "Could not extract source from BedrockRuntimeClientTypes.DocumentBlock" + ) + } + guard let name = sdkDocumentBlock.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.DocumentBlock" + ) + } + guard let sdkFormat = sdkDocumentBlock.format else { + throw BedrockServiceError.decodingError( + "Could not extract format from BedrockRuntimeClientTypes.DocumentBlock" + ) + } + let format = try DocumentBlock.Format(from: sdkFormat) + switch sdkDocumentSource { + case .bytes(let data): + self = DocumentBlock(name: name, format: format, source: data.base64EncodedString()) + case .sdkUnknown(let unknownImageSource): + throw BedrockServiceError.notImplemented( + "ImageSource \(unknownImageSource) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + public func getSDKDocumentBlock() throws -> BedrockRuntimeClientTypes.DocumentBlock { + guard let data = Data(base64Encoded: source) else { + throw BedrockServiceError.decodingError( + "Could not decode document source from base64 string. String: \(source)" + ) + } + return BedrockRuntimeClientTypes.DocumentBlock( + format: format.getSDKDocumentFormat(), + name: name, + source: BedrockRuntimeClientTypes.DocumentSource.bytes(data) + ) + } + + public enum Format: Codable { + case csv + case doc + case docx + case html + case md + case pdf + case txt + case xls + case xlsx + + public init(from sdkDocumentFormat: BedrockRuntimeClientTypes.DocumentFormat) throws { + switch sdkDocumentFormat { + case .csv: self = .csv + case .doc: self = .doc + case .docx: self = .docx + case .html: self = .html + case .md: self = .md + case .pdf: self = .pdf + case .txt: self = .txt + case .xls: self = .xls + case .xlsx: self = .xlsx + case .sdkUnknown(let unknownDocumentFormat): + throw BedrockServiceError.notImplemented( + "DocumentFormat \(unknownDocumentFormat) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + public func getSDKDocumentFormat() -> BedrockRuntimeClientTypes.DocumentFormat { + switch self { + case .csv: return .csv + case .doc: return .doc + case .docx: return .docx + case .html: return .html + case .md: return .md + case .pdf: return .pdf + case .txt: return .txt + case .xls: return .xls + case .xlsx: return .xlsx + } + } + } +} diff --git a/backend/Sources/BedrockTypes/Converse/DocumentToJSON.swift b/backend/Sources/BedrockTypes/Converse/DocumentToJSON.swift new file mode 100644 index 00000000..a1545eca --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/DocumentToJSON.swift @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation +import Smithy + +// FIXME: avoid extensions on structs you do not control +extension SmithyDocument { + public func toJSON() throws -> JSON { + switch self.type { + case .string: + return JSON(try self.asString()) + case .boolean: + return JSON(try self.asBoolean()) + case .integer: + return JSON(try self.asInteger()) + case .double, .float: + return JSON(try self.asDouble()) + case .list: + let array = try self.asList().map { try $0.toJSON() } + return JSON(array) + case .map: + let map = try self.asStringMap() + var result: [String: JSON] = [:] + for (key, value) in map { + result[key] = try value.toJSON() + } + return JSON(result) + case .blob: + let data = try self.asBlob() + return JSON(data) + default: + throw DocumentError.typeMismatch("Unsupported type for JSON conversion: \(self.type)") + } + } +} diff --git a/backend/Sources/BedrockTypes/Converse/ImageBlock.swift b/backend/Sources/BedrockTypes/Converse/ImageBlock.swift new file mode 100644 index 00000000..c56d6444 --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/ImageBlock.swift @@ -0,0 +1,90 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public struct ImageBlock: Codable { + public let format: Format + public let source: String // 64 encoded + + public init(format: Format, source: String) { + self.format = format + self.source = source + } + + public init(from sdkImageBlock: BedrockRuntimeClientTypes.ImageBlock) throws { + guard let sdkFormat = sdkImageBlock.format else { + throw BedrockServiceError.decodingError( + "Could not extract format from BedrockRuntimeClientTypes.ImageBlock" + ) + } + guard let sdkImageSource = sdkImageBlock.source else { + throw BedrockServiceError.decodingError( + "Could not extract source from BedrockRuntimeClientTypes.ImageBlock" + ) + } + let format = try ImageBlock.Format(from: sdkFormat) + switch sdkImageSource { + case .bytes(let data): + self = ImageBlock(format: format, source: data.base64EncodedString()) + case .sdkUnknown(let unknownImageSource): + throw BedrockServiceError.notImplemented( + "ImageSource \(unknownImageSource) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + public func getSDKImageBlock() throws -> BedrockRuntimeClientTypes.ImageBlock { + guard let data = Data(base64Encoded: source) else { + throw BedrockServiceError.decodingError( + "Could not decode image source from base64 string. String: \(source)" + ) + } + return BedrockRuntimeClientTypes.ImageBlock( + format: format.getSDKImageFormat(), + source: BedrockRuntimeClientTypes.ImageSource.bytes(data) + ) + } + + public enum Format: Codable { + case gif + case jpeg + case png + case webp + + public init(from sdkImageFormat: BedrockRuntimeClientTypes.ImageFormat) throws { + switch sdkImageFormat { + case .gif: self = .gif + case .jpeg: self = .jpeg + case .png: self = .png + case .webp: self = .webp + case .sdkUnknown(let unknownImageFormat): + throw BedrockServiceError.notImplemented( + "ImageFormat \(unknownImageFormat) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + public func getSDKImageFormat() -> BedrockRuntimeClientTypes.ImageFormat { + switch self { + case .gif: return .gif + case .jpeg: return .jpeg + case .png: return .png + case .webp: return .webp + } + } + } +} diff --git a/backend/Sources/BedrockTypes/Converse/JSON.swift b/backend/Sources/BedrockTypes/Converse/JSON.swift new file mode 100644 index 00000000..edbe68e3 --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/JSON.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct JSON: Codable { + var value: Any? + + public init(_ value: Any?) { + self.value = value + } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { + self.value = nil + } else if let intValue = try? container.decode(Int.self) { + self.value = intValue + } else if let doubleValue = try? container.decode(Double.self) { + self.value = doubleValue + } else if let stringValue = try? container.decode(String.self) { + self.value = stringValue + } else if let boolValue = try? container.decode(Bool.self) { + self.value = boolValue + } else if let arrayValue = try? container.decode([JSON].self) { + self.value = arrayValue.map { $0.value } + } else if let dictionaryValue = try? container.decode([String: JSON].self) { + self.value = dictionaryValue.mapValues { $0.value } + } else { + throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported type") + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + if let jsonValue = value as? JSON { + try jsonValue.encode(to: encoder) + } else if let intValue = value as? Int { + try container.encode(intValue) + } else if let doubleValue = value as? Double { + try container.encode(doubleValue) + } else if let stringValue = value as? String { + try container.encode(stringValue) + } else if let boolValue = value as? Bool { + try container.encode(boolValue) + } else if let arrayValue = value as? [Any] { + let jsonArray = arrayValue.map { JSON($0) } + try container.encode(jsonArray) + } else if let dictionaryValue = value as? [String: Any] { + let jsonDictionary = dictionaryValue.mapValues { JSON($0) } + try container.encode(jsonDictionary) + } else { + // try container.encode(String(describing: value ?? "nil")) + throw EncodingError.invalidValue( + value ?? "nil", + EncodingError.Context(codingPath: encoder.codingPath, debugDescription: "Unsupported type") + ) + } + } +} diff --git a/backend/Sources/BedrockTypes/Content.swift b/backend/Sources/BedrockTypes/Converse/JSONtoDocument.swift similarity index 74% rename from backend/Sources/BedrockTypes/Content.swift rename to backend/Sources/BedrockTypes/Converse/JSONtoDocument.swift index 3fedbf9d..b29beb20 100644 --- a/backend/Sources/BedrockTypes/Content.swift +++ b/backend/Sources/BedrockTypes/Converse/JSONtoDocument.swift @@ -14,7 +14,12 @@ //===----------------------------------------------------------------------===// import Foundation +import Smithy -public enum Content: Codable { - case text(String) -} \ No newline at end of file +extension JSON { + public func toDocument() throws -> Document { + let encoder = JSONEncoder() + let encoded = try encoder.encode(self) + return try Document.make(from: encoded) + } +} diff --git a/backend/Sources/BedrockTypes/Converse/Message.swift b/backend/Sources/BedrockTypes/Converse/Message.swift new file mode 100644 index 00000000..af8967d2 --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/Message.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public struct Message: Codable { + public let role: Role + public let content: [Content] + + package init(from role: Role, content: [Content]) { + self.role = role + self.content = content + } + + /// convenience initializer for message with only a user prompt + public init(_ prompt: String) { + self.init(from: .user, content: [.text(prompt)]) + } + + /// convenience initializer for message with only a ToolResultBlock + public init(_ toolResult: ToolResultBlock) { + self.init(from: .user, content: [.toolResult(toolResult)]) + } + + /// convenience initializer for message with only an ImageBlock + public init(_ imageBlock: ImageBlock) { + self.init(from: .user, content: [.image(imageBlock)]) + } + + /// convenience initializer for message with an ImageBlock.Format and imageBytes + public init(imageFormat: ImageBlock.Format, imageBytes: String) { + self.init(from: .user, content: [.image(ImageBlock(format: imageFormat, source: imageBytes))]) + } + + /// convenience initializer for message with an ImageBlock and a user prompt + public init(prompt: String, imageBlock: ImageBlock) { + self.init(from: .user, content: [.text(prompt), .image(imageBlock)]) + } + + /// convenience initializer for message with a user prompt, an ImageBlock.Format and imageBytes + public init(prompt: String, imageFormat: ImageBlock.Format, imageBytes: String) { + self.init(from: .user, content: [.text(prompt), .image(ImageBlock(format: imageFormat, source: imageBytes))]) + } + + public init(from sdkMessage: BedrockRuntimeClientTypes.Message) throws { + guard let sdkRole = sdkMessage.role else { + throw BedrockServiceError.decodingError("Could not extract role from BedrockRuntimeClientTypes.Message") + } + guard let sdkContent = sdkMessage.content else { + throw BedrockServiceError.decodingError("Could not extract content from BedrockRuntimeClientTypes.Message") + } + let content: [Content] = try sdkContent.map { try Content(from: $0) } + self = Message(from: try Role(from: sdkRole), content: content) + } + + public func getSDKMessage() throws -> BedrockRuntimeClientTypes.Message { + let contentBlocks: [BedrockRuntimeClientTypes.ContentBlock] = try content.map { + content -> BedrockRuntimeClientTypes.ContentBlock in + return try content.getSDKContentBlock() + } + return BedrockRuntimeClientTypes.Message( + content: contentBlocks, + role: role.getSDKConversationRole() + ) + } +} diff --git a/backend/Sources/BedrockTypes/Converse/S3Location.swift b/backend/Sources/BedrockTypes/Converse/S3Location.swift new file mode 100644 index 00000000..d244cd51 --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/S3Location.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public struct S3Location: Codable, Sendable { + public let bucketOwner: String? + public let uri: String + + public init(bucketOwner: String? = nil, uri: String) { + self.bucketOwner = bucketOwner + self.uri = uri + } + + public init(from sdkS3Location: BedrockRuntimeClientTypes.S3Location) throws { + guard let uri = sdkS3Location.uri else { + throw BedrockServiceError.decodingError( + "Could not extract URI from BedrockRuntimeClientTypes.S3Location" + ) + } + guard uri.hasPrefix("") else { + throw BedrockServiceError.invalidURI("URI should start with \"s3://\". Your URI: \(uri)") + } + self = S3Location(bucketOwner: sdkS3Location.bucketOwner, uri: uri) + } + + public func getSDKS3Location() -> BedrockRuntimeClientTypes.S3Location { + BedrockRuntimeClientTypes.S3Location(bucketOwner: self.bucketOwner, uri: self.uri) + } +} diff --git a/backend/Sources/BedrockTypes/Converse/Tool.swift b/backend/Sources/BedrockTypes/Converse/Tool.swift new file mode 100644 index 00000000..b3707afa --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/Tool.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation +import Smithy + +public struct Tool: Codable { + public let name: String + public let inputSchema: JSON + public let description: String? + + public init(name: String, inputSchema: JSON, description: String? = nil) { + self.name = name + self.inputSchema = inputSchema + self.description = description + } + + public init(from sdkToolSpecification: BedrockRuntimeClientTypes.ToolSpecification) throws { + guard let name = sdkToolSpecification.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard let sdkInputSchema = sdkToolSpecification.inputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract inputSchema from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard case .json(let smithyDocument) = sdkInputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract JSON from BedrockRuntimeClientTypes.ToolSpecification.inputSchema" + ) + } + let inputSchema = try smithyDocument.toJSON() + self = Tool( + name: name, + inputSchema: inputSchema, + description: sdkToolSpecification.description + ) + } + + public func getSDKToolSpecification() throws -> BedrockRuntimeClientTypes.ToolSpecification { + BedrockRuntimeClientTypes.ToolSpecification( + description: description, + inputSchema: .json(try inputSchema.toDocument()), + name: name + ) + } +} diff --git a/backend/Sources/BedrockTypes/Converse/ToolResultBlock.swift b/backend/Sources/BedrockTypes/Converse/ToolResultBlock.swift new file mode 100644 index 00000000..d615c893 --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/ToolResultBlock.swift @@ -0,0 +1,170 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public struct ToolResultBlock: Codable { + public let id: String + public let content: [Content] + public let status: Status? // currently only supported by Anthropic Claude 3 models + + public init(id: String, content: [Content], status: Status? = nil) { + self.id = id + self.content = content + self.status = status + } + + public init(from sdkToolResultBlock: BedrockRuntimeClientTypes.ToolResultBlock) throws { + guard let sdkToolResultContent = sdkToolResultBlock.content else { + throw BedrockServiceError.decodingError( + "Could not extract content from BedrockRuntimeClientTypes.ToolResultBlock" + ) + } + guard let id = sdkToolResultBlock.toolUseId else { + throw BedrockServiceError.decodingError( + "Could not extract toolUseId from BedrockRuntimeClientTypes.ToolResultBlock" + ) + } + let sdkToolStatus: BedrockRuntimeClientTypes.ToolResultStatus? = sdkToolResultBlock.status + var status: Status? = nil + if let sdkToolStatus = sdkToolStatus { + status = try Status(from: sdkToolStatus) + } + let toolContents = try sdkToolResultContent.map { try Content(from: $0) } + self = ToolResultBlock(id: id, content: toolContents, status: status) + } + + public func getSDKToolResultBlock() throws -> BedrockRuntimeClientTypes.ToolResultBlock { + BedrockRuntimeClientTypes.ToolResultBlock( + content: try content.map { try $0.getSDKToolResultContentBlock() }, + status: status?.getSDKToolStatus(), + toolUseId: id + ) + } + + public enum Status: Codable { + case success + case error + + init(from sdkToolStatus: BedrockRuntimeClientTypes.ToolResultStatus) throws { + switch sdkToolStatus { + case .success: self = .success + case .error: self = .error + case .sdkUnknown(let unknownToolStatus): + throw BedrockServiceError.notImplemented( + "ToolResultStatus \(unknownToolStatus) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKToolStatus() -> BedrockRuntimeClientTypes.ToolResultStatus { + switch self { + case .success: .success + case .error: .error + } + } + } + + public enum Content { + case json(JSON) + case text(String) + case image(ImageBlock) // currently only supported by Anthropic Claude 3 models + case document(DocumentBlock) + case video(VideoBlock) + + init(from sdkToolResultContent: BedrockRuntimeClientTypes.ToolResultContentBlock) throws { + switch sdkToolResultContent { + case .document(let sdkDocumentBlock): + self = .document(try DocumentBlock(from: sdkDocumentBlock)) + case .image(let sdkImageBlock): + self = .image(try ImageBlock(from: sdkImageBlock)) + case .text(let text): + self = .text(text) + case .video(let sdkVideoBlock): + self = .video(try VideoBlock(from: sdkVideoBlock)) + case .json(let document): + self = .json(try document.toJSON()) + case .sdkUnknown(let unknownToolResultContent): + throw BedrockServiceError.notImplemented( + "ToolResultContentBlock \(unknownToolResultContent) is not implemented by BedrockRuntimeClientTypes" + ) + // default: + // throw BedrockServiceError.notImplemented( + // "ToolResultContentBlock \(sdkToolResultContent) is not implemented by BedrockTypes" + // ) + } + } + + func getSDKToolResultContentBlock() throws -> BedrockRuntimeClientTypes.ToolResultContentBlock { + switch self { + case .json(let json): + .json(try json.toDocument()) + case .document(let documentBlock): + .document(try documentBlock.getSDKDocumentBlock()) + case .image(let imageBlock): + .image(try imageBlock.getSDKImageBlock()) + case .text(let text): + .text(text) + case .video(let videoBlock): + .video(try videoBlock.getSDKVideoBlock()) + } + } + } +} + +extension ToolResultBlock.Content: Codable { + private enum CodingKeys: String, CodingKey { + case json, text, image, document, video + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .json(let json): + try container.encode(json, forKey: .json) + case .text(let text): + try container.encode(text, forKey: .text) + case .image(let image): + try container.encode(image, forKey: .image) + case .document(let doc): + try container.encode(doc, forKey: .document) + case .video(let video): + try container.encode(video, forKey: .video) + } + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let json = try container.decodeIfPresent(JSON.self, forKey: .json) { + self = .json(json) + } else if let text = try container.decodeIfPresent(String.self, forKey: .text) { + self = .text(text) + } else if let image = try container.decodeIfPresent(ImageBlock.self, forKey: .image) { + self = .image(image) + } else if let doc = try container.decodeIfPresent(DocumentBlock.self, forKey: .document) { + self = .document(doc) + } else if let video = try container.decodeIfPresent(VideoBlock.self, forKey: .video) { + self = .video(video) + } else { + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Invalid tool result content" + ) + ) + } + } +} diff --git a/backend/Sources/BedrockTypes/Converse/ToolUseBlock.swift b/backend/Sources/BedrockTypes/Converse/ToolUseBlock.swift new file mode 100644 index 00000000..c736233f --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/ToolUseBlock.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public struct ToolUseBlock: Codable { + public let id: String + public let name: String + public let input: JSON + + public init(id: String, name: String, input: JSON) { + self.id = id + self.name = name + self.input = input + } + + public init(from sdkToolUseBlock: BedrockRuntimeClientTypes.ToolUseBlock) throws { + guard let sdkId = sdkToolUseBlock.toolUseId else { + throw BedrockServiceError.decodingError( + "Could not extract toolUseId from BedrockRuntimeClientTypes.ToolUseBlock" + ) + } + guard let sdkName = sdkToolUseBlock.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.ToolUseBlock" + ) + } + guard let sdkInput = sdkToolUseBlock.input else { + throw BedrockServiceError.decodingError( + "Could not extract input from BedrockRuntimeClientTypes.ToolUseBlock" + ) + } + self = ToolUseBlock( + id: sdkId, + name: sdkName, + input: try sdkInput.toJSON() + ) + } + + public func getSDKToolUseBlock() throws -> BedrockRuntimeClientTypes.ToolUseBlock { + .init( + input: try input.toDocument(), + name: name, + toolUseId: id + ) + } +} diff --git a/backend/Sources/BedrockTypes/Converse/VideoBlock.swift b/backend/Sources/BedrockTypes/Converse/VideoBlock.swift new file mode 100644 index 00000000..cafb711a --- /dev/null +++ b/backend/Sources/BedrockTypes/Converse/VideoBlock.swift @@ -0,0 +1,131 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import Foundation + +public struct VideoBlock: Codable { + public let format: Format + public let source: Source + + public init(format: Format, source: Source) { + self.format = format + self.source = source + } + + public init(from sdkVideoBlock: BedrockRuntimeClientTypes.VideoBlock) throws { + guard let sdkFormat = sdkVideoBlock.format else { + throw BedrockServiceError.decodingError( + "Could not extract format from BedrockRuntimeClientTypes.VideoBlock" + ) + } + guard let sdkSource = sdkVideoBlock.source else { + throw BedrockServiceError.decodingError( + "Could not extract source from BedrockRuntimeClientTypes.VideoBlock" + ) + } + self = VideoBlock( + format: try VideoBlock.Format(from: sdkFormat), + source: try VideoBlock.Source(from: sdkSource) + ) + } + + public func getSDKVideoBlock() throws -> BedrockRuntimeClientTypes.VideoBlock { + BedrockRuntimeClientTypes.VideoBlock( + format: try format.getSDKVideoFormat(), + source: try source.getSDKVideoSource() + ) + } + + public enum Source: Codable { + case bytes(String) // base64 + case s3(S3Location) + + public init(from sdkVideoSource: BedrockRuntimeClientTypes.VideoSource) throws { + switch sdkVideoSource { + case .bytes(let data): + self = .bytes(data.base64EncodedString()) + case .s3location(let sdkS3Location): + self = .s3(try S3Location(from: sdkS3Location)) + case .sdkUnknown(let unknownVideoSource): + throw BedrockServiceError.notImplemented( + "VideoSource \(unknownVideoSource) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + public func getSDKVideoSource() throws -> BedrockRuntimeClientTypes.VideoSource { + switch self { + case .bytes(let data): + guard let sdkData = Data(base64Encoded: data) else { + throw BedrockServiceError.decodingError( + "Could not decode video source from base64 string. String: \(data)" + ) + } + return .bytes(sdkData) + case .s3(let s3Location): + return .s3location(s3Location.getSDKS3Location()) + } + } + } + + public enum Format: Codable { + case flv + case mkv + case mov + case mp4 + case mpeg + case mpg + case threeGp + case webm + case wmv + + public init(from sdkVideoFormat: BedrockRuntimeClientTypes.VideoFormat) throws { + switch sdkVideoFormat { + case .flv: self = .flv + case .mkv: self = .mkv + case .mov: self = .mov + case .mp4: self = .mp4 + case .mpeg: self = .mpeg + case .mpg: self = .mpg + case .threeGp: self = .threeGp + case .webm: self = .webm + case .wmv: self = .wmv + case .sdkUnknown(let unknownVideoFormat): + throw BedrockServiceError.notImplemented( + "VideoFormat \(unknownVideoFormat) is not implemented by BedrockRuntimeClientTypes" + ) + // default: // in case new video formats get added to the sdk + // throw BedrockServiceError.notSupported( + // "VideoFormat \(sdkVideoFormat) is not supported by BedrockTypes" + // ) + } + } + + public func getSDKVideoFormat() throws -> BedrockRuntimeClientTypes.VideoFormat { + switch self { + case .flv: return .flv + case .mkv: return .mkv + case .mov: return .mov + case .mp4: return .mp4 + case .mpeg: return .mpeg + case .mpg: return .mpg + case .threeGp: return .threeGp + case .webm: return .webm + case .wmv: return .wmv + } + } + } +} diff --git a/backend/Sources/BedrockTypes/ContentType.swift b/backend/Sources/BedrockTypes/InvokeModel/ContentType.swift similarity index 100% rename from backend/Sources/BedrockTypes/ContentType.swift rename to backend/Sources/BedrockTypes/InvokeModel/ContentType.swift diff --git a/backend/Sources/BedrockTypes/ImageGenerationOutput.swift b/backend/Sources/BedrockTypes/InvokeModel/ImageGenerationOutput.swift similarity index 100% rename from backend/Sources/BedrockTypes/ImageGenerationOutput.swift rename to backend/Sources/BedrockTypes/InvokeModel/ImageGenerationOutput.swift diff --git a/backend/Sources/BedrockTypes/ImageResolution.swift b/backend/Sources/BedrockTypes/InvokeModel/ImageResolution.swift similarity index 100% rename from backend/Sources/BedrockTypes/ImageResolution.swift rename to backend/Sources/BedrockTypes/InvokeModel/ImageResolution.swift diff --git a/backend/Sources/BedrockTypes/Protocols.swift b/backend/Sources/BedrockTypes/InvokeModel/Protocols.swift similarity index 100% rename from backend/Sources/BedrockTypes/Protocols.swift rename to backend/Sources/BedrockTypes/InvokeModel/Protocols.swift diff --git a/backend/Sources/BedrockTypes/TextCompletion.swift b/backend/Sources/BedrockTypes/InvokeModel/TextCompletion.swift similarity index 100% rename from backend/Sources/BedrockTypes/TextCompletion.swift rename to backend/Sources/BedrockTypes/InvokeModel/TextCompletion.swift diff --git a/backend/Sources/BedrockService/ModelSummary.swift b/backend/Sources/BedrockTypes/ListModels/ModelSummary.swift similarity index 94% rename from backend/Sources/BedrockService/ModelSummary.swift rename to backend/Sources/BedrockTypes/ListModels/ModelSummary.swift index da77cd27..4123174b 100644 --- a/backend/Sources/BedrockService/ModelSummary.swift +++ b/backend/Sources/BedrockTypes/ListModels/ModelSummary.swift @@ -15,9 +15,7 @@ @preconcurrency import AWSBedrock import Foundation -import BedrockTypes -// comment to explain difference public struct ModelSummary: Encodable { let modelName: String let providerName: String @@ -42,7 +40,9 @@ public struct ModelSummary: Encodable { throw BedrockServiceError.notFound("BedrockClientTypes.FoundationModelSummary does not have a modelArn") } guard let modelLifecycle = sdkModelSummary.modelLifecycle else { - throw BedrockServiceError.notFound("BedrockClientTypes.FoundationModelSummary does not have a modelLifecycle") + throw BedrockServiceError.notFound( + "BedrockClientTypes.FoundationModelSummary does not have a modelLifecycle" + ) } guard let sdkStatus = modelLifecycle.status else { throw BedrockServiceError.notFound( diff --git a/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift new file mode 100644 index 00000000..1c40e309 --- /dev/null +++ b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +// Text +public protocol ConverseModality: Modality { + var converseParameters: ConverseParameters { get } + var converseFeatures: [ConverseFeature] { get } + + func getConverseParameters() -> ConverseParameters + func getConverseFeatures() -> [ConverseFeature] +} + +// https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html +public enum ConverseFeature: String, Codable, Sendable { + case textGeneration = "text-generation" + case vision = "vision" + case document = "document" + case toolUse = "tool-use" + case systemPrompts = "system-prompts" +} + +// default implementation +extension ConverseModality { + + func getConverseParameters() -> ConverseParameters { + converseParameters + } + + func getConverseFeatures() -> [ConverseFeature] { + converseFeatures + } +} diff --git a/backend/Sources/BedrockTypes/Message.swift b/backend/Sources/BedrockTypes/Modalities/StandardConverse.swift similarity index 53% rename from backend/Sources/BedrockTypes/Message.swift rename to backend/Sources/BedrockTypes/Modalities/StandardConverse.swift index fa38b181..2687eba8 100644 --- a/backend/Sources/BedrockTypes/Message.swift +++ b/backend/Sources/BedrockTypes/Modalities/StandardConverse.swift @@ -15,16 +15,17 @@ import Foundation -public struct Message: Codable { - public let role: Role - public let content: [Content] +public struct StandardConverse: ConverseModality { + public func getName() -> String { "Standard Converse Modality" } - // public static func createTextMessage(from role: Role, text: String) -> Self { - // Message(from: role, content: [.text(text)]) - // } + public let converseParameters: ConverseParameters + public let converseFeatures: [ConverseFeature] - public init(from role: Role, content: [Content]) { - self.role = role - self.content = content + public init(parameters: ConverseParameters, features: [ConverseFeature]) { + self.converseParameters = parameters + self.converseFeatures = features } -} \ No newline at end of file + + public func getConverseParameters() -> ConverseParameters { converseParameters } + public func getConverseFeatures() -> [ConverseFeature] { converseFeatures } +} diff --git a/backend/Sources/BedrockTypes/Modalities/TextModality.swift b/backend/Sources/BedrockTypes/Modalities/TextModality.swift index 2995b795..82c12196 100644 --- a/backend/Sources/BedrockTypes/Modalities/TextModality.swift +++ b/backend/Sources/BedrockTypes/Modalities/TextModality.swift @@ -17,7 +17,6 @@ import Foundation public protocol TextModality: Modality { - init(parameters: TextGenerationParameters) func getParameters() -> TextGenerationParameters func getTextRequestBody( diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift index 8525b357..5386aa91 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift @@ -15,13 +15,17 @@ import Foundation -struct NovaText: TextModality { +struct NovaText: TextModality, ConverseModality { func getName() -> String { "Nova Text Generation" } - + let parameters: TextGenerationParameters + let converseFeatures: [ConverseFeature] + let converseParameters: ConverseParameters - init(parameters: TextGenerationParameters) { + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration]) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift index 2a624534..59cdb05d 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift @@ -22,7 +22,8 @@ typealias NovaMicro = NovaText extension BedrockModel { public static let nova_micro: BedrockModel = BedrockModel( - id: "amazon.nova-micro-v1:0", name: "Nova Micro", + id: "amazon.nova-micro-v1:0", + name: "Nova Micro", modality: NovaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), @@ -31,7 +32,38 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: nil, defaultValue: 50), stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .toolUse] + ) + ) + public static let nova_lite: BedrockModel = BedrockModel( + id: "amazon.nova-lite-v1:0", + name: "Nova Lite", + modality: NovaText( + parameters: TextGenerationParameters( + temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 5_000, defaultValue: 5_000), + topP: Parameter(.topP, minValue: 0, maxValue: 1.0, defaultValue: 0.9), + topK: Parameter(.topK, minValue: 0, maxValue: nil, defaultValue: 50), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .vision, .systemPrompts, .document, .toolUse] + ) + ) + public static let nova_pro: BedrockModel = BedrockModel( + id: "amazon.nova-pro-v1:0", + name: "Nova Pro", + modality: NovaText( + parameters: TextGenerationParameters( + temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 5_000, defaultValue: 5_000), + topP: Parameter(.topP, minValue: 0, maxValue: 1.0, defaultValue: 0.9), + topK: Parameter(.topK, minValue: 0, maxValue: nil, defaultValue: 50), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) } @@ -42,7 +74,8 @@ typealias NovaCanvas = AmazonImage extension BedrockModel { public static let nova_canvas: BedrockModel = BedrockModel( - id: "amazon.nova-canvas-v1:0", name: "Nova Canvas", + id: "amazon.nova-canvas-v1:0", + name: "Nova Canvas", modality: NovaCanvas( parameters: ImageGenerationParameters( nrOfImages: Parameter(.nrOfImages, minValue: 1, maxValue: 5, defaultValue: 1), diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift b/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift index 7ac6d4a5..109ab7b0 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift @@ -15,19 +15,27 @@ import Foundation -struct TitanText: TextModality { +struct TitanText: TextModality, ConverseModality { func getName() -> String { "Titan Text Generation" } let parameters: TextGenerationParameters + let converseParameters: ConverseParameters + let converseFeatures: [ConverseFeature] - init(parameters: TextGenerationParameters) { + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration, .document]) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { parameters } + func getConverseParameters() -> ConverseParameters { + ConverseParameters(textGenerationParameters: parameters) + } + func getTextRequestBody( prompt: String, maxTokens: Int?, diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift index 48560d83..a3c276ba 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift @@ -33,7 +33,8 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), maxPromptSize: nil - ) + ), + features: [.textGeneration] ) ) public static let titan_text_g1_express: BedrockModel = BedrockModel( diff --git a/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift b/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift index 092c1c2c..93eb94b2 100644 --- a/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift +++ b/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift @@ -15,19 +15,27 @@ import Foundation -struct AnthropicText: TextModality { +struct AnthropicText: TextModality, ConverseModality { let parameters: TextGenerationParameters + let converseParameters: ConverseParameters + let converseFeatures: [ConverseFeature] func getName() -> String { "Anthropic Text Generation" } - init(parameters: TextGenerationParameters) { + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration, .systemPrompts, .document]) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { parameters } + func getConverseParameters() -> ConverseParameters { + ConverseParameters(textGenerationParameters: parameters) + } + func getTextRequestBody( prompt: String, maxTokens: Int?, diff --git a/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift index f3d20831..dd9d2e90 100644 --- a/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift @@ -39,7 +39,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [] ) ) public static let claudev1: BedrockModel = BedrockModel( @@ -52,7 +53,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [] ) ) public static let claudev2: BedrockModel = BedrockModel( @@ -65,7 +67,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let claudev2_1: BedrockModel = BedrockModel( @@ -78,7 +81,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let claudev3_opus: BedrockModel = BedrockModel( @@ -91,7 +95,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_haiku: BedrockModel = BedrockModel( @@ -104,7 +109,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_5_haiku: BedrockModel = BedrockModel( @@ -117,7 +123,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] ) ) public static let claudev3_5_sonnet: BedrockModel = BedrockModel( @@ -130,7 +137,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_5_sonnet_v2: BedrockModel = BedrockModel( @@ -143,7 +151,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_7_sonnet: BedrockModel = BedrockModel( @@ -156,7 +165,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) } diff --git a/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift b/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift index 4e65c4e5..44da21e9 100644 --- a/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift +++ b/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift @@ -15,13 +15,20 @@ import Foundation -struct DeepSeekText: TextModality { +struct DeepSeekText: TextModality, ConverseModality { let parameters: TextGenerationParameters + let converseFeatures: [ConverseFeature] + let converseParameters: ConverseParameters func getName() -> String { "DeepSeek Text Generation" } - init(parameters: TextGenerationParameters) { + init( + parameters: TextGenerationParameters, + features: [ConverseFeature] = [.textGeneration, .systemPrompts, .document] + ) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { diff --git a/backend/Sources/BedrockTypes/Models/Llama/Llama.swift b/backend/Sources/BedrockTypes/Models/Llama/Llama.swift index cfcf2d8c..a8fa1c16 100644 --- a/backend/Sources/BedrockTypes/Models/Llama/Llama.swift +++ b/backend/Sources/BedrockTypes/Models/Llama/Llama.swift @@ -15,13 +15,20 @@ import Foundation -struct LlamaText: TextModality { +struct LlamaText: TextModality, ConverseModality { func getName() -> String { "Llama Text Generation" } let parameters: TextGenerationParameters + let converseParameters: ConverseParameters + let converseFeatures: [ConverseFeature] - init(parameters: TextGenerationParameters) { + init( + parameters: TextGenerationParameters, + features: [ConverseFeature] = [.textGeneration, .systemPrompts, .document] + ) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { diff --git a/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift index 3e0a7de4..e0963a23 100644 --- a/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift @@ -19,7 +19,8 @@ import Foundation extension BedrockModel { public static let llama_3_8b_instruct: BedrockModel = BedrockModel( - id: "meta.llama3-8b-instruct-v1:0", name: "Llama 3 8B Instruct", + id: "meta.llama3-8b-instruct-v1:0", + name: "Llama 3 8B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -28,11 +29,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_70b_instruct: BedrockModel = BedrockModel( - id: "meta.llama3-70b-instruct-v1:0", name: "Llama 3 70B Instruct", + id: "meta.llama3-70b-instruct-v1:0", + name: "Llama 3 70B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -41,11 +44,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_1_8b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-1-8b-instruct-v1:0", name: "Llama 3.1 8B Instruct", + id: "us.meta.llama3-1-8b-instruct-v1:0", + name: "Llama 3.1 8B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -54,11 +59,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] ) ) public static let llama3_1_70b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-1-70b-instruct-v1:0", name: "Llama 3.1 70B Instruct", + id: "us.meta.llama3-1-70b-instruct-v1:0", + name: "Llama 3.1 70B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -67,11 +74,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] ) ) public static let llama3_2_1b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-2-1b-instruct-v1:0", name: "Llama 3.2 1B Instruct", + id: "us.meta.llama3-2-1b-instruct-v1:0", + name: "Llama 3.2 1B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -80,11 +89,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_2_3b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-2-3b-instruct-v1:0", name: "Llama 3.2 3B Instruct", + id: "us.meta.llama3-2-3b-instruct-v1:0", + name: "Llama 3.2 3B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -93,11 +104,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_3_70b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-3-70b-instruct-v1:0", name: "Llama 3.3 70B Instruct", + id: "us.meta.llama3-3-70b-instruct-v1:0", + name: "Llama 3.3 70B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -106,20 +119,8 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) - ) - ) - public static let llama3_8b_instruct: BedrockModel = BedrockModel( - id: "meta.llama3-8b-instruct-v1:0", name: "Llama 3 8B Instruct", - modality: LlamaText( - parameters: TextGenerationParameters( - temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), - maxTokens: Parameter(.maxTokens, minValue: 0, maxValue: 2_048, defaultValue: 512), - topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 0.9), - topK: Parameter.notSupported(.topK), - stopSequences: StopSequenceParams.notSupported(), - maxPromptSize: nil - ) + ), + features: [] ) ) } diff --git a/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift new file mode 100644 index 00000000..85aaef4e --- /dev/null +++ b/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +// MARK: converse only +// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral-text-completion.html +// https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html + +typealias MistralConverse = StandardConverse + +extension BedrockModel { + public static let mistral_large_2402 = BedrockModel( + id: "mistral.mistral-large-2402-v1:0", + name: "Mistral Large (24.02)", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 8_192, defaultValue: 8_192), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 1), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] + ) + ) + public static let mistral_small_2402 = BedrockModel( + id: "mistral.mistral-small-2402-v1:0", + name: "Mistral Small (24.02)", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 8_192, defaultValue: 8_192), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 1), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .toolUse] + ) + ) + public static let mistral_7B_instruct = BedrockModel( + id: "mistral.mistral-7b-instruct-v0:2", + name: "Mistral 7B Instruct", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 8_192, defaultValue: 512), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 0.9), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .document] + ) + ) + public static let mistral_8x7B_instruct = BedrockModel( + id: "mistral.mixtral-8x7b-instruct-v0:1", + name: "Mixtral 8x7B Instruct", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 4_096, defaultValue: 512), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 0.9), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .document] + ) + ) +} diff --git a/backend/Sources/BedrockTypes/Parameters/ConverseParameters.swift b/backend/Sources/BedrockTypes/Parameters/ConverseParameters.swift new file mode 100644 index 00000000..94859d12 --- /dev/null +++ b/backend/Sources/BedrockTypes/Parameters/ConverseParameters.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct ConverseParameters: Parameters { + public let temperature: Parameter + public let maxTokens: Parameter + public let topP: Parameter + public let prompt: PromptParams + public let stopSequences: StopSequenceParams + + public init( + temperature: Parameter, + maxTokens: Parameter, + topP: Parameter, + stopSequences: StopSequenceParams, + maxPromptSize: Int? + ) { + self.temperature = temperature + self.maxTokens = maxTokens + self.topP = topP + self.prompt = PromptParams(maxSize: maxPromptSize) + self.stopSequences = stopSequences + } + + public init(textGenerationParameters: TextGenerationParameters) { + self.temperature = textGenerationParameters.temperature + self.maxTokens = textGenerationParameters.maxTokens + self.topP = textGenerationParameters.topP + self.prompt = textGenerationParameters.prompt + self.stopSequences = textGenerationParameters.stopSequences + } +} diff --git a/backend/Sources/BedrockTypes/Role.swift b/backend/Sources/BedrockTypes/Role.swift index d2ce14d8..582e4de6 100644 --- a/backend/Sources/BedrockTypes/Role.swift +++ b/backend/Sources/BedrockTypes/Role.swift @@ -13,9 +13,28 @@ // //===----------------------------------------------------------------------===// +@preconcurrency import AWSBedrockRuntime import Foundation public enum Role: String, Codable { case user case assistant + + public init(from sdkConversationRole: BedrockRuntimeClientTypes.ConversationRole) throws { + switch sdkConversationRole { + case .user: self = .user + case .assistant: self = .assistant + case .sdkUnknown(let unknownRole): + throw BedrockServiceError.notImplemented( + "Role \(unknownRole) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + public func getSDKConversationRole() -> BedrockRuntimeClientTypes.ConversationRole { + switch self { + case .user: return .user + case .assistant: return .assistant + } + } } diff --git a/frontend/components/chatPlayground/ChatComponent.jsx b/frontend/components/chatPlayground/ChatComponent.jsx index 4383880f..9fe020de 100644 --- a/frontend/components/chatPlayground/ChatComponent.jsx +++ b/frontend/components/chatPlayground/ChatComponent.jsx @@ -8,6 +8,7 @@ import GlobalConfig from "@/app/app.config"; import ChatModelSelector from "./ChatModelSelector"; import { defaultModel } from "@/helpers/modelData"; import NumericInput from "../NumericInput"; +import Image from 'next/image'; export default function ChatContainer() { @@ -21,7 +22,8 @@ export default function ChatContainer() { const [topP, setTopP] = useState(0.9); const [stopSequences, setStopSequences] = useState([]); const [stopSequenceInput, setStopSequenceInput] = useState(''); - + const [referenceImage, setReferenceImage] = useState(null); + const [previewImage, setPreviewImage] = useState('/placeholder.png'); const endpoint = `/foundation-models/chat/${selectedModel.modelId}`; const api = `${GlobalConfig.apiHost}:${GlobalConfig.apiPort}${endpoint}`; @@ -66,6 +68,63 @@ export default function ChatContainer() { setHistory([]); }; + const handleFileUpload = async (event) => { + const file = event.target.files[0]; + if (file) { + // Create preview for the UI + const reader = new FileReader(); + reader.onload = (e) => { + setPreviewImage(e.target.result); + }; + reader.readAsDataURL(file); + + // Compress and store the image + const compressedBase64 = await compressImage(file); + console.log('Compressed image size:', compressedBase64.length); + setReferenceImage(compressedBase64); + } + }; + + const compressImage = (file) => { + return new Promise((resolve) => { + const reader = new FileReader(); + reader.onload = (event) => { + const img = document.createElement('img'); // Use regular img element instead of Next.js Image + img.onload = () => { + const canvas = document.createElement('canvas'); + // Reduce max dimensions to 512x512 + const maxSize = 512; + let width = img.width; + let height = img.height; + + if (width > height) { + if (width > maxSize) { + height *= maxSize / width; + width = maxSize; + } + } else { + if (height > maxSize) { + width *= maxSize / height; + height = maxSize; + } + } + + canvas.width = width; + canvas.height = height; + + const ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0, width, height); + + // Increase compression by reducing quality to 0.5 (50%) + const compressedBase64 = canvas.toDataURL('image/jpeg', 0.5); + resolve(compressedBase64.split(',')[1]); + }; + img.src = event.target.result; + }; + reader.readAsDataURL(file); + }); + }; + const sendMessage = async () => { const newMessage = { sender: "Human", message: inputValue }; setConversation(prevConversation => [...prevConversation, newMessage]); @@ -80,6 +139,7 @@ export default function ChatContainer() { body: JSON.stringify({ prompt: inputValue, history: history, + imageBytes: referenceImage, maxTokens: parseInt(maxTokens, 10), temperature: parseFloat(temperature), topP: parseFloat(topP), @@ -236,6 +296,7 @@ export default function ChatContainer() { {/* Input window */}
+ {/* Text input */}
+ {/* Reference image with preview */} +
+ + Reference image +
+ + {/* Send button */}
+
diff --git a/frontend/helpers/modelData.js b/frontend/helpers/modelData.js index 600b5a07..48594f97 100644 --- a/frontend/helpers/modelData.js +++ b/frontend/helpers/modelData.js @@ -87,6 +87,48 @@ export const models = [ default: 200 } }, + { + modelName: "Amazon Nova Lite", + modelId: "amazon.nova-lite-v1:0", + temperatureRange: { + min: 0, + max: 1, + default: 0.5 + }, + maxTokenRange: { + min: 0, + max: 8191, + default: 200 + } + }, + { + modelName: "Mistral Large (24.02)", + modelId: "mistral.mistral-large-2402-v1:0", + temperatureRange: { + min: 0, + max: 1, + default: 0.7 + }, + maxTokenRange: { + min: 0, + max: 8191, + default: 8191 + } + }, + { + modelName: "Mistral Small (24.02)", + modelId: "mistral.mistral-small-2402-v1:0", + temperatureRange: { + min: 0, + max: 1, + default: 0.7 + }, + maxTokenRange: { + min: 0, + max: 8191, + default: 8191 + } + }, { modelName: "Amazon Titan Text Express", modelId: "amazon.titan-text-express-v1",