From 915735432dda4708d87ad2083e7ef911689a8601 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Fri, 28 Mar 2025 13:14:29 +0100 Subject: [PATCH 01/35] initial week 14 commit --- .../BedrockService/Converse/ConversionExtensions.swift | 4 ++++ backend/Sources/BedrockTypes/Content.swift | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions.swift index d52a11c1..b5e38617 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions.swift @@ -48,6 +48,8 @@ extension Content { switch sdkContentBlock { case .text(let text): self = .text(text) + // case .image(let image): // TODO + // self = .image(image) case .sdkUnknown(let unknownContentBlock): throw BedrockServiceError.notImplemented( "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" @@ -63,6 +65,8 @@ extension Content { switch self { case .text(let text): return BedrockRuntimeClientTypes.ContentBlock.text(text) + // case .image(let format, let source): + // return BedrockRuntimeClientTypes.ContentBlock.image(BedrockRuntimeClientTypes.ImageBlock(format: format, source: source)) } } } diff --git a/backend/Sources/BedrockTypes/Content.swift b/backend/Sources/BedrockTypes/Content.swift index 3fedbf9d..dcdc7435 100644 --- a/backend/Sources/BedrockTypes/Content.swift +++ b/backend/Sources/BedrockTypes/Content.swift @@ -17,4 +17,12 @@ import Foundation public enum Content: Codable { case text(String) + // case image(ImageFormat, String) //string are the bytes + + public enum ImageFormat: Codable { + case gif + case jpeg + case png + case webp + } } \ No newline at end of file From 9e4ddd87b0e9bd43b921f269e5bef995f3d71756 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 10:34:19 +0200 Subject: [PATCH 02/35] backend: converse image --- .../Converse/ConverseRequest.swift | 8 +-- .../Converse/ConversionExtensions.swift | 71 ++++++++++++++++--- .../BedrockService/ParameterValidation.swift | 4 +- .../Sources/BedrockService/SwiftBedrock.swift | 11 ++- .../Sources/BedrockTypes/BedrockModel.swift | 29 ++++++++ backend/Sources/BedrockTypes/Content.swift | 2 +- .../Modalities/ConverseModality.swift | 23 ++++++ .../Models/Amazon/Nova/Nova.swift | 6 +- .../Parameters/ConverseParameters.swift | 46 ++++++++++++ 9 files changed, 181 insertions(+), 19 deletions(-) create mode 100644 backend/Sources/BedrockTypes/Modalities/ConverseModality.swift create mode 100644 backend/Sources/BedrockTypes/Parameters/ConverseParameters.swift diff --git a/backend/Sources/BedrockService/Converse/ConverseRequest.swift b/backend/Sources/BedrockService/Converse/ConverseRequest.swift index c11f8488..913a6c56 100644 --- a/backend/Sources/BedrockService/Converse/ConverseRequest.swift +++ b/backend/Sources/BedrockService/Converse/ConverseRequest.swift @@ -40,7 +40,7 @@ public struct ConverseRequest { ) } - func getConverseInput() -> ConverseInput { + func getConverseInput() throws -> ConverseInput { let sdkInferenceConfig: BedrockRuntimeClientTypes.InferenceConfiguration? if inferenceConfig != nil { sdkInferenceConfig = inferenceConfig!.getSDKInferenceConfig() @@ -49,13 +49,13 @@ public struct ConverseRequest { } return ConverseInput( inferenceConfig: sdkInferenceConfig, - messages: getSDKMessages(), + messages: try getSDKMessages(), modelId: model.id ) } - private func getSDKMessages() -> [BedrockRuntimeClientTypes.Message] { - messages.map { $0.getSDKMessage() } + private func getSDKMessages() throws -> [BedrockRuntimeClientTypes.Message] { + try messages.map { try $0.getSDKMessage() } } struct InferenceConfig { diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions.swift index b5e38617..c76c7814 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions.swift @@ -14,8 +14,8 @@ //===----------------------------------------------------------------------===// @preconcurrency import AWSBedrockRuntime -import Foundation import BedrockTypes +import Foundation extension Message { @@ -30,10 +30,10 @@ extension Message { self = Message(from: try Role(from: sdkRole), content: content) } - func getSDKMessage() -> BedrockRuntimeClientTypes.Message { - let contentBlocks: [BedrockRuntimeClientTypes.ContentBlock] = content.map { + func getSDKMessage() throws -> BedrockRuntimeClientTypes.Message { + let contentBlocks: [BedrockRuntimeClientTypes.ContentBlock] = try content.map { content -> BedrockRuntimeClientTypes.ContentBlock in - return content.getSDKContentBlock() + return try content.getSDKContentBlock() } return BedrockRuntimeClientTypes.Message( content: contentBlocks, @@ -48,8 +48,25 @@ extension Content { switch sdkContentBlock { case .text(let text): self = .text(text) - // case .image(let image): // TODO - // self = .image(image) + case .image(let sdkImage): + guard let sdkFormat = sdkImage.format else { + throw BedrockServiceError.decodingError( + "Could not extract format from BedrockRuntimeClientTypes.ImageBlock" + ) + } + guard let sdkImageSource = sdkImage.source else { + throw BedrockServiceError.decodingError( + "Could not extract source from BedrockRuntimeClientTypes.ImageBlock" + ) + } + switch sdkImageSource { + case .bytes(let data): + self = .image(format: try Content.ImageFormat(from: sdkFormat), source: data.base64EncodedString()) + case .sdkUnknown(let unknownImageSource): + throw BedrockServiceError.notImplemented( + "ImageSource \(unknownImageSource) is not implemented by BedrockRuntimeClientTypes" + ) + } case .sdkUnknown(let unknownContentBlock): throw BedrockServiceError.notImplemented( "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" @@ -61,14 +78,50 @@ extension Content { } } - func getSDKContentBlock() -> BedrockRuntimeClientTypes.ContentBlock { + func getSDKContentBlock() throws -> BedrockRuntimeClientTypes.ContentBlock { switch self { case .text(let text): return BedrockRuntimeClientTypes.ContentBlock.text(text) - // case .image(let format, let source): - // return BedrockRuntimeClientTypes.ContentBlock.image(BedrockRuntimeClientTypes.ImageBlock(format: format, source: source)) + case .image(let format, let source): + guard let data = Data(base64Encoded: source) else { + throw BedrockServiceError.decodingError( + "Could not decode image source from base64 string. String: \(source)" + ) + } + return BedrockRuntimeClientTypes.ContentBlock.image( + BedrockRuntimeClientTypes.ImageBlock( + format: format.getSDKImageFormat(), + source: BedrockRuntimeClientTypes.ImageSource.bytes(data) + ) + ) + } + } +} + +extension Content.ImageFormat { + + init(from sdkImageFormat: BedrockRuntimeClientTypes.ImageFormat) throws { + switch sdkImageFormat { + case .gif: self = .gif + case .jpeg: self = .jpeg + case .png: self = .png + case .webp: self = .webp + case .sdkUnknown(let unknownImageFormat): + throw BedrockServiceError.notImplemented( + "ImageFormat \(unknownImageFormat) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKImageFormat() -> BedrockRuntimeClientTypes.ImageFormat { + switch self { + case .gif: return .gif + case .jpeg: return .jpeg + case .png: return .png + case .webp: return .webp } } + } extension Role { diff --git a/backend/Sources/BedrockService/ParameterValidation.swift b/backend/Sources/BedrockService/ParameterValidation.swift index a097c44a..429a6849 100644 --- a/backend/Sources/BedrockService/ParameterValidation.swift +++ b/backend/Sources/BedrockService/ParameterValidation.swift @@ -108,7 +108,7 @@ extension BedrockService { /// Validate parameters for a converse request public func validateConverseParams( - modality: any TextModality, + modality: any ConverseModality, prompt: String, history: [Message], maxTokens: Int?, @@ -116,7 +116,7 @@ extension BedrockService { topP: Double?, stopSequences: [String]? ) throws { - let parameters = modality.getParameters() + let parameters = modality.getConverseParameters() try validatePrompt(prompt, maxPromptTokens: parameters.prompt.maxSize) if maxTokens != nil { try validateParameterValue(maxTokens!, parameter: parameters.maxTokens) diff --git a/backend/Sources/BedrockService/SwiftBedrock.swift b/backend/Sources/BedrockService/SwiftBedrock.swift index 9e6df845..29f2b4b1 100644 --- a/backend/Sources/BedrockService/SwiftBedrock.swift +++ b/backend/Sources/BedrockService/SwiftBedrock.swift @@ -460,6 +460,8 @@ public struct BedrockService: Sendable { public func converse( with model: BedrockModel, prompt: String, + imageFormat: Content.ImageFormat? = nil, + imageBytes: String? = nil, history: [Message] = [], maxTokens: Int? = nil, temperature: Double? = nil, @@ -475,7 +477,7 @@ public struct BedrockService: Sendable { ] ) do { - let modality = try model.getTextModality() // FIXME later: ConverseModality? + let modality: ConverseModality = try model.getConverseModality() try validateConverseParams( modality: modality, prompt: prompt, @@ -488,6 +490,11 @@ public struct BedrockService: Sendable { var messages = history messages.append(Message(from: .user, content: [.text(prompt)])) + if let imageFormat: Content.ImageFormat = imageFormat, + let imageBytes: String = imageBytes + { + messages.append(Message(from: .user, content: [.image(format: imageFormat, source: imageBytes)])) + } let converseRequest = ConverseRequest( model: model, @@ -497,7 +504,7 @@ public struct BedrockService: Sendable { topP: topP, stopSequences: stopSequences ) - let input = converseRequest.getConverseInput() + let input = try converseRequest.getConverseInput() logger.trace( "Created ConverseInput", metadata: ["messages.count": "\(messages.count)", "model": "\(model.id)"] diff --git a/backend/Sources/BedrockTypes/BedrockModel.swift b/backend/Sources/BedrockTypes/BedrockModel.swift index fafea5b3..78f24eb4 100644 --- a/backend/Sources/BedrockTypes/BedrockModel.swift +++ b/backend/Sources/BedrockTypes/BedrockModel.swift @@ -113,6 +113,35 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return textModality } + /// Checks if the model supports text generation and returns ConverseModality + /// - Returns: ConverseModality if the model supports text modality + public func getConverseModality() throws -> any ConverseModality { + guard let modality = modality as? any ConverseModality else { + throw BedrockServiceError.invalidModality( + self, + modality, + "Model \(id) does not support text generation" + ) + } + return modality + } + + // FIXME: this would be cleaner + // Error: Only concrete types such as structs, enums and classes can conform to protocols + public func getModality() throws -> M { + guard let modality = modality as? M else { + throw BedrockServiceError.invalidModality( + self, + modality, + "Model \(id) does not support \(M.self)" + ) + } + return modality + } + public func hasModality(_ type: M.Type) -> Bool { + modality as? M != nil + } + /// Checks if the model supports image generation /// - Returns: True if the model supports image generation public func hasImageModality() -> Bool { diff --git a/backend/Sources/BedrockTypes/Content.swift b/backend/Sources/BedrockTypes/Content.swift index dcdc7435..2e2200ff 100644 --- a/backend/Sources/BedrockTypes/Content.swift +++ b/backend/Sources/BedrockTypes/Content.swift @@ -17,7 +17,7 @@ import Foundation public enum Content: Codable { case text(String) - // case image(ImageFormat, String) //string are the bytes + case image(format: ImageFormat, source: String) // String are the 64 encoded bytes public enum ImageFormat: Codable { case gif diff --git a/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift new file mode 100644 index 00000000..3745c608 --- /dev/null +++ b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public protocol ConverseModality: Modality { + // init(textGenerationParameters: TextGenerationParameters?, imageGenerationparameters: ImageGenerationParameters?) + // init(parameters: ConverseParameters) + + func getConverseParameters() -> ConverseParameters +} diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift index 8525b357..10c82f43 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift @@ -15,7 +15,7 @@ import Foundation -struct NovaText: TextModality { +struct NovaText: TextModality, ConverseModality { func getName() -> String { "Nova Text Generation" } let parameters: TextGenerationParameters @@ -28,6 +28,10 @@ struct NovaText: TextModality { parameters } + func getConverseParameters() -> ConverseParameters { + ConverseParameters(textGenerationParameters: parameters) + } + func getTextRequestBody( prompt: String, maxTokens: Int?, diff --git a/backend/Sources/BedrockTypes/Parameters/ConverseParameters.swift b/backend/Sources/BedrockTypes/Parameters/ConverseParameters.swift new file mode 100644 index 00000000..94859d12 --- /dev/null +++ b/backend/Sources/BedrockTypes/Parameters/ConverseParameters.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct ConverseParameters: Parameters { + public let temperature: Parameter + public let maxTokens: Parameter + public let topP: Parameter + public let prompt: PromptParams + public let stopSequences: StopSequenceParams + + public init( + temperature: Parameter, + maxTokens: Parameter, + topP: Parameter, + stopSequences: StopSequenceParams, + maxPromptSize: Int? + ) { + self.temperature = temperature + self.maxTokens = maxTokens + self.topP = topP + self.prompt = PromptParams(maxSize: maxPromptSize) + self.stopSequences = stopSequences + } + + public init(textGenerationParameters: TextGenerationParameters) { + self.temperature = textGenerationParameters.temperature + self.maxTokens = textGenerationParameters.maxTokens + self.topP = textGenerationParameters.topP + self.prompt = textGenerationParameters.prompt + self.stopSequences = textGenerationParameters.stopSequences + } +} From beee69bb7ba07ad06e25c8686dcce40f131af638 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 11:42:15 +0200 Subject: [PATCH 03/35] frontend: converse vision + backend: modality changes --- backend/Sources/App/Application+build.swift | 6 +- backend/Sources/App/Types/Chat.swift | 2 + .../Sources/BedrockService/SwiftBedrock.swift | 8 ++ .../Sources/BedrockTypes/BedrockModel.swift | 28 ++++++- .../Modalities/ConverseModality.swift | 13 ++- .../Models/Amazon/Nova/Nova.swift | 14 ++++ .../Amazon/Nova/NovaBedrockModels.swift | 19 ++++- .../chatPlayground/ChatComponent.jsx | 80 ++++++++++++++++++- frontend/helpers/modelData.js | 14 ++++ 9 files changed, 173 insertions(+), 11 deletions(-) diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index 4e807eae..d896b650 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -168,13 +168,15 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router Bool { modality as? any ConditionedTextToImageModality != nil } + + /// Checks if the model supports converse + /// - Returns: True if the model supports converse + public func hasConverseModality() -> Bool { + modality as? any ConverseModality != nil + } + + /// Checks if the model supports converse vision + /// - Returns: True if the model supports converse vision + public func hasConverseVisionModality() -> Bool { + modality as? any ConverseVisionModality != nil + } + + public func hasConverseDocumentModality() -> Bool { + modality as? any ConverseDocumentModality != nil + } + + public func hasConverseToolModality() -> Bool { + modality as? any ConverseToolModality != nil + } } extension BedrockModel: Encodable { diff --git a/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift index 3745c608..cde2c5cd 100644 --- a/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift +++ b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift @@ -15,9 +15,16 @@ import Foundation +// Text public protocol ConverseModality: Modality { - // init(textGenerationParameters: TextGenerationParameters?, imageGenerationparameters: ImageGenerationParameters?) - // init(parameters: ConverseParameters) - func getConverseParameters() -> ConverseParameters } + +// Vision +public protocol ConverseVisionModality: ConverseModality {} + +// Document +public protocol ConverseDocumentModality: ConverseModality {} + +// Tool use +public protocol ConverseToolModality: ConverseModality {} diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift index 10c82f43..8be92506 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift @@ -15,6 +15,20 @@ import Foundation +struct NovaConverse: ConverseVisionModality, ConverseDocumentModality, ConverseToolModality { + func getName() -> String { "Nova Lite Converse Modality" } + + let converseParameters: ConverseParameters + + init(parameters: ConverseParameters) { + self.converseParameters = parameters + } + + func getConverseParameters() -> ConverseParameters { + converseParameters + } +} + struct NovaText: TextModality, ConverseModality { func getName() -> String { "Nova Text Generation" } diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift index 2a624534..b0a7317d 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift @@ -22,7 +22,8 @@ typealias NovaMicro = NovaText extension BedrockModel { public static let nova_micro: BedrockModel = BedrockModel( - id: "amazon.nova-micro-v1:0", name: "Nova Micro", + id: "amazon.nova-micro-v1:0", + name: "Nova Micro", modality: NovaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), @@ -34,6 +35,19 @@ extension BedrockModel { ) ) ) + public static let nova_lite: BedrockModel = BedrockModel( + id: "amazon.nova-lite-v1:0", + name: "Nova Lite", + modality: NovaConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 5_000, defaultValue: 5_000), + topP: Parameter(.topP, minValue: 0, maxValue: 1.0, defaultValue: 0.9), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ) + ) + ) } // MARK: image generation @@ -42,7 +56,8 @@ typealias NovaCanvas = AmazonImage extension BedrockModel { public static let nova_canvas: BedrockModel = BedrockModel( - id: "amazon.nova-canvas-v1:0", name: "Nova Canvas", + id: "amazon.nova-canvas-v1:0", + name: "Nova Canvas", modality: NovaCanvas( parameters: ImageGenerationParameters( nrOfImages: Parameter(.nrOfImages, minValue: 1, maxValue: 5, defaultValue: 1), diff --git a/frontend/components/chatPlayground/ChatComponent.jsx b/frontend/components/chatPlayground/ChatComponent.jsx index 4383880f..9fe020de 100644 --- a/frontend/components/chatPlayground/ChatComponent.jsx +++ b/frontend/components/chatPlayground/ChatComponent.jsx @@ -8,6 +8,7 @@ import GlobalConfig from "@/app/app.config"; import ChatModelSelector from "./ChatModelSelector"; import { defaultModel } from "@/helpers/modelData"; import NumericInput from "../NumericInput"; +import Image from 'next/image'; export default function ChatContainer() { @@ -21,7 +22,8 @@ export default function ChatContainer() { const [topP, setTopP] = useState(0.9); const [stopSequences, setStopSequences] = useState([]); const [stopSequenceInput, setStopSequenceInput] = useState(''); - + const [referenceImage, setReferenceImage] = useState(null); + const [previewImage, setPreviewImage] = useState('/placeholder.png'); const endpoint = `/foundation-models/chat/${selectedModel.modelId}`; const api = `${GlobalConfig.apiHost}:${GlobalConfig.apiPort}${endpoint}`; @@ -66,6 +68,63 @@ export default function ChatContainer() { setHistory([]); }; + const handleFileUpload = async (event) => { + const file = event.target.files[0]; + if (file) { + // Create preview for the UI + const reader = new FileReader(); + reader.onload = (e) => { + setPreviewImage(e.target.result); + }; + reader.readAsDataURL(file); + + // Compress and store the image + const compressedBase64 = await compressImage(file); + console.log('Compressed image size:', compressedBase64.length); + setReferenceImage(compressedBase64); + } + }; + + const compressImage = (file) => { + return new Promise((resolve) => { + const reader = new FileReader(); + reader.onload = (event) => { + const img = document.createElement('img'); // Use regular img element instead of Next.js Image + img.onload = () => { + const canvas = document.createElement('canvas'); + // Reduce max dimensions to 512x512 + const maxSize = 512; + let width = img.width; + let height = img.height; + + if (width > height) { + if (width > maxSize) { + height *= maxSize / width; + width = maxSize; + } + } else { + if (height > maxSize) { + width *= maxSize / height; + height = maxSize; + } + } + + canvas.width = width; + canvas.height = height; + + const ctx = canvas.getContext('2d'); + ctx.drawImage(img, 0, 0, width, height); + + // Increase compression by reducing quality to 0.5 (50%) + const compressedBase64 = canvas.toDataURL('image/jpeg', 0.5); + resolve(compressedBase64.split(',')[1]); + }; + img.src = event.target.result; + }; + reader.readAsDataURL(file); + }); + }; + const sendMessage = async () => { const newMessage = { sender: "Human", message: inputValue }; setConversation(prevConversation => [...prevConversation, newMessage]); @@ -80,6 +139,7 @@ export default function ChatContainer() { body: JSON.stringify({ prompt: inputValue, history: history, + imageBytes: referenceImage, maxTokens: parseInt(maxTokens, 10), temperature: parseFloat(temperature), topP: parseFloat(topP), @@ -236,6 +296,7 @@ export default function ChatContainer() { {/* Input window */}
+ {/* Text input */}
+ {/* Reference image with preview */} +
+ + Reference image +
+ + {/* Send button */}
+
diff --git a/frontend/helpers/modelData.js b/frontend/helpers/modelData.js index 600b5a07..d8ae6881 100644 --- a/frontend/helpers/modelData.js +++ b/frontend/helpers/modelData.js @@ -87,6 +87,20 @@ export const models = [ default: 200 } }, + { + modelName: "Amazon Nova Lite", + modelId: "amazon.nova-lite-v1:0", + temperatureRange: { + min: 0, + max: 1, + default: 0.5 + }, + maxTokenRange: { + min: 0, + max: 8191, + default: 200 + } + }, { modelName: "Amazon Titan Text Express", modelId: "amazon.titan-text-express-v1", From bebf86a642a1c6ae3741d07ba80be82f2f6848c9 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 12:43:21 +0200 Subject: [PATCH 04/35] backend: converse modalitity added to all supported models --- .../Sources/BedrockService/SwiftBedrock.swift | 2 +- .../Sources/BedrockTypes/BedrockModel.swift | 26 +++++---- .../Modalities/ConverseModality.swift | 51 +++++++++++++++-- .../Modalities/TextModality.swift | 1 - .../Models/Amazon/Nova/Nova.swift | 25 +++------ .../Amazon/Nova/NovaBedrockModels.swift | 17 +++++- .../Models/Amazon/Titan/Titan.swift | 12 +++- .../Amazon/Titan/TitanBedrockModels.swift | 3 +- .../Models/Anthropic/Anthropic.swift | 12 +++- .../Anthropic/AnthropicBedrockModels.swift | 30 ++++++---- .../Models/DeepSeek/DeepSeek.swift | 11 +++- .../BedrockTypes/Models/Llama/Llama.swift | 11 +++- .../Models/Llama/LlamaBedrockModels.swift | 55 ++++++++++--------- 13 files changed, 175 insertions(+), 81 deletions(-) diff --git a/backend/Sources/BedrockService/SwiftBedrock.swift b/backend/Sources/BedrockService/SwiftBedrock.swift index 99097294..bd2d5869 100644 --- a/backend/Sources/BedrockService/SwiftBedrock.swift +++ b/backend/Sources/BedrockService/SwiftBedrock.swift @@ -494,7 +494,7 @@ public struct BedrockService: Sendable { if let imageFormat: Content.ImageFormat = imageFormat, let imageBytes: String = imageBytes { - guard model.hasConverseVisionModality() else { + guard model.hasConverseModality(.vision) else { throw BedrockServiceError.invalidModality( model, modality, "This model does not support converse vision." diff --git a/backend/Sources/BedrockTypes/BedrockModel.swift b/backend/Sources/BedrockTypes/BedrockModel.swift index cd060627..417e694f 100644 --- a/backend/Sources/BedrockTypes/BedrockModel.swift +++ b/backend/Sources/BedrockTypes/BedrockModel.swift @@ -78,6 +78,8 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { self = BedrockModel.nova_micro case BedrockModel.nova_lite.id: self = BedrockModel.nova_lite + case BedrockModel.nova_pro.id: + self = BedrockModel.nova_pro case BedrockModel.nova_canvas.id: self = BedrockModel.nova_canvas // deepseek @@ -213,19 +215,23 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { modality as? any ConverseModality != nil } - /// Checks if the model supports converse vision - /// - Returns: True if the model supports converse vision - public func hasConverseVisionModality() -> Bool { - modality as? any ConverseVisionModality != nil + // /// Checks if the model supports converse vision + // /// - Returns: True if the model supports converse vision + public func hasConverseModality(_ feature: ConverseFeature = .textGeneration) -> Bool { + if let converseModality = modality as? any ConverseModality { + let features = converseModality.getConverseFeatures() + return features.contains(feature) + } + return false } - public func hasConverseDocumentModality() -> Bool { - modality as? any ConverseDocumentModality != nil - } + // public func hasConverseDocumentModality() -> Bool { + // modality as? any ConverseDocumentModality != nil + // } - public func hasConverseToolModality() -> Bool { - modality as? any ConverseToolModality != nil - } + // public func hasConverseToolModality() -> Bool { + // modality as? any ConverseToolModality != nil + // } } extension BedrockModel: Encodable { diff --git a/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift index cde2c5cd..5793bd15 100644 --- a/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift +++ b/backend/Sources/BedrockTypes/Modalities/ConverseModality.swift @@ -17,14 +17,53 @@ import Foundation // Text public protocol ConverseModality: Modality { + var converseParameters: ConverseParameters { get } + var converseFeatures: [ConverseFeature] { get } + func getConverseParameters() -> ConverseParameters + func getConverseFeatures() -> [ConverseFeature] + + init(parameters: ConverseParameters, features: [ConverseFeature]) } -// Vision -public protocol ConverseVisionModality: ConverseModality {} +// https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html +public enum ConverseFeature: String, Codable, Sendable { + case textGeneration = "text-generation" + case vision = "vision" + case document = "document" + case toolUse = "tool-use" + case systemPrompts = "system-prompts" +} + +// defualt implementation +extension ConverseModality { + init(parameters: ConverseParameters, features: [ConverseFeature]) { + self = .init(parameters: parameters, features: features) + } + + func getConverseParameters() -> ConverseParameters { + converseParameters + } + + func getConverseFeatures() -> [ConverseFeature] { + converseFeatures + } +} +// extension ConverseModality { +// // func getConverseParameters() -> ConverseParameters { +// // ConverseParameters(textGenerationParameters: parameters) +// // } + +// func getConverseFeatures() -> [ConverseFeature] { +// [.textGeneration] +// } +// } + +// // Vision +// public protocol ConverseVisionModality: ConverseModality {} -// Document -public protocol ConverseDocumentModality: ConverseModality {} +// // Document +// public protocol ConverseDocumentModality: ConverseModality {} -// Tool use -public protocol ConverseToolModality: ConverseModality {} +// // Tool use +// public protocol ConverseToolModality: ConverseModality {} diff --git a/backend/Sources/BedrockTypes/Modalities/TextModality.swift b/backend/Sources/BedrockTypes/Modalities/TextModality.swift index 2995b795..82c12196 100644 --- a/backend/Sources/BedrockTypes/Modalities/TextModality.swift +++ b/backend/Sources/BedrockTypes/Modalities/TextModality.swift @@ -17,7 +17,6 @@ import Foundation public protocol TextModality: Modality { - init(parameters: TextGenerationParameters) func getParameters() -> TextGenerationParameters func getTextRequestBody( diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift index 8be92506..bc0367d7 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift @@ -15,37 +15,30 @@ import Foundation -struct NovaConverse: ConverseVisionModality, ConverseDocumentModality, ConverseToolModality { - func getName() -> String { "Nova Lite Converse Modality" } +struct NovaConverse: ConverseModality { + func getName() -> String { "Nova Lite and Nova Pro Converse Modality" } let converseParameters: ConverseParameters - - init(parameters: ConverseParameters) { - self.converseParameters = parameters - } - - func getConverseParameters() -> ConverseParameters { - converseParameters - } + let converseFeatures: [ConverseFeature] } struct NovaText: TextModality, ConverseModality { func getName() -> String { "Nova Text Generation" } - + let parameters: TextGenerationParameters + let converseFeatures: [ConverseFeature] + let converseParameters: ConverseParameters - init(parameters: TextGenerationParameters) { + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration]) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { parameters } - func getConverseParameters() -> ConverseParameters { - ConverseParameters(textGenerationParameters: parameters) - } - func getTextRequestBody( prompt: String, maxTokens: Int?, diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift index b0a7317d..4e82dbdc 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift @@ -45,7 +45,22 @@ extension BedrockModel { topP: Parameter(.topP, minValue: 0, maxValue: 1.0, defaultValue: 0.9), stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), maxPromptSize: nil - ) + ), + features: [.textGeneration, .vision, .systemPrompts, .document, .toolUse] + ) + ) + public static let nova_pro: BedrockModel = BedrockModel( + id: "amazon.nova-pro-v1:0", + name: "Nova Pro", + modality: NovaConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 5_000, defaultValue: 5_000), + topP: Parameter(.topP, minValue: 0, maxValue: 1.0, defaultValue: 0.9), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) } diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift b/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift index 7ac6d4a5..109ab7b0 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Titan/Titan.swift @@ -15,19 +15,27 @@ import Foundation -struct TitanText: TextModality { +struct TitanText: TextModality, ConverseModality { func getName() -> String { "Titan Text Generation" } let parameters: TextGenerationParameters + let converseParameters: ConverseParameters + let converseFeatures: [ConverseFeature] - init(parameters: TextGenerationParameters) { + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration, .document]) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { parameters } + func getConverseParameters() -> ConverseParameters { + ConverseParameters(textGenerationParameters: parameters) + } + func getTextRequestBody( prompt: String, maxTokens: Int?, diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift index 48560d83..a3c276ba 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Titan/TitanBedrockModels.swift @@ -33,7 +33,8 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), maxPromptSize: nil - ) + ), + features: [.textGeneration] ) ) public static let titan_text_g1_express: BedrockModel = BedrockModel( diff --git a/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift b/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift index 092c1c2c..93eb94b2 100644 --- a/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift +++ b/backend/Sources/BedrockTypes/Models/Anthropic/Anthropic.swift @@ -15,19 +15,27 @@ import Foundation -struct AnthropicText: TextModality { +struct AnthropicText: TextModality, ConverseModality { let parameters: TextGenerationParameters + let converseParameters: ConverseParameters + let converseFeatures: [ConverseFeature] func getName() -> String { "Anthropic Text Generation" } - init(parameters: TextGenerationParameters) { + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration, .systemPrompts, .document]) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { parameters } + func getConverseParameters() -> ConverseParameters { + ConverseParameters(textGenerationParameters: parameters) + } + func getTextRequestBody( prompt: String, maxTokens: Int?, diff --git a/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift index f3d20831..dd9d2e90 100644 --- a/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Anthropic/AnthropicBedrockModels.swift @@ -39,7 +39,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [] ) ) public static let claudev1: BedrockModel = BedrockModel( @@ -52,7 +53,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [] ) ) public static let claudev2: BedrockModel = BedrockModel( @@ -65,7 +67,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let claudev2_1: BedrockModel = BedrockModel( @@ -78,7 +81,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let claudev3_opus: BedrockModel = BedrockModel( @@ -91,7 +95,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_haiku: BedrockModel = BedrockModel( @@ -104,7 +109,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_5_haiku: BedrockModel = BedrockModel( @@ -117,7 +123,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] ) ) public static let claudev3_5_sonnet: BedrockModel = BedrockModel( @@ -130,7 +137,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_5_sonnet_v2: BedrockModel = BedrockModel( @@ -143,7 +151,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) public static let claudev3_7_sonnet: BedrockModel = BedrockModel( @@ -156,7 +165,8 @@ extension BedrockModel { topK: Parameter(.topK, minValue: 0, maxValue: 500, defaultValue: 0), stopSequences: StopSequenceParams(maxSequences: 8191, defaultValue: []), maxPromptSize: 200_000 - ) + ), + features: [.textGeneration, .systemPrompts, .document, .vision, .toolUse] ) ) } diff --git a/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift b/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift index 4e65c4e5..44da21e9 100644 --- a/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift +++ b/backend/Sources/BedrockTypes/Models/DeepSeek/DeepSeek.swift @@ -15,13 +15,20 @@ import Foundation -struct DeepSeekText: TextModality { +struct DeepSeekText: TextModality, ConverseModality { let parameters: TextGenerationParameters + let converseFeatures: [ConverseFeature] + let converseParameters: ConverseParameters func getName() -> String { "DeepSeek Text Generation" } - init(parameters: TextGenerationParameters) { + init( + parameters: TextGenerationParameters, + features: [ConverseFeature] = [.textGeneration, .systemPrompts, .document] + ) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { diff --git a/backend/Sources/BedrockTypes/Models/Llama/Llama.swift b/backend/Sources/BedrockTypes/Models/Llama/Llama.swift index cfcf2d8c..a8fa1c16 100644 --- a/backend/Sources/BedrockTypes/Models/Llama/Llama.swift +++ b/backend/Sources/BedrockTypes/Models/Llama/Llama.swift @@ -15,13 +15,20 @@ import Foundation -struct LlamaText: TextModality { +struct LlamaText: TextModality, ConverseModality { func getName() -> String { "Llama Text Generation" } let parameters: TextGenerationParameters + let converseParameters: ConverseParameters + let converseFeatures: [ConverseFeature] - init(parameters: TextGenerationParameters) { + init( + parameters: TextGenerationParameters, + features: [ConverseFeature] = [.textGeneration, .systemPrompts, .document] + ) { self.parameters = parameters + self.converseFeatures = features + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) } func getParameters() -> TextGenerationParameters { diff --git a/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift index 3e0a7de4..e0963a23 100644 --- a/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Llama/LlamaBedrockModels.swift @@ -19,7 +19,8 @@ import Foundation extension BedrockModel { public static let llama_3_8b_instruct: BedrockModel = BedrockModel( - id: "meta.llama3-8b-instruct-v1:0", name: "Llama 3 8B Instruct", + id: "meta.llama3-8b-instruct-v1:0", + name: "Llama 3 8B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -28,11 +29,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_70b_instruct: BedrockModel = BedrockModel( - id: "meta.llama3-70b-instruct-v1:0", name: "Llama 3 70B Instruct", + id: "meta.llama3-70b-instruct-v1:0", + name: "Llama 3 70B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -41,11 +44,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_1_8b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-1-8b-instruct-v1:0", name: "Llama 3.1 8B Instruct", + id: "us.meta.llama3-1-8b-instruct-v1:0", + name: "Llama 3.1 8B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -54,11 +59,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] ) ) public static let llama3_1_70b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-1-70b-instruct-v1:0", name: "Llama 3.1 70B Instruct", + id: "us.meta.llama3-1-70b-instruct-v1:0", + name: "Llama 3.1 70B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -67,11 +74,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] ) ) public static let llama3_2_1b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-2-1b-instruct-v1:0", name: "Llama 3.2 1B Instruct", + id: "us.meta.llama3-2-1b-instruct-v1:0", + name: "Llama 3.2 1B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -80,11 +89,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_2_3b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-2-3b-instruct-v1:0", name: "Llama 3.2 3B Instruct", + id: "us.meta.llama3-2-3b-instruct-v1:0", + name: "Llama 3.2 3B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -93,11 +104,13 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) + ), + features: [.textGeneration, .systemPrompts, .document] ) ) public static let llama3_3_70b_instruct: BedrockModel = BedrockModel( - id: "us.meta.llama3-3-70b-instruct-v1:0", name: "Llama 3.3 70B Instruct", + id: "us.meta.llama3-3-70b-instruct-v1:0", + name: "Llama 3.3 70B Instruct", modality: LlamaText( parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), @@ -106,20 +119,8 @@ extension BedrockModel { topK: Parameter.notSupported(.topK), stopSequences: StopSequenceParams.notSupported(), maxPromptSize: nil - ) - ) - ) - public static let llama3_8b_instruct: BedrockModel = BedrockModel( - id: "meta.llama3-8b-instruct-v1:0", name: "Llama 3 8B Instruct", - modality: LlamaText( - parameters: TextGenerationParameters( - temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), - maxTokens: Parameter(.maxTokens, minValue: 0, maxValue: 2_048, defaultValue: 512), - topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 0.9), - topK: Parameter.notSupported(.topK), - stopSequences: StopSequenceParams.notSupported(), - maxPromptSize: nil - ) + ), + features: [] ) ) } From 3e1d72418c04b1ac9c56779fe61b402cfd62b2d1 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:10:14 +0200 Subject: [PATCH 05/35] Mistral support only for converse added --- backend/Sources/App/Application+build.swift | 4 + .../Sources/BedrockTypes/BedrockModel.swift | 21 +++-- .../Amazon/Nova/NovaBedrockModels.swift | 3 +- .../BedrockTypes/Models/Mistral/Mistral.swift | 28 +++++++ .../Models/Mistral/MistralBedrockModels.swift | 79 +++++++++++++++++++ frontend/helpers/modelData.js | 28 +++++++ 6 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 backend/Sources/BedrockTypes/Models/Mistral/Mistral.swift create mode 100644 backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index d896b650..d284bb4b 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -165,9 +165,13 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router String { "Mistral Converse Modality" } + + let converseParameters: ConverseParameters + let converseFeatures: [ConverseFeature] + + init(parameters: ConverseParameters, features: [ConverseFeature]) { + self.converseParameters = parameters + self.converseFeatures = features + } +} diff --git a/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift new file mode 100644 index 00000000..b6b29273 --- /dev/null +++ b/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +// MARK: converse only +// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral-text-completion.html +// https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html + +extension BedrockModel { + public static let mistral_large_2402 = BedrockModel( + id: "mistral.mistral-large-2402-v1:0", + name: "Mistral Large (24.02)", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 8_192, defaultValue: 8_192), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 1), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] + ) + ) + public static let mistral_small_2402 = BedrockModel( + id: "mistral.mistral-small-2402-v1:0", + name: "Mistral Small (24.02)", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.7), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 8_192, defaultValue: 8_192), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 1), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .toolUse] + ) + ) + public static let mistral_7B_instruct = BedrockModel( + id: "mistral.mistral-7b-instruct-v0:2", + name: "Mistral 7B Instruct", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 8_192, defaultValue: 512), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 0.9), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .document] + ) + ) + public static let mistral_8x7B_instruct = BedrockModel( + id: "mistral.mixtral-8x7b-instruct-v0:1", + name: "Mixtral 8x7B Instruct", + modality: MistralConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.5), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 4_096, defaultValue: 512), + topP: Parameter(.topP, minValue: 0, maxValue: 1, defaultValue: 0.9), + stopSequences: StopSequenceParams(maxSequences: 10, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .document] + ) + ) +} diff --git a/frontend/helpers/modelData.js b/frontend/helpers/modelData.js index d8ae6881..48594f97 100644 --- a/frontend/helpers/modelData.js +++ b/frontend/helpers/modelData.js @@ -101,6 +101,34 @@ export const models = [ default: 200 } }, + { + modelName: "Mistral Large (24.02)", + modelId: "mistral.mistral-large-2402-v1:0", + temperatureRange: { + min: 0, + max: 1, + default: 0.7 + }, + maxTokenRange: { + min: 0, + max: 8191, + default: 8191 + } + }, + { + modelName: "Mistral Small (24.02)", + modelId: "mistral.mistral-small-2402-v1:0", + temperatureRange: { + min: 0, + max: 1, + default: 0.7 + }, + maxTokenRange: { + min: 0, + max: 8191, + default: 8191 + } + }, { modelName: "Amazon Titan Text Express", modelId: "amazon.titan-text-express-v1", From 71456294abba964cd1c2906495bd14db83e1d53a Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:14:55 +0200 Subject: [PATCH 06/35] converse image input --- backend/Sources/App/Application+build.swift | 6 +---- .../Sources/BedrockTypes/BedrockModel.swift | 18 +++++-------- .../Modalities/ConverseModality.swift | 25 +------------------ .../Models/Amazon/Nova/Nova.swift | 5 ++++ 4 files changed, 13 insertions(+), 41 deletions(-) diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index d284bb4b..bdda426f 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -165,13 +165,9 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router Router ConverseParameters func getConverseFeatures() -> [ConverseFeature] - - init(parameters: ConverseParameters, features: [ConverseFeature]) } // https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html @@ -35,11 +33,8 @@ public enum ConverseFeature: String, Codable, Sendable { case systemPrompts = "system-prompts" } -// defualt implementation +// default implementation extension ConverseModality { - init(parameters: ConverseParameters, features: [ConverseFeature]) { - self = .init(parameters: parameters, features: features) - } func getConverseParameters() -> ConverseParameters { converseParameters @@ -49,21 +44,3 @@ extension ConverseModality { converseFeatures } } -// extension ConverseModality { -// // func getConverseParameters() -> ConverseParameters { -// // ConverseParameters(textGenerationParameters: parameters) -// // } - -// func getConverseFeatures() -> [ConverseFeature] { -// [.textGeneration] -// } -// } - -// // Vision -// public protocol ConverseVisionModality: ConverseModality {} - -// // Document -// public protocol ConverseDocumentModality: ConverseModality {} - -// // Tool use -// public protocol ConverseToolModality: ConverseModality {} diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift index bc0367d7..5a30bf3a 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift @@ -20,6 +20,11 @@ struct NovaConverse: ConverseModality { let converseParameters: ConverseParameters let converseFeatures: [ConverseFeature] + + init(parameters: ConverseParameters, features: [ConverseFeature]) { + self.converseParameters = parameters + self.converseFeatures = features + } } struct NovaText: TextModality, ConverseModality { From f5e8bae6b4fde526d8114036344de611d40c07b5 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 16:37:38 +0200 Subject: [PATCH 07/35] backend: converse refactor using Blocks --- backend/Sources/App/Types/Chat.swift | 2 +- .../Converse/ConverseRequest.swift | 1 + .../Converse/ConversionExtensions.swift | 145 ------------------ .../ContentConversionExtension.swift | 50 ++++++ .../ImageConversionExtension.swift | 79 ++++++++++ .../MessageConversionExtension.swift | 43 ++++++ .../RoleConversionExtension.swift | 38 +++++ .../ToolConversionExtension.swift | 46 ++++++ .../Sources/BedrockService/SwiftBedrock.swift | 7 +- .../{ => ConverseContent}/Content.swift | 14 +- .../ConverseContent/DocumentBlock.swift | 34 ++++ .../ConverseContent/ImageBlock.swift | 33 ++++ .../ConverseContent/S3Location.swift | 21 +++ .../ConverseContent/ToolBlocks.swift | 46 ++++++ .../ConverseContent/VideoBlock.swift | 38 +++++ 15 files changed, 438 insertions(+), 159 deletions(-) delete mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions.swift create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/MessageConversionExtension.swift create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/RoleConversionExtension.swift create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift rename backend/Sources/BedrockTypes/{ => ConverseContent}/Content.swift (77%) create mode 100644 backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift create mode 100644 backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift create mode 100644 backend/Sources/BedrockTypes/ConverseContent/S3Location.swift create mode 100644 backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift create mode 100644 backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift diff --git a/backend/Sources/App/Types/Chat.swift b/backend/Sources/App/Types/Chat.swift index 7dcf14a5..d397b637 100644 --- a/backend/Sources/App/Types/Chat.swift +++ b/backend/Sources/App/Types/Chat.swift @@ -22,7 +22,7 @@ extension Message: ResponseCodable {} struct ChatInput: Codable { let prompt: String let history: [Message] - let imageFormat: Content.ImageFormat? + let imageFormat: ImageFormat? let imageBytes: String? let maxTokens: Int? let temperature: Double? diff --git a/backend/Sources/BedrockService/Converse/ConverseRequest.swift b/backend/Sources/BedrockService/Converse/ConverseRequest.swift index 913a6c56..c584ac8b 100644 --- a/backend/Sources/BedrockService/Converse/ConverseRequest.swift +++ b/backend/Sources/BedrockService/Converse/ConverseRequest.swift @@ -21,6 +21,7 @@ public struct ConverseRequest { let model: BedrockModel let messages: [Message] let inferenceConfig: InferenceConfig? + // let toolConfig: ToolConfig? init( model: BedrockModel, diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions.swift deleted file mode 100644 index c76c7814..00000000 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions.swift +++ /dev/null @@ -1,145 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Foundation Models Playground open source project -// -// Copyright (c) 2025 Amazon.com, Inc. or its affiliates -// and the Swift Foundation Models Playground project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -@preconcurrency import AWSBedrockRuntime -import BedrockTypes -import Foundation - -extension Message { - - init(from sdkMessage: BedrockRuntimeClientTypes.Message) throws { - guard let sdkRole = sdkMessage.role else { - throw BedrockServiceError.decodingError("Could not extract role from BedrockRuntimeClientTypes.Message") - } - guard let sdkContent = sdkMessage.content else { - throw BedrockServiceError.decodingError("Could not extract content from BedrockRuntimeClientTypes.Message") - } - let content: [Content] = try sdkContent.map { try Content(from: $0) } - self = Message(from: try Role(from: sdkRole), content: content) - } - - func getSDKMessage() throws -> BedrockRuntimeClientTypes.Message { - let contentBlocks: [BedrockRuntimeClientTypes.ContentBlock] = try content.map { - content -> BedrockRuntimeClientTypes.ContentBlock in - return try content.getSDKContentBlock() - } - return BedrockRuntimeClientTypes.Message( - content: contentBlocks, - role: role.getSDKConversationRole() - ) - } -} - -extension Content { - - init(from sdkContentBlock: BedrockRuntimeClientTypes.ContentBlock) throws { - switch sdkContentBlock { - case .text(let text): - self = .text(text) - case .image(let sdkImage): - guard let sdkFormat = sdkImage.format else { - throw BedrockServiceError.decodingError( - "Could not extract format from BedrockRuntimeClientTypes.ImageBlock" - ) - } - guard let sdkImageSource = sdkImage.source else { - throw BedrockServiceError.decodingError( - "Could not extract source from BedrockRuntimeClientTypes.ImageBlock" - ) - } - switch sdkImageSource { - case .bytes(let data): - self = .image(format: try Content.ImageFormat(from: sdkFormat), source: data.base64EncodedString()) - case .sdkUnknown(let unknownImageSource): - throw BedrockServiceError.notImplemented( - "ImageSource \(unknownImageSource) is not implemented by BedrockRuntimeClientTypes" - ) - } - case .sdkUnknown(let unknownContentBlock): - throw BedrockServiceError.notImplemented( - "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" - ) - default: - throw BedrockServiceError.notImplemented( - "\(sdkContentBlock.self) is not implemented by this library" - ) - } - } - - func getSDKContentBlock() throws -> BedrockRuntimeClientTypes.ContentBlock { - switch self { - case .text(let text): - return BedrockRuntimeClientTypes.ContentBlock.text(text) - case .image(let format, let source): - guard let data = Data(base64Encoded: source) else { - throw BedrockServiceError.decodingError( - "Could not decode image source from base64 string. String: \(source)" - ) - } - return BedrockRuntimeClientTypes.ContentBlock.image( - BedrockRuntimeClientTypes.ImageBlock( - format: format.getSDKImageFormat(), - source: BedrockRuntimeClientTypes.ImageSource.bytes(data) - ) - ) - } - } -} - -extension Content.ImageFormat { - - init(from sdkImageFormat: BedrockRuntimeClientTypes.ImageFormat) throws { - switch sdkImageFormat { - case .gif: self = .gif - case .jpeg: self = .jpeg - case .png: self = .png - case .webp: self = .webp - case .sdkUnknown(let unknownImageFormat): - throw BedrockServiceError.notImplemented( - "ImageFormat \(unknownImageFormat) is not implemented by BedrockRuntimeClientTypes" - ) - } - } - - func getSDKImageFormat() -> BedrockRuntimeClientTypes.ImageFormat { - switch self { - case .gif: return .gif - case .jpeg: return .jpeg - case .png: return .png - case .webp: return .webp - } - } - -} - -extension Role { - init(from sdkConversationRole: BedrockRuntimeClientTypes.ConversationRole) throws { - switch sdkConversationRole { - case .user: self = .user - case .assistant: self = .assistant - case .sdkUnknown(let unknownRole): - throw BedrockServiceError.notImplemented( - "Role \(unknownRole) is not implemented by BedrockRuntimeClientTypes" - ) - } - } - - func getSDKConversationRole() -> BedrockRuntimeClientTypes.ConversationRole { - switch self { - case .user: return .user - case .assistant: return .assistant - } - } -} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift new file mode 100644 index 00000000..2ba3cf90 --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +extension Content { + + init(from sdkContentBlock: BedrockRuntimeClientTypes.ContentBlock) throws { + switch sdkContentBlock { + case .text(let text): + self = .text(text) + case .image(let sdkImage): + self = .image(try ImageBlock(from: sdkImage)) + case .sdkUnknown(let unknownContentBlock): + throw BedrockServiceError.notImplemented( + "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" + ) + default: + throw BedrockServiceError.notImplemented( + "\(sdkContentBlock.self) is not implemented by this library" + ) + } + } + + func getSDKContentBlock() throws -> BedrockRuntimeClientTypes.ContentBlock { + switch self { + case .text(let text): + return BedrockRuntimeClientTypes.ContentBlock.text(text) + case .image(let imageBlock): + return BedrockRuntimeClientTypes.ContentBlock.image(try imageBlock.getSDKImageBlock()) + default: + print("TODO") + return BedrockRuntimeClientTypes.ContentBlock.text("TODO") + } + } +} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift new file mode 100644 index 00000000..59be2206 --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +extension ImageBlock { + init(from sdkImageBlock: BedrockRuntimeClientTypes.ImageBlock) throws { + guard let sdkFormat = sdkImageBlock.format else { + throw BedrockServiceError.decodingError( + "Could not extract format from BedrockRuntimeClientTypes.ImageBlock" + ) + } + guard let sdkImageSource = sdkImageBlock.source else { + throw BedrockServiceError.decodingError( + "Could not extract source from BedrockRuntimeClientTypes.ImageBlock" + ) + } + let format = try ImageFormat(from: sdkFormat) + switch sdkImageSource { + case .bytes(let data): + self = ImageBlock(format: format, source: data.base64EncodedString()) + case .sdkUnknown(let unknownImageSource): + throw BedrockServiceError.notImplemented( + "ImageSource \(unknownImageSource) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKImageBlock() throws -> BedrockRuntimeClientTypes.ImageBlock { + guard let data = Data(base64Encoded: source) else { + throw BedrockServiceError.decodingError( + "Could not decode image source from base64 string. String: \(source)" + ) + } + return BedrockRuntimeClientTypes.ImageBlock( + format: format.getSDKImageFormat(), + source: BedrockRuntimeClientTypes.ImageSource.bytes(data) + ) + } +} + +extension ImageFormat { + + init(from sdkImageFormat: BedrockRuntimeClientTypes.ImageFormat) throws { + switch sdkImageFormat { + case .gif: self = .gif + case .jpeg: self = .jpeg + case .png: self = .png + case .webp: self = .webp + case .sdkUnknown(let unknownImageFormat): + throw BedrockServiceError.notImplemented( + "ImageFormat \(unknownImageFormat) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKImageFormat() -> BedrockRuntimeClientTypes.ImageFormat { + switch self { + case .gif: return .gif + case .jpeg: return .jpeg + case .png: return .png + case .webp: return .webp + } + } +} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/MessageConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/MessageConversionExtension.swift new file mode 100644 index 00000000..b3d575f7 --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/MessageConversionExtension.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +extension Message { + + init(from sdkMessage: BedrockRuntimeClientTypes.Message) throws { + guard let sdkRole = sdkMessage.role else { + throw BedrockServiceError.decodingError("Could not extract role from BedrockRuntimeClientTypes.Message") + } + guard let sdkContent = sdkMessage.content else { + throw BedrockServiceError.decodingError("Could not extract content from BedrockRuntimeClientTypes.Message") + } + let content: [Content] = try sdkContent.map { try Content(from: $0) } + self = Message(from: try Role(from: sdkRole), content: content) + } + + func getSDKMessage() throws -> BedrockRuntimeClientTypes.Message { + let contentBlocks: [BedrockRuntimeClientTypes.ContentBlock] = try content.map { + content -> BedrockRuntimeClientTypes.ContentBlock in + return try content.getSDKContentBlock() + } + return BedrockRuntimeClientTypes.Message( + content: contentBlocks, + role: role.getSDKConversationRole() + ) + } +} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/RoleConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/RoleConversionExtension.swift new file mode 100644 index 00000000..7c9680ad --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/RoleConversionExtension.swift @@ -0,0 +1,38 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +extension Role { + init(from sdkConversationRole: BedrockRuntimeClientTypes.ConversationRole) throws { + switch sdkConversationRole { + case .user: self = .user + case .assistant: self = .assistant + case .sdkUnknown(let unknownRole): + throw BedrockServiceError.notImplemented( + "Role \(unknownRole) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKConversationRole() -> BedrockRuntimeClientTypes.ConversationRole { + switch self { + case .user: return .user + case .assistant: return .assistant + } + } +} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift new file mode 100644 index 00000000..ce44cad1 --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +extension ToolUseBlock { + init(from sdkToolUseBlock: BedrockRuntimeClientTypes.ToolUseBlock) throws { + guard let sdkId = sdkToolUseBlock.toolUseId else { + throw BedrockServiceError.decodingError( + "Could not extract toolUseId from BedrockRuntimeClientTypes.ToolUseBlock" + ) + } + guard let sdkName = sdkToolUseBlock.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.ToolUseBlock" + ) + } + self = ToolUseBlock( + id: sdkId, + name: sdkName + // input: sdkToolUseBlock.input + ) + } + + func getSDKToolUseBlock() throws -> BedrockRuntimeClientTypes.ToolUseBlock { + .init( + name: name, + toolUseId: id + // input: input + ) + } +} \ No newline at end of file diff --git a/backend/Sources/BedrockService/SwiftBedrock.swift b/backend/Sources/BedrockService/SwiftBedrock.swift index bd2d5869..a8a5a95f 100644 --- a/backend/Sources/BedrockService/SwiftBedrock.swift +++ b/backend/Sources/BedrockService/SwiftBedrock.swift @@ -460,7 +460,7 @@ public struct BedrockService: Sendable { public func converse( with model: BedrockModel, prompt: String, - imageFormat: Content.ImageFormat? = nil, + imageFormat: ImageFormat? = nil, imageBytes: String? = nil, history: [Message] = [], maxTokens: Int? = nil, @@ -491,7 +491,7 @@ public struct BedrockService: Sendable { var messages = history messages.append(Message(from: .user, content: [.text(prompt)])) - if let imageFormat: Content.ImageFormat = imageFormat, + if let imageFormat: ImageFormat = imageFormat, let imageBytes: String = imageBytes { guard model.hasConverseModality(.vision) else { @@ -500,8 +500,7 @@ public struct BedrockService: Sendable { "This model does not support converse vision." ) } - messages.append(Message(from: .user, content: [.image(format: imageFormat, source: imageBytes)])) - print("HIER!") + messages.append(Message(from: .user, content: [.image(ImageBlock(format: imageFormat, source: imageBytes))])) } let converseRequest = ConverseRequest( diff --git a/backend/Sources/BedrockTypes/Content.swift b/backend/Sources/BedrockTypes/ConverseContent/Content.swift similarity index 77% rename from backend/Sources/BedrockTypes/Content.swift rename to backend/Sources/BedrockTypes/ConverseContent/Content.swift index 2e2200ff..59c9ce48 100644 --- a/backend/Sources/BedrockTypes/Content.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/Content.swift @@ -17,12 +17,8 @@ import Foundation public enum Content: Codable { case text(String) - case image(format: ImageFormat, source: String) // String are the 64 encoded bytes - - public enum ImageFormat: Codable { - case gif - case jpeg - case png - case webp - } -} \ No newline at end of file + case image(ImageBlock) // String are the 64 encoded bytes + case toolUse(ToolUseBlock) + case toolresult(ToolResultBlock) + // case reasoningcontent(String) +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift new file mode 100644 index 00000000..79b48328 --- /dev/null +++ b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +struct DocumentBlock: Codable { + let name: String + let format: DocumentFormat + let source: String // 64 encoded +} + +enum DocumentFormat: Codable { + case csv + case doc + case docx + case html + case md + case pdf + case txt + case xls + case xlsx +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift new file mode 100644 index 00000000..aac21ce1 --- /dev/null +++ b/backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct ImageBlock: Codable { + public let format: ImageFormat + public let source: String // 64 encoded + + public init(format: ImageFormat, source: String) { + self.format = format + self.source = source + } +} + +public enum ImageFormat: Codable { + case gif + case jpeg + case png + case webp +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift b/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift new file mode 100644 index 00000000..d4021446 --- /dev/null +++ b/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct S3Location: Codable { + let bucketOwner: String + let uri: String +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift new file mode 100644 index 00000000..9d2ca1e1 --- /dev/null +++ b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift @@ -0,0 +1,46 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct ToolUseBlock: Codable { + public let id: String + public let name: String + // let input: [String: Any] + + public init(id: String, name: String) { + self.id = id + self.name = name + } +} + +public struct ToolResultBlock: Codable { + let content: String + let status: Status // currently only supported by Anthropic Claude 3 models + let toolUseId: String + + enum Status: Codable { + case success + case error + } + + enum ToolResultContent: Codable { + // case json([String: Any]) + case text(String) + case image(ImageBlock) // currently only supported by Anthropic Claude 3 models + case document(DocumentBlock) + case video(VideoBlock) + } +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift new file mode 100644 index 00000000..4770441b --- /dev/null +++ b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift @@ -0,0 +1,38 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct VideoBlock: Codable { + let format: Format + let source: Source + + enum Source: Codable { + case bytes(String) // base64 + case s3(S3Location) + } + + enum Format: Codable { + case flv + case mkv + case mov + case mp4 + case mpeg + case mpg + case threeGp + case webm + case wmv + } +} From 94bf47e86087fb065116c1acdb1c5be4a8a6d866 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 16:47:30 +0200 Subject: [PATCH 08/35] DocumentFormat --- .../DocumentConversionExtension.swift | 48 +++++++++++++++++++ .../ConverseContent/DocumentBlock.swift | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift new file mode 100644 index 00000000..c1a9356a --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +// extension DocumentBlock { +// init(from sdkDocumentBlock: BedrockRuntimeClientTypes.DocumentBlock) throws { +// return BedrockRuntimeClientTypes.DocumentBlock( +// format: format, +// name: , +// source: , +// ) +// } +// } + +extension DocumentFormat { + init(from sdkDocumentFormat: BedrockRuntimeClientTypes.DocumentFormat) throws { + switch sdkDocumentFormat { + case .csv: self = .csv + case .doc: self = .doc + case .docx: self = .docx + case .html: self = .html + case .md: self = .md + case .pdf: self = .pdf + case .txt: self = .txt + case .xls: self = .xls + case .xlsx: self = .xlsx + case .sdkUnknown(let unknownDocumentFormat): + throw BedrockServiceError.notImplemented( + "DocumentFormat \(unknownDocumentFormat) is not implemented by BedrockRuntimeClientTypes" + ) + } + } +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift index 79b48328..2bb0d82d 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift @@ -21,7 +21,7 @@ struct DocumentBlock: Codable { let source: String // 64 encoded } -enum DocumentFormat: Codable { +public enum DocumentFormat: Codable { case csv case doc case docx From d5b0575188a06f29b5c9406413a77ac0c8116bc0 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 16:48:29 +0200 Subject: [PATCH 09/35] DocumentFormat --- .../DocumentConversionExtension.swift | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift index c1a9356a..875b650d 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift @@ -45,4 +45,18 @@ extension DocumentFormat { ) } } + + func getSDKDocumentFormat() -> BedrockRuntimeClientTypes.DocumentFormat { + switch self { + case .csv: return .csv + case .doc: return .doc + case .docx: return .docx + case .html: return .html + case .md: return .md + case .pdf: return .pdf + case .txt: return .txt + case .xls: return .xls + case .xlsx: return .xlsx + } + } } From 193896d26b09b37a3e9b841708bb23682ae72c40 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 16:58:31 +0200 Subject: [PATCH 10/35] DocumentBlock --- .../DocumentConversionExtension.swift | 50 +++++++++++++++---- .../ConverseContent/DocumentBlock.swift | 14 ++++-- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift index 875b650d..59f58fcb 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift @@ -17,15 +17,47 @@ import BedrockTypes import Foundation -// extension DocumentBlock { -// init(from sdkDocumentBlock: BedrockRuntimeClientTypes.DocumentBlock) throws { -// return BedrockRuntimeClientTypes.DocumentBlock( -// format: format, -// name: , -// source: , -// ) -// } -// } +extension DocumentBlock { + init(from sdkDocumentBlock: BedrockRuntimeClientTypes.DocumentBlock) throws { + guard let sdkDocumentSource = sdkDocumentBlock.source else { + throw BedrockServiceError.decodingError( + "Could not extract source from BedrockRuntimeClientTypes.DocumentBlock" + ) + } + guard let name = sdkDocumentBlock.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.DocumentBlock" + ) + } + guard let sdkFormat = sdkDocumentBlock.format else { + throw BedrockServiceError.decodingError( + "Could not extract format from BedrockRuntimeClientTypes.DocumentBlock" + ) + } + let format = try DocumentFormat(from: sdkFormat) + switch sdkDocumentSource { + case .bytes(let data): + self = DocumentBlock(name: name, format: format, source: data.base64EncodedString()) + case .sdkUnknown(let unknownImageSource): + throw BedrockServiceError.notImplemented( + "ImageSource \(unknownImageSource) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKDocumentBlock() throws -> BedrockRuntimeClientTypes.DocumentBlock { + guard let data = Data(base64Encoded: source) else { + throw BedrockServiceError.decodingError( + "Could not decode document source from base64 string. String: \(source)" + ) + } + return BedrockRuntimeClientTypes.DocumentBlock( + format: format.getSDKDocumentFormat(), + name: name, + source: BedrockRuntimeClientTypes.DocumentSource.bytes(data) + ) + } +} extension DocumentFormat { init(from sdkDocumentFormat: BedrockRuntimeClientTypes.DocumentFormat) throws { diff --git a/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift index 2bb0d82d..237a3427 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift @@ -15,10 +15,16 @@ import Foundation -struct DocumentBlock: Codable { - let name: String - let format: DocumentFormat - let source: String // 64 encoded +public struct DocumentBlock: Codable { + public let name: String + public let format: DocumentFormat + public let source: String // 64 encoded + + public init(name: String, format: DocumentFormat, source: String) { + self.name = name + self.format = format + self.source = source + } } public enum DocumentFormat: Codable { From ac5cb6c529cfb933f00943894ff50a525b5a507d Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:02:12 +0200 Subject: [PATCH 11/35] DocumentBlock --- .../ConversionExtensions/ContentConversionExtension.swift | 4 ++++ backend/Sources/BedrockTypes/ConverseContent/Content.swift | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift index 2ba3cf90..56ae882c 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift @@ -25,6 +25,8 @@ extension Content { self = .text(text) case .image(let sdkImage): self = .image(try ImageBlock(from: sdkImage)) + case .document(let sdkDocumentBlock): + self = .document(try DocumentBlock(from: sdkDocumentBlock)) case .sdkUnknown(let unknownContentBlock): throw BedrockServiceError.notImplemented( "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" @@ -42,6 +44,8 @@ extension Content { return BedrockRuntimeClientTypes.ContentBlock.text(text) case .image(let imageBlock): return BedrockRuntimeClientTypes.ContentBlock.image(try imageBlock.getSDKImageBlock()) + case .document(let documentBlock): + return BedrockRuntimeClientTypes.ContentBlock.document(try documentBlock.getSDKDocumentBlock()) default: print("TODO") return BedrockRuntimeClientTypes.ContentBlock.text("TODO") diff --git a/backend/Sources/BedrockTypes/ConverseContent/Content.swift b/backend/Sources/BedrockTypes/ConverseContent/Content.swift index 59c9ce48..3699392e 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/Content.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/Content.swift @@ -20,5 +20,6 @@ public enum Content: Codable { case image(ImageBlock) // String are the 64 encoded bytes case toolUse(ToolUseBlock) case toolresult(ToolResultBlock) - // case reasoningcontent(String) + case document(DocumentBlock) + // case reasoningcontent(ReasoningBlock) } From fec96e22775e883784eb7adf8dd8d9808ad4b190 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:17:45 +0200 Subject: [PATCH 12/35] S3Location --- .../S3LocationConversionExtension.swift | 33 +++++++++++++++++ .../VideoConversionExtension.swift | 37 +++++++++++++++++++ .../BedrockTypes/BedrockServiceError.swift | 1 + .../ConverseContent/S3Location.swift | 9 ++++- .../ConverseContent/VideoBlock.swift | 34 ++++++++--------- 5 files changed, 95 insertions(+), 19 deletions(-) create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift new file mode 100644 index 00000000..2d9931e2 --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +extension S3Location { + init(from sdkS3Location: BedrockRuntimeClientTypes.S3Location) throws { + guard let uri = sdkS3Location.uri else { + throw BedrockServiceError.decodingError( + "Could not extract URI from BedrockRuntimeClientTypes.S3Location" + ) + } + guard uri.hasPrefix("") else { + throw BedrockServiceError.invalidURI("URI should start with \"s3://\". Your URI: \(uri)") + } + self = S3Location(bucketOwner: sdkS3Location.bucketOwner, uri: uri) + } +} + diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift new file mode 100644 index 00000000..2091e403 --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +extension VideoBlock { + // init(from sdkVideoBlock: BedrockRuntimeClientTypes.VideoBlock) throws + // func getSDKVideoBlock() throws -> BedrockRuntimeClientTypes.VideoBlock +} + +extension VideoFormat { + // init(from sdkVideoFormat: BedrockRuntimeClientTypes.VideoFormat) throws + // func getSDKVideoFormat() throws -> BedrockRuntimeClientTypes.VideoFormat +} + +extension VideoSource { + init(from sdkVideoSource: BedrockRuntimeClientTypes.VideoSource) throws { + switch sdkVideoSource { + case .bytes(let data): self = .bytes(data.base64EncodedString()) + case .s3location(let sdkS3Location): .s3() + } + } +} diff --git a/backend/Sources/BedrockTypes/BedrockServiceError.swift b/backend/Sources/BedrockTypes/BedrockServiceError.swift index e5c56fef..24193dac 100644 --- a/backend/Sources/BedrockTypes/BedrockServiceError.swift +++ b/backend/Sources/BedrockTypes/BedrockServiceError.swift @@ -21,6 +21,7 @@ public enum BedrockServiceError: Error { // case invalidModel(BedrockModel, String) case invalidPrompt(String) case invalidStopSequences([String], String) + case invalidURI(String) case invalidSDKResponse(String) case invalidSDKResponseBody(Data?) case completionNotFound(String) diff --git a/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift b/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift index d4021446..81e23703 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift @@ -15,7 +15,12 @@ import Foundation -public struct S3Location: Codable { - let bucketOwner: String +public struct S3Location: Codable, Sendable { + let bucketOwner: String? let uri: String + + public init(bucketOwner: String? = nil, uri: String) { + self.bucketOwner = bucketOwner + self.uri = uri + } } diff --git a/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift index 4770441b..902acc5b 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift @@ -16,23 +16,23 @@ import Foundation public struct VideoBlock: Codable { - let format: Format - let source: Source + public let format: VideoFormat + public let source: VideoSource +} - enum Source: Codable { - case bytes(String) // base64 - case s3(S3Location) - } +public enum VideoSource: Codable { + case bytes(String) // base64 + case s3(S3Location) +} - enum Format: Codable { - case flv - case mkv - case mov - case mp4 - case mpeg - case mpg - case threeGp - case webm - case wmv - } +public enum VideoFormat: Codable { + case flv + case mkv + case mov + case mp4 + case mpeg + case mpg + case threeGp + case webm + case wmv } From 3fef779d362b90878272a7c2c51f327777158d69 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:26:29 +0200 Subject: [PATCH 13/35] VideoSource --- .../DocumentConversionExtension.swift | 2 +- .../S3LocationConversionExtension.swift | 4 ++++ .../VideoConversionExtension.swift | 24 +++++++++++++++++-- .../ConverseContent/S3Location.swift | 4 ++-- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift index 59f58fcb..82b27414 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift @@ -37,7 +37,7 @@ extension DocumentBlock { let format = try DocumentFormat(from: sdkFormat) switch sdkDocumentSource { case .bytes(let data): - self = DocumentBlock(name: name, format: format, source: data.base64EncodedString()) + self = DocumentBlock(name: name,format: format, source: data.base64EncodedString()) case .sdkUnknown(let unknownImageSource): throw BedrockServiceError.notImplemented( "ImageSource \(unknownImageSource) is not implemented by BedrockRuntimeClientTypes" diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift index 2d9931e2..9f750ffa 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift @@ -29,5 +29,9 @@ extension S3Location { } self = S3Location(bucketOwner: sdkS3Location.bucketOwner, uri: uri) } + + func getSDKS3Location() -> BedrockRuntimeClientTypes.S3Location { + BedrockRuntimeClientTypes.S3Location(bucketOwner: self.bucketOwner, uri: self.uri) + } } diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift index 2091e403..d912c753 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift @@ -30,8 +30,28 @@ extension VideoFormat { extension VideoSource { init(from sdkVideoSource: BedrockRuntimeClientTypes.VideoSource) throws { switch sdkVideoSource { - case .bytes(let data): self = .bytes(data.base64EncodedString()) - case .s3location(let sdkS3Location): .s3() + case .bytes(let data): + self = .bytes(data.base64EncodedString()) + case .s3location(let sdkS3Location): + self = .s3(try S3Location(from: sdkS3Location)) + case .sdkUnknown(let unknownVideoSource): + throw BedrockServiceError.notImplemented( + "VideoSource \(unknownVideoSource) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKVideoSource() throws -> BedrockRuntimeClientTypes.VideoSource { + switch self { + case .bytes(let data): + guard let sdkData = Data(base64Encoded: data) else { + throw BedrockServiceError.decodingError( + "Could not decode video source from base64 string. String: \(data)" + ) + } + return .bytes(sdkData) + case .s3(let s3Location): + return .s3location(s3Location.getSDKS3Location()) } } } diff --git a/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift b/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift index 81e23703..ecf11d46 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/S3Location.swift @@ -16,8 +16,8 @@ import Foundation public struct S3Location: Codable, Sendable { - let bucketOwner: String? - let uri: String + public let bucketOwner: String? + public let uri: String public init(bucketOwner: String? = nil, uri: String) { self.bucketOwner = bucketOwner From da385b2e051e34fe37953fb1c9db7c02deb13c1a Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:32:02 +0200 Subject: [PATCH 14/35] VideoFormat --- .../VideoConversionExtension.swift | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift index d912c753..92a6ad57 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift @@ -23,8 +23,41 @@ extension VideoBlock { } extension VideoFormat { - // init(from sdkVideoFormat: BedrockRuntimeClientTypes.VideoFormat) throws - // func getSDKVideoFormat() throws -> BedrockRuntimeClientTypes.VideoFormat + init(from sdkVideoFormat: BedrockRuntimeClientTypes.VideoFormat) throws { + switch sdkVideoFormat { + case .flv: self = .flv + case .mkv: self = .mkv + case .mov: self = .mov + case .mp4: self = .mp4 + case .mpeg: self = .mpeg + case .mpg: self = .mpg + case .threeGp: self = .threeGp + case .webm: self = .webm + case .wmv: self = .wmv + case .sdkUnknown(let unknownVideoFormat): + throw BedrockServiceError.notImplemented( + "VideoFormat \(unknownVideoFormat) is not implemented by BedrockRuntimeClientTypes" + ) + // default: // in case new video formats get added to the sdk + // throw BedrockServiceError.notSupported( + // "VideoFormat \(sdkVideoFormat) is not supported by BedrockTypes" + // ) + } + } + + func getSDKVideoFormat() throws -> BedrockRuntimeClientTypes.VideoFormat { + switch self { + case .flv: return .flv + case .mkv: return .mkv + case .mov: return .mov + case .mp4: return .mp4 + case .mpeg: return .mpeg + case .mpg: return .mpg + case .threeGp: return .threeGp + case .webm: return .webm + case .wmv: return .wmv + } + } } extension VideoSource { From 00faaf46335d8c181212f380635d4dc62450dfd2 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:35:27 +0200 Subject: [PATCH 15/35] VideoBlock --- .../VideoConversionExtension.swift | 25 +++++++++++++++++-- .../ConverseContent/VideoBlock.swift | 5 ++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift index 92a6ad57..72ac1095 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift @@ -18,8 +18,29 @@ import BedrockTypes import Foundation extension VideoBlock { - // init(from sdkVideoBlock: BedrockRuntimeClientTypes.VideoBlock) throws - // func getSDKVideoBlock() throws -> BedrockRuntimeClientTypes.VideoBlock + init(from sdkVideoBlock: BedrockRuntimeClientTypes.VideoBlock) throws { + guard let sdkFormat = sdkVideoBlock.format else { + throw BedrockServiceError.decodingError( + "Could not extract format from BedrockRuntimeClientTypes.VideoBlock" + ) + } + guard let sdkSource = sdkVideoBlock.source else { + throw BedrockServiceError.decodingError( + "Could not extract source from BedrockRuntimeClientTypes.VideoBlock" + ) + } + self = VideoBlock( + format: try VideoFormat(from: sdkFormat), + source: try VideoSource(from: sdkSource) + ) + } + + func getSDKVideoBlock() throws -> BedrockRuntimeClientTypes.VideoBlock { + BedrockRuntimeClientTypes.VideoBlock( + format: try format.getSDKVideoFormat(), + source: try source.getSDKVideoSource() + ) + } } extension VideoFormat { diff --git a/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift index 902acc5b..5aa24c00 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift @@ -18,6 +18,11 @@ import Foundation public struct VideoBlock: Codable { public let format: VideoFormat public let source: VideoSource + + public init(format: VideoFormat, source: VideoSource) { + self.format = format + self.source = source + } } public enum VideoSource: Codable { From 1e8f0bebe362dca4743274cc4cd44346a8fc0e8c Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:43:06 +0200 Subject: [PATCH 16/35] ToolStatus --- .../ToolConversionExtension.swift | 26 +++++++++++++++-- .../ConverseContent/ToolBlocks.swift | 28 +++++++++---------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift index ce44cad1..2d0bf863 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift @@ -32,7 +32,7 @@ extension ToolUseBlock { self = ToolUseBlock( id: sdkId, name: sdkName - // input: sdkToolUseBlock.input + // input: sdkToolUseBlock.input ) } @@ -40,7 +40,27 @@ extension ToolUseBlock { .init( name: name, toolUseId: id - // input: input + // input: input ) } -} \ No newline at end of file +} + +extension ToolStatus { + init(from sdkToolStatus: BedrockRuntimeClientTypes.ToolResultStatus) throws { + switch sdkToolStatus { + case .success: self = .success + case .error: self = .error + case .sdkUnknown(let unknownToolStatus): + throw BedrockServiceError.notImplemented( + "ToolResultStatus \(unknownToolStatus) is not implemented by BedrockRuntimeClientTypes" + ) + } + } + + func getSDKToolStatus() -> BedrockRuntimeClientTypes.ToolResultStatus { + switch self { + case .success: .success + case .error: .error + } + } +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift index 9d2ca1e1..e3d47b2a 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift @@ -27,20 +27,20 @@ public struct ToolUseBlock: Codable { } public struct ToolResultBlock: Codable { - let content: String - let status: Status // currently only supported by Anthropic Claude 3 models - let toolUseId: String + public let content: String + public let status: ToolStatus // currently only supported by Anthropic Claude 3 models + public let toolUseId: String +} - enum Status: Codable { - case success - case error - } +public enum ToolStatus: Codable { + case success + case error +} - enum ToolResultContent: Codable { - // case json([String: Any]) - case text(String) - case image(ImageBlock) // currently only supported by Anthropic Claude 3 models - case document(DocumentBlock) - case video(VideoBlock) - } +public enum ToolResultContent: Codable { + // case json([String: Any]) + case text(String) + case image(ImageBlock) // currently only supported by Anthropic Claude 3 models + case document(DocumentBlock) + case video(VideoBlock) } From 8ac47e1c16b1d299cd54fb8ee2e90879fab69358 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 17:50:31 +0200 Subject: [PATCH 17/35] ToolResultContent --- .../ToolConversionExtension.swift | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift index 2d0bf863..d2756aff 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift @@ -45,6 +45,7 @@ extension ToolUseBlock { } } + extension ToolStatus { init(from sdkToolStatus: BedrockRuntimeClientTypes.ToolResultStatus) throws { switch sdkToolStatus { @@ -59,8 +60,33 @@ extension ToolStatus { func getSDKToolStatus() -> BedrockRuntimeClientTypes.ToolResultStatus { switch self { - case .success: .success - case .error: .error + case .success: .success + case .error: .error + } + } +} + +extension ToolResultContent { + init(from sdkToolResultContent: BedrockRuntimeClientTypes.ToolResultContentBlock) throws { + switch sdkToolResultContent { + case .document(let sdkDocumentBlock): + self = .document(try DocumentBlock(from: sdkDocumentBlock)) + case .image(let sdkImageBlock): + self = .image(try ImageBlock(from: sdkImageBlock)) + case .text(let text): + self = .text(text) + case .video(let sdkVideoBlock): + self = .video(try VideoBlock(from: sdkVideoBlock)) + // case .json(let sdkJSON): + // self = .json() + case .sdkUnknown(let unknownToolResultContent): + throw BedrockServiceError.notImplemented( + "ToolResultContentBlock \(unknownToolResultContent) is not implemented by BedrockRuntimeClientTypes" + ) + default: + throw BedrockServiceError.notImplemented( + "ToolResultContentBlock \(sdkToolResultContent) is not implemented by BedrockTypes" + ) } } } From 3b1fd1aa6c101240445b9f3b70a5ccf412dfb052 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 18:05:57 +0200 Subject: [PATCH 18/35] ToolResultBlock --- .../ToolConversionExtension.swift | 21 +++++++++++++++++++ .../ConverseContent/ToolBlocks.swift | 12 ++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift index d2756aff..fce1304b 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift @@ -45,6 +45,27 @@ extension ToolUseBlock { } } +extension ToolResultBlock { + init(from sdkToolResultBlock: BedrockRuntimeClientTypes.ToolResultBlock) throws { + guard let sdkToolResultContent = sdkToolResultBlock.content else { + throw BedrockServiceError.decodingError( + "Could not extract content from BedrockRuntimeClientTypes.ToolResultBlock" + ) + } + guard let id = sdkToolResultBlock.toolUseId else { + throw BedrockServiceError.decodingError( + "Could not extract toolUseId from BedrockRuntimeClientTypes.ToolResultBlock" + ) + } + let sdkToolStatus: BedrockRuntimeClientTypes.ToolResultStatus? = sdkToolResultBlock.status + var status: ToolStatus? = nil + if let sdkToolStatus = sdkToolStatus { + status = try ToolStatus(from: sdkToolStatus) + } + let toolContents = try sdkToolResultContent.map { try ToolResultContent(from: $0) } + self = ToolResultBlock(id: id, content: toolContents, status: status) + } +} extension ToolStatus { init(from sdkToolStatus: BedrockRuntimeClientTypes.ToolResultStatus) throws { diff --git a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift index e3d47b2a..c2913f0e 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift @@ -27,9 +27,15 @@ public struct ToolUseBlock: Codable { } public struct ToolResultBlock: Codable { - public let content: String - public let status: ToolStatus // currently only supported by Anthropic Claude 3 models - public let toolUseId: String + public let id: String + public let content: [ToolResultContent] + public let status: ToolStatus? // currently only supported by Anthropic Claude 3 models + + public init(id: String, content: [ToolResultContent], status: ToolStatus? = nil) { + self.id = id + self.content = content + self.status = status + } } public enum ToolStatus: Codable { From b8c20a270814b9ab28e6b86025e1350036f9e3ad Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 18:15:00 +0200 Subject: [PATCH 19/35] Content --- .../ConversionExtensions/ContentConversionExtension.swift | 6 ++++++ backend/Sources/BedrockTypes/ConverseContent/Content.swift | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift index 56ae882c..ad98d6d3 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift @@ -27,6 +27,12 @@ extension Content { self = .image(try ImageBlock(from: sdkImage)) case .document(let sdkDocumentBlock): self = .document(try DocumentBlock(from: sdkDocumentBlock)) + case .tooluse(let sdkToolUseBlock): + self = .toolUse(try ToolUseBlock(from: sdkToolUseBlock)) + case .toolresult(let sdkToolResultBlock): + self = .toolResult(try ToolResultBlock(from: sdkToolResultBlock)) + case .video(let sdkVideoBlock): + self = .video(try VideoBlock(from: sdkVideoBlock)) case .sdkUnknown(let unknownContentBlock): throw BedrockServiceError.notImplemented( "ContentBlock \(unknownContentBlock) is not implemented by BedrockRuntimeClientTypes" diff --git a/backend/Sources/BedrockTypes/ConverseContent/Content.swift b/backend/Sources/BedrockTypes/ConverseContent/Content.swift index 3699392e..70d63b72 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/Content.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/Content.swift @@ -19,7 +19,8 @@ public enum Content: Codable { case text(String) case image(ImageBlock) // String are the 64 encoded bytes case toolUse(ToolUseBlock) - case toolresult(ToolResultBlock) + case toolResult(ToolResultBlock) case document(DocumentBlock) + case video(VideoBlock) // case reasoningcontent(ReasoningBlock) } From 8b9a7b92c5c1595ed64529caf7cd26e0972b4283 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Mon, 31 Mar 2025 18:47:15 +0200 Subject: [PATCH 20/35] base for toolconfig --- .../Converse/ConverseRequest.swift | 55 ++++++++++++++++++- .../ContentConversionExtension.swift | 12 +++- .../ToolConversionExtension.swift | 22 ++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConverseRequest.swift b/backend/Sources/BedrockService/Converse/ConverseRequest.swift index c584ac8b..e68206d2 100644 --- a/backend/Sources/BedrockService/Converse/ConverseRequest.swift +++ b/backend/Sources/BedrockService/Converse/ConverseRequest.swift @@ -14,14 +14,14 @@ //===----------------------------------------------------------------------===// @preconcurrency import AWSBedrockRuntime -import Foundation import BedrockTypes +import Foundation public struct ConverseRequest { let model: BedrockModel let messages: [Message] let inferenceConfig: InferenceConfig? - // let toolConfig: ToolConfig? + let toolConfig: ToolConfig? init( model: BedrockModel, @@ -39,6 +39,7 @@ public struct ConverseRequest { topP: topP, stopSequences: stopSequences ) + self.toolConfig = nil } func getConverseInput() throws -> ConverseInput { @@ -87,3 +88,53 @@ public struct ConverseRequest { } } } + +public struct ToolConfig { + // let toolChoice: ToolChoice? + let tools: [ToolSpecification] +} + +// public enum ToolChoice { +// /// (Default). The Model automatically decides if a tool should be called or whether to generate text instead. +// case auto(_) +// /// The model must request at least one tool (no text is generated). +// case any(_) +// /// The Model must request the specified tool. Only supported by Anthropic Claude 3 models. +// case tool(String) +// } + +public struct ToolSpecification { + public let name: String + public let inputSchema: [String: Any] + public let description: String? + + init(name: String, inputSchema: [String: Any], description: String? = nil) { + self.name = name + self.inputSchema = inputSchema + self.description = description + } + + init(from sdkToolSpecification: BedrockRuntimeClientTypes.ToolSpecification) throws { + guard let name = sdkToolSpecification.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard let sdkInputSchema = sdkToolSpecification.inputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract inputSchema from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard case .json(let smithyDocument) = sdkInputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract JSON from BedrockRuntimeClientTypes.ToolSpecification.inputSchema" + ) + } + let inputSchema = try smithyDocument.asStringMap() + self = ToolSpecification( + name: name, + inputSchema: inputSchema, + description: sdkToolSpecification.description + ) + } +} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift index ad98d6d3..8b3e995e 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift @@ -52,9 +52,15 @@ extension Content { return BedrockRuntimeClientTypes.ContentBlock.image(try imageBlock.getSDKImageBlock()) case .document(let documentBlock): return BedrockRuntimeClientTypes.ContentBlock.document(try documentBlock.getSDKDocumentBlock()) - default: - print("TODO") - return BedrockRuntimeClientTypes.ContentBlock.text("TODO") + case .toolResult(let toolResultBlock): + return BedrockRuntimeClientTypes.ContentBlock.toolresult(try toolResultBlock.getSDKToolResultBlock()) + case .toolUse(let toolUseBlock): + return BedrockRuntimeClientTypes.ContentBlock.tooluse(try toolUseBlock.getSDKToolUseBlock()) + case .video(let videoBlock): + return BedrockRuntimeClientTypes.ContentBlock.video(try videoBlock.getSDKVideoBlock()) + // default: + // print("TODO") + // return BedrockRuntimeClientTypes.ContentBlock.text("TODO") } } } diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift index fce1304b..4b757120 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift @@ -65,6 +65,15 @@ extension ToolResultBlock { let toolContents = try sdkToolResultContent.map { try ToolResultContent(from: $0) } self = ToolResultBlock(id: id, content: toolContents, status: status) } + + func getSDKToolResultBlock() throws -> BedrockRuntimeClientTypes.ToolResultBlock { + BedrockRuntimeClientTypes.ToolResultBlock( + content: try content.map { try $0.getSDKToolResultContentBlock() }, + status: status?.getSDKToolStatus(), + toolUseId: id + ) + + } } extension ToolStatus { @@ -110,4 +119,17 @@ extension ToolResultContent { ) } } + + func getSDKToolResultContentBlock() throws -> BedrockRuntimeClientTypes.ToolResultContentBlock { + switch self { + case .document(let documentBlock): + .document(try documentBlock.getSDKDocumentBlock()) + case .image(let imageBlock): + .image(try imageBlock.getSDKImageBlock()) + case .text(let text): + .text(text) + case .video(let videoBlock): + .video(try videoBlock.getSDKVideoBlock()) + } + } } From 226ab12db39ed38c2d7f7c2e5a6cbd588e8e2c4f Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Tue, 1 Apr 2025 10:15:55 +0200 Subject: [PATCH 21/35] nested types for Blocks --- backend/Sources/App/Types/Chat.swift | 2 +- .../Converse/ConverseRequest.swift | 43 ++------------- .../DocumentConversionExtension.swift | 4 +- .../ImageConversionExtension.swift | 4 +- .../VideoConversionExtension.swift | 8 +-- .../BedrockService/Converse/Tool.swift | 54 +++++++++++++++++++ .../Sources/BedrockService/SwiftBedrock.swift | 7 +-- .../Sources/BedrockTypes/BedrockModel.swift | 8 --- .../ConverseContent/Content.swift | 2 +- .../ConverseContent/DocumentBlock.swift | 26 ++++----- .../ConverseContent/ImageBlock.swift | 8 +-- .../ConverseContent/ToolBlocks.swift | 2 +- .../ConverseContent/VideoBlock.swift | 36 ++++++------- 13 files changed, 108 insertions(+), 96 deletions(-) create mode 100644 backend/Sources/BedrockService/Converse/Tool.swift diff --git a/backend/Sources/App/Types/Chat.swift b/backend/Sources/App/Types/Chat.swift index d397b637..c98b9920 100644 --- a/backend/Sources/App/Types/Chat.swift +++ b/backend/Sources/App/Types/Chat.swift @@ -22,7 +22,7 @@ extension Message: ResponseCodable {} struct ChatInput: Codable { let prompt: String let history: [Message] - let imageFormat: ImageFormat? + let imageFormat: ImageBlock.Format? let imageBytes: String? let maxTokens: Int? let temperature: Double? diff --git a/backend/Sources/BedrockService/Converse/ConverseRequest.swift b/backend/Sources/BedrockService/Converse/ConverseRequest.swift index e68206d2..b90daddb 100644 --- a/backend/Sources/BedrockService/Converse/ConverseRequest.swift +++ b/backend/Sources/BedrockService/Converse/ConverseRequest.swift @@ -21,7 +21,7 @@ public struct ConverseRequest { let model: BedrockModel let messages: [Message] let inferenceConfig: InferenceConfig? - let toolConfig: ToolConfig? + // let toolConfig: ToolConfig? init( model: BedrockModel, @@ -30,6 +30,7 @@ public struct ConverseRequest { temperature: Double?, topP: Double?, stopSequences: [String]? + // tools: [Tool] ) { self.messages = messages self.model = model @@ -39,7 +40,7 @@ public struct ConverseRequest { topP: topP, stopSequences: stopSequences ) - self.toolConfig = nil + // self.toolConfig = ToolConfig(tools: tools) } func getConverseInput() throws -> ConverseInput { @@ -91,7 +92,7 @@ public struct ConverseRequest { public struct ToolConfig { // let toolChoice: ToolChoice? - let tools: [ToolSpecification] + let tools: [Tool] } // public enum ToolChoice { @@ -102,39 +103,3 @@ public struct ToolConfig { // /// The Model must request the specified tool. Only supported by Anthropic Claude 3 models. // case tool(String) // } - -public struct ToolSpecification { - public let name: String - public let inputSchema: [String: Any] - public let description: String? - - init(name: String, inputSchema: [String: Any], description: String? = nil) { - self.name = name - self.inputSchema = inputSchema - self.description = description - } - - init(from sdkToolSpecification: BedrockRuntimeClientTypes.ToolSpecification) throws { - guard let name = sdkToolSpecification.name else { - throw BedrockServiceError.decodingError( - "Could not extract name from BedrockRuntimeClientTypes.ToolSpecification" - ) - } - guard let sdkInputSchema = sdkToolSpecification.inputSchema else { - throw BedrockServiceError.decodingError( - "Could not extract inputSchema from BedrockRuntimeClientTypes.ToolSpecification" - ) - } - guard case .json(let smithyDocument) = sdkInputSchema else { - throw BedrockServiceError.decodingError( - "Could not extract JSON from BedrockRuntimeClientTypes.ToolSpecification.inputSchema" - ) - } - let inputSchema = try smithyDocument.asStringMap() - self = ToolSpecification( - name: name, - inputSchema: inputSchema, - description: sdkToolSpecification.description - ) - } -} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift index 82b27414..440cadcb 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift @@ -34,7 +34,7 @@ extension DocumentBlock { "Could not extract format from BedrockRuntimeClientTypes.DocumentBlock" ) } - let format = try DocumentFormat(from: sdkFormat) + let format = try DocumentBlock.Format(from: sdkFormat) switch sdkDocumentSource { case .bytes(let data): self = DocumentBlock(name: name,format: format, source: data.base64EncodedString()) @@ -59,7 +59,7 @@ extension DocumentBlock { } } -extension DocumentFormat { +extension DocumentBlock.Format { init(from sdkDocumentFormat: BedrockRuntimeClientTypes.DocumentFormat) throws { switch sdkDocumentFormat { case .csv: self = .csv diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift index 59be2206..35b22505 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift @@ -29,7 +29,7 @@ extension ImageBlock { "Could not extract source from BedrockRuntimeClientTypes.ImageBlock" ) } - let format = try ImageFormat(from: sdkFormat) + let format = try ImageBlock.Format(from: sdkFormat) switch sdkImageSource { case .bytes(let data): self = ImageBlock(format: format, source: data.base64EncodedString()) @@ -53,7 +53,7 @@ extension ImageBlock { } } -extension ImageFormat { +extension ImageBlock.Format { init(from sdkImageFormat: BedrockRuntimeClientTypes.ImageFormat) throws { switch sdkImageFormat { diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift index 72ac1095..0476a947 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift @@ -30,8 +30,8 @@ extension VideoBlock { ) } self = VideoBlock( - format: try VideoFormat(from: sdkFormat), - source: try VideoSource(from: sdkSource) + format: try VideoBlock.Format(from: sdkFormat), + source: try VideoBlock.Source(from: sdkSource) ) } @@ -43,7 +43,7 @@ extension VideoBlock { } } -extension VideoFormat { +extension VideoBlock.Format { init(from sdkVideoFormat: BedrockRuntimeClientTypes.VideoFormat) throws { switch sdkVideoFormat { case .flv: self = .flv @@ -81,7 +81,7 @@ extension VideoFormat { } } -extension VideoSource { +extension VideoBlock.Source { init(from sdkVideoSource: BedrockRuntimeClientTypes.VideoSource) throws { switch sdkVideoSource { case .bytes(let data): diff --git a/backend/Sources/BedrockService/Converse/Tool.swift b/backend/Sources/BedrockService/Converse/Tool.swift new file mode 100644 index 00000000..9facd50d --- /dev/null +++ b/backend/Sources/BedrockService/Converse/Tool.swift @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@preconcurrency import AWSBedrockRuntime +import BedrockTypes +import Foundation + +public struct Tool { + public let name: String + public let inputSchema: [String: Any] + public let description: String? + + init(name: String, inputSchema: [String: Any], description: String? = nil) { + self.name = name + self.inputSchema = inputSchema + self.description = description + } + + init(from sdkToolSpecification: BedrockRuntimeClientTypes.ToolSpecification) throws { + guard let name = sdkToolSpecification.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard let sdkInputSchema = sdkToolSpecification.inputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract inputSchema from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard case .json(let smithyDocument) = sdkInputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract JSON from BedrockRuntimeClientTypes.ToolSpecification.inputSchema" + ) + } + let inputSchema = try smithyDocument.asStringMap() + self = Tool( + name: name, + inputSchema: inputSchema, + description: sdkToolSpecification.description + ) + } +} \ No newline at end of file diff --git a/backend/Sources/BedrockService/SwiftBedrock.swift b/backend/Sources/BedrockService/SwiftBedrock.swift index a8a5a95f..8c97c48c 100644 --- a/backend/Sources/BedrockService/SwiftBedrock.swift +++ b/backend/Sources/BedrockService/SwiftBedrock.swift @@ -460,13 +460,14 @@ public struct BedrockService: Sendable { public func converse( with model: BedrockModel, prompt: String, - imageFormat: ImageFormat? = nil, + imageFormat: ImageBlock.Format? = nil, imageBytes: String? = nil, history: [Message] = [], maxTokens: Int? = nil, temperature: Double? = nil, topP: Double? = nil, - stopSequences: [String]? = nil + stopSequences: [String]? = nil, + tools: [Tool]? = nil ) async throws -> (String, [Message]) { logger.trace( "Conversing", @@ -491,7 +492,7 @@ public struct BedrockService: Sendable { var messages = history messages.append(Message(from: .user, content: [.text(prompt)])) - if let imageFormat: ImageFormat = imageFormat, + if let imageFormat: ImageBlock.Format = imageFormat, let imageBytes: String = imageBytes { guard model.hasConverseModality(.vision) else { diff --git a/backend/Sources/BedrockTypes/BedrockModel.swift b/backend/Sources/BedrockTypes/BedrockModel.swift index b7561b22..8f8226c6 100644 --- a/backend/Sources/BedrockTypes/BedrockModel.swift +++ b/backend/Sources/BedrockTypes/BedrockModel.swift @@ -229,14 +229,6 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { } return false } - - // public func hasConverseDocumentModality() -> Bool { - // modality as? any ConverseDocumentModality != nil - // } - - // public func hasConverseToolModality() -> Bool { - // modality as? any ConverseToolModality != nil - // } } extension BedrockModel: Encodable { diff --git a/backend/Sources/BedrockTypes/ConverseContent/Content.swift b/backend/Sources/BedrockTypes/ConverseContent/Content.swift index 70d63b72..4fca0366 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/Content.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/Content.swift @@ -17,7 +17,7 @@ import Foundation public enum Content: Codable { case text(String) - case image(ImageBlock) // String are the 64 encoded bytes + case image(ImageBlock) case toolUse(ToolUseBlock) case toolResult(ToolResultBlock) case document(DocumentBlock) diff --git a/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift index 237a3427..f34c97c2 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift @@ -17,24 +17,24 @@ import Foundation public struct DocumentBlock: Codable { public let name: String - public let format: DocumentFormat + public let format: Format public let source: String // 64 encoded - public init(name: String, format: DocumentFormat, source: String) { + public init(name: String, format: Format, source: String) { self.name = name self.format = format self.source = source } -} -public enum DocumentFormat: Codable { - case csv - case doc - case docx - case html - case md - case pdf - case txt - case xls - case xlsx + public enum Format: Codable { + case csv + case doc + case docx + case html + case md + case pdf + case txt + case xls + case xlsx + } } diff --git a/backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift index aac21ce1..ba66561f 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift @@ -16,18 +16,18 @@ import Foundation public struct ImageBlock: Codable { - public let format: ImageFormat + public let format: Format public let source: String // 64 encoded - public init(format: ImageFormat, source: String) { + public init(format: Format, source: String) { self.format = format self.source = source } -} -public enum ImageFormat: Codable { + public enum Format: Codable { case gif case jpeg case png case webp } +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift index c2913f0e..2809688a 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift @@ -44,7 +44,7 @@ public enum ToolStatus: Codable { } public enum ToolResultContent: Codable { - // case json([String: Any]) + // case json([String: Any]) // Just Data case text(String) case image(ImageBlock) // currently only supported by Anthropic Claude 3 models case document(DocumentBlock) diff --git a/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift index 5aa24c00..ded89b64 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift @@ -16,28 +16,28 @@ import Foundation public struct VideoBlock: Codable { - public let format: VideoFormat - public let source: VideoSource + public let format: Format + public let source: Source - public init(format: VideoFormat, source: VideoSource) { + public init(format: Format, source: Source) { self.format = format self.source = source } -} -public enum VideoSource: Codable { - case bytes(String) // base64 - case s3(S3Location) -} + public enum Source: Codable { + case bytes(String) // base64 + case s3(S3Location) + } -public enum VideoFormat: Codable { - case flv - case mkv - case mov - case mp4 - case mpeg - case mpg - case threeGp - case webm - case wmv + public enum Format: Codable { + case flv + case mkv + case mov + case mp4 + case mpeg + case mpg + case threeGp + case webm + case wmv + } } From 8201a9e3f7be468876765a316ea1854348ff3cf6 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Tue, 1 Apr 2025 10:37:37 +0200 Subject: [PATCH 22/35] backend: converse systemprompts --- backend/Sources/App/Application+build.swift | 3 ++- backend/Sources/App/Types/Chat.swift | 1 + .../Converse/ConverseRequest.swift | 16 +++++++++++--- .../Sources/BedrockService/SwiftBedrock.swift | 21 ++++++++++++------- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index bdda426f..b91b30f8 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -181,7 +181,8 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router [BedrockRuntimeClientTypes.SystemContentBlock]? { + return systemPrompts?.map { + BedrockRuntimeClientTypes.SystemContentBlock.text($0) + } + } + struct InferenceConfig { let maxTokens: Int? let temperature: Double? diff --git a/backend/Sources/BedrockService/SwiftBedrock.swift b/backend/Sources/BedrockService/SwiftBedrock.swift index 8c97c48c..5bd43a2b 100644 --- a/backend/Sources/BedrockService/SwiftBedrock.swift +++ b/backend/Sources/BedrockService/SwiftBedrock.swift @@ -17,9 +17,9 @@ @preconcurrency import AWSBedrockRuntime import AWSClientRuntime import AWSSDKIdentity +import BedrockTypes import Foundation import Logging -import BedrockTypes public struct BedrockService: Sendable { let region: Region @@ -461,12 +461,13 @@ public struct BedrockService: Sendable { with model: BedrockModel, prompt: String, imageFormat: ImageBlock.Format? = nil, - imageBytes: String? = nil, + imageBytes: String? = nil, history: [Message] = [], maxTokens: Int? = nil, temperature: Double? = nil, topP: Double? = nil, stopSequences: [String]? = nil, + systemPrompts: [String]? = nil, tools: [Tool]? = nil ) async throws -> (String, [Message]) { logger.trace( @@ -482,26 +483,31 @@ public struct BedrockService: Sendable { try validateConverseParams( modality: modality, prompt: prompt, - // FIXME: add image + // FIXME: add image history: history, maxTokens: maxTokens, temperature: temperature, topP: topP, stopSequences: stopSequences + // FIXME: add systemPrompts + // FIXME: add tools ) var messages = history messages.append(Message(from: .user, content: [.text(prompt)])) if let imageFormat: ImageBlock.Format = imageFormat, - let imageBytes: String = imageBytes + let imageBytes: String = imageBytes { guard model.hasConverseModality(.vision) else { throw BedrockServiceError.invalidModality( - model, modality, + model, + modality, "This model does not support converse vision." ) } - messages.append(Message(from: .user, content: [.image(ImageBlock(format: imageFormat, source: imageBytes))])) + messages.append( + Message(from: .user, content: [.image(ImageBlock(format: imageFormat, source: imageBytes))]) + ) } let converseRequest = ConverseRequest( @@ -510,7 +516,8 @@ public struct BedrockService: Sendable { maxTokens: maxTokens, temperature: temperature, topP: topP, - stopSequences: stopSequences + stopSequences: stopSequences, + systemPrompts: systemPrompts ) let input = try converseRequest.getConverseInput() logger.trace( From 74d8c57188298b0d3c6c1fee3209c73bce3bf81d Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Wed, 2 Apr 2025 15:59:20 +0200 Subject: [PATCH 23/35] JSON for tools --- backend/Sources/App/Application+build.swift | 3 +- backend/Sources/App/Types/Chat.swift | 1 + .../ConversionExtensions/DocumentToJSON.swift | 48 +++++++++++++ .../ConversionExtensions/JSONtoDocument.swift | 26 +++++++ .../ToolConversionExtension.swift | 66 +++++++++++++++--- .../BedrockService/Converse/Tool.swift | 54 --------------- .../BedrockTypes/ConverseContent/JSON.swift | 69 +++++++++++++++++++ .../BedrockTypes/ConverseContent/Tool.swift | 28 ++++++++ .../ConverseContent/ToolBlocks.swift | 7 +- 9 files changed, 235 insertions(+), 67 deletions(-) create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/DocumentToJSON.swift create mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/JSONtoDocument.swift delete mode 100644 backend/Sources/BedrockService/Converse/Tool.swift create mode 100644 backend/Sources/BedrockTypes/ConverseContent/JSON.swift create mode 100644 backend/Sources/BedrockTypes/ConverseContent/Tool.swift diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index b91b30f8..462d4e0d 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -182,7 +182,8 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router JSON { + switch self.type { + case .string: + return JSON(try self.asString()) + case .boolean: + return JSON(try self.asBoolean()) + case .integer: + return JSON(try self.asInteger()) + case .double, .float: + return JSON(try self.asDouble()) + case .list: + let array = try self.asList().map { try $0.toJSON() } + return JSON(array) + case .map: + let map = try self.asStringMap() + var result: [String: JSON] = [:] + for (key, value) in map { + result[key] = try value.toJSON() + } + return JSON(result) + case .blob: + let data = try self.asBlob() + return JSON(data) + default: + throw DocumentError.typeMismatch("Unsupported type for JSON conversion: \(self.type)") + } + } +} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/JSONtoDocument.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/JSONtoDocument.swift new file mode 100644 index 00000000..5d22d981 --- /dev/null +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/JSONtoDocument.swift @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import BedrockTypes +import Foundation +import Smithy + +extension JSON { + func toDocument() throws -> Document { + let encoder = JSONEncoder() + let encoded = try encoder.encode(self) + return try Document.make(from: encoded) + } +} diff --git a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift index 4b757120..8a54ed6f 100644 --- a/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift +++ b/backend/Sources/BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift @@ -16,6 +16,41 @@ @preconcurrency import AWSBedrockRuntime import BedrockTypes import Foundation +import Smithy + +extension Tool { + init(from sdkToolSpecification: BedrockRuntimeClientTypes.ToolSpecification) throws { + guard let name = sdkToolSpecification.name else { + throw BedrockServiceError.decodingError( + "Could not extract name from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard let sdkInputSchema = sdkToolSpecification.inputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract inputSchema from BedrockRuntimeClientTypes.ToolSpecification" + ) + } + guard case .json(let smithyDocument) = sdkInputSchema else { + throw BedrockServiceError.decodingError( + "Could not extract JSON from BedrockRuntimeClientTypes.ToolSpecification.inputSchema" + ) + } + let inputSchema = try smithyDocument.toJSON() + self = Tool( + name: name, + inputSchema: inputSchema, + description: sdkToolSpecification.description + ) + } + + func getSDKToolSpecification() throws -> BedrockRuntimeClientTypes.ToolSpecification { + BedrockRuntimeClientTypes.ToolSpecification( + description: description, + inputSchema: .json(try inputSchema.toDocument()), + name: name + ) + } +} extension ToolUseBlock { init(from sdkToolUseBlock: BedrockRuntimeClientTypes.ToolUseBlock) throws { @@ -29,18 +64,23 @@ extension ToolUseBlock { "Could not extract name from BedrockRuntimeClientTypes.ToolUseBlock" ) } + guard let sdkInput = sdkToolUseBlock.input else { + throw BedrockServiceError.decodingError( + "Could not extract input from BedrockRuntimeClientTypes.ToolUseBlock" + ) + } self = ToolUseBlock( id: sdkId, - name: sdkName - // input: sdkToolUseBlock.input + name: sdkName, + input: try sdkInput.toJSON() ) } func getSDKToolUseBlock() throws -> BedrockRuntimeClientTypes.ToolUseBlock { .init( + input: try input.toDocument(), name: name, toolUseId: id - // input: input ) } } @@ -72,7 +112,7 @@ extension ToolResultBlock { status: status?.getSDKToolStatus(), toolUseId: id ) - + } } @@ -107,21 +147,27 @@ extension ToolResultContent { self = .text(text) case .video(let sdkVideoBlock): self = .video(try VideoBlock(from: sdkVideoBlock)) - // case .json(let sdkJSON): - // self = .json() + case .json(let document): + self = .json(try document.toJSON()) + // case .json(let document): + // self = .json(document.data(using: .utf8)) case .sdkUnknown(let unknownToolResultContent): throw BedrockServiceError.notImplemented( "ToolResultContentBlock \(unknownToolResultContent) is not implemented by BedrockRuntimeClientTypes" ) - default: - throw BedrockServiceError.notImplemented( - "ToolResultContentBlock \(sdkToolResultContent) is not implemented by BedrockTypes" - ) + // default: + // throw BedrockServiceError.notImplemented( + // "ToolResultContentBlock \(sdkToolResultContent) is not implemented by BedrockTypes" + // ) } } func getSDKToolResultContentBlock() throws -> BedrockRuntimeClientTypes.ToolResultContentBlock { switch self { + // case .json(let data): + // .json(try Document.make(from: data)) + case .json(let json): + .json(try json.toDocument()) case .document(let documentBlock): .document(try documentBlock.getSDKDocumentBlock()) case .image(let imageBlock): diff --git a/backend/Sources/BedrockService/Converse/Tool.swift b/backend/Sources/BedrockService/Converse/Tool.swift deleted file mode 100644 index 9facd50d..00000000 --- a/backend/Sources/BedrockService/Converse/Tool.swift +++ /dev/null @@ -1,54 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Foundation Models Playground open source project -// -// Copyright (c) 2025 Amazon.com, Inc. or its affiliates -// and the Swift Foundation Models Playground project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -@preconcurrency import AWSBedrockRuntime -import BedrockTypes -import Foundation - -public struct Tool { - public let name: String - public let inputSchema: [String: Any] - public let description: String? - - init(name: String, inputSchema: [String: Any], description: String? = nil) { - self.name = name - self.inputSchema = inputSchema - self.description = description - } - - init(from sdkToolSpecification: BedrockRuntimeClientTypes.ToolSpecification) throws { - guard let name = sdkToolSpecification.name else { - throw BedrockServiceError.decodingError( - "Could not extract name from BedrockRuntimeClientTypes.ToolSpecification" - ) - } - guard let sdkInputSchema = sdkToolSpecification.inputSchema else { - throw BedrockServiceError.decodingError( - "Could not extract inputSchema from BedrockRuntimeClientTypes.ToolSpecification" - ) - } - guard case .json(let smithyDocument) = sdkInputSchema else { - throw BedrockServiceError.decodingError( - "Could not extract JSON from BedrockRuntimeClientTypes.ToolSpecification.inputSchema" - ) - } - let inputSchema = try smithyDocument.asStringMap() - self = Tool( - name: name, - inputSchema: inputSchema, - description: sdkToolSpecification.description - ) - } -} \ No newline at end of file diff --git a/backend/Sources/BedrockTypes/ConverseContent/JSON.swift b/backend/Sources/BedrockTypes/ConverseContent/JSON.swift new file mode 100644 index 00000000..1eb50956 --- /dev/null +++ b/backend/Sources/BedrockTypes/ConverseContent/JSON.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct JSON: Codable { + var value: Any? + + public init(_ value: Any?) { + self.value = value + } + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + if container.decodeNil() { + self.value = nil + } else if let intValue = try? container.decode(Int.self) { + self.value = intValue + } else if let doubleValue = try? container.decode(Double.self) { + self.value = doubleValue + } else if let stringValue = try? container.decode(String.self) { + self.value = stringValue + } else if let boolValue = try? container.decode(Bool.self) { + self.value = boolValue + } else if let arrayValue = try? container.decode([JSON].self) { + self.value = arrayValue.map { $0.value } + } else if let dictionaryValue = try? container.decode([String: JSON].self) { + self.value = dictionaryValue.mapValues { $0.value } + } else { + throw DecodingError.dataCorruptedError(in: container, debugDescription: "Unsupported type") + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + if let intValue = value as? Int { + try container.encode(intValue) + } else if let doubleValue = value as? Double { + try container.encode(doubleValue) + } else if let stringValue = value as? String { + try container.encode(stringValue) + } else if let boolValue = value as? Bool { + try container.encode(boolValue) + } else if let arrayValue = value as? [Any] { + let jsonArray = arrayValue.map { JSON($0) } + try container.encode(jsonArray) + } else if let dictionaryValue = value as? [String: Any] { + let jsonDictionary = dictionaryValue.mapValues { JSON($0) } + try container.encode(jsonDictionary) + } else { + throw EncodingError.invalidValue( + value ?? "nil", + EncodingError.Context(codingPath: encoder.codingPath, debugDescription: "Unsupported type") + ) + } + } +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/Tool.swift b/backend/Sources/BedrockTypes/ConverseContent/Tool.swift new file mode 100644 index 00000000..2d49bced --- /dev/null +++ b/backend/Sources/BedrockTypes/ConverseContent/Tool.swift @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +public struct Tool: Codable { + public let name: String + public let inputSchema: JSON + public let description: String? + + public init(name: String, inputSchema: JSON, description: String? = nil) { + self.name = name + self.inputSchema = inputSchema + self.description = description + } +} diff --git a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift index 2809688a..616fd104 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift @@ -18,11 +18,12 @@ import Foundation public struct ToolUseBlock: Codable { public let id: String public let name: String - // let input: [String: Any] + public let input: JSON - public init(id: String, name: String) { + public init(id: String, name: String, input: JSON) { self.id = id self.name = name + self.input = input } } @@ -45,6 +46,8 @@ public enum ToolStatus: Codable { public enum ToolResultContent: Codable { // case json([String: Any]) // Just Data + // case json(Data) + case json(JSON) case text(String) case image(ImageBlock) // currently only supported by Anthropic Claude 3 models case document(DocumentBlock) From 5376807f32fdea6d36ff759efea01a2a98b9c2fb Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Wed, 2 Apr 2025 16:12:20 +0200 Subject: [PATCH 24/35] backend: tools --- .../Converse/ConverseRequest.swift | 41 +++++++++++-------- .../Sources/BedrockService/SwiftBedrock.swift | 3 +- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/backend/Sources/BedrockService/Converse/ConverseRequest.swift b/backend/Sources/BedrockService/Converse/ConverseRequest.swift index 904522f7..b9a37795 100644 --- a/backend/Sources/BedrockService/Converse/ConverseRequest.swift +++ b/backend/Sources/BedrockService/Converse/ConverseRequest.swift @@ -21,7 +21,7 @@ public struct ConverseRequest { let model: BedrockModel let messages: [Message] let inferenceConfig: InferenceConfig? - // let toolConfig: ToolConfig? + let toolConfig: ToolConfig? let systemPrompts: [String]? init( @@ -31,8 +31,8 @@ public struct ConverseRequest { temperature: Double?, topP: Double?, stopSequences: [String]?, - systemPrompts: [String]? - // tools: [Tool] + systemPrompts: [String]?, + tools: [Tool]? ) { self.messages = messages self.model = model @@ -43,21 +43,20 @@ public struct ConverseRequest { stopSequences: stopSequences ) self.systemPrompts = systemPrompts - // self.toolConfig = ToolConfig(tools: tools) + if tools != nil { + self.toolConfig = ToolConfig(tools: tools!) + } else { + self.toolConfig = nil + } } func getConverseInput() throws -> ConverseInput { - let sdkInferenceConfig: BedrockRuntimeClientTypes.InferenceConfiguration? - if inferenceConfig != nil { - sdkInferenceConfig = inferenceConfig!.getSDKInferenceConfig() - } else { - sdkInferenceConfig = nil - } - return ConverseInput( - inferenceConfig: sdkInferenceConfig, + ConverseInput( + inferenceConfig: inferenceConfig?.getSDKInferenceConfig(), messages: try getSDKMessages(), modelId: model.id, - system: getSDKSystemPrompts() + system: getSDKSystemPrompts(), + toolConfig: try toolConfig?.getSDKToolConfig() ) } @@ -66,7 +65,7 @@ public struct ConverseRequest { } private func getSDKSystemPrompts() -> [BedrockRuntimeClientTypes.SystemContentBlock]? { - return systemPrompts?.map { + systemPrompts?.map { BedrockRuntimeClientTypes.SystemContentBlock.text($0) } } @@ -98,11 +97,17 @@ public struct ConverseRequest { ) } } -} -public struct ToolConfig { - // let toolChoice: ToolChoice? - let tools: [Tool] + public struct ToolConfig { + // let toolChoice: ToolChoice? + let tools: [Tool] + + func getSDKToolConfig() throws -> BedrockRuntimeClientTypes.ToolConfiguration { + BedrockRuntimeClientTypes.ToolConfiguration( + tools: try tools.map { .toolspec(try $0.getSDKToolSpecification()) } + ) + } + } } // public enum ToolChoice { diff --git a/backend/Sources/BedrockService/SwiftBedrock.swift b/backend/Sources/BedrockService/SwiftBedrock.swift index 5bd43a2b..ff28a661 100644 --- a/backend/Sources/BedrockService/SwiftBedrock.swift +++ b/backend/Sources/BedrockService/SwiftBedrock.swift @@ -517,7 +517,8 @@ public struct BedrockService: Sendable { temperature: temperature, topP: topP, stopSequences: stopSequences, - systemPrompts: systemPrompts + systemPrompts: systemPrompts, + tools: tools ) let input = try converseRequest.getConverseInput() logger.trace( From b527ae39c7f486894de052b2ce99ac104b149089 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Wed, 2 Apr 2025 17:25:06 +0200 Subject: [PATCH 25/35] tool use --- backend/Sources/App/Application+build.swift | 21 +++++++++++++------ backend/Sources/App/Types/Chat.swift | 2 +- ...wiftBedrock.swift => BedrockService.swift} | 17 ++++++++++++++- .../BedrockTypes/ConverseContent/JSON.swift | 5 ++++- .../{ => ConverseContent}/Message.swift | 0 5 files changed, 36 insertions(+), 9 deletions(-) rename backend/Sources/BedrockService/{SwiftBedrock.swift => BedrockService.swift} (97%) rename backend/Sources/BedrockTypes/{ => ConverseContent}/Message.swift (100%) diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index 462d4e0d..473d2487 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -13,10 +13,11 @@ // //===----------------------------------------------------------------------===// -import Hummingbird -import Logging import BedrockService import BedrockTypes +import Hummingbird +import Logging +import Foundation /// Application arguments protocol. We use a protocol so we can call /// `buildApplication` inside Tests as well as in the App executable. @@ -175,9 +176,9 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router Router Router Date: Wed, 2 Apr 2025 17:57:40 +0200 Subject: [PATCH 26/35] Tool result --- backend/Sources/App/Application+build.swift | 13 ++--- backend/Sources/App/Types/Chat.swift | 1 + .../BedrockService/BedrockService.swift | 50 ++++++++++++------- .../BedrockService/ParameterValidation.swift | 10 ++-- .../ConverseContent/ToolBlocks.swift | 48 +++++++++++++++++- 5 files changed, 89 insertions(+), 33 deletions(-) diff --git a/backend/Sources/App/Application+build.swift b/backend/Sources/App/Application+build.swift index 473d2487..2e6536c6 100644 --- a/backend/Sources/App/Application+build.swift +++ b/backend/Sources/App/Application+build.swift @@ -184,17 +184,10 @@ func buildRouter(useSSO: Bool, logger: Logger) async throws -> Router (String, [Message]) { logger.trace( "Conversing", metadata: [ "model.id": .string(model.id), "model.modality": .string(model.modality.getName()), - "prompt": .string(prompt), + "prompt": .string(prompt ?? "No prompt"), ] ) do { @@ -489,35 +490,50 @@ public struct BedrockService: Sendable { temperature: temperature, topP: topP, stopSequences: stopSequences - // FIXME: add systemPrompts - // FIXME: add tools + // FIXME: add systemPrompts + // FIXME: add tools ) var messages = history - messages.append(Message(from: .user, content: [.text(prompt)])) - if let imageFormat: ImageBlock.Format = imageFormat, - let imageBytes: String = imageBytes - { - guard model.hasConverseModality(.vision) else { + + if tools != nil || toolResult != nil { + guard model.hasConverseModality(.toolUse) else { throw BedrockServiceError.invalidModality( model, modality, - "This model does not support converse vision." + "This model does not support converse tool." ) } - messages.append( - Message(from: .user, content: [.image(ImageBlock(format: imageFormat, source: imageBytes))]) - ) } - if tools != nil { - guard model.hasConverseModality(.toolUse) else { + if toolResult != nil { + guard let _: [Tool] = tools else { + throw BedrockServiceError.invalidPrompt("Tool result is defined but tools are not.") + } + guard case .toolUse(_) = messages.last?.content.last else { + throw BedrockServiceError.invalidPrompt("Tool result is defined but last message is not tool use.") + } + messages.append(Message(from: .user, content: [.toolResult(toolResult!)])) + } else { + guard let prompt = prompt else { + throw BedrockServiceError.invalidPrompt("Prompt is not defined.") + } + messages.append(Message(from: .user, content: [.text(prompt)])) + } + + if let imageFormat = imageFormat, + let imageBytes = imageBytes + { + guard model.hasConverseModality(.vision) else { throw BedrockServiceError.invalidModality( model, modality, - "This model does not support converse tool." + "This model does not support converse vision." ) } + messages.append( + Message(from: .user, content: [.image(ImageBlock(format: imageFormat, source: imageBytes))]) + ) } let converseRequest = ConverseRequest( diff --git a/backend/Sources/BedrockService/ParameterValidation.swift b/backend/Sources/BedrockService/ParameterValidation.swift index 429a6849..6989a814 100644 --- a/backend/Sources/BedrockService/ParameterValidation.swift +++ b/backend/Sources/BedrockService/ParameterValidation.swift @@ -13,8 +13,8 @@ // //===----------------------------------------------------------------------===// -import Foundation import BedrockTypes +import Foundation extension BedrockService { @@ -109,7 +109,7 @@ extension BedrockService { /// Validate parameters for a converse request public func validateConverseParams( modality: any ConverseModality, - prompt: String, + prompt: String?, history: [Message], maxTokens: Int?, temperature: Double?, @@ -117,7 +117,9 @@ extension BedrockService { stopSequences: [String]? ) throws { let parameters = modality.getConverseParameters() - try validatePrompt(prompt, maxPromptTokens: parameters.prompt.maxSize) + if prompt != nil { + try validatePrompt(prompt!, maxPromptTokens: parameters.prompt.maxSize) + } if maxTokens != nil { try validateParameterValue(maxTokens!, parameter: parameters.maxTokens) } @@ -177,7 +179,7 @@ extension BedrockService { metadata: [ "parameter": "\(parameter.name)", "value": "\(value)", "value.min": "\(String(describing: parameter.minValue))", - "value.max": "\(String(describing: parameter.maxValue))" + "value.max": "\(String(describing: parameter.maxValue))", ] ) } diff --git a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift index 616fd104..cc7eebb3 100644 --- a/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift +++ b/backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift @@ -44,8 +44,8 @@ public enum ToolStatus: Codable { case error } -public enum ToolResultContent: Codable { - // case json([String: Any]) // Just Data +public enum ToolResultContent { + // case json([String: Any]) // Just Data // case json(Data) case json(JSON) case text(String) @@ -53,3 +53,47 @@ public enum ToolResultContent: Codable { case document(DocumentBlock) case video(VideoBlock) } + +extension ToolResultContent: Codable { + private enum CodingKeys: String, CodingKey { + case json, text, image, document, video + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .json(let json): + try container.encode(json, forKey: .json) + case .text(let text): + try container.encode(text, forKey: .text) + case .image(let image): + try container.encode(image, forKey: .image) + case .document(let doc): + try container.encode(doc, forKey: .document) + case .video(let video): + try container.encode(video, forKey: .video) + } + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let json = try container.decodeIfPresent(JSON.self, forKey: .json) { + self = .json(json) + } else if let text = try container.decodeIfPresent(String.self, forKey: .text) { + self = .text(text) + } else if let image = try container.decodeIfPresent(ImageBlock.self, forKey: .image) { + self = .image(image) + } else if let doc = try container.decodeIfPresent(DocumentBlock.self, forKey: .document) { + self = .document(doc) + } else if let video = try container.decodeIfPresent(VideoBlock.self, forKey: .video) { + self = .video(video) + } else { + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Invalid tool result content" + ) + ) + } + } +} From 1b2cf7cd851e44df39560eaf783a5fcb7bc34da7 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Wed, 2 Apr 2025 18:01:10 +0200 Subject: [PATCH 27/35] Tool result --- backend/Sources/App/Types/Chat.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Sources/App/Types/Chat.swift b/backend/Sources/App/Types/Chat.swift index c5e1f411..c57c5a74 100644 --- a/backend/Sources/App/Types/Chat.swift +++ b/backend/Sources/App/Types/Chat.swift @@ -20,7 +20,7 @@ import BedrockTypes extension Message: ResponseCodable {} struct ChatInput: Codable { - let prompt: String + let prompt: String? let history: [Message]? let imageFormat: ImageBlock.Format? let imageBytes: String? From 97a7beabd35f5878ff6ca3a974ed18d295065123 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Wed, 2 Apr 2025 18:11:35 +0200 Subject: [PATCH 28/35] Deleted Converse only modality for Nova --- README.md | 3 ++- .../BedrockTypes/Models/Amazon/Nova/Nova.swift | 18 +++++++++--------- .../Models/Amazon/Nova/NovaBedrockModels.swift | 10 ++++++---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b2a110b4..80f19a20 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ Work in progress, feel free to open issue, but do not use in your projects. + diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift index 5a30bf3a..b74480c5 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift @@ -15,17 +15,17 @@ import Foundation -struct NovaConverse: ConverseModality { - func getName() -> String { "Nova Lite and Nova Pro Converse Modality" } +// struct NovaConverse: ConverseModality { +// func getName() -> String { "Nova Lite and Nova Pro Converse Modality" } - let converseParameters: ConverseParameters - let converseFeatures: [ConverseFeature] +// let converseParameters: ConverseParameters +// let converseFeatures: [ConverseFeature] - init(parameters: ConverseParameters, features: [ConverseFeature]) { - self.converseParameters = parameters - self.converseFeatures = features - } -} +// init(parameters: ConverseParameters, features: [ConverseFeature]) { +// self.converseParameters = parameters +// self.converseFeatures = features +// } +// } struct NovaText: TextModality, ConverseModality { func getName() -> String { "Nova Text Generation" } diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift index 237526c8..59cdb05d 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/NovaBedrockModels.swift @@ -39,11 +39,12 @@ extension BedrockModel { public static let nova_lite: BedrockModel = BedrockModel( id: "amazon.nova-lite-v1:0", name: "Nova Lite", - modality: NovaConverse( - parameters: ConverseParameters( + modality: NovaText( + parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 5_000, defaultValue: 5_000), topP: Parameter(.topP, minValue: 0, maxValue: 1.0, defaultValue: 0.9), + topK: Parameter(.topK, minValue: 0, maxValue: nil, defaultValue: 50), stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), maxPromptSize: nil ), @@ -53,11 +54,12 @@ extension BedrockModel { public static let nova_pro: BedrockModel = BedrockModel( id: "amazon.nova-pro-v1:0", name: "Nova Pro", - modality: NovaConverse( - parameters: ConverseParameters( + modality: NovaText( + parameters: TextGenerationParameters( temperature: Parameter(.temperature, minValue: 0.00001, maxValue: 1, defaultValue: 0.7), maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: 5_000, defaultValue: 5_000), topP: Parameter(.topP, minValue: 0, maxValue: 1.0, defaultValue: 0.9), + topK: Parameter(.topK, minValue: 0, maxValue: nil, defaultValue: 50), stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), maxPromptSize: nil ), From ed422139fc8dabdb5cb647f1dab1cbcb2c5a4f24 Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Thu, 3 Apr 2025 10:54:00 +0200 Subject: [PATCH 29/35] refactor: moved converse transforamtions to BedrockTypes --- README.md | 4 +- backend/Package.swift | 4 + .../BedrockService/BedrockService.swift | 6 +- .../RoleConversionExtension.swift | 38 ----- .../VideoConversionExtension.swift | 111 -------------- .../Sources/BedrockService/ModelSummary.swift | 6 +- .../Converse/Content.swift} | 14 +- .../Converse/DocumentBlock.swift} | 87 +++++++---- .../Converse}/DocumentToJSON.swift | 4 +- .../Converse/ImageBlock.swift} | 57 ++++--- .../{ConverseContent => Converse}/JSON.swift | 0 .../Converse}/JSONtoDocument.swift | 3 +- .../Converse/Message.swift} | 27 +++- .../Converse/S3Location.swift} | 16 +- .../Sources/BedrockTypes/Converse/Tool.swift | 62 ++++++++ .../Converse/ToolResultBlock.swift} | 145 +++++++++--------- .../BedrockTypes/Converse/ToolUseBlock.swift | 60 ++++++++ .../BedrockTypes/Converse/VideoBlock.swift | 131 ++++++++++++++++ .../ConverseContent/Content.swift | 26 ---- .../ConverseContent/DocumentBlock.swift | 40 ----- .../ConverseContent/ImageBlock.swift | 33 ---- .../ConverseContent/Message.swift | 30 ---- .../ConverseContent/S3Location.swift | 26 ---- .../BedrockTypes/ConverseContent/Tool.swift | 28 ---- .../ConverseContent/ToolBlocks.swift | 99 ------------ .../ConverseContent/VideoBlock.swift | 43 ------ .../{ => InvokeModel}/ContentType.swift | 0 .../ImageGenerationOutput.swift | 0 .../{ => InvokeModel}/ImageResolution.swift | 0 .../{ => InvokeModel}/Protocols.swift | 0 .../{ => InvokeModel}/TextCompletion.swift | 0 backend/Sources/BedrockTypes/Role.swift | 19 +++ 32 files changed, 489 insertions(+), 630 deletions(-) delete mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/RoleConversionExtension.swift delete mode 100644 backend/Sources/BedrockService/Converse/ConversionExtensions/VideoConversionExtension.swift rename backend/Sources/{BedrockService/Converse/ConversionExtensions/ContentConversionExtension.swift => BedrockTypes/Converse/Content.swift} (86%) rename backend/Sources/{BedrockService/Converse/ConversionExtensions/DocumentConversionExtension.swift => BedrockTypes/Converse/DocumentBlock.swift} (53%) rename backend/Sources/{BedrockService/Converse/ConversionExtensions => BedrockTypes/Converse}/DocumentToJSON.swift (94%) rename backend/Sources/{BedrockService/Converse/ConversionExtensions/ImageConversionExtension.swift => BedrockTypes/Converse/ImageBlock.swift} (61%) rename backend/Sources/BedrockTypes/{ConverseContent => Converse}/JSON.swift (100%) rename backend/Sources/{BedrockService/Converse/ConversionExtensions => BedrockTypes/Converse}/JSONtoDocument.swift (92%) rename backend/Sources/{BedrockService/Converse/ConversionExtensions/MessageConversionExtension.swift => BedrockTypes/Converse/Message.swift} (67%) rename backend/Sources/{BedrockService/Converse/ConversionExtensions/S3LocationConversionExtension.swift => BedrockTypes/Converse/S3Location.swift} (75%) create mode 100644 backend/Sources/BedrockTypes/Converse/Tool.swift rename backend/Sources/{BedrockService/Converse/ConversionExtensions/ToolConversionExtension.swift => BedrockTypes/Converse/ToolResultBlock.swift} (56%) create mode 100644 backend/Sources/BedrockTypes/Converse/ToolUseBlock.swift create mode 100644 backend/Sources/BedrockTypes/Converse/VideoBlock.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/Content.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/DocumentBlock.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/ImageBlock.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/Message.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/S3Location.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/Tool.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/ToolBlocks.swift delete mode 100644 backend/Sources/BedrockTypes/ConverseContent/VideoBlock.swift rename backend/Sources/BedrockTypes/{ => InvokeModel}/ContentType.swift (100%) rename backend/Sources/BedrockTypes/{ => InvokeModel}/ImageGenerationOutput.swift (100%) rename backend/Sources/BedrockTypes/{ => InvokeModel}/ImageResolution.swift (100%) rename backend/Sources/BedrockTypes/{ => InvokeModel}/Protocols.swift (100%) rename backend/Sources/BedrockTypes/{ => InvokeModel}/TextCompletion.swift (100%) diff --git a/README.md b/README.md index 80f19a20..376fde83 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # SwiftBedrockService Work in progress, feel free to open issue, but do not use in your projects. - +1. How to use the library? +2. Example for every functionality +3. How to add a model(family) +Note that the minimum, maximum and default values for each parameter are model specific and defined when the BedrockModel is created. Some parameters might not be supported by certain models. + +## How to chat using the Converse API + +### Text prompt + +### Vision + +### Tools + +## How to add a BedrockModel + +### Text + +### Image + +### Converse + + diff --git a/backend/Sources/BedrockService/BedrockService.swift b/backend/Sources/BedrockService/BedrockService.swift index e886dd6a..792c528b 100644 --- a/backend/Sources/BedrockService/BedrockService.swift +++ b/backend/Sources/BedrockService/BedrockService.swift @@ -44,7 +44,7 @@ public struct BedrockService: Sendable { bedrockRuntimeClient: BedrockRuntimeClientProtocol? = nil, useSSO: Bool = false ) async throws { - self.logger = logger ?? BedrockService.createLogger("swiftbedrock.service") + self.logger = logger ?? BedrockService.createLogger("bedrock.service") self.logger.trace( "Initializing SwiftBedrock", metadata: ["region": .string(region.rawValue)] diff --git a/backend/Sources/BedrockTypes/BedrockModel.swift b/backend/Sources/BedrockTypes/BedrockModel.swift index 8f8226c6..2c315540 100644 --- a/backend/Sources/BedrockTypes/BedrockModel.swift +++ b/backend/Sources/BedrockTypes/BedrockModel.swift @@ -103,6 +103,10 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { } } + // MARK: Modality checks + + // MARK - Text completion + /// Checks if the model supports text generation /// - Returns: True if the model supports text generation public func hasTextModality() -> Bool { @@ -122,34 +126,7 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return textModality } - /// Checks if the model supports text generation and returns ConverseModality - /// - Returns: ConverseModality if the model supports text modality - public func getConverseModality() throws -> any ConverseModality { - guard let modality = modality as? any ConverseModality else { - throw BedrockServiceError.invalidModality( - self, - modality, - "Model \(id) does not support text generation" - ) - } - return modality - } - - // FIXME: this would be cleaner - // Error: Only concrete types such as structs, enums and classes can conform to protocols - public func getModality() throws -> M { - guard let modality = modality as? M else { - throw BedrockServiceError.invalidModality( - self, - modality, - "Model \(id) does not support \(M.self)" - ) - } - return modality - } - public func hasModality(_ type: M.Type) -> Bool { - modality as? M != nil - } + // MARK - Image generation /// Checks if the model supports image generation /// - Returns: True if the model supports image generation @@ -170,6 +147,12 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return imageModality } + /// Checks if the model supports text to image generation + /// - Returns: True if the model supports text to image generation + public func hasTextToImageModality() -> Bool { + modality as? any TextToImageModality != nil + } + /// Checks if the model supports text to image generation and returns TextToImageModality /// - Returns: TextToImageModality if the model supports image modality public func getTextToImageModality() throws -> any TextToImageModality { @@ -183,6 +166,12 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return textToImageModality } + /// Checks if the model supports image variation + /// - Returns: True if the model supports image variation + public func hasImageVariationModality() -> Bool { + modality as? any ImageVariationModality != nil + } + /// Checks if the model supports image variation and returns ImageVariationModality /// - Returns: ImageVariationModality if the model supports image modality public func getImageVariationModality() throws -> any ImageVariationModality { @@ -196,32 +185,24 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { return modality } - /// Checks if the model supports text to image generation - /// - Returns: True if the model supports text to image generation - public func hasTextToImageModality() -> Bool { - modality as? any TextToImageModality != nil - } - - /// Checks if the model supports image variation - /// - Returns: True if the model supports image variation - public func hasImageVariationModality() -> Bool { - modality as? any ImageVariationModality != nil - } - /// Checks if the model supports conditioned text to image generation /// - Returns: True if the model supports conditioned text to image generation public func hasConditionedTextToImageModality() -> Bool { modality as? any ConditionedTextToImageModality != nil } + // MARK - Converse + /// Checks if the model supports converse /// - Returns: True if the model supports converse public func hasConverseModality() -> Bool { modality as? any ConverseModality != nil } - // /// Checks if the model supports converse vision - // /// - Returns: True if the model supports converse vision + /// Checks if the model supports a specific converse feature + /// - Parameters: + /// - feature: the ConverseFeature that will be checked + /// - Returns: True if the model supports the converse feature public func hasConverseModality(_ feature: ConverseFeature = .textGeneration) -> Bool { if let converseModality = modality as? any ConverseModality { let features = converseModality.getConverseFeatures() @@ -229,6 +210,19 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { } return false } + + /// Checks if the model supports text generation and returns ConverseModality + /// - Returns: ConverseModality if the model supports text modality + public func getConverseModality() throws -> any ConverseModality { + guard let modality = modality as? any ConverseModality else { + throw BedrockServiceError.invalidModality( + self, + modality, + "Model \(id) does not support text generation" + ) + } + return modality + } } extension BedrockModel: Encodable { From d7aa092004782d14795670ffa3d0fa1b9db37d8b Mon Sep 17 00:00:00 2001 From: monadierickx <126071495+monadierickx@users.noreply.github.com> Date: Thu, 3 Apr 2025 21:33:48 +0200 Subject: [PATCH 35/35] README + StanderdConverse --- README.md | 177 ++++++++++++++++++ backend/Package.swift | 2 +- .../Types/Cohere/CohereBedrockModels.swift | 52 +++++ .../BedrockService/BedrockService.swift | 2 +- .../Sources/BedrockTypes/BedrockModel.swift | 3 + .../StandardConverse.swift} | 13 +- .../Models/Amazon/Nova/Nova.swift | 12 -- .../Models/Mistral/MistralBedrockModels.swift | 2 + 8 files changed, 244 insertions(+), 19 deletions(-) create mode 100644 backend/Sources/App/Types/Cohere/CohereBedrockModels.swift rename backend/Sources/BedrockTypes/{Models/Mistral/Mistral.swift => Modalities/StandardConverse.swift} (60%) diff --git a/README.md b/README.md index dd69ee25..8bc63a69 100644 --- a/README.md +++ b/README.md @@ -141,16 +141,193 @@ Note that the minimum, maximum and default values for each parameter are model s ### Text prompt +```swift +let model = .nova_lite + +guard model.hasConverseModality() else { + print("\(model.name) does not support converse") +} + +var (reply, history) = try await bedrock.converse( + with: model, + prompt: "Tell me about rainbows", + history: [], + maxTokens: 512, +) + +print("Assistant: \(reply)") + +(reply, history) = try await bedrock.converse( + with: model, + prompt: "Do you think birds can see them too?", + history: history, + maxTokens: 512, +) + +print("Assistant: \(reply)") +``` + + ### Vision +```swift +let model = .nova_lite + +guard model.hasConverseModality(.vision) else { + print("\(model.name) does not support converse") +} + +let (reply, history) = try await bedrock.converse( + with model: model, + prompt: "Can you tell me about this plant?", + imageFormat: .jpeg, + imageBytes: base64EncodedImage, + temperature: 0.8, +) + +print("Assistant: \(reply)") +``` + ### Tools +```swift +let model = .nova_lite + +// verify that the model supports tool usage +guard model.hasConverseModality(.toolUse) else { + print("\(model.name) does not support converse tools") +} + +// define the inputschema for your tool +let inputSchema = JSON([ + "type": "object", + "properties": [ + "sign": [ + "type": "string", + "description": "The call sign for the radio station for which you want the most popular song. Example calls signs are WZPZ and WKRP." + ] + ], + "required": [ + "sign" + ] +]) + +// create a Tool object +let tool = Tool(name: "top_song", inputSchema: inputSchema, description: "Get the most popular song played on a radio station.") + +// pass a prompt and the tool to converse +var (reply, history) = try await bedrock.converse( + with model: model, + prompt: "What is the most popular song on WZPZ?", + tools: [tool] +) + +print("Assistant: \(reply)") +// The reply will be similar to this: "I need to use the \"top_song\" tool to find the most popular song on the radio station WZPZ. I will input the call sign \"WZPZ\" into the tool to get the required information." +// The last message in the history will contain the tool use request + +if case .toolUse(let toolUse) = history.last?.content.last { + let id = toolUse.id + let name = toolUse.name + let input = toolUse.input + + // Logic to use the tool here + + let toolResult = ToolResultBlock(id: id, content: [.text("The Best Song Ever")], status: .success) + + // Send the toolResult back to the model + (reply, history) = try await bedrock.converse( + with: model, + history: history, + tools: [tool], + toolResult: toolResult +) +} + +print("Assistant: \(reply)") +// The final reply will be similar to: "The most popular song currently played on WZPZ is \"The Best Song Ever\". If you need more information or have another request, feel free to ask!" +``` + +### Make your own `Message` + +Alternatively use the `converse` function that does not take a `prompt`, `toolResult` or `image` and construct the `Message` yourself. + +```swift +// Message with prompt +let (reply, history) = try await bedrock.converse( + with model: model, + conversation: [Message("What day of the week is it?")], + maxTokens: 512, + temperature: 1, + stopSequences: ["THE END"], + systemPrompts: ["Today is Wednesday, make sure to mention that."] +) + +// Message with an image and prompt +let (reply, history) = try await bedrock.converse( + with model: model, + conversation: [Message(prompt: "What is in the this teacup?", imageFormat: .jpeg, imageBytes: base64EncodedImage)], +) + +// Message with toolResult +let (reply, history) = try await bedrock.converse( + with model: model, + conversation: [Message(toolResult)], + tools: [toolA, toolB] +) +``` + ## How to add a BedrockModel ### Text +-- Under Construction -- + ### Image +-- Under Construction -- + ### Converse +To add a new model that only needs the ConverseModality, simply use the `StandardConverse` and add the correct [inferece parameters](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html) and [supported converse features](https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html). + +```swift +extension BedrockModel { + public static let new_bedrock_model = BedrockModel( + id: "family.model-id-v1:0", + name: "New Model Name", + modality: StandardConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.3), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: nil, defaultValue: nil), + topP: Parameter(.topP, minValue: 0.01, maxValue: 0.99, defaultValue: 0.75), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] + ) + ) +} +``` + +If the model also implements other modalities you might need to create you own `Modality` and make sure it conforms to `ConverseModality` by implementing the `getConverseParameters` and `getConverseFeatures` functions. Note that the `ConverseParameters` can be extracted from `TextGenerationParameters` by using the public initializer. +```swift +struct ModelFamilyModality: TextModality, ConverseModality { + func getName() -> String { "Model Family Text and Converse Modality" } + + let parameters: TextGenerationParameters + let converseFeatures: [ConverseFeature] + let converseParameters: ConverseParameters + + init(parameters: TextGenerationParameters, features: [ConverseFeature] = [.textGeneration]) { + self.parameters = parameters + self.converseFeatures = features + + // public initializer to extract `ConverseParameters` from `TextGenerationParameters` + self.converseParameters = ConverseParameters(textGenerationParameters: parameters) + } + + // ... +} +``` diff --git a/backend/Package.swift b/backend/Package.swift index 9f22df4f..81a719cd 100644 --- a/backend/Package.swift +++ b/backend/Package.swift @@ -42,7 +42,7 @@ let package = Package( .target( name: "BedrockTypes", dependencies: [ - .product(name: "AWSClientRuntime", package: "aws-sdk-swift"), + .product(name: "AWSBedrockRuntime", package: "aws-sdk-swift"), .product(name: "Smithy", package: "smithy-swift"), ], path: "Sources/BedrockTypes" diff --git a/backend/Sources/App/Types/Cohere/CohereBedrockModels.swift b/backend/Sources/App/Types/Cohere/CohereBedrockModels.swift new file mode 100644 index 00000000..580c2c32 --- /dev/null +++ b/backend/Sources/App/Types/Cohere/CohereBedrockModels.swift @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Foundation Models Playground open source project +// +// Copyright (c) 2025 Amazon.com, Inc. or its affiliates +// and the Swift Foundation Models Playground project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift Foundation Models Playground project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import BedrockTypes +import Foundation + +// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html +typealias CohereConverse = StandardConverse + +extension BedrockModel { + public static let cohere_command_R_plus = BedrockModel( + id: "cohere.command-r-plus-v1:0", + name: "Cohere Command R+", + modality: CohereConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.3), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: nil, defaultValue: nil), + topP: Parameter(.topP, minValue: 0.01, maxValue: 0.99, defaultValue: 0.75), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] + ) + ) + + public static let cohere_command_R = BedrockModel( + id: "cohere.command-r-v1:0", + name: "Cohere Command R", + modality: CohereConverse( + parameters: ConverseParameters( + temperature: Parameter(.temperature, minValue: 0, maxValue: 1, defaultValue: 0.3), + maxTokens: Parameter(.maxTokens, minValue: 1, maxValue: nil, defaultValue: nil), + topP: Parameter(.topP, minValue: 0.01, maxValue: 0.99, defaultValue: 0.75), + stopSequences: StopSequenceParams(maxSequences: nil, defaultValue: []), + maxPromptSize: nil + ), + features: [.textGeneration, .systemPrompts, .document, .toolUse] + ) + ) +} diff --git a/backend/Sources/BedrockService/BedrockService.swift b/backend/Sources/BedrockService/BedrockService.swift index 792c528b..96285116 100644 --- a/backend/Sources/BedrockService/BedrockService.swift +++ b/backend/Sources/BedrockService/BedrockService.swift @@ -603,7 +603,7 @@ public struct BedrockService: Sendable { /// - Returns: Tuple containing the model's response text and updated message history public func converse( with model: BedrockModel, - prompt: String?, + prompt: String? = nil, imageFormat: ImageBlock.Format? = nil, imageBytes: String? = nil, history: [Message] = [], diff --git a/backend/Sources/BedrockTypes/BedrockModel.swift b/backend/Sources/BedrockTypes/BedrockModel.swift index 2c315540..0d3d778a 100644 --- a/backend/Sources/BedrockTypes/BedrockModel.swift +++ b/backend/Sources/BedrockTypes/BedrockModel.swift @@ -98,6 +98,9 @@ public struct BedrockModel: Hashable, Sendable, Equatable, RawRepresentable { case BedrockModel.mistral_small_2402.id: self = BedrockModel.mistral_small_2402 case BedrockModel.mistral_7B_instruct.id: self = BedrockModel.mistral_7B_instruct case BedrockModel.mistral_8x7B_instruct.id: self = BedrockModel.mistral_8x7B_instruct + //cohere + // case BedrockModel.cohere_command_R_plus.id: self = BedrockModel.cohere_command_R_plus + // case BedrockModel.cohere_command_R.id: self = BedrockModel.cohere_command_R default: return nil } diff --git a/backend/Sources/BedrockTypes/Models/Mistral/Mistral.swift b/backend/Sources/BedrockTypes/Modalities/StandardConverse.swift similarity index 60% rename from backend/Sources/BedrockTypes/Models/Mistral/Mistral.swift rename to backend/Sources/BedrockTypes/Modalities/StandardConverse.swift index f1353a16..2687eba8 100644 --- a/backend/Sources/BedrockTypes/Models/Mistral/Mistral.swift +++ b/backend/Sources/BedrockTypes/Modalities/StandardConverse.swift @@ -15,14 +15,17 @@ import Foundation -struct MistralConverse: ConverseModality { - func getName() -> String { "Mistral Converse Modality" } +public struct StandardConverse: ConverseModality { + public func getName() -> String { "Standard Converse Modality" } - let converseParameters: ConverseParameters - let converseFeatures: [ConverseFeature] + public let converseParameters: ConverseParameters + public let converseFeatures: [ConverseFeature] - init(parameters: ConverseParameters, features: [ConverseFeature]) { + public init(parameters: ConverseParameters, features: [ConverseFeature]) { self.converseParameters = parameters self.converseFeatures = features } + + public func getConverseParameters() -> ConverseParameters { converseParameters } + public func getConverseFeatures() -> [ConverseFeature] { converseFeatures } } diff --git a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift index b74480c5..5386aa91 100644 --- a/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift +++ b/backend/Sources/BedrockTypes/Models/Amazon/Nova/Nova.swift @@ -15,18 +15,6 @@ import Foundation -// struct NovaConverse: ConverseModality { -// func getName() -> String { "Nova Lite and Nova Pro Converse Modality" } - -// let converseParameters: ConverseParameters -// let converseFeatures: [ConverseFeature] - -// init(parameters: ConverseParameters, features: [ConverseFeature]) { -// self.converseParameters = parameters -// self.converseFeatures = features -// } -// } - struct NovaText: TextModality, ConverseModality { func getName() -> String { "Nova Text Generation" } diff --git a/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift b/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift index b6b29273..85aaef4e 100644 --- a/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift +++ b/backend/Sources/BedrockTypes/Models/Mistral/MistralBedrockModels.swift @@ -19,6 +19,8 @@ import Foundation // https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mistral-text-completion.html // https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html +typealias MistralConverse = StandardConverse + extension BedrockModel { public static let mistral_large_2402 = BedrockModel( id: "mistral.mistral-large-2402-v1:0",