diff --git a/extensions/image-processing-api/functions/src/operations/genkit.ts b/extensions/image-processing-api/functions/src/operations/genkit.ts new file mode 100644 index 00000000..b3b31108 --- /dev/null +++ b/extensions/image-processing-api/functions/src/operations/genkit.ts @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016-present Invertase Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sharp from 'sharp'; +import superstruct from 'superstruct'; +import { OperationBuilder, ValidatedOperation } from '../types'; + +const name = 'genkit'; + +const struct = superstruct.object({ + operation: superstruct.literal(name), + prompt: superstruct.string(), // require a prompt string +}); + +export const operationGenkit: OperationBuilder = { + name, + struct, + async build( + operation: ValidatedOperation, + fileMetadata: sharp.Metadata | null, + ) { + const options = operation.options as { prompt: string }; + // Return a custom action with the prompt parameter. + return [ + { + method: 'genkitCall', + arguments: [options.prompt], + }, + ]; + }, +}; + +// TODO: Genkit implementation +export const callGenkit: (any) => Promise = async (x: any) => x; diff --git a/extensions/image-processing-api/functions/src/operations/index.ts b/extensions/image-processing-api/functions/src/operations/index.ts index e1ba6e5e..90b76223 100644 --- a/extensions/image-processing-api/functions/src/operations/index.ts +++ b/extensions/image-processing-api/functions/src/operations/index.ts @@ -15,7 +15,7 @@ */ import { AssertionError } from 'assert'; -import sharp, { SharpOptions } from 'sharp'; +import sharp from 'sharp'; import superstruct from 'superstruct'; import { omitKey, omitUndefinedValues } from '../utils'; @@ -56,6 +56,7 @@ import { operationTint } from './tint'; import { operationGrayScale } from './grayscale'; import { operationModulate } from './modulate'; import { operationLinear } from './linear'; +import { callGenkit, operationGenkit } from './genkit'; export * from './input'; export * from './output'; @@ -115,6 +116,7 @@ const builders: { [key: string]: OperationBuilder } = { grayscale: operationGrayScale, modulate: operationModulate, linear: operationLinear, + genkit: operationGenkit, }; export function builderForOperation(operation: Operation): OperationBuilder { @@ -259,8 +261,27 @@ export async function applyValidatedOperation( ); for (let i = 0; i < builtOperation.actions.length; i++) { const action = builtOperation.actions[i]; + + // Handle our custom external API action. + if (action.method === 'genkitCall' && currentInstance) { + // Retrieve the prompt from the action arguments. + const promptParam = action.arguments[0]; + const buffer = await currentInstance.toBuffer(); + const base64Image = buffer.toString('base64'); + // Call the external API with the image and prompt. + const newBase64 = await callGenkit({ + image: base64Image, + prompt: promptParam, + }); + const newBuffer = Buffer.from(newBase64, 'base64'); + currentInstance = sharp(newBuffer); + continue; + } + if (action.method == 'constructor') { - currentInstance = sharp(...(action.arguments as SharpOptions[])); + currentInstance = sharp( + ...(action.arguments as Parameters), + ); } else if (currentInstance != null) { currentInstance = ( currentInstance[action.method] as (...args: unknown[]) => sharp.Sharp