Image to text (#131)

Co-authored-by: Eliott C <[email protected]>

Author: vvmnnnkv

README.md

@@ -97,6 +97,11 @@ await inference.textToImage({
     negative_prompt: 'blurry',
   }
 })
+
+await inference.imageToText({
+  data: await (await fetch('https://picsum.photos/300/300')).blob(),
+  model: 'nlpconnect/vit-gpt2-image-captioning',  
+})
 ```
 
 There are more features of course, check each library's README!

packages/inference/README.md

@@ -157,6 +157,11 @@ await hf.textToImage({
     negative_prompt: 'blurry',
   }
 })
+
+await hf.imageToText({
+  data: readFileSync('test/cats.png'),
+  model: 'nlpconnect/vit-gpt2-image-captioning'
+})
 ```
 
 ## Supported Tasks
@@ -188,6 +193,7 @@ await hf.textToImage({
 - [x] Object detection
 - [x] Image segmentation
 - [x] Text to image
+- [x] Image to text
 
 ## Running tests
 

packages/inference/package.json

@@ -45,15 +45,15 @@
 		"format": "prettier --write .",
 		"format:check": "prettier --check .",
 		"prepublishOnly": "pnpm run build",
-		"test": "vitest run",
-		"test:browser": "vitest run --browser.name=chrome --browser.headless",
+		"test": "vitest run --config vitest.config.ts",
+		"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest.config.ts",
 		"type-check": "tsc"
 	},
 	"devDependencies": {
 		"@types/node": "18.13.0",
 		"typescript": "4.9.5",
 		"vite": "^4.1.4",
-		"vitest": "^0.29.2"
+		"vitest": "^0.29.8"
 	},
 	"resolutions": {}
 }

packages/inference/pnpm-lock.yaml

@@ -608,6 +608,20 @@ export type TextToImageArgs = Args & {
 
 export type TextToImageReturn = Blob;
 
+export type ImageToTextArgs = Args & {
+	/**
+	 * Binary image data
+	 */
+	data: Blob | ArrayBuffer;
+};
+
+export interface ImageToTextReturn {
+	/**
+	 * The generated caption
+	 */
+	generated_text: string;
+}
+
 export class HfInference {
 	private readonly apiKey: string;
 	private readonly defaultOptions: Options;
@@ -946,6 +960,18 @@ export class HfInference {
 		return res;
 	}
 
+	/**
+	 * This task reads some image input and outputs the text caption.
+	 */
+	public async imageToText(args: ImageToTextArgs, options?: Options): Promise<ImageToTextReturn> {
+		return (
+			await this.request<[ImageToTextReturn]>(args, {
+				...options,
+				binary: true,
+			})
+		)?.[0];
+	}
+
 	/**
 	 * Helper that prepares request arguments
 	 */