Merge pull request #251 from martinRenou/put_image_data_improvements

Improve put_image_data performances

Author: martinRenou

examples/binary_image.ipynb

@@ -119,7 +119,6 @@
     "    data = np.stack((np.zeros_like(z), vecscale(z), vecscale(z)), axis=2)\n",
     "\n",
     "    with hold_canvas():\n",
-    "        canvas.clear()\n",
     "        canvas.put_image_data(data, 0, 0)\n",
     "\n",
     "\n",
@@ -131,18 +130,11 @@
     "\n",
     "VBox((canvas, HBox((play, progress))))"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -156,7 +148,14 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.5"
+   "version": "3.10.4"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
   }
  },
  "nbformat": 4,

ipycanvas/canvas.py

@@ -1371,9 +1371,9 @@ def put_image_data(self, image_data, x=0, y=0):
         draw. Unlike the CanvasRenderingContext2D.putImageData method, this method **is** affected by the canvas transformation
         matrix, and supports transparency.
         """
-        image_metadata, image_buffer = binary_image(image_data)
+        image_buffer = binary_image(image_data)
         _CANVAS_MANAGER.send_draw_command(
-            self, COMMANDS["putImageData"], [image_metadata, x, y], [image_buffer]
+            self, COMMANDS["putImageData"], [x, y], [image_buffer]
         )
 
     def create_image_data(self, width, height):

ipycanvas/utils.py

@@ -24,28 +24,12 @@ def image_bytes_to_array(im_bytes):
     return np.array(im)
 
 
-def binary_image(ar):
-    """Turn a NumPy array representing an array of pixels into a binary buffer."""
-    if ar is None:
-        return None
-    if ar.dtype != np.uint8:
-        ar = ar.astype(np.uint8)
-    if ar.ndim == 1:
-        ar = ar[np.newaxis, :]
-    if ar.ndim == 2:
-        # extend grayscale to RGBA
-        add_alpha = np.full((ar.shape[0], ar.shape[1], 4), 255, dtype=np.uint8)
-        add_alpha[:, :, :3] = np.repeat(ar[:, :, np.newaxis], repeats=3, axis=2)
-        ar = add_alpha
-    if ar.ndim != 3:
-        raise ValueError("Please supply an RGBA array with shape (width, height, 4).")
-    if ar.shape[2] != 4 and ar.shape[2] == 3:
-        add_alpha = np.full((ar.shape[0], ar.shape[1], 4), 255, dtype=np.uint8)
-        add_alpha[:, :, :3] = ar
-        ar = add_alpha
-    if not ar.flags["C_CONTIGUOUS"]:  # make sure it's contiguous
-        ar = np.ascontiguousarray(ar, dtype=np.uint8)
-    return {"shape": ar.shape, "dtype": str(ar.dtype)}, memoryview(ar)
+def binary_image(ar, quality=75):
+    f = BytesIO()
+    PILImage.fromarray(ar.astype(np.uint8), "RGB" if ar.shape[2] == 3 else "RGBA").save(
+        f, "JPEG", quality=quality
+    )
+    return f.getvalue()
 
 
 def array_to_binary(ar):

src/utils.ts

@@ -151,3 +151,18 @@ export async function fromBytes(
     img.src = URL.createObjectURL(blob);
   });
 }
+
+export async function bufferToImage(buffer: any): Promise<HTMLImageElement> {
+  let url: string;
+
+  const blob = new Blob([buffer], { type: 'image/jpeg' });
+  url = URL.createObjectURL(blob);
+
+  const img = new Image();
+  return new Promise(resolve => {
+    img.onload = () => {
+      resolve(img);
+    };
+    img.src = url;
+  });
+}

src/widget.ts

@@ -16,7 +16,13 @@ import { RoughCanvas } from 'roughjs/bin/canvas';
 
 import { MODULE_NAME, MODULE_VERSION } from './version';
 
-import { getArg, toBytes, fromBytes, getTypedArray } from './utils';
+import {
+  getArg,
+  toBytes,
+  fromBytes,
+  getTypedArray,
+  bufferToImage
+} from './utils';
 
 function getContext(canvas: HTMLCanvasElement) {
   const context = canvas.getContext('2d');
@@ -280,7 +286,7 @@ export class CanvasManagerModel extends WidgetModel {
         await this.currentCanvas.drawImage(args, buffers);
         break;
       case 'putImageData':
-        this.currentCanvas.putImageData(args, buffers);
+        await this.currentCanvas.putImageData(args, buffers);
         break;
       case 'set':
         await this.currentCanvas.setAttr(args[0], args[1]);
@@ -1050,8 +1056,8 @@ export class CanvasModel extends DOMWidgetModel {
     image: HTMLCanvasElement | HTMLImageElement,
     x: number,
     y: number,
-    width: number | undefined,
-    height: number | undefined
+    width?: number,
+    height?: number
   ) {
     if (width === undefined || height === undefined) {
       this.ctx.drawImage(image, x, y);
@@ -1060,22 +1066,12 @@ export class CanvasModel extends DOMWidgetModel {
     }
   }
 
-  putImageData(args: any[], buffers: any) {
-    const [bufferMetadata, dx, dy] = args;
-
-    const width = bufferMetadata.shape[1];
-    const height = bufferMetadata.shape[0];
-
-    const data = new Uint8ClampedArray(buffers[0].buffer);
-    const imageData = new ImageData(data, width, height);
+  async putImageData(args: any[], buffers: any) {
+    const [x, y] = args;
 
-    // Draw on a temporary off-screen canvas. This is a workaround for `putImageData` to support transparency.
-    const offscreenCanvas = document.createElement('canvas');
-    offscreenCanvas.width = width;
-    offscreenCanvas.height = height;
-    getContext(offscreenCanvas).putImageData(imageData, 0, 0);
+    const image = await bufferToImage(buffers[0]);
 
-    this.ctx.drawImage(offscreenCanvas, dx, dy);
+    this._drawImage(image, x, y);
   }
 
   async setAttr(attr: number, value: any) {