Merge pull request #406 from martinRenou/image_embedding_refactor

Refactor image embedding logic

Author: martinRenou

ipympl/backend_nbagg.py

@@ -51,6 +51,7 @@
     Float,
     Instance,
     List,
+    Tuple,
     Unicode,
     default,
 )
@@ -184,8 +185,7 @@ class Canvas(DOMWidget, FigureCanvasWebAggCore):
     # This will still be used by ipywidgets in the case of embedding.
     _data_url = Any(None).tag(sync=True)
 
-    _width = CInt().tag(sync=True)
-    _height = CInt().tag(sync=True)
+    _size = Tuple([0, 0]).tag(sync=True)
 
     _figure_label = Unicode('Figure').tag(sync=True)
     _message = Unicode().tag(sync=True)
@@ -265,9 +265,11 @@ def send_json(self, content):
             self._figure_label = content['label']
 
         elif content['type'] == 'resize':
-            self._width = content['size'][0]
-            self._height = content['size'][1]
-            # Send resize message anyway
+            self._size = content['size']
+            # Send resize message anyway:
+            # We absolutely need this instead of a `_size` trait change listening
+            # on the front-end, otherwise ipywidgets might squash multiple changes
+            # and the resizing protocol is not respected anymore
             self.send({'data': json.dumps(content)})
 
         elif content['type'] == 'image_mode':
@@ -306,28 +308,36 @@ def _repr_mimebundle_(self, **kwargs):
 
         buf = io.BytesIO()
         self.figure.savefig(buf, format='png', dpi='figure')
-        self._data_url = b64encode(buf.getvalue()).decode('utf-8')
-        # Figure width in pixels
+
+        base64_image = b64encode(buf.getvalue()).decode('utf-8')
+        self._data_url = f'data:image/png;base64,{base64_image}'
+        # Figure size in pixels
         pwidth = self.figure.get_figwidth() * self.figure.get_dpi()
+        pheight = self.figure.get_figheight() * self.figure.get_dpi()
         # Scale size to match widget on HiDPI monitors.
         if hasattr(self, 'device_pixel_ratio'):  # Matplotlib 3.5+
             width = pwidth / self.device_pixel_ratio
+            height = pheight / self.device_pixel_ratio
         else:
             width = pwidth / self._dpi_ratio
+            height = pheight / self._dpi_ratio
         html = """
             <div style="display: inline-block;">
                 <div class="jupyter-widgets widget-label" style="text-align: center;">
                     {}
                 </div>
-                <img src='data:image/png;base64,{}' width={}/>
+                <img src='{}' width={}/>
             </div>
         """.format(
             self._figure_label, self._data_url, width
         )
 
+        # Update the widget model properly for HTML embedding
+        self._size = (width, height)
+
         data = {
             'text/plain': plaintext,
-            'image/png': self._data_url,
+            'image/png': base64_image,
             'text/html': html,
             'application/vnd.jupyter.widget-view+json': {
                 'version_major': 2,

src/mpl_widget.ts

@@ -39,8 +39,7 @@ export class MPLCanvasModel extends DOMWidgetModel {
             capture_scroll: false,
             pan_zoom_throttle: 33,
             _data_url: null,
-            _width: 0,
-            _height: 0,
+            _size: [0, 0],
             _figure_label: 'Figure',
             _message: '',
             _cursor: 'pointer',
@@ -78,7 +77,7 @@ export class MPLCanvasModel extends DOMWidgetModel {
         this.resize_requested = false;
         this.ratio = (window.devicePixelRatio || 1) / backingStore;
 
-        this.resize_canvas(this.get('_width'), this.get('_height'));
+        this.resize_canvas();
 
         this._init_image();
 
@@ -88,13 +87,21 @@ export class MPLCanvasModel extends DOMWidgetModel {
                 view.update_canvas();
             });
         });
+        this.on('change:_size', () => {
+            this.resize_canvas();
+            this.offscreen_context.drawImage(this.image, 0, 0);
+        });
         this.on('comm_live_update', this.update_disabled.bind(this));
 
         this.update_disabled();
 
         this.send_initialization_message();
     }
 
+    get size(): [number, number] {
+        return this.get('_size');
+    }
+
     get disabled(): boolean {
         return !this.comm_live;
     }
@@ -149,13 +156,12 @@ export class MPLCanvasModel extends DOMWidgetModel {
     }
 
     handle_resize(msg: { [index: string]: any }) {
-        const size = msg['size'];
-        this.resize_canvas(size[0], size[1]);
+        this.resize_canvas();
         this.offscreen_context.drawImage(this.image, 0, 0);
 
         if (!this.resize_requested) {
             this._for_each_view((view: MPLCanvasView) => {
-                view.resize_canvas(size[0], size[1]);
+                view.resize_and_update_canvas(this.size);
             });
         }
 
@@ -169,6 +175,9 @@ export class MPLCanvasModel extends DOMWidgetModel {
         }
     }
 
+    /*
+     * Request a resize to the backend
+     */
     resize(width: number, height: number) {
         // Do not request a super small size, as it seems to break the back-end
         if (width <= 5 || height <= 5) {
@@ -177,7 +186,7 @@ export class MPLCanvasModel extends DOMWidgetModel {
 
         this._for_each_view((view: MPLCanvasView) => {
             // Do an initial resize of each view, stretching the old canvas.
-            view.resize_canvas(width, height);
+            view.resize_and_update_canvas([width, height]);
         });
 
         if (this.resize_requested) {
@@ -189,9 +198,12 @@ export class MPLCanvasModel extends DOMWidgetModel {
         }
     }
 
-    resize_canvas(width: number, height: number) {
-        this.offscreen_canvas.width = width * this.ratio;
-        this.offscreen_canvas.height = height * this.ratio;
+    /*
+     * Resize the offscreen canvas
+     */
+    resize_canvas() {
+        this.offscreen_canvas.width = this.size[0] * this.ratio;
+        this.offscreen_canvas.height = this.size[1] * this.ratio;
     }
 
     handle_rubberband(msg: any) {
@@ -275,17 +287,49 @@ export class MPLCanvasModel extends DOMWidgetModel {
         this.image = new Image();
 
         this.image.onload = () => {
+            // In case of an embedded widget, the initial size is not correct
+            // and we are not receiving any resize event from the server
+            if (this.disabled) {
+                this.offscreen_canvas.width = this.image.width;
+                this.offscreen_canvas.height = this.image.height;
+
+                this.offscreen_context.drawImage(this.image, 0, 0);
+
+                this._for_each_view((view: MPLCanvasView) => {
+                    // TODO Make this part of the CanvasView API?
+                    // It feels out of place in the model
+                    view.canvas.width = this.image.width / this.ratio;
+                    view.canvas.height = this.image.height / this.ratio;
+                    view.canvas.style.width = view.canvas.width + 'px';
+                    view.canvas.style.height = view.canvas.height + 'px';
+
+                    view.top_canvas.width = this.image.width / this.ratio;
+                    view.top_canvas.height = this.image.height / this.ratio;
+                    view.top_canvas.style.width = view.top_canvas.width + 'px';
+                    view.top_canvas.style.height =
+                        view.top_canvas.height + 'px';
+
+                    view.canvas_div.style.width = view.canvas.width + 'px';
+                    view.canvas_div.style.height = view.canvas.height + 'px';
+
+                    view.update_canvas(true);
+                });
+
+                return;
+            }
+
+            // Full images could contain transparency (where diff images
+            // almost always do), so we need to clear the canvas so that
+            // there is no ghosting.
             if (this.get('_image_mode') === 'full') {
-                // Full images could contain transparency (where diff images
-                // almost always do), so we need to clear the canvas so that
-                // there is no ghosting.
                 this.offscreen_context.clearRect(
                     0,
                     0,
                     this.offscreen_canvas.width,
                     this.offscreen_canvas.height
                 );
             }
+
             this.offscreen_context.drawImage(this.image, 0, 0);
 
             this._for_each_view((view: MPLCanvasView) => {
@@ -556,19 +600,32 @@ export class MPLCanvasView extends DOMWidgetView {
             return false;
         });
 
-        this.resize_canvas(this.model.get('_width'), this.model.get('_height'));
-        this.update_canvas();
+        this.resize_and_update_canvas(this.model.size);
     }
 
-    update_canvas() {
+    /*
+     * Update the canvas view
+     */
+    update_canvas(stretch = false) {
         if (this.canvas.width === 0 || this.canvas.height === 0) {
             return;
         }
 
         this.top_context.save();
 
         this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
-        this.context.drawImage(this.model.offscreen_canvas, 0, 0);
+
+        if (stretch) {
+            this.context.drawImage(
+                this.model.offscreen_canvas,
+                0,
+                0,
+                this.canvas.width,
+                this.canvas.height
+            );
+        } else {
+            this.context.drawImage(this.model.offscreen_canvas, 0, 0);
+        }
 
         this.top_context.clearRect(
             0,
@@ -651,18 +708,18 @@ export class MPLCanvasView extends DOMWidgetView {
         this.footer.textContent = this.model.get('_message');
     }
 
-    resize_canvas(width: number, height: number) {
+    resize_and_update_canvas(size: [number, number]) {
         // Keep the size of the canvas, and rubber band canvas in sync.
-        this.canvas.setAttribute('width', `${width * this.model.ratio}`);
-        this.canvas.setAttribute('height', `${height * this.model.ratio}`);
-        this.canvas.style.width = width + 'px';
-        this.canvas.style.height = height + 'px';
+        this.canvas.setAttribute('width', `${size[0] * this.model.ratio}`);
+        this.canvas.setAttribute('height', `${size[1] * this.model.ratio}`);
+        this.canvas.style.width = size[0] + 'px';
+        this.canvas.style.height = size[1] + 'px';
 
-        this.top_canvas.setAttribute('width', String(width));
-        this.top_canvas.setAttribute('height', String(height));
+        this.top_canvas.setAttribute('width', String(size[0]));
+        this.top_canvas.setAttribute('height', String(size[1]));
 
-        this.canvas_div.style.width = width + 'px';
-        this.canvas_div.style.height = height + 'px';
+        this.canvas_div.style.width = size[0] + 'px';
+        this.canvas_div.style.height = size[1] + 'px';
 
         this.update_canvas();
     }