Add HTML-in-Canvas support with texElementImage2D and interactive hit testing (#8519)

* Add HTML-in-Canvas support with texElementImage2D and interactive hit testing

Support using HTML elements as live WebGL texture sources via the HTML-in-Canvas
proposal. Add GraphicsDevice.supportsHtmlTextures public API, texElementImage2D
upload path, and two examples demonstrating the feature.

Made-with: Cursor

* Fix ESLint no-use-before-define and quotes errors in examples

Move app.on('destroy') handlers after the variables they reference.

Made-with: Cursor

* updates

---------

Co-authored-by: Martin Valigursky <mvaligursky@snapchat.com>

Author: mvaligursky

examples/assets/models/MaterialsVariantsShoe.glb

@@ -0,0 +1,199 @@
+// @config DESCRIPTION <div style="color:black">Renders live HTML content directly as a WebGL texture via the <b>HTML-in-Canvas</b> API (<code>texElementImage2D</code>).<br>Includes animated CSS gradients, text glow, and a pulsing circle — all driven by standard CSS.</div>
+//
+// This example demonstrates the HTML-in-Canvas API: a styled HTML element with
+// CSS animations is appended to a canvas marked with the "layoutsubtree"
+// attribute, then captured into a WebGL texture via texElementImage2D.
+//
+// Fallback: when device.supportsHtmlTextures is false, a static 2D canvas with
+// hand-drawn placeholder graphics is used as the texture source instead.
+//
+import { deviceType } from 'examples/utils';
+import * as pc from 'playcanvas';
+
+const canvas = /** @type {HTMLCanvasElement} */ (document.getElementById('application-canvas'));
+
+// Enable layoutsubtree for HTML-in-Canvas support
+canvas.setAttribute('layoutsubtree', 'true');
+
+window.focus();
+
+const gfxOptions = {
+    deviceTypes: [deviceType]
+};
+
+const device = await pc.createGraphicsDevice(canvas, gfxOptions);
+device.maxPixelRatio = Math.min(window.devicePixelRatio, 2);
+
+const createOptions = new pc.AppOptions();
+createOptions.graphicsDevice = device;
+
+createOptions.componentSystems = [pc.RenderComponentSystem, pc.CameraComponentSystem, pc.LightComponentSystem];
+createOptions.resourceHandlers = [pc.TextureHandler, pc.ContainerHandler];
+
+const app = new pc.AppBase(canvas);
+app.init(createOptions);
+app.start();
+
+app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW);
+app.setCanvasResolution(pc.RESOLUTION_AUTO);
+
+const resize = () => app.resizeCanvas();
+window.addEventListener('resize', resize);
+
+// Create an HTML element to use as texture source.
+// Per the HTML-in-Canvas proposal, the element must be a direct child of the canvas.
+// The 'inert' attribute prevents hit testing on the element.
+const htmlElement = document.createElement('div');
+htmlElement.setAttribute('inert', '');
+htmlElement.style.width = '512px';
+htmlElement.style.height = '512px';
+htmlElement.style.padding = '10px';
+htmlElement.style.background = 'linear-gradient(45deg, #ff6b6b, #4ecdc4, #45b7d1, #f9ca24, #ff6b6b, #4ecdc4, #45b7d1, #f9ca24)';
+htmlElement.style.backgroundSize = '400% 400%';
+htmlElement.style.animation = 'gradient-shift 4s ease infinite';
+htmlElement.style.borderRadius = '0';
+htmlElement.style.fontFamily = 'Arial, sans-serif';
+htmlElement.style.fontSize = '24px';
+htmlElement.style.color = 'white';
+htmlElement.style.textAlign = 'center';
+htmlElement.style.display = 'flex';
+htmlElement.style.flexDirection = 'column';
+htmlElement.style.justifyContent = 'center';
+htmlElement.style.alignItems = 'center';
+htmlElement.innerHTML = `
+    <h1 style="margin: 0 0 20px 0; animation: glow 3s ease-in-out infinite;">HTML in Canvas!</h1>
+    <p style="margin: 0; text-shadow: 1px 1px 2px rgba(0,0,0,0.5);">This texture is rendered from HTML using texElementImage2D</p>
+    <div style="margin-top: 20px; width: 50px; height: 50px; border-radius: 50%; animation: pulse 2s infinite;"></div>
+`;
+
+const style = document.createElement('style');
+style.textContent = `
+    @keyframes glow {
+        0%, 100% { color: white;   text-shadow: 0 0 10px rgba(0,0,0,0.8), 0 0 20px rgba(0,0,0,0.4); font-size: 42px; }
+        50%      { color: #f9ca24; text-shadow: 0 0 15px rgba(0,0,0,0.8), 0 0 30px #f9ca24, 0 0 60px #f9ca24, 0 0 90px rgba(249,202,36,0.4); font-size: 48px; }
+    }
+    @keyframes gradient-shift {
+        0%   { background-position: 0% 50%; }
+        50%  { background-position: 100% 50%; }
+        100% { background-position: 0% 50%; }
+    }
+    @keyframes pulse {
+        0%   { transform: scale(1);   background: #ff6b6b; }
+        25%  { transform: scale(1.2); background: #f9ca24; }
+        50%  { transform: scale(1);   background: #4ecdc4; }
+        75%  { transform: scale(1.2); background: #45b7d1; }
+        100% { transform: scale(1);   background: #ff6b6b; }
+    }
+`;
+document.head.appendChild(style);
+
+canvas.appendChild(htmlElement);
+
+// Create texture
+const htmlTexture = new pc.Texture(device, {
+    width: 512,
+    height: 512,
+    format: pc.PIXELFORMAT_RGBA8,
+    name: 'htmlTexture'
+});
+
+// Fallback canvas texture for browsers without texElementImage2D support
+const createFallbackTexture = () => {
+    const fallbackCanvas = document.createElement('canvas');
+    fallbackCanvas.width = 512;
+    fallbackCanvas.height = 512;
+    const ctx = fallbackCanvas.getContext('2d');
+    if (!ctx) return null;
+
+    const gradient = ctx.createLinearGradient(0, 0, 512, 512);
+    gradient.addColorStop(0, '#ff6b6b');
+    gradient.addColorStop(0.33, '#4ecdc4');
+    gradient.addColorStop(0.66, '#45b7d1');
+    gradient.addColorStop(1, '#f9ca24');
+    ctx.fillStyle = gradient;
+    ctx.fillRect(0, 0, 512, 512);
+
+    ctx.fillStyle = 'white';
+    ctx.font = 'bold 36px Arial';
+    ctx.textAlign = 'center';
+    ctx.shadowColor = 'rgba(0,0,0,0.5)';
+    ctx.shadowBlur = 4;
+    ctx.shadowOffsetX = 2;
+    ctx.shadowOffsetY = 2;
+    ctx.fillText('HTML in Canvas!', 256, 180);
+
+    ctx.font = '20px Arial';
+    ctx.fillText('(Canvas Fallback)', 256, 220);
+    ctx.fillText('texElementImage2D not available', 256, 260);
+
+    ctx.beginPath();
+    ctx.arc(256, 320, 25, 0, 2 * Math.PI);
+    ctx.fillStyle = 'white';
+    ctx.fill();
+
+    return fallbackCanvas;
+};
+
+// Start with fallback texture, then switch to HTML source once the paint record is ready
+const fallbackCanvas = createFallbackTexture();
+if (fallbackCanvas) {
+    htmlTexture.setSource(fallbackCanvas);
+}
+
+const onPaintUpload = () => {
+    if (!app.graphicsDevice) return;
+    htmlTexture.upload();
+};
+
+app.on('destroy', () => {
+    window.removeEventListener('resize', resize);
+    canvas.removeEventListener('paint', onPaintUpload);
+    if (htmlElement.parentNode) htmlElement.parentNode.removeChild(htmlElement);
+    if (style.parentNode) style.parentNode.removeChild(style);
+});
+
+if (device.supportsHtmlTextures) {
+    // The browser must paint the HTML element before texElementImage2D can use it.
+    // Wait for the 'paint' event, then set the HTML element as the texture source.
+    canvas.addEventListener('paint', () => {
+        htmlTexture.setSource(/** @type {any} */ (htmlElement));
+    }, { once: true });
+    canvas.requestPaint();
+
+    // Re-upload the texture whenever the browser repaints the HTML children
+    canvas.addEventListener('paint', onPaintUpload);
+} else {
+    console.warn('HTML textures are not supported - using canvas fallback');
+}
+
+// Create material with the HTML texture
+const material = new pc.StandardMaterial();
+material.diffuseMap = htmlTexture;
+material.update();
+
+const box = new pc.Entity('cube');
+box.addComponent('render', {
+    type: 'box',
+    material: material
+});
+app.root.addChild(box);
+
+const camera = new pc.Entity('camera');
+camera.addComponent('camera', {
+    clearColor: new pc.Color(1, 1, 1)
+});
+app.root.addChild(camera);
+camera.setPosition(0, 0, 3);
+
+app.scene.ambientLight = new pc.Color(0.3, 0.3, 0.3);
+
+const light = new pc.Entity('light');
+light.addComponent('light');
+app.root.addChild(light);
+light.setEulerAngles(45, 0, 0);
+
+app.on('update', (/** @type {number} */ dt) => {
+    box.rotate(3 * dt, 5 * dt, 6 * dt);
+});
+
+export { app };

examples/assets/textures/background_shoes.png

@@ -400,6 +400,16 @@ class GraphicsDevice extends EventHandler {
      */
     supportsShaderF16 = false;
 
+    /**
+     * True if HTML elements (e.g. `<div>`) can be used as texture sources via the HTML-in-Canvas
+     * API. When supported, an HTML element appended to a canvas with the `layoutsubtree` attribute
+     * can be passed to {@link Texture#setSource} and rendered as a live texture in the 3D scene.
+     *
+     * @type {boolean}
+     * @readonly
+     */
+    supportsHtmlTextures = false;
+
     /**
      * True if 32-bit floating-point textures can be used as a frame buffer.
      *
@@ -1054,17 +1064,18 @@ class GraphicsDevice extends EventHandler {
     }
 
     /**
-     * Reports whether a texture source is a canvas, image, video or ImageBitmap.
+     * Reports whether a texture source is a canvas, image, video, ImageBitmap, or HTML element.
      *
      * @param {*} texture - Texture source data.
-     * @returns {boolean} True if the texture is a canvas, image, video or ImageBitmap and false
-     * otherwise.
+     * @returns {boolean} True if the texture is a canvas, image, video, ImageBitmap, or HTML
+     * element and false otherwise.
      * @ignore
      */
     _isBrowserInterface(texture) {
         return this._isImageBrowserInterface(texture) ||
                 this._isImageCanvasInterface(texture) ||
-                this._isImageVideoInterface(texture);
+                this._isImageVideoInterface(texture) ||
+                this._isHTMLElementInterface(texture);
     }
 
     _isImageBrowserInterface(texture) {
@@ -1080,6 +1091,22 @@ class GraphicsDevice extends EventHandler {
         return (typeof HTMLVideoElement !== 'undefined' && texture instanceof HTMLVideoElement);
     }
 
+    /**
+     * Reports whether a texture source is a generic HTML element (not image, canvas, or video).
+     * Used for the HTML-in-Canvas proposal (texElementImage2D).
+     *
+     * @param {*} texture - Texture source data.
+     * @returns {boolean} True if the texture is an HTMLElement that is not an image, canvas, or
+     * video.
+     * @ignore
+     */
+    _isHTMLElementInterface(texture) {
+        return (typeof HTMLElement !== 'undefined' && texture instanceof HTMLElement &&
+                !(typeof HTMLImageElement !== 'undefined' && texture instanceof HTMLImageElement) &&
+                !(typeof HTMLCanvasElement !== 'undefined' && texture instanceof HTMLCanvasElement) &&
+                !(typeof HTMLVideoElement !== 'undefined' && texture instanceof HTMLVideoElement));
+    }
+
     /**
      * Sets the width and height of the canvas, then fires the `resizecanvas` event. Note that the
      * specified width and height values will be multiplied by the value of

examples/src/examples/misc/html-texture-configurator.example.mjs

@@ -1052,16 +1052,31 @@ class Texture {
     }
 
     /**
-     * Set the pixel data of the texture from a canvas, image, video DOM element. If the texture is
-     * a cubemap, the supplied source must be an array of 6 canvases, images or videos.
+     * Set the pixel data of the texture from a canvas, image, video, or HTML DOM element. If the
+     * texture is a cubemap, the supplied source must be an array of 6 canvases, images or videos.
      *
-     * @param {HTMLCanvasElement|HTMLImageElement|HTMLVideoElement|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]} source - A
-     * canvas, image or video element, or an array of 6 canvas, image or video elements.
+     * Note: using an HTML element (e.g. `<div>`) as a source requires
+     * {@link GraphicsDevice#supportsHtmlTextures} to be true.
+     *
+     * @param {HTMLCanvasElement|HTMLImageElement|HTMLVideoElement|HTMLElement|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|HTMLElement[]} source - A
+     * canvas, image, video, or HTML element, or an array of 6 canvas, image, video, or HTML
+     * elements.
      * @param {number} [mipLevel] - A non-negative integer specifying the image level of detail.
      * Defaults to 0, which represents the base image source. A level value of N, that is greater
      * than 0, represents the image source for the Nth mipmap reduction level.
      */
     setSource(source, mipLevel = 0) {
+        if (this.device._isHTMLElementInterface(source)) {
+            if (!this.device.supportsHtmlTextures) {
+                Debug.error('Texture#setSource: HTML element textures are not supported on this device. Check device.supportsHtmlTextures before calling setSource with an HTML element.');
+                return;
+            }
+            if (this._cubemap || this._volume) {
+                Debug.error('Texture#setSource: HTML element textures can only be used with 2D textures, not cubemaps or volume textures.');
+                return;
+            }
+        }
+
         let invalid = false;
         let width, height;
 
@@ -1110,6 +1125,10 @@ class Texture {
                 if (source instanceof HTMLVideoElement) {
                     width = source.videoWidth;
                     height = source.videoHeight;
+                } else if (this.device._isHTMLElementInterface(source)) {
+                    const rect = source.getBoundingClientRect();
+                    width = Math.floor(rect.width) || 1;
+                    height = Math.floor(rect.height) || 1;
                 } else {
                     width = source.width;
                     height = source.height;