WebXR Image Tracking

externs.js

@@ -16,6 +16,7 @@ var WebAssembly = {};
 var XRWebGLLayer = {};
 var XRRay = {};
 var XRHand = {};
+var XRImageTrackingResult = {};
 var DOMPoint = {};
 
 // extras requires this

src/index.js

@@ -275,6 +275,8 @@ export { XrLightEstimation } from './xr/xr-light-estimation.js';
 export { XrManager } from './xr/xr-manager.js';
 export { XrHitTest } from './xr/xr-hit-test.js';
 export { XrHitTestSource } from './xr/xr-hit-test-source.js';
+export { XrImageTracking } from './xr/xr-image-tracking.js';
+export { XrTrackedImage } from './xr/xr-tracked-image.js';
 export { XrDomOverlay } from './xr/xr-dom-overlay.js';
 
 // BACKWARDS COMPATIBILITY

src/xr/xr-image-tracking.js

@@ -0,0 +1,162 @@
+import { EventHandler } from '../core/event-handler.js';
+import { XrTrackedImage } from './xr-tracked-image.js';
+
+/**
+ * @class
+ * @name pc.XrImageTracking
+ * @classdesc Image Tracking provides the ability to track real world images by provided image samples and their estimate sizes.
+ * @description Image Tracking provides the ability to track real world images by provided image samples and their estimate sizes.
+ * @param {pc.XrManager} manager - WebXR Manager.
+ * @property {boolean} supported True if Image Tracking is supported.
+ * @property {boolean} available True if Image Tracking is available. This property will be false if no images were provided for the AR session or there was an error processing the provided images.
+ * @property {pc.XrTrackedImage[]} images List of {@link pc.XrTrackedImage} that contain tracking information.
+ */
+function XrImageTracking(manager) {
+    EventHandler.call(this);
+
+    this._manager = manager;
+    this._supported = !! window.XRImageTrackingResult;
+    this._available = false;
+
+    this._images = [];
+
+    if (this._supported) {
+        this._manager.on('start', this._onSessionStart, this);
+        this._manager.on('end', this._onSessionEnd, this);
+    }
+}
+XrImageTracking.prototype = Object.create(EventHandler.prototype);
+XrImageTracking.prototype.constructor = XrImageTracking;
+
+/**
+ * @event
+ * @name pc.XrImageTracking#error
+ * @param {Error} error - Error object related to a failure of image tracking.
+ * @description Fired when the XR session is started, but image tracking failed to process the provided images.
+ */
+
+/**
+ * @function
+ * @name pc.XrImageTracking#add
+ * @description Add an image for image tracking. A width can also be provided to help the underlying system estimate the appropriate transformation. Modifying the tracked images list is only possible before an AR session is started.
+ * @param {object} image - Image that is matching real world image as close as possible. Resolution of images should be at least 300x300. High resolution does NOT improve tracking performance. Color of image is irelevant, so greyscale images can be used. Images with too many geometric features or repeating patterns will reduce tracking stability.
+ * @param {number} width - Width (in meters) of image in the real world. Providing this value as close to the real value will improve tracking quality.
+ * @returns {pc.XrTrackedImage} Tracked image object that will contain tracking information.
+ * @example
+ * // image with width of 20cm (0.2m)
+ * app.xr.imageTracking.add(bookCoverImg, 0.2);
+ */
+XrImageTracking.prototype.add = function (image, width) {
+    if (! this._supported || this._manager.active) return null;
+
+    var trackedImage = new XrTrackedImage(image, width);
+    this._images.push(trackedImage);
+    return trackedImage;
+};
+
+/**
+ * @function
+ * @name pc.XrImageTracking#remove
+ * @description Remove an image from image tracking.
+ * @param {pc.XrTrackedImage} trackedImage - Tracked image to be removed. Modifying the tracked images list is only possible before an AR session is started.
+ */
+XrImageTracking.prototype.remove = function (trackedImage) {
+    if (this._manager.active) return;
+
+    var ind = this._images.indexOf(trackedImage);
+    if (ind !== -1) {
+        trackedImage.destroy();
+        this._images.splice(ind, 1);
+    }
+};
+
+XrImageTracking.prototype._onSessionStart = function () {
+    var self = this;
+
+    this._manager.session.getTrackedImageScores().then(function (images) {
+        self._available = true;
+
+        for (var i = 0; i < images.length; i++) {
+            self._images[i]._trackable = images[i] === 'trackable';
+        }
+    }).catch(function (err) {
+        self._available = false;
+        self.fire('error', err);
+    });
+};
+
+XrImageTracking.prototype._onSessionEnd = function () {
+    this._available = false;
+
+    for (var i = 0; i < this._images.length; i++) {
+        this._images[i]._pose = null;
+        this._images[i]._measuredWidth = 0;
+
+        if (this._images[i]._tracking) {
+            this._images[i]._tracking = false;
+            this._images[i].fire('untracked');
+        }
+    }
+};
+
+XrImageTracking.prototype.prepareImages = function (callback) {
+    if (this._images.length) {
+        Promise.all(this._images.map(function (trackedImage) {
+            return trackedImage.prepare();
+        })).then(function (bitmaps) {
+            callback(null, bitmaps);
+        }).catch(function (err) {
+            callback(err, null);
+        });
+    } else {
+        callback(null, null);
+    }
+};
+
+XrImageTracking.prototype.update = function (frame) {
+    if (! this._available) return;
+
+    var results = frame.getImageTrackingResults();
+    var index = { };
+    var i;
+
+    for (i = 0; i < results.length; i++) {
+        index[results[i].index] = results[i];
+
+        var trackedImage = this._images[results[i].index];
+        trackedImage._emulated = results[i].trackingState === 'emulated';
+        trackedImage._measuredWidth = results[i].measuredWidthInMeters;
+        trackedImage._dirtyTransform = true;
+        trackedImage._pose = frame.getPose(results[i].imageSpace, this._manager._referenceSpace);
+    }
+
+    for (i = 0; i < this._images.length; i++) {
+        if (this._images[i]._tracking && ! index[i]) {
+            this._images[i]._tracking = false;
+            this._images[i].fire('untracked');
+        } else if (! this._images[i]._tracking && index[i]) {
+            this._images[i]._tracking = true;
+            this._images[i].fire('tracked');
+        }
+    }
+};
+
+Object.defineProperty(XrImageTracking.prototype, 'supported', {
+    get: function () {
+        return this._supported;
+    }
+});
+
+Object.defineProperty(XrImageTracking.prototype, 'available', {
+    get: function () {
+        return this._available;
+    }
+});
+
+Object.defineProperty(XrImageTracking.prototype, 'images', {
+    get: function () {
+        return this._images;
+    }
+});
+
+export { XrImageTracking };

src/xr/xr-light-estimation.js

@@ -194,7 +194,7 @@ Object.defineProperty(XrLightEstimation.prototype, 'supported', {
  */
 Object.defineProperty(XrLightEstimation.prototype, 'available', {
     get: function () {
-        return !! this._available;
+        return this._available;
     }
 });
 

src/xr/xr-manager.js

@@ -10,6 +10,7 @@ import { XRTYPE_INLINE, XRTYPE_VR, XRTYPE_AR } from './constants.js';
 import { XrHitTest } from './xr-hit-test.js';
 import { XrInput } from './xr-input.js';
 import { XrLightEstimation } from './xr-light-estimation.js';
+import { XrImageTracking } from './xr-image-tracking.js';
 import { XrDomOverlay } from './xr-dom-overlay.js';
 
 /**
@@ -55,6 +56,7 @@ function XrManager(app) {
     this.input = new XrInput(this);
     this.hitTest = new XrHitTest(this);
     this.lightEstimation = new XrLightEstimation(this);
+    this.imageTracking = new XrImageTracking(this);
     this.domOverlay = new XrDomOverlay(this);
 
     this._camera = null;
@@ -138,7 +140,6 @@ XrManager.prototype.constructor = XrManager;
  * });
  */
 
-
 /**
  * @event
  * @name pc.XrManager#error
@@ -218,6 +219,10 @@ XrManager.prototype.start = function (camera, type, spaceType, options) {
         opts.optionalFeatures.push('light-estimation');
         opts.optionalFeatures.push('hit-test');
 
+        if (options && options.imageTracking) {
+            opts.optionalFeatures.push('image-tracking');
+        }
+
         if (this.domOverlay.root) {
             opts.optionalFeatures.push('dom-overlay');
             opts.domOverlay = { root: this.domOverlay.root };
@@ -226,11 +231,31 @@ XrManager.prototype.start = function (camera, type, spaceType, options) {
         opts.optionalFeatures.push('hand-tracking');
     }
 
-    if (options && options.optionalFeatures) {
+    if (options && options.optionalFeatures)
         opts.optionalFeatures = opts.optionalFeatures.concat(options.optionalFeatures);
+
+    if (this.imageTracking.images.length) {
+        this.imageTracking.prepareImages(function (err, trackedImages) {
+            if (err) {
+                if (callback) callback(err);
+                self.fire('error', err);
+                return;
+            }
+
+            if (trackedImages !== null)
+                opts.trackedImages = trackedImages;
+
+            self._onStartOptionsReady(type, spaceType, opts, callback);
+        });
+    } else {
+        self._onStartOptionsReady(type, spaceType, opts, callback);
     }
+};
 
-    navigator.xr.requestSession(type, opts).then(function (session) {
+XrManager.prototype._onStartOptionsReady = function (type, spaceType, options, callback) {
+    var self = this;
+
+    navigator.xr.requestSession(type, options).then(function (session) {
         self._onSessionStart(session, spaceType, callback);
     }).catch(function (ex) {
         self._camera.camera.xr = null;
@@ -484,6 +509,9 @@ XrManager.prototype.update = function (frame) {
         if (this.lightEstimation.supported) {
             this.lightEstimation.update(frame);
         }
+        if (this.imageTracking.supported) {
+            this.imageTracking.update(frame);
+        }
     }
 
     this.fire('update', frame);