Merge pull request #7561 from bojidar-bg/7342-screenToWorld

Add screenToWorld function mirroring worldToScreen

Author: davepagurek

src/accessibility/outputs.js

@@ -540,14 +540,9 @@ function outputs(p5, fn){
 
   //gets position of shape in the canvas
   fn._getPos = function (x, y) {
-    const untransformedPosition = new DOMPointReadOnly(x, y);
-    const currentTransform = this._renderer.isP3D ?
-      new DOMMatrix(this._renderer.calculateCombinedMatrix()) :
-      this.drawingContext.getTransform();
-    const { x: transformedX, y: transformedY } = untransformedPosition
-      .matrixTransform(currentTransform);
-    const canvasWidth = this.width * this._renderer._pixelDensity;
-    const canvasHeight = this.height * this._renderer._pixelDensity;
+    const { x: transformedX, y: transformedY } = this.worldToScreen(new this.Vector(x, y));
+    const canvasWidth = this.width;
+    const canvasHeight = this.height;
     if (transformedX < 0.4 * canvasWidth) {
       if (transformedY < 0.4 * canvasHeight) {
         return 'top left';

src/core/environment.js

@@ -1268,7 +1268,9 @@ function environment(p5, fn){
    * of 3D objects.
    *
    * @method worldToScreen
-   * @param {p5.Vector} worldPosition The 3D coordinates in the world space.
+   * @param {Number|p5.Vector} x The x coordinate in world space. (Or a vector for all three coordinates.)
+   * @param {Number} y The y coordinate in world space.
+   * @param {Number} [z] The z coordinate in world space.
    * @return {p5.Vector} A vector containing the 2D screen coordinates.
    * @example
    * <div>
@@ -1380,27 +1382,77 @@ function environment(p5, fn){
    *
    */
   fn.worldToScreen = function(worldPosition) {
-    const renderer = this._renderer;
-    if (renderer.drawingContext instanceof CanvasRenderingContext2D) {
-      // Handle 2D context
-      const transformMatrix = new DOMMatrix()
-        .scale(1 / renderer._pInst.pixelDensity())
-        .multiply(renderer.drawingContext.getTransform());
-      const screenCoordinates = transformMatrix.transformPoint(
-        new DOMPoint(worldPosition.x, worldPosition.y)
-      );
-      return new p5.Vector(screenCoordinates.x, screenCoordinates.y);
-    } else {
-          // Handle WebGL context (3D)
-          const modelViewMatrix = renderer.calculateCombinedMatrix();
-          const cameraCoordinates = modelViewMatrix.multiplyPoint(worldPosition);
-          const normalizedDeviceCoordinates =
-            renderer.states.uPMatrix.multiplyAndNormalizePoint(cameraCoordinates);
-          const screenX = (0.5 + 0.5 * normalizedDeviceCoordinates.x) * this.width;
-          const screenY = (0.5 - 0.5 * normalizedDeviceCoordinates.y) * this.height;
-          const screenZ = 0.5 + 0.5 * normalizedDeviceCoordinates.z;
-          return new Vector(screenX, screenY, screenZ);
+    if (typeof worldPosition === "number") {
+      // We got passed numbers, convert to vector
+      worldPosition = this.createVector(...arguments);
+    }
+
+    const matrix = this._renderer.getWorldToScreenMatrix();
+    const screenPosition = matrix.multiplyAndNormalizePoint(worldPosition);
+    return screenPosition;
+  };
+  /**
+   * Converts 2D screen coordinates to 3D world coordinates.
+   *
+   * This function takes a vector and converts its coordinates from coordinates
+   * on the screen to coordinates in the currently drawn object. This can be
+   * useful for determining the mouse position relative to a 2D or 3D object.
+   *
+   * If given, the Z component of the input coordinates is treated as "depth",
+   * or distance from the camera.
+   *
+   * @method screenToWorld
+   * @param {Number|p5.Vector} x The x coordinate in screen space. (Or a vector for all three coordinates.)
+   * @param {Number} y The y coordinate in screen space.
+   * @param {Number} [z] The z coordinate in screen space.
+   * @return {p5.Vector} A vector containing the 3D world space coordinates.
+   * @example
+   * <div>
+   * <code>
+   *
+   * function setup() {
+   *   createCanvas(100, 100);
+   *   describe('A rotating square with a line passing through the mouse drawn across it.');
+   * }
+   *
+   * function draw() {
+   *   background(220);
+   *
+   *   // Move to center and rotate
+   *   translate(width/2, height/2);
+   *   rotate(millis() / 1000);
+   *   rect(-30, -30, 60);
+   *
+   *   // Compute the location of the mouse in the coordinates of the square
+   *   let localMouse = screenToWorld(createVector(mouseX, mouseY));
+   *
+   *   // Draw a line parallel to the local Y axis, passing through the mouse
+   *   line(localMouse.x, -30, localMouse.x, 30);
+   * }
+   *
+   * </code>
+   * </div>
+   *
+   */
+  fn.screenToWorld = function(screenPosition) {
+    if (typeof screenPosition === "number") {
+      // We got passed numbers, convert to vector
+      screenPosition = this.createVector(...arguments);
+    }
+
+    const matrix = this._renderer.getWorldToScreenMatrix();
+
+    if (screenPosition.dimensions == 2) {
+      // Calculate a sensible Z value for the current camera projection that
+      // will result in 0 once converted to world coordinates
+      let z = matrix.mat4[14] / matrix.mat4[15];
+      screenPosition = this.createVector(screenPosition.x, screenPosition.y, z);
     }
+
+    const matrixInverse = matrix.invert(matrix);
+
+    const worldPosition = matrixInverse.multiplyAndNormalizePoint(screenPosition);
+    return worldPosition;
   };
 }
 

src/core/p5.Renderer2D.js

@@ -7,6 +7,7 @@ import { Element } from '../dom/p5.Element';
 import { MediaElement } from '../dom/p5.MediaElement';
 import { RGBHDR } from '../color/creating_reading';
 import FilterRenderer2D from '../image/filterRenderer2D';
+import { Matrix } from '../math/p5.Matrix';
 import { PrimitiveToPath2DConverter } from '../shape/custom_shapes';
 
 
@@ -999,6 +1000,13 @@ class Renderer2D extends Renderer {
     this.drawingContext.transform(a, b, c, d, e, f);
   }
 
+  getWorldToScreenMatrix() {
+    let domMatrix = new DOMMatrix()
+      .scale(1 / this._pixelDensity)
+      .multiply(this.drawingContext.getTransform());
+    return new Matrix(domMatrix.toFloat32Array());
+  }
+
   resetMatrix() {
     this.drawingContext.setTransform(1, 0, 0, 1, 0, 0);
     this.drawingContext.scale(

src/math/p5.Vector.js

@@ -34,14 +34,14 @@ class Vector {
   // This is how it comes in with createVector()
   // This check if the first argument is a function
   constructor(...args) {
-    let dimensions = args.length; // TODO: make default 3 if no arguments
     let values = args.map((arg) => arg || 0);
     if (typeof args[0] === "function") {
       this.isPInst = true;
       this._fromRadians = args[0];
       this._toRadians = args[1];
       values = args.slice(2).map((arg) => arg || 0);
     }
+    let dimensions = values.length; // TODO: make default 3 if no arguments
     if (dimensions === 0) {
       this.dimensions = 2;
       this._values = [0, 0, 0];

src/webgl/p5.RendererGL.js

@@ -1024,12 +1024,23 @@ class RendererGL extends Renderer {
     this.clear(..._col._getRGBA());
   }
 
-  // Combines the model and view matrices to get the uMVMatrix
-  // This method will be reusable wherever you need to update the combined matrix.
-  calculateCombinedMatrix() {
+
+  /**
+   * Get a matrix from world-space to screen-space
+   */
+  getWorldToScreenMatrix() {
     const modelMatrix = this.states.uModelMatrix;
     const viewMatrix = this.states.uViewMatrix;
-    return modelMatrix.copy().mult(viewMatrix);
+    const projectionMatrix = this.states.uPMatrix;
+    const projectedToScreenMatrix = new Matrix(4);
+    projectedToScreenMatrix.scale(this.width, this.height, 1);
+    projectedToScreenMatrix.translate([0.5, 0.5, 0.5]);
+    projectedToScreenMatrix.scale(0.5, -0.5, 0.5);
+
+    const modelViewMatrix = modelMatrix.copy().mult(viewMatrix);
+    const modelViewProjectionMatrix = modelViewMatrix.mult(projectionMatrix);
+    const worldToScreenMatrix = modelViewProjectionMatrix.mult(projectedToScreenMatrix);
+    return worldToScreenMatrix;
   }
 
   //////////////////////////////////////////////
@@ -2204,7 +2215,7 @@ class RendererGL extends Renderer {
     const modelMatrix = this.states.uModelMatrix;
     const viewMatrix = this.states.uViewMatrix;
     const projectionMatrix = this.states.uPMatrix;
-    const modelViewMatrix = this.calculateCombinedMatrix();
+    const modelViewMatrix = modelMatrix.copy().mult(viewMatrix);
 
     shader.setUniform(
       "uPerspective",

test/unit/core/environment.js

@@ -249,6 +249,15 @@ suite('Environment', function() {
       assert.closeTo(screenPos.y, 50, 0.1);
     });
 
+    test('worldToScreen accepts numbers with translation in 2D', function() {
+      myp5.push();
+      myp5.translate(50, 50);
+      let screenPos = myp5.worldToScreen(0, 0);
+      myp5.pop();
+      assert.closeTo(screenPos.x, 50, 0.1);
+      assert.closeTo(screenPos.y, 50, 0.1);
+    });
+
     test('worldToScreen with rotation in 2D', function() {
       myp5.push();
       myp5.translate(50, 50);
@@ -259,6 +268,46 @@ suite('Environment', function() {
       assert.closeTo(screenPos.x, 50, 0.1);
       assert.closeTo(screenPos.y, 60, 0.1);
     });
+
+    test('screenToWorld for 2D context', function() {
+      let screenPos = myp5.createVector(50, 50);
+      let worldPos = myp5.screenToWorld(screenPos);
+      assert.closeTo(worldPos.x, 50, 0.1);
+      assert.closeTo(worldPos.y, 50, 0.1);
+    });
+
+    test('screenToWorld accepts numbers with translation in 2D', function() {
+      myp5.push();
+      myp5.translate(50, 50);
+      let worldPos = myp5.screenToWorld(50, 50);
+      myp5.pop();
+      assert.closeTo(worldPos.x, 0, 0.1);
+      assert.closeTo(worldPos.y, 0, 0.1);
+    });
+
+    test('screenToWorld with rotation in 2D', function() {
+      myp5.push();
+      myp5.translate(50, 50);
+      myp5.rotate(myp5.PI / 2);
+      let screenPos = myp5.createVector(50, 60);
+      let worldPos = myp5.screenToWorld(screenPos);
+      myp5.pop();
+      assert.closeTo(worldPos.x, 10, 0.1);
+      assert.closeTo(worldPos.y, 0, 0.1);
+    });
+
+    test('screenToWorld is inverse of worldToScreen in 2D', function() {
+      myp5.push();
+      // Arbitrary rotation
+      myp5.translate(10, 10);
+      myp5.rotate(myp5.PI / 8);
+      let origWorldPos = myp5.createVector(20, 20);
+      let screenPos = myp5.worldToScreen(origWorldPos);
+      let worldPos = myp5.screenToWorld(screenPos);
+      myp5.pop();
+      assert.closeTo(worldPos.x, origWorldPos.x, 0.1);
+      assert.closeTo(worldPos.y, origWorldPos.y, 0.1);
+    });
   });
 
   suite('3D context test', function() {
@@ -273,6 +322,15 @@ suite('Environment', function() {
       assert.closeTo(screenPos.y, 50, 0.1);
     });
 
+    test('worldToScreen accepts numbers with translation in 3D', function() {
+      myp5.push();
+      myp5.translate(50, 50, 0);
+      let screenPos = myp5.worldToScreen(0, 0, 0);
+      myp5.pop();
+      assert.closeTo(screenPos.x, 100, 0.1);
+      assert.closeTo(screenPos.y, 100, 0.1);
+    });
+
     test('worldToScreen with rotation in 3D around Y-axis', function() {
       myp5.push();
       myp5.rotateY(myp5.PI / 2);
@@ -292,5 +350,57 @@ suite('Environment', function() {
       assert.closeTo(screenPos.x, 50, 0.1);
       assert.closeTo(screenPos.y, 60, 0.1);
     });
+
+    test('screenToWorld for 3D context', function() {
+      let screenPos = myp5.createVector(50, 50);
+      let worldPos = myp5.screenToWorld(screenPos);
+      assert.closeTo(worldPos.x, 0, 0.1);
+      assert.closeTo(worldPos.y, 0, 0.1);
+      assert.closeTo(worldPos.z, 0, 0.1);
+    });
+
+    test('screenToWorld accepts numbers with translation in 3D', function() {
+      myp5.push();
+      myp5.translate(50, 50);
+      let worldPos = myp5.screenToWorld(100, 100);
+      myp5.pop();
+      assert.closeTo(worldPos.x, 0, 0.1);
+      assert.closeTo(worldPos.y, 0, 0.1);
+    });
+
+    test('screenToWorld with rotation in 3D around Y-axis', function() {
+      myp5.push();
+      myp5.rotateY(myp5.PI / 2);
+      let screenPos = myp5.createVector(50, 50);
+      let worldPos = myp5.screenToWorld(screenPos);
+      myp5.pop();
+      assert.closeTo(worldPos.y, 0, 0.1);
+    });
+
+    test('screenToWorld with rotation in 3D around Z-axis', function() {
+      myp5.push();
+      myp5.rotateZ(myp5.PI / 2);
+      let screenPos = myp5.createVector(50, 60);
+      let worldPos = myp5.screenToWorld(screenPos);
+      myp5.pop();
+      assert.closeTo(worldPos.x, 10, 0.1);
+      assert.closeTo(worldPos.y, 0, 0.1);
+    });
+
+    test('screenToWorld is inverse of worldToScreen in 3D', function() {
+      myp5.push();
+      // Arbitrary rotation
+      myp5.translate(10, 10);
+      myp5.rotateX(myp5.PI / 3);
+      myp5.rotateY(myp5.PI / 4);
+      myp5.rotateZ(myp5.PI / 8);
+      let origWorldPos = myp5.createVector(20, 20, 0);
+      let screenPos = myp5.worldToScreen(origWorldPos);
+      let worldPos = myp5.screenToWorld(screenPos);
+      myp5.pop();
+      assert.closeTo(worldPos.x, origWorldPos.x, 0.1);
+      assert.closeTo(worldPos.y, origWorldPos.y, 0.1);
+      assert.closeTo(worldPos.z, origWorldPos.z, 0.1);
+    });
   });
 });

test/unit/math/p5.Vector.js

@@ -52,6 +52,10 @@ suite("p5.Vector", function () {
       assert.equal(v.y, 0);
       assert.equal(v.z, 0);
     });
+
+    test("should have dimensions initialized to 2", function () {
+      assert.equal(v.dimensions, 2);
+    });
   });
 
   suite.todo("p5.prototype.createVector(1, 2, 3)", function () {
@@ -64,6 +68,10 @@ suite("p5.Vector", function () {
       assert.equal(v.y, 2);
       assert.equal(v.z, 3);
     });
+
+    test("should have dimensions initialized to 3", function () {
+      assert.equal(v.dimensions, 3);
+    });
   });
 
   suite("new p5.Vector()", function () {