diff --git a/.eslintignore b/.eslintignore
new file mode 100644
index 00000000..a9001463
--- /dev/null
+++ b/.eslintignore
@@ -0,0 +1 @@
+webrtc-adapter.js
diff --git a/.eslintrc.js b/.eslintrc.js
new file mode 100644
index 00000000..fc0c2935
--- /dev/null
+++ b/.eslintrc.js
@@ -0,0 +1,51 @@
+/* eslint-env node */
+module.exports = {
+ 'env': {
+ 'es2022': true,
+ },
+ 'extends': 'eslint:recommended',
+ 'globals': {
+ 'Stats': 'readonly',
+
+ 'realityEditor': 'writable',
+ 'createNameSpace': 'writable',
+ 'globalStates': 'writable',
+ 'objects': 'writable',
+ 'overlayDiv': 'writable',
+ },
+ 'parserOptions': {
+ 'ecmaVersion': 2022,
+ 'sourceType': 'module',
+ },
+ 'rules': {
+ 'no-shadow': 'off',
+ 'no-useless-escape': 'off',
+ 'no-prototype-builtins': 'off',
+ 'no-redeclare': [
+ 'error',
+ {'builtinGlobals': false}
+ ],
+ 'no-unused-vars': [
+ 'error',
+ {
+ 'varsIgnorePattern': '^_',
+ 'argsIgnorePattern': '^_',
+ },
+ ],
+ 'no-inner-declarations': 'off',
+ },
+ 'overrides': [{
+ 'files': [
+ 'content_scripts/**/*.js',
+ 'tools/**/*.js',
+ ],
+ 'env': {
+ 'browser': true,
+ }
+ }, {
+ 'files': ['interfaces/**/*.js'],
+ 'env': {
+ 'node': true,
+ }
+ }],
+};
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000..290ad028
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: npm
+ directory: "/"
+ schedule:
+ interval: daily
+ time: "10:00"
+ open-pull-requests-limit: 10
diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
new file mode 100644
index 00000000..1c755c66
--- /dev/null
+++ b/.github/workflows/nodejs.yml
@@ -0,0 +1,24 @@
+name: Node.js CI
+
+on: [push]
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+
+ strategy:
+ matrix:
+ node-version: [20.x]
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v1
+ with:
+ node-version: ${{ matrix.node-version }}
+ - run: npm install
+ - run: npm run build --if-present
+ - run: npm test
+ env:
+ CI: true
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..6f365606
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,12 @@
+# idea editing tools
+.idea
+
+# osx directory files
+.DS_Store
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+
+#Https certs
+*.pem
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..e87a115e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,363 @@
+Mozilla Public License, version 2.0
+
+1. Definitions
+
+1.1. "Contributor"
+
+ means each individual or legal entity that creates, contributes to the
+ creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+
+ means the combination of the Contributions of others (if any) used by a
+ Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+
+ means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+
+ means Source Code Form to which the initial Contributor has attached the
+ notice in Exhibit A, the Executable Form of such Source Code Form, and
+ Modifications of such Source Code Form, in each case including portions
+ thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+ means
+
+ a. that the initial Contributor has attached the notice described in
+ Exhibit B to the Covered Software; or
+
+ b. that the Covered Software was made available under the terms of
+ version 1.1 or earlier of the License, but not also under the terms of
+ a Secondary License.
+
+1.6. "Executable Form"
+
+ means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+
+ means a work that combines Covered Software with other material, in a
+ separate file or files, that is not Covered Software.
+
+1.8. "License"
+
+ means this document.
+
+1.9. "Licensable"
+
+ means having the right to grant, to the maximum extent possible, whether
+ at the time of the initial grant or subsequently, any and all of the
+ rights conveyed by this License.
+
+1.10. "Modifications"
+
+ means any of the following:
+
+ a. any file in Source Code Form that results from an addition to,
+ deletion from, or modification of the contents of Covered Software; or
+
+ b. any new file in Source Code Form that contains any Covered Software.
+
+1.11. "Patent Claims" of a Contributor
+
+ means any patent claim(s), including without limitation, method,
+ process, and apparatus claims, in any patent Licensable by such
+ Contributor that would be infringed, but for the grant of the License,
+ by the making, using, selling, offering for sale, having made, import,
+ or transfer of either its Contributions or its Contributor Version.
+
+1.12. "Secondary License"
+
+ means either the GNU General Public License, Version 2.0, the GNU Lesser
+ General Public License, Version 2.1, the GNU Affero General Public
+ License, Version 3.0, or any later versions of those licenses.
+
+1.13. "Source Code Form"
+
+ means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+
+ means an individual or a legal entity exercising rights under this
+ License. For legal entities, "You" includes any entity that controls, is
+ controlled by, or is under common control with You. For purposes of this
+ definition, "control" means (a) the power, direct or indirect, to cause
+ the direction or management of such entity, whether by contract or
+ otherwise, or (b) ownership of more than fifty percent (50%) of the
+ outstanding shares or beneficial ownership of such entity.
+
+
+2. License Grants and Conditions
+
+2.1. Grants
+
+ Each Contributor hereby grants You a world-wide, royalty-free,
+ non-exclusive license:
+
+ a. under intellectual property rights (other than patent or trademark)
+ Licensable by such Contributor to use, reproduce, make available,
+ modify, display, perform, distribute, and otherwise exploit its
+ Contributions, either on an unmodified basis, with Modifications, or
+ as part of a Larger Work; and
+
+ b. under Patent Claims of such Contributor to make, use, sell, offer for
+ sale, have made, import, and otherwise transfer either its
+ Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+ The licenses granted in Section 2.1 with respect to any Contribution
+ become effective for each Contribution on the date the Contributor first
+ distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+ The licenses granted in this Section 2 are the only rights granted under
+ this License. No additional rights or licenses will be implied from the
+ distribution or licensing of Covered Software under this License.
+ Notwithstanding Section 2.1(b) above, no patent license is granted by a
+ Contributor:
+
+ a. for any code that a Contributor has removed from Covered Software; or
+
+ b. for infringements caused by: (i) Your and any other third party's
+ modifications of Covered Software, or (ii) the combination of its
+ Contributions with other software (except as part of its Contributor
+ Version); or
+
+ c. under Patent Claims infringed by Covered Software in the absence of
+ its Contributions.
+
+ This License does not grant any rights in the trademarks, service marks,
+ or logos of any Contributor (except as may be necessary to comply with
+ the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+ No Contributor makes additional grants as a result of Your choice to
+ distribute the Covered Software under a subsequent version of this
+ License (see Section 10.2) or under the terms of a Secondary License (if
+ permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+ Each Contributor represents that the Contributor believes its
+ Contributions are its original creation(s) or it has sufficient rights to
+ grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+ This License is not intended to limit any rights You have under
+ applicable copyright doctrines of fair use, fair dealing, or other
+ equivalents.
+
+2.7. Conditions
+
+ Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
+ Section 2.1.
+
+
+3. Responsibilities
+
+3.1. Distribution of Source Form
+
+ All distribution of Covered Software in Source Code Form, including any
+ Modifications that You create or to which You contribute, must be under
+ the terms of this License. You must inform recipients that the Source
+ Code Form of the Covered Software is governed by the terms of this
+ License, and how they can obtain a copy of this License. You may not
+ attempt to alter or restrict the recipients' rights in the Source Code
+ Form.
+
+3.2. Distribution of Executable Form
+
+ If You distribute Covered Software in Executable Form then:
+
+ a. such Covered Software must also be made available in Source Code Form,
+ as described in Section 3.1, and You must inform recipients of the
+ Executable Form how they can obtain a copy of such Source Code Form by
+ reasonable means in a timely manner, at a charge no more than the cost
+ of distribution to the recipient; and
+
+ b. You may distribute such Executable Form under the terms of this
+ License, or sublicense it under different terms, provided that the
+ license for the Executable Form does not attempt to limit or alter the
+ recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+ You may create and distribute a Larger Work under terms of Your choice,
+ provided that You also comply with the requirements of this License for
+ the Covered Software. If the Larger Work is a combination of Covered
+ Software with a work governed by one or more Secondary Licenses, and the
+ Covered Software is not Incompatible With Secondary Licenses, this
+ License permits You to additionally distribute such Covered Software
+ under the terms of such Secondary License(s), so that the recipient of
+ the Larger Work may, at their option, further distribute the Covered
+ Software under the terms of either this License or such Secondary
+ License(s).
+
+3.4. Notices
+
+ You may not remove or alter the substance of any license notices
+ (including copyright notices, patent notices, disclaimers of warranty, or
+ limitations of liability) contained within the Source Code Form of the
+ Covered Software, except that You may alter any license notices to the
+ extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+ You may choose to offer, and to charge a fee for, warranty, support,
+ indemnity or liability obligations to one or more recipients of Covered
+ Software. However, You may do so only on Your own behalf, and not on
+ behalf of any Contributor. You must make it absolutely clear that any
+ such warranty, support, indemnity, or liability obligation is offered by
+ You alone, and You hereby agree to indemnify every Contributor for any
+ liability incurred by such Contributor as a result of warranty, support,
+ indemnity or liability terms You offer. You may include additional
+ disclaimers of warranty and limitations of liability specific to any
+ jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+
+ If it is impossible for You to comply with any of the terms of this License
+ with respect to some or all of the Covered Software due to statute,
+ judicial order, or regulation then You must: (a) comply with the terms of
+ this License to the maximum extent possible; and (b) describe the
+ limitations and the code they affect. Such description must be placed in a
+ text file included with all distributions of the Covered Software under
+ this License. Except to the extent prohibited by statute or regulation,
+ such description must be sufficiently detailed for a recipient of ordinary
+ skill to be able to understand it.
+
+5. Termination
+
+5.1. The rights granted under this License will terminate automatically if You
+ fail to comply with any of its terms. However, if You become compliant,
+ then the rights granted under this License from a particular Contributor
+ are reinstated (a) provisionally, unless and until such Contributor
+ explicitly and finally terminates Your grants, and (b) on an ongoing
+ basis, if such Contributor fails to notify You of the non-compliance by
+ some reasonable means prior to 60 days after You have come back into
+ compliance. Moreover, Your grants from a particular Contributor are
+ reinstated on an ongoing basis if such Contributor notifies You of the
+ non-compliance by some reasonable means, this is the first time You have
+ received notice of non-compliance with this License from such
+ Contributor, and You become compliant prior to 30 days after Your receipt
+ of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+ infringement claim (excluding declaratory judgment actions,
+ counter-claims, and cross-claims) alleging that a Contributor Version
+ directly or indirectly infringes any patent, then the rights granted to
+ You by any and all Contributors for the Covered Software under Section
+ 2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
+ license agreements (excluding distributors and resellers) which have been
+ validly granted by You or Your distributors under this License prior to
+ termination shall survive termination.
+
+6. Disclaimer of Warranty
+
+ Covered Software is provided under this License on an "as is" basis,
+ without warranty of any kind, either expressed, implied, or statutory,
+ including, without limitation, warranties that the Covered Software is free
+ of defects, merchantable, fit for a particular purpose or non-infringing.
+ The entire risk as to the quality and performance of the Covered Software
+ is with You. Should any Covered Software prove defective in any respect,
+ You (not any Contributor) assume the cost of any necessary servicing,
+ repair, or correction. This disclaimer of warranty constitutes an essential
+ part of this License. No use of any Covered Software is authorized under
+ this License except under this disclaimer.
+
+7. Limitation of Liability
+
+ Under no circumstances and under no legal theory, whether tort (including
+ negligence), contract, or otherwise, shall any Contributor, or anyone who
+ distributes Covered Software as permitted above, be liable to You for any
+ direct, indirect, special, incidental, or consequential damages of any
+ character including, without limitation, damages for lost profits, loss of
+ goodwill, work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses, even if such party shall have been
+ informed of the possibility of such damages. This limitation of liability
+ shall not apply to liability for death or personal injury resulting from
+ such party's negligence to the extent applicable law prohibits such
+ limitation. Some jurisdictions do not allow the exclusion or limitation of
+ incidental or consequential damages, so this exclusion and limitation may
+ not apply to You.
+
+8. Litigation
+
+ Any litigation relating to this License may be brought only in the courts
+ of a jurisdiction where the defendant maintains its principal place of
+ business and such litigation shall be governed by laws of that
+ jurisdiction, without reference to its conflict-of-law provisions. Nothing
+ in this Section shall prevent a party's ability to bring cross-claims or
+ counter-claims.
+
+9. Miscellaneous
+
+ This License represents the complete agreement concerning the subject
+ matter hereof. If any provision of this License is held to be
+ unenforceable, such provision shall be reformed only to the extent
+ necessary to make it enforceable. Any law or regulation which provides that
+ the language of a contract shall be construed against the drafter shall not
+ be used to construe this License against a Contributor.
+
+
+10. Versions of the License
+
+10.1. New Versions
+
+ Mozilla Foundation is the license steward. Except as provided in Section
+ 10.3, no one other than the license steward has the right to modify or
+ publish new versions of this License. Each version will be given a
+ distinguishing version number.
+
+10.2. Effect of New Versions
+
+ You may distribute the Covered Software under the terms of the version
+ of the License under which You originally received the Covered Software,
+ or under the terms of any subsequent version published by the license
+ steward.
+
+10.3. Modified Versions
+
+ If you create software not governed by this License, and you want to
+ create a new license for such software, you may create and use a
+ modified version of this License if you rename the license and remove
+ any references to the name of the license steward (except to note that
+ such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+ Licenses If You choose to distribute Source Code Form that is
+ Incompatible With Secondary Licenses under the terms of this version of
+ the License, the notice described in Exhibit B of this License must be
+ attached.
+
+Exhibit A - Source Code Form License Notice
+
+ This Source Code Form is subject to the
+ terms of the Mozilla Public License, v.
+ 2.0. If a copy of the MPL was not
+ distributed with this file, You can
+ obtain one at
+ http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular file,
+then You may include the notice in a location (such as a LICENSE file in a
+relevant directory) where a recipient would be likely to look for such a
+notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+
+ This Source Code Form is "Incompatible
+ With Secondary Licenses", as defined by
+ the Mozilla Public License, v. 2.0.
+
diff --git a/README.md b/README.md
index 879ce2c4..58e0f2cd 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,35 @@
-# vuforia-spatial-remote-operator-addon
\ No newline at end of file
+## Read First
+The Vuforia Spatial Toolbox and Vuforia Spatial Edge Server make up a shared research platform for exploring spatial computing as a community. This research platform is not an out of the box production-ready enterprise solution. Please read the [MPL 2.0 license](LICENSE) before use.
+
+Join the conversations in our [discourse forum](https://forum.spatialtoolbox.vuforia.com) if you have questions, ideas want to collaborate or just say hi.
+
+
+# vuforia-spatial-remote-operator-addon
+
+The Remote Operator is an add-on for [Vuforia Spatial Toolbox](https://github.com/ptcrealitylab/vuforia-spatial-toolbox-ios) that makes it compatible with the [Vuforia Spatial Toolbox Virtualizer](https://github.com/ptcrealitylab/vuforia-spatial-toolbox-virtualizer) Unity project. The result is a browser-based web app that combines a live volumetric capture of a space with mixed reality content.
+
+Full installation instructions are in the Vuforia Spatial Toolbox Virtualizer [README](https://github.com/ptcrealitylab/Vuforia-Spatial-Toolbox-Virtualizer#vuforia-spatial-toolbox-virtualizer).
+
+This add-on contains two hardware interfaces:
+1. `virtualizer`: This interface provides a websocket-based communication layer between the Vuforia Spatial Edge Server and the Unity application.
+2. `remoteOperatorUI`: This interface will serve the Remote Operator web app on `localhost:8081`.
+
+The add-on also contains some `content_scripts` that will modify the [Vuforia Spatial Toolbox User Interface](https://github.com/ptcrealitylab/vuforia-spatial-toolbox-userinterface) to be able to run in a desktop browser environment in addition to on a mobile AR device.
+
+**Special Setup instructions:**
+1. Make sure to turn on both of these hardware interfaces in the "Manage Hardware Interfaces" tab of your Edge Server's web interface (`localhost:8080`)
+2. The `remoteOperatorUI` interface needs to be configured with a path to a local copy of the [Vuforia Spatial Toolbox User Interface](https://github.com/ptcrealitylab/vuforia-spatial-toolbox-userinterface) repository. Click on the yellow gear on the `remoteOperatorUI` interface to view its configurations, and type in the path (e.g. `/Users/Benjamin/Documents/vuforia-spatial-toolbox-userinterface`) and hit save.
+3. After configuring, restart your edge server to serve the web app on port 8081.
+4. In order to see any tools in the pocket menu of the Remote Operator, you need to have an activated "World Object" on your server. Click "Add World Object" on localhost:8080 and give it a target to activate it.
+
+**Using the Remote Operator:**
+1. Objects, Tools, and Nodes will only appear in the Remote Operator if they have been localized against a World Object. To do this, make sure you have a World Object set up on your edge server and give it an Image Target or Area Target. With your Vuforia Spatial Toolbox iOS app, look at that target first and then look at the other Objects in your space. This will save their locations (relative to the World Object's origin) in the edge server. The World Object's position will be treated as the (0,0,0) position in your Unity project, and all localized Object, Tools, and Nodes will be rendered relative to that.
+2. The Remote Operator background will be blank until you connect to an Virtualizer. It will attempt to discover the possible Virtualizers in this network. Make sure your Unity project is running. Click on the drop-down menu in the top left to select a Virtualizer to connect to. Once connected, this Virtualizer must be designated as the "Primary Virtualizer". Go into the Settings menu (click the gear icon) and type in the IP address of the computer running the Unity project into the "Primary Virtualizer IP" field (e.g. `192.168.0.12`) and toggle this mode on. You should now see the video stream in the background.
+3. To control the camera of the Remote Operator:
+ - `Right-click`: Rotate Camera
+ - `Shift + Right-click`: Pan Camera
+ - `Scroll Wheel`: Zoom
+ - `V`: Toggle visibility of extra UI
+4. You can select multiple Virtualizers running on two different computers to receive a combined volumetric video for a larger area. Click on both Virtualizers in the drop-down menu to connect to them both, and ensure that one of them is set as the Primary Virtualizer.
+
+Please use the [forum](https://forum.spatialtoolbox.vuforia.com) for any questions.
diff --git a/content_resources/calendarButton.svg b/content_resources/calendarButton.svg
new file mode 100644
index 00000000..e35e60b1
--- /dev/null
+++ b/content_resources/calendarButton.svg
@@ -0,0 +1,51 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/cameraPan.svg b/content_resources/cameraPan.svg
new file mode 100644
index 00000000..6f659864
--- /dev/null
+++ b/content_resources/cameraPan.svg
@@ -0,0 +1,35 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/cameraRotate.svg b/content_resources/cameraRotate.svg
new file mode 100644
index 00000000..e85620a2
--- /dev/null
+++ b/content_resources/cameraRotate.svg
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/cameraZoom.svg b/content_resources/cameraZoom.svg
new file mode 100644
index 00000000..7b5ebf72
--- /dev/null
+++ b/content_resources/cameraZoom.svg
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/hideTimelineButton.svg b/content_resources/hideTimelineButton.svg
new file mode 100644
index 00000000..ae13b864
--- /dev/null
+++ b/content_resources/hideTimelineButton.svg
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
diff --git a/content_resources/pauseButton.svg b/content_resources/pauseButton.svg
new file mode 100644
index 00000000..b27b4ed4
--- /dev/null
+++ b/content_resources/pauseButton.svg
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/playButton.svg b/content_resources/playButton.svg
new file mode 100644
index 00000000..6d589e52
--- /dev/null
+++ b/content_resources/playButton.svg
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
diff --git a/content_resources/seekButton.svg b/content_resources/seekButton.svg
new file mode 100644
index 00000000..f6f34394
--- /dev/null
+++ b/content_resources/seekButton.svg
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/showTimelineButton.svg b/content_resources/showTimelineButton.svg
new file mode 100644
index 00000000..71eaa7ca
--- /dev/null
+++ b/content_resources/showTimelineButton.svg
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_128x.svg b/content_resources/speedButton_128x.svg
new file mode 100644
index 00000000..45cf0403
--- /dev/null
+++ b/content_resources/speedButton_128x.svg
@@ -0,0 +1,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_16x.svg b/content_resources/speedButton_16x.svg
new file mode 100644
index 00000000..77ed70d9
--- /dev/null
+++ b/content_resources/speedButton_16x.svg
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_1x.svg b/content_resources/speedButton_1x.svg
new file mode 100644
index 00000000..2386cf9e
--- /dev/null
+++ b/content_resources/speedButton_1x.svg
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_256x.svg b/content_resources/speedButton_256x.svg
new file mode 100644
index 00000000..7b10f3b0
--- /dev/null
+++ b/content_resources/speedButton_256x.svg
@@ -0,0 +1,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_2x.svg b/content_resources/speedButton_2x.svg
new file mode 100644
index 00000000..13130439
--- /dev/null
+++ b/content_resources/speedButton_2x.svg
@@ -0,0 +1,17 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_32x.svg b/content_resources/speedButton_32x.svg
new file mode 100644
index 00000000..5c52e56f
--- /dev/null
+++ b/content_resources/speedButton_32x.svg
@@ -0,0 +1,22 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_4x.svg b/content_resources/speedButton_4x.svg
new file mode 100644
index 00000000..560eb675
--- /dev/null
+++ b/content_resources/speedButton_4x.svg
@@ -0,0 +1,15 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_64x.svg b/content_resources/speedButton_64x.svg
new file mode 100644
index 00000000..08e2691a
--- /dev/null
+++ b/content_resources/speedButton_64x.svg
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/speedButton_8x.svg b/content_resources/speedButton_8x.svg
new file mode 100644
index 00000000..09354438
--- /dev/null
+++ b/content_resources/speedButton_8x.svg
@@ -0,0 +1,21 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/content_resources/timelinePlayhead.svg b/content_resources/timelinePlayhead.svg
new file mode 100644
index 00000000..e2d3901b
--- /dev/null
+++ b/content_resources/timelinePlayhead.svg
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
diff --git a/content_resources/touch-control-zoom-dynamic.svg b/content_resources/touch-control-zoom-dynamic.svg
new file mode 100644
index 00000000..b9a56c61
--- /dev/null
+++ b/content_resources/touch-control-zoom-dynamic.svg
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/content_resources/touch-control-zoom-static.svg b/content_resources/touch-control-zoom-static.svg
new file mode 100644
index 00000000..2afb233e
--- /dev/null
+++ b/content_resources/touch-control-zoom-static.svg
@@ -0,0 +1,33 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/content_resources/touch-controls-white-pan.svg b/content_resources/touch-controls-white-pan.svg
new file mode 100644
index 00000000..24d2f4dd
--- /dev/null
+++ b/content_resources/touch-controls-white-pan.svg
@@ -0,0 +1,36 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/content_resources/touch-controls-white-pointer.svg b/content_resources/touch-controls-white-pointer.svg
new file mode 100644
index 00000000..53bdd7e7
--- /dev/null
+++ b/content_resources/touch-controls-white-pointer.svg
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/content_resources/touch-controls-white-rotate.svg b/content_resources/touch-controls-white-rotate.svg
new file mode 100644
index 00000000..75ec4d6a
--- /dev/null
+++ b/content_resources/touch-controls-white-rotate.svg
@@ -0,0 +1,34 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/content_resources/touch-controls-white-zoom.svg b/content_resources/touch-controls-white-zoom.svg
new file mode 100644
index 00000000..5a867a60
--- /dev/null
+++ b/content_resources/touch-controls-white-zoom.svg
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/content_resources/zoomSliderBackground.svg b/content_resources/zoomSliderBackground.svg
new file mode 100644
index 00000000..c4e8c2e8
--- /dev/null
+++ b/content_resources/zoomSliderBackground.svg
@@ -0,0 +1 @@
+– +
\ No newline at end of file
diff --git a/content_resources/zoomSliderHandle.svg b/content_resources/zoomSliderHandle.svg
new file mode 100644
index 00000000..4640a76e
--- /dev/null
+++ b/content_resources/zoomSliderHandle.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/content_scripts/Calendar.js b/content_scripts/Calendar.js
new file mode 100644
index 00000000..75e156fe
--- /dev/null
+++ b/content_scripts/Calendar.js
@@ -0,0 +1,304 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+ class Calendar {
+ constructor(parent, initiallyVisible) {
+ this.dateNow = new Date(Date.now());
+ this.selectedDate = {
+ month: this.dateNow.getMonth(),
+ year: this.dateNow.getFullYear(),
+ day: this.dateNow.getDate() // use getDate. getDay returns index of weekday, e.g. Tues = 2
+ };
+ this.dateWhenSelected = {
+ month: null,
+ year: null,
+ day: null
+ };
+ this.highlightedDates = [];
+ this.padding = 10;
+ this.weekDayNames = ['Su', 'M', 'T', 'W', 'Th', 'F', 'Sa'];
+ this.monthNames = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+ this.selectedDateDiv = null;
+ this.callbacks = {
+ onDateSelected: []
+ };
+ // do this after initializing state
+ this.buildDom();
+ if (!initiallyVisible) {
+ this.hide();
+ }
+ parent.appendChild(this.dom);
+
+ this.updateDomForMonth(this.selectedDate.month, this.selectedDate.year);
+ }
+ buildDom() {
+ this.dom = document.createElement('div');
+ this.dom.classList.add('calendar');
+ this.dom.style.padding = this.padding + 'px';
+
+ let header = document.createElement('div');
+ header.id = 'calHeader';
+ header.style.top = this.padding + 'px';
+ let prevMonth = document.createElement('div');
+ prevMonth.id = 'calPrevMonth';
+ prevMonth.style.left = this.padding + 'px';
+ prevMonth.innerText = '<';
+ let nextMonth = document.createElement('div');
+ nextMonth.id = 'calNextMonth';
+ nextMonth.style.right = this.padding + 'px';
+ nextMonth.innerText = '>';
+ let monthName = document.createElement('div');
+ monthName.id = 'calMonthName';
+ monthName.innerText = 'March 2022';
+ header.appendChild(prevMonth);
+ header.appendChild(monthName);
+ header.appendChild(nextMonth);
+ this.dom.appendChild(header);
+
+ let labels = document.createElement('div');
+ labels.id = 'calLabels';
+ labels.style.top = 30 + this.padding + 'px';
+ for (let i = 0; i < 7; i++) {
+ let label = document.createElement('div');
+ let x = (300 / 7) * i + this.padding;
+ label.style.width = (300 / 7) + 'px';
+ label.style.left = JSON.stringify(x) + 'px';
+ label.innerText = this.weekDayNames[i];
+ labels.appendChild(label);
+ }
+ this.dom.appendChild(labels);
+
+ let dates = document.createElement('div');
+ dates.id = 'calDates';
+ dates.style.top = 60 + this.padding + 'px';
+ for (let r = 0; r < 6; r++) {
+ for (let c = 0; c < 7; c++) {
+ let date = document.createElement('div');
+ date.classList.add('calDate');
+ let dayNumber = 1 + r * 7 + c;
+ // let weekDayName = this.weekDayNames[c];
+
+ let x = (300 / 7) * c + this.padding;
+ let y = (240 / 6) * r;
+ date.style.width = (300 / 7) + 'px';
+ date.style.left = JSON.stringify(x) + 'px';
+ date.style.height = (240 / 6) + 'px';
+ date.style.lineHeight = (240 / 6) + 'px';
+ date.style.borderRadius = (300 / 7) * 0.5 + 'px';
+ date.style.top = JSON.stringify(y) + 'px';
+
+ date.setAttribute('dayNumber', JSON.stringify(dayNumber));
+ // date.setAttribute('weekDayName', weekDayName);
+ dates.appendChild(date);
+
+ date.addEventListener('pointerup', _ => {
+ this.selectDate(date);
+ });
+ }
+ }
+ this.dom.appendChild(dates);
+
+ prevMonth.addEventListener('pointerup', _ => {
+ this.scrollMonth(-1);
+ });
+ nextMonth.addEventListener('pointerup', _ => {
+ this.scrollMonth(1);
+ });
+ monthName.addEventListener('pointerup', _ => {
+ this.scrollToToday();
+ });
+ }
+ updateDomForMonth(monthIndex, year) {
+ let date = new Date(year, monthIndex);
+ console.debug('new date = ' + date.toString());
+ let monthName = this.monthNames[monthIndex];
+ document.getElementById('calMonthName').innerText = monthName + ' ' + year;
+ let calDates = document.getElementById('calDates');
+ let numDays = this.daysInMonth(monthIndex, year);
+ console.debug(monthName + ' has ' + numDays + ' days');
+ console.debug(date.getDate() + ' is a ' + this.weekDayNames[date.getDay()]);
+ let dayOneIndex = date.getDay();
+ let searching = true;
+ let isNextMonth = false;
+ let i = 0;
+ let currentDayNumber = 1;
+ let firstDayOffset = 0;
+ for (let r = 0; r < 6; r++) {
+ for (let c = 0; c < 7; c++) {
+ if (searching) {
+ if (c === dayOneIndex) {
+ calDates.children[i].innerText = currentDayNumber;
+ calDates.children[i].setAttribute('dayNumber', JSON.stringify(currentDayNumber));
+ currentDayNumber += 1;
+ searching = false;
+ firstDayOffset = i;
+ } else {
+ calDates.children[i].innerText = '_';
+ }
+ } else if (currentDayNumber <= numDays) {
+ calDates.children[i].innerText = currentDayNumber;
+ calDates.children[i].setAttribute('dayNumber', JSON.stringify(currentDayNumber));
+ currentDayNumber += 1;
+ } else {
+ calDates.children[i].innerText = '';
+ }
+ if (isNextMonth) {
+ calDates.children[i].classList.add('otherMonthDate');
+ } else {
+ calDates.children[i].classList.remove('otherMonthDate');
+ }
+ if (currentDayNumber > numDays) {
+ currentDayNumber = 1;
+ isNextMonth = true;
+ }
+ i++;
+ }
+ }
+
+ // back-fill dates from previous month
+ if (firstDayOffset > 0) {
+ Array.from(calDates.children).forEach((elt, idx) => {
+ if (elt.innerText !== '_') { return; }
+ let relativeIndex = idx - firstDayOffset;
+ elt.innerText = new Date(year, monthIndex, relativeIndex + 1).getDate();
+ calDates.children[idx].setAttribute('dayNumber', elt.innerText);
+ elt.classList.add('otherMonthDate');
+ });
+ }
+
+ // reset highlights, and re-select a date if you scrolled the month since it was selected
+ if (this.selectedDate.month === this.dateWhenSelected.month &&
+ this.selectedDate.year === this.dateWhenSelected.year) {
+ Array.from(calDates.children).forEach((elt) => {
+ let dayNumber = parseInt(elt.getAttribute('dayNumber'));
+ if (dayNumber === this.dateWhenSelected.day) {
+ this.selectDate(elt);
+ }
+ });
+ }
+
+ // reset highlights
+ Array.from(calDates.children).forEach((elt) => {
+ elt.classList.remove('highlightedDate');
+ });
+ this.highlightDates(this.highlightedDates);
+ }
+ selectDate(elt) {
+ if (!elt) { return; }
+ if (elt.classList.contains('otherMonthDate')) {
+ let dayNumber = parseInt(elt.getAttribute('dayNumber'));
+ if (dayNumber > 14) {
+ this.scrollMonth(-1);
+ } else {
+ this.scrollMonth(1);
+ }
+ this.selectDate(this.getDateElementForDay(dayNumber));
+ return;
+ }
+
+ this.selectedDate.day = parseInt(elt.getAttribute('dayNumber'));
+ console.debug('selected day: ' + this.selectedDate.day);
+
+ this.dateWhenSelected.day = this.selectedDate.day;
+ this.dateWhenSelected.month = this.selectedDate.month;
+ this.dateWhenSelected.year = this.selectedDate.year;
+
+ this.unselectPreviousDate();
+ this.selectedDateDiv = elt;
+ this.selectedDateDiv.classList.add('selectedDate');
+
+ this.callbacks.onDateSelected.forEach(callback => {
+ callback(new Date(this.selectedDate.year, this.selectedDate.month, this.selectedDate.day));
+ });
+ }
+ unselectPreviousDate() {
+ if (this.selectedDateDiv) {
+ this.selectedDateDiv.classList.remove('selectedDate');
+ this.selectedDateDiv = null;
+ }
+ }
+ getDateElementForDay(number) {
+ let match = null;
+ Array.from(document.getElementById('calDates').children).forEach(elt => {
+ if (parseInt(elt.getAttribute('dayNumber')) === number &&
+ !elt.classList.contains('otherMonthDate')) {
+ match = elt;
+ }
+ });
+ return match;
+ }
+ getDateElement(dateObject) {
+ let year = dateObject.getFullYear();
+ let month = dateObject.getMonth();
+ let day = dateObject.getDate();
+ if (this.selectedDate.year !== year) { return null; }
+ if (this.selectedDate.month !== month) { return null; }
+ return this.getDateElementForDay(day);
+ }
+ scrollMonth(increment) {
+ this.selectedDate.month += increment;
+ if (this.selectedDate.month < 0) {
+ this.selectedDate.year -= 1;
+ this.selectedDate.month += 12;
+ } else if (this.selectedDate.month >= 12) {
+ this.selectedDate.year += 1;
+ this.selectedDate.month -= 12;
+ }
+ this.unselectPreviousDate();
+ this.updateDomForMonth(this.selectedDate.month, this.selectedDate.year);
+ }
+ scrollToToday() {
+ this.unselectPreviousDate();
+ this.selectedDate = {
+ month: this.dateNow.getMonth(),
+ year: this.dateNow.getFullYear(),
+ day: this.dateNow.getDate() // use getDate. getDay returns index of weekday, e.g. Tues = 2
+ };
+ this.updateDomForMonth(this.selectedDate.month, this.selectedDate.year);
+ }
+ // https://stackoverflow.com/a/1184359
+ // Month in JavaScript is 0-indexed (January is 0, February is 1, etc),
+ // but by using 0 as the day it will give us the last day of the prior
+ // month. So passing in 1 as the month number will return the last day
+ daysInMonth(month, year) {
+ return new Date(year, month + 1, 0).getDate();
+ }
+ selectToday() {
+ this.scrollToToday();
+ this.selectDate(this.getDateElementForDay(this.selectedDate.day));
+ }
+ selectDay(timestamp) {
+ this.unselectPreviousDate();
+ let thisDate = new Date(timestamp);
+ this.selectedDate = {
+ month: thisDate.getMonth(),
+ year: thisDate.getFullYear(),
+ day: thisDate.getDate() // use getDate. getDay returns index of weekday, e.g. Tues = 2
+ };
+ this.updateDomForMonth(this.selectedDate.month, this.selectedDate.year);
+ this.selectDate(this.getDateElementForDay(this.selectedDate.day));
+ }
+ onDateSelected(callback) {
+ this.callbacks.onDateSelected.push(callback);
+ }
+ highlightDates(datesList) {
+ this.highlightedDates = datesList;
+ datesList.forEach(dateObject => {
+ this.highlightDate(this.getDateElement(dateObject));
+ });
+ }
+ highlightDate(dateElement) {
+ if (!dateElement) { return; }
+ dateElement.classList.add('highlightedDate');
+ }
+ show() {
+ this.dom.classList.add('timelineCalendarVisible');
+ this.dom.classList.remove('timelineCalendarHidden');
+ }
+ hide() {
+ this.dom.classList.remove('timelineCalendarVisible');
+ this.dom.classList.add('timelineCalendarHidden');
+ }
+ }
+ exports.Calendar = Calendar;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/CameraFollowCoordinator.js b/content_scripts/CameraFollowCoordinator.js
new file mode 100644
index 00000000..6cc1f2fb
--- /dev/null
+++ b/content_scripts/CameraFollowCoordinator.js
@@ -0,0 +1,269 @@
+const followPerspectiveMenuText = 'Follow Perspective';
+const PERSPECTIVES = [
+ {
+ keyboardShortcut: '_1',
+ menuBarName: 'Follow 1st-Person',
+ distanceToCamera: 0,
+ },
+ {
+ keyboardShortcut: '_2',
+ menuBarName: 'Follow 1st-Person (Wide)',
+ distanceToCamera: 1500,
+ },
+ {
+ keyboardShortcut: '_3',
+ menuBarName: 'Follow 3rd-Person',
+ distanceToCamera: 3000,
+ },
+ {
+ keyboardShortcut: '_4',
+ menuBarName: 'Follow 3rd-Person (Wide)',
+ distanceToCamera: 4500,
+ },
+ {
+ keyboardShortcut: '_5',
+ menuBarName: 'Follow Aerial',
+ distanceToCamera: 6000,
+ }
+];
+
+const changeTargetButtons = [
+ { name: 'Follow Next Target', shortcutKey: 'RIGHT', dIndex: 1 },
+ { name: 'Follow Previous Target', shortcutKey: 'LEFT', dIndex: -1 }
+];
+
+/**
+ * Wraps a reference to a followable element in a class that we add/delete
+ * without accidentally deleting the referenced class instance
+ */
+class CameraFollowTarget {
+ constructor(followable) {
+ this.followable = followable;
+ this.id = this.followable.id;
+ this.displayName = this.followable.displayName;
+ }
+}
+
+/**
+ * Adding CameraFollowTargets to a CameraFollowCoordinator allows it to control
+ * its virtualCamera and make it follow the followable target.
+ */
+export class CameraFollowCoordinator {
+ constructor(virtualCamera) {
+ this.virtualCamera = virtualCamera;
+ this.followTargets = {};
+ this.currentFollowTarget = null;
+ this.followDistance = 3000;
+ this.currentFollowIndex = 0;
+
+ this.virtualCamera.onFirstPersonDistanceToggled((isFirstPerson, currentDistance) => {
+ if (!this.currentFollowTarget) return;
+ this.currentFollowTarget.followable.onFollowDistanceUpdated(currentDistance);
+ this.currentFollowTarget.isFollowing2D = isFirstPerson;
+ if (isFirstPerson) {
+ this.currentFollowTarget.followable.enableFirstPersonMode();
+ } else if (!isFirstPerson) {
+ this.currentFollowTarget.followable.disableFirstPersonMode();
+ }
+ });
+
+ this.virtualCamera.onStopFollowing(() => {
+ this.unfollow();
+ });
+ }
+ addFollowTarget(followable) {
+ this.followTargets[followable.id] = new CameraFollowTarget(followable);
+ this.updateFollowMenu();
+ }
+ removeFollowTarget(id) {
+ delete this.followTargets[id];
+ this.updateFollowMenu();
+ }
+ follow(targetId, followDistance) {
+ if (this.currentFollowTarget && targetId !== this.currentFollowTarget.id) {
+ this.unfollow();
+ }
+ this.currentFollowTarget = this.followTargets[targetId];
+ // make sure the follow index updates if we manually select a follow target
+ this.currentFollowIndex = Object.keys(this.followTargets).indexOf(targetId);
+ if (!this.currentFollowTarget) return;
+ if (this.currentFollowTarget.followable) {
+ this.currentFollowTarget.followable.onCameraStartedFollowing();
+ }
+ if (typeof followDistance !== 'undefined') {
+ this.followDistance = followDistance;
+ }
+ this.virtualCamera.follow(this.currentFollowTarget.followable.sceneNode, this.followDistance);
+ this.updateFollowMenu();
+ }
+ unfollow() {
+ if (!this.currentFollowTarget) return;
+
+ this.currentFollowTarget.followable.onCameraStoppedFollowing();
+ this.currentFollowTarget.followable.disableFirstPersonMode();
+ this.currentFollowTarget = null;
+ this.virtualCamera.stopFollowing();
+ this.updateFollowMenu();
+ }
+ followNext() {
+ if (!this.currentFollowTarget) return;
+ this.close2DUI();
+ let numTargets = Object.keys(this.followTargets).length;
+ this.currentFollowIndex = (this.currentFollowIndex + 1) % numTargets;
+ this.followTargetAtIndex(this.currentFollowIndex);
+ }
+ followPrevious() {
+ if (!this.currentFollowTarget) return;
+ this.close2DUI();
+ let numTargets = Object.keys(this.followTargets).length;
+ this.currentFollowIndex = (this.currentFollowIndex - 1) % numTargets;
+ if (this.currentFollowIndex < 0) { this.currentFollowIndex += numTargets; }
+ this.followTargetAtIndex(this.currentFollowIndex);
+ }
+ followTargetAtIndex(index) {
+ let followTarget = Object.values(this.followTargets)[index];
+ if (!followTarget) {
+ console.warn('Can\'t find a virtualizer to follow');
+ return;
+ }
+ realityEditor.envelopeManager.focusEnvelope(followTarget.followable.frameKey);
+ this.follow(followTarget.id, this.followDistance);
+ }
+ close2DUI() {
+ // if the followable specifies a frameKey, try to stop focusing
+ if (typeof this.currentFollowTarget.followable.frameKey !== 'undefined') {
+ realityEditor.envelopeManager.blurEnvelope(this.currentFollowTarget.followable.frameKey );
+ }
+ }
+ update() {
+ Object.values(this.followTargets).forEach(followTarget => {
+ try {
+ followTarget.followable.updateSceneNode();
+ } catch (_e) {
+ // console.warn('error in updateSceneNode for one of the followTargets')
+ }
+ });
+ }
+ addMenuItems() {
+ let menuBar = realityEditor.gui.getMenuBar();
+ let numTargets = Object.keys(this.followTargets).length;
+
+ const perspectiveItemMenu = new realityEditor.gui.MenuItemSubmenu(followPerspectiveMenuText, { toggle: false, disabled: false });
+ menuBar.addItemToMenu(realityEditor.gui.MENU.Follow, perspectiveItemMenu);
+
+ // Setup Following Menu Items for each perspective
+ PERSPECTIVES.forEach(info => {
+ const followItem = new realityEditor.gui.MenuItem(info.menuBarName, { shortcutKey: info.keyboardShortcut, toggle: false, disabled: (numTargets === 0) }, () => {
+ if (Object.values(this.followTargets).length === 0) {
+ console.warn('Can\'t find a virtualizer to follow');
+ return;
+ }
+
+ if (this.currentFollowIndex >= Object.keys(this.followTargets).length) {
+ this.currentFollowIndex = 0;
+ }
+ let thisTarget = Object.values(this.followTargets)[this.currentFollowIndex];
+
+ this.followDistance = info.distanceToCamera;
+
+ this.follow(thisTarget.id, this.followDistance);
+ });
+ perspectiveItemMenu.addItemToSubmenu(followItem);
+ });
+
+ changeTargetButtons.forEach(itemInfo => {
+ const item = new realityEditor.gui.MenuItem(itemInfo.name, { shortcutKey: itemInfo.shortcutKey, toggle: false, disabled: (numTargets === 0) }, () => {
+ if (Object.values(this.followTargets).length === 0) return; // can't swap targets if not following anything
+ if (!this.currentFollowTarget) return;
+ (itemInfo.dIndex > 0) ? this.followNext() : this.followPrevious();
+ });
+ menuBar.addItemToMenu(realityEditor.gui.MENU.Follow, item);
+ });
+
+ // move this one to the bottom of the menu by adding it again
+ menuBar.addItemToMenu(realityEditor.gui.MENU.Follow, realityEditor.gui.stopFollowingItem);
+
+ // adds a horizontal rule to the menu to visually separate these items from the list of followables
+ let separator2 = new realityEditor.gui.MenuItem('', { isSeparator: true });
+ menuBar.addItemToMenu(realityEditor.gui.MENU.Follow, separator2);
+ }
+ // Updates the Follow menu to contain a menu item for each available follow target
+ updateFollowMenu() {
+ let menuBar = realityEditor.gui.getMenuBar();
+ let numTargets = Object.keys(this.followTargets).length;
+
+ // show/hide Follow menu and enable/disable following buttons if >0 targets
+ if (numTargets === 0) {
+ menuBar.disableMenu(realityEditor.gui.followMenu);
+ realityEditor.gui.getMenuBar().setItemEnabled(realityEditor.gui.ITEM.StopFollowing, false);
+ Object.values(PERSPECTIVES).forEach(info => {
+ realityEditor.gui.getMenuBar().setItemEnabled(info.menuBarName, false);
+ });
+ } else {
+ menuBar.enableMenu(realityEditor.gui.followMenu);
+ realityEditor.gui.getMenuBar().setItemEnabled(realityEditor.gui.ITEM.StopFollowing, true);
+ Object.values(PERSPECTIVES).forEach(info => {
+ realityEditor.gui.getMenuBar().setItemEnabled(info.menuBarName, true);
+ });
+ }
+
+ // only enable the change-target buttons if following and there's another
+ let changeTargetsEnabled = numTargets >= 2 && this.currentFollowTarget;
+ changeTargetButtons.forEach(info => {
+ realityEditor.gui.getMenuBar().setItemEnabled(info.name, changeTargetsEnabled);
+ });
+
+ // don't remove the Stop Following or Prev/Next items
+ let itemsToSkip = [
+ realityEditor.gui.ITEM.StopFollowing,
+ followPerspectiveMenuText
+ ];
+ changeTargetButtons.forEach(itemInfo => {
+ itemsToSkip.push(itemInfo.name);
+ });
+ PERSPECTIVES.forEach(itemInfo => {
+ itemsToSkip.push(itemInfo.menuBarName);
+ });
+
+ let itemsToRemove = [];
+ // remove items that don't match current set of follow targets
+ realityEditor.gui.followMenu.items.forEach(menuItem => {
+ if (itemsToSkip.includes(menuItem.text)) return;
+ if (menuItem.options.isSeparator) return;
+
+ itemsToRemove.push(menuItem.text);
+ });
+
+ itemsToRemove.forEach(itemText => {
+ menuBar.removeItemFromMenu(realityEditor.gui.MENU.Follow, itemText);
+ });
+
+ let itemsToAdd = [];
+
+ // add follow targets that don't exist yet in menu items
+ Object.values(this.followTargets).forEach(followTarget => {
+ itemsToAdd.push(followTarget.displayName);
+ });
+
+ itemsToAdd.forEach(displayName => {
+ let itemText = `Follow ${displayName}`;
+ this.addTargetToFollowMenu(displayName, itemText);
+ });
+ }
+ addTargetToFollowMenu(displayName, menuItemText) {
+ let menuBar = realityEditor.gui.getMenuBar();
+ const targetItem = new realityEditor.gui.MenuItem(menuItemText, { toggle: false, disabled: false }, () => {
+ if (Object.values(this.followTargets).length === 0) {
+ console.warn('Can\'t find a target to follow');
+ return;
+ }
+ // search the targets for one whose displayName matches the item text
+ let targetDisplayNames = Object.values(this.followTargets).map(target => target.displayName);
+ let index = targetDisplayNames.indexOf(displayName);
+ let thisTarget = Object.values(this.followTargets)[index];
+ if (!thisTarget) return;
+ this.follow(thisTarget.id, this.followDistance);
+ });
+ menuBar.addItemToMenu(realityEditor.gui.MENU.Follow, targetItem);
+ }
+}
diff --git a/content_scripts/CameraVis.js b/content_scripts/CameraVis.js
new file mode 100644
index 00000000..6b07f581
--- /dev/null
+++ b/content_scripts/CameraVis.js
@@ -0,0 +1,520 @@
+import * as THREE from '../../thirdPartyCode/three/three.module.js';
+import {Spaghetti} from '../../src/humanPose/spaghetti.js';
+import {CameraVisPatch} from '../../src/spatialCapture/CameraVisPatch.js';
+import {
+ createPointCloud,
+ createPointCloudMaterial,
+ DEPTH_WIDTH,
+ DEPTH_HEIGHT,
+ ShaderMode
+} from '../../src/spatialCapture/Shaders.js';
+import {VisualDiff} from '../../src/spatialCapture/VisualDiff.js';
+import {Followable} from '../../src/gui/ar/Followable.js';
+
+const debug = false;
+// TODO(jhobin): re-enable when Safari behaves
+const enableFastTextureUpload = false;
+
+function setMatrixFromArray(matrix, array) {
+ matrix.set(
+ array[0], array[4], array[8], array[12],
+ array[1], array[5], array[9], array[13],
+ array[2], array[6], array[10], array[14],
+ array[3], array[7], array[11], array[15]
+ );
+}
+
+export class CameraVis extends Followable {
+ static count = 0;
+
+ constructor(id, floorOffset, color) {
+ // first we must set up the Followable so that the remote operator
+ // camera system will be able to follow this video...
+ CameraVis.count++;
+ let parentNode = realityEditor.sceneGraph.getVisualElement('CameraGroupContainer');
+ if (!parentNode) {
+ let gpNode = realityEditor.sceneGraph.getGroundPlaneNode();
+ let cameraGroupContainerId = realityEditor.sceneGraph.addVisualElement('CameraGroupContainer', gpNode);
+ parentNode = realityEditor.sceneGraph.getSceneNodeById(cameraGroupContainerId);
+ let transformationMatrix = realityEditor.gui.ar.utilities.makeGroundPlaneRotationX(0);
+ transformationMatrix[13] = -1 * floorOffset;
+ parentNode.setLocalMatrix(transformationMatrix);
+ }
+ // count (e.g. 1) is more user-friendly than the id (e.g. prov123)
+ let menuItemName = `Live Video ${CameraVis.count}`;
+ super('CameraVisFollowable_' + id, menuItemName, parentNode);
+
+ // then the CameraVis can initialize as usual...
+ this.id = id;
+ this.firstPersonMode = false;
+ this.shaderMode = ShaderMode.SOLID;
+ this.container = new THREE.Group();
+ // this.container.scale.set(0.001, 0.001, 0.001);
+ // this.container.rotation.y = Math.PI;
+ this.container.position.y = -floorOffset;
+ this.container.rotation.x = Math.PI / 2;
+
+ this.container.updateMatrix();
+ this.container.updateMatrixWorld(true);
+ this.container.matrixAutoUpdate = false;
+
+ this.container.name = 'CameraVisContainer_' + id;
+ this.lastUpdate = Date.now();
+ this.phone = new THREE.Group();
+ this.phone.matrixAutoUpdate = false;
+ this.phone.frustumCulled = false;
+ this.container.add(this.phone);
+
+ this.maxDepthMeters = 5; // this goes down if lidar is pointed at a wall/floor/object closer than 5 meters
+
+ this.cameraMeshGroup = new THREE.Group();
+
+ const geo = new THREE.BoxGeometry(100, 100, 80);
+ if (!color) {
+ let colorId = id;
+ if (typeof id === 'string') {
+ colorId = 0;
+ for (let i = 0; i < id.length; i++) {
+ colorId ^= id.charCodeAt(i);
+ }
+ }
+ let hue = ((colorId / 29) % Math.PI) * 360 / Math.PI;
+ const colorStr = `hsl(${hue}, 100%, 50%)`;
+ this.color = new THREE.Color(colorStr);
+ } else {
+ this.color = color;
+ }
+ this.colorRGB = [
+ 255 * this.color.r,
+ 255 * this.color.g,
+ 255 * this.color.b,
+ ];
+ this.cameraMeshGroupMat = new THREE.MeshBasicMaterial({color: this.color});
+ const box = new THREE.Mesh(geo, this.cameraMeshGroupMat);
+ box.name = 'cameraVisCamera';
+ box.cameraVisId = this.id;
+ this.cameraMeshGroup.add(box);
+
+ const geoCone = new THREE.ConeGeometry(60, 180, 16, 1);
+ const cone = new THREE.Mesh(geoCone, this.cameraMeshGroupMat);
+ cone.rotation.x = -Math.PI / 2;
+ cone.rotation.y = Math.PI / 8;
+ cone.position.z = 65;
+ cone.name = 'cameraVisCamera';
+ cone.cameraVisId = this.id;
+ this.cameraMeshGroup.add(cone);
+
+ this.phone.add(this.cameraMeshGroup);
+
+ this.texture = new THREE.Texture();
+ this.texture.minFilter = THREE.LinearFilter;
+ this.texture.magFilter = THREE.LinearFilter;
+ this.texture.generateMipmaps = false;
+ if (enableFastTextureUpload) {
+ this.texture.isVideoTexture = true;
+ this.texture.update = function() {
+ };
+ }
+
+ this.textureDepth = new THREE.Texture();
+ this.textureDepth.minFilter = THREE.LinearFilter;
+ this.textureDepth.magFilter = THREE.LinearFilter;
+ this.textureDepth.generateMipmaps = false;
+ if (enableFastTextureUpload) {
+ this.textureDepth.isVideoTexture = true;
+ this.textureDepth.update = function() {
+ };
+ }
+
+ this.material = null;
+ this.mesh = null;
+
+ /**
+ * this material will overwrite the the scene depth and will not render color
+ * @type {THREE.Material}
+ */
+ this.maskMeshMaterial = null;
+ /**
+ * this shallow copy of the mesh will erase the background depth values, so the original mesh renders ontop of the background
+ * @type {THREE.Object3D}
+ */
+ this.maskMesh = null;
+
+ if (debug) {
+ this.setupDebugCubes();
+ }
+
+ this.setupPointCloud();
+
+ this.time = performance.now();
+ this.matrices = [];
+ this.loading = {};
+
+ this.historyPoints = [];
+ // note: we will color the path in each point, rather than in the constructor
+ this.historyMesh = new Spaghetti(this.historyPoints, null, 'Camera Spaghetti Line', {
+ widthMm: 30,
+ heightMm: 30,
+ usePerVertexColors: true,
+ wallBrightness: 0.6
+ });
+
+ // we add the historyMesh to scene because crossing up vector gets messed up by rotation if added to this.container
+ realityEditor.gui.threejsScene.addToScene(this.historyMesh);
+ }
+
+ /**
+ * Clone the current state of the mesh rendering part of this CameraVis
+ * @param {ShaderMode} shaderMode - initial shader mode to set on the patches
+ * @return {{key: string, patch: CameraVisPatch}} unique key for patch and object containing all relevant meshes
+ */
+ clonePatch(shaderMode) {
+ let now = Date.now();
+
+ let utils = realityEditor.gui.ar.utilities;
+ let worldMatrix = realityEditor.sceneGraph.getSceneNodeById(realityEditor.sceneGraph.getWorldId()).worldMatrix;
+ let patchContainerMatrix = realityEditor.gui.ar.utilities.newIdentityMatrix();
+
+ let updatedPatchContainerMatrix = new THREE.Matrix4();
+ setMatrixFromArray(updatedPatchContainerMatrix, patchContainerMatrix);
+
+ // this works because the world is parented to the same object that the phone is parented to
+ let phoneRelativeToWorldMatrix = [];
+ utils.multiplyMatrix(Array.from(this.phone.matrixWorld.elements), utils.invertMatrix(worldMatrix), phoneRelativeToWorldMatrix);
+
+ let updatedPatchPhoneMatrix = new THREE.Matrix4();
+ setMatrixFromArray(updatedPatchPhoneMatrix, phoneRelativeToWorldMatrix);
+
+ let serialization = {
+ key: '',
+ id: this.id,
+ container: Array.from(updatedPatchContainerMatrix.elements),
+ phone: Array.from(updatedPatchPhoneMatrix.elements),
+ texture: this.texture.image.toDataURL('image/jpeg', 0.7),
+ textureDepth: this.textureDepth.image.toDataURL(),
+ creationTime: now,
+ };
+
+ const textureImage = document.createElement('img');
+ textureImage.src = serialization.texture;
+ const textureDepthImage = document.createElement('img');
+ textureDepthImage.src = serialization.textureDepth;
+
+ const frameKey = CameraVisPatch.createToolForPatchSerialization(serialization, shaderMode);
+
+ return {
+ key: frameKey,
+ patch: CameraVisPatch.createPatch(
+ updatedPatchContainerMatrix,
+ updatedPatchPhoneMatrix,
+ textureImage,
+ textureDepthImage,
+ now,
+ shaderMode
+ ),
+ };
+ }
+
+ setupDebugCubes() {
+ let debugDepth = new THREE.MeshBasicMaterial({
+ map: this.textureDepth,
+ });
+ let debugDepthCube = new THREE.Mesh(new THREE.PlaneGeometry(500, 500 * DEPTH_HEIGHT / DEPTH_WIDTH), debugDepth);
+ this.container.add(debugDepthCube);
+ debugDepthCube.position.set(400, 250, -1000);
+
+ let debugColor = new THREE.MeshBasicMaterial({
+ map: this.texture,
+ });
+ this.debugColorCube = new THREE.Mesh(new THREE.PlaneGeometry(100, 100 * 1080 / 1920), debugColor);
+ // this.container.add(debugColorCube);
+ this.debugColorCube.position.set(-180 * window.innerWidth / window.innerHeight, 140, -1000);
+ this.debugColorCube.rotation.z = Math.PI;
+ }
+
+ toggleColorCube(i) {
+ if (!this.debugColorCube || !this.debugColorCube.parent) {
+ this.addColorCube(i);
+ } else {
+ this.removeColorCube();
+ }
+ }
+
+ addColorCube(i) {
+ if (!this.debugColorCube) {
+ let debugColor = new THREE.MeshBasicMaterial({
+ map: this.texture,
+ });
+ this.debugColorCube = new THREE.Mesh(new THREE.PlaneGeometry(100, 100 * 1080 / 1920), debugColor);
+ // this.container.add(debugColorCube);
+ this.debugColorCube.rotation.z = Math.PI;
+ }
+ let x = -180 * window.innerWidth / window.innerHeight;
+ let y = 140 - i * 100;
+ this.debugColorCube.position.set(x, y, -1000);
+ realityEditor.gui.threejsScene.addToScene(this.debugColorCube, {parentToCamera: true});
+ }
+
+ removeColorCube() {
+ realityEditor.gui.threejsScene.removeFromScene(this.debugColorCube);
+ }
+
+ #updateMaskMaterial() {
+ if (this.maskMaterial) this.maskMaterial.dispose();
+
+ this.maskMaterial = this.mesh.material.clone();
+ this.maskMaterial.colorWrite = false;
+ this.maskMaterial.uniforms.useFarDepth.value = true;
+ this.maskMaterial.depthFunc = THREE.AlwaysDepth;
+ this.maskMesh.material = this.maskMaterial;
+ }
+
+ setupPointCloud() {
+ const mesh = createPointCloud(this.texture, this.textureDepth, this.shaderMode, this.color);
+
+ this.mesh = mesh;
+ this.material = mesh.material;
+
+ this.phone.add(mesh);
+
+ this.maskMesh = this.mesh.clone();
+ this.maskMesh.renderOrder = realityEditor.gui.threejsScene.RENDER_ORDER_DEPTH_REPLACEMENT;
+ this.mesh.parent.add(this.maskMesh);
+ this.#updateMaskMaterial();
+ }
+
+ update(mat, delayed, rawMatricesMsg) {
+ let now = performance.now();
+ if (this.shaderMode === ShaderMode.HOLO) {
+ this.material.uniforms.time.value = window.performance.now();
+ this.maskMaterial.uniforms.time.value = this.material.uniforms.time.value;
+ }
+ this.lastUpdate = now;
+
+
+ if (rawMatricesMsg) {
+ let width = this.material.uniforms.width.value;
+ let height = this.material.uniforms.height.value;
+ let rawWidth = rawMatricesMsg.imageSize[0];
+ let rawHeight = rawMatricesMsg.imageSize[1];
+
+ this.material.uniforms.focalLength.value = new THREE.Vector2(
+ rawMatricesMsg.focalLength[0] / rawWidth * width,
+ rawMatricesMsg.focalLength[1] / rawHeight * height,
+ );
+ this.maskMaterial.uniforms.focalLength.value = this.material.uniforms.focalLength.value;
+ // convert principal point from image Y-axis bottom-to-top in Vuforia to top-to-bottom in OpenGL
+ this.material.uniforms.principalPoint.value = new THREE.Vector2(
+ rawMatricesMsg.principalPoint[0] / rawWidth * width,
+ (rawHeight - rawMatricesMsg.principalPoint[1]) / rawHeight * height,
+ );
+ this.maskMaterial.uniforms.principalPoint.value = this.material.uniforms.principalPoint.value;
+ }
+
+ if (this.time > now || !delayed) {
+ this.setMatrix(mat);
+ return;
+ }
+ this.matrices.push({
+ matrix: mat,
+ time: now,
+ });
+ }
+
+ setTime(time) {
+ this.time = time;
+ if (this.matrices.length === 0) {
+ return;
+ }
+ let latest = this.matrices[0];
+ if (latest.time > time) {
+ return;
+ }
+ let latestI = 0;
+ for (let i = 1; i < this.matrices.length; i++) {
+ let mat = this.matrices[i];
+ if (mat.time > time) {
+ break;
+ }
+ latest = mat;
+ latestI = i;
+ }
+ this.matrices.splice(0, latestI + 1);
+
+ this.setMatrix(latest.matrix);
+ }
+
+ getSceneNodeMatrix() {
+ let matrix = this.phone.matrixWorld.clone();
+
+ let initialVehicleMatrix = new THREE.Matrix4().fromArray([
+ -1, 0, 0, 0,
+ 0, 1, 0, 0,
+ 0, 0, -1, 0,
+ 0, 0, 0, 1,
+ ]);
+ matrix.multiply(initialVehicleMatrix);
+
+ return matrix;
+ }
+
+ setMatrix(newMatrix) {
+ setMatrixFromArray(this.phone.matrix, newMatrix);
+ this.phone.updateMatrixWorld(true);
+ this.texture.needsUpdate = true;
+ this.textureDepth.needsUpdate = true;
+
+ if (this.cutoutViewFrustum) {
+ realityEditor.gui.ar.desktopRenderer.updateAreaGltfForCamera(this.id, this.phone.matrixWorld, this.maxDepthMeters);
+ }
+
+ this.hideNearCamera(newMatrix[12], newMatrix[13], newMatrix[14]);
+ let localHistoryPoint = new THREE.Vector3( newMatrix[12], newMatrix[13], newMatrix[14]);
+
+ // history point needs to be transformed into the groundPlane coordinate system
+ let worldHistoryPoint = this.container.localToWorld(localHistoryPoint);
+ let rootNode = realityEditor.sceneGraph.getSceneNodeById('ROOT');
+ let gpNode = realityEditor.sceneGraph.getGroundPlaneNode();
+ let gpHistoryPoint = realityEditor.sceneGraph.convertToNewCoordSystem(worldHistoryPoint, rootNode, gpNode);
+
+ let nextHistoryPoint = {
+ x: gpHistoryPoint.x,
+ y: gpHistoryPoint.y,
+ z: gpHistoryPoint.z,
+ color: this.colorRGB,
+ timestamp: Date.now()
+ };
+
+ let addToHistory = this.historyPoints.length === 0;
+ if (this.historyPoints.length > 0) {
+ let lastHistoryPoint = this.historyPoints[this.historyPoints.length - 1];
+ let diffSq = (lastHistoryPoint.x - nextHistoryPoint.x) * (lastHistoryPoint.x - nextHistoryPoint.x) +
+ (lastHistoryPoint.y - nextHistoryPoint.y) * (lastHistoryPoint.y - nextHistoryPoint.y) +
+ (lastHistoryPoint.z - nextHistoryPoint.z) * (lastHistoryPoint.z - nextHistoryPoint.z);
+
+ addToHistory = diffSq > 100 * 100;
+ }
+
+ if (addToHistory) {
+ this.historyPoints.push(nextHistoryPoint);
+ this.historyMesh.setPoints(this.historyPoints);
+ }
+
+ if (this.sceneNode) {
+ this.sceneNode.setLocalMatrix(newMatrix);
+ }
+
+ if (this.firstPersonMode) {
+ let matrix = this.getSceneNodeMatrix();
+ let eye = new THREE.Vector3(0, 0, 0);
+ eye.applyMatrix4(matrix);
+ let target = new THREE.Vector3(0, 0, -1);
+ target.applyMatrix4(matrix);
+ matrix.lookAt(eye, target, new THREE.Vector3(0, 1, 0));
+ realityEditor.sceneGraph.setCameraPosition(matrix.elements);
+ }
+
+ if (this.shaderMode === ShaderMode.DIFF) {
+ this.visualDiff.showCameraVisDiff(this);
+ }
+ }
+
+ hideNearCamera() {
+ let mat = this.phone.matrix.clone();
+ mat.premultiply(this.container.matrix);
+ const x = mat.elements[12];
+ const y = mat.elements[13];
+ const z = mat.elements[14];
+
+ let cameraNode = realityEditor.sceneGraph.getSceneNodeById('CAMERA');
+ const cameraX = cameraNode.worldMatrix[12];
+ const cameraY = cameraNode.worldMatrix[13];
+ const cameraZ = cameraNode.worldMatrix[14];
+
+ let diffSq = (cameraX - x) * (cameraX - x) +
+ (cameraY - y) * (cameraY - y) +
+ (cameraZ - z) * (cameraZ - z);
+
+ if (diffSq < 3000 * 3000) {
+ if (this.cameraMeshGroup.visible) {
+ this.cameraMeshGroup.visible = false;
+ }
+ } else if (!this.cameraMeshGroup.visible) {
+ this.cameraMeshGroup.visible = true;
+ }
+ }
+
+ setShaderMode(shaderMode) {
+ if (shaderMode !== this.shaderMode) {
+ this.shaderMode = shaderMode;
+
+ if (this.matDiff) {
+ this.matDiff.dispose();
+ this.matDiff = null;
+ }
+
+ if (this.shaderMode === ShaderMode.DIFF && !this.visualDiff) {
+ this.visualDiff = new VisualDiff();
+ }
+ this.material = createPointCloudMaterial(this.texture, this.textureDepth, this.shaderMode, this.color);
+ this.mesh.material = this.material;
+ this.#updateMaskMaterial();
+ }
+ }
+
+ /* ---------------- Override Followable Functions ---------------- */
+
+ doesOverrideCameraUpdatesInFirstPerson() {
+ return true;
+ }
+
+ enableFirstPersonMode() {
+ this.firstPersonMode = true;
+ if (this.shaderMode === ShaderMode.SOLID) {
+ this.setShaderMode(ShaderMode.FIRST_PERSON);
+ }
+ }
+
+ disableFirstPersonMode() {
+ this.firstPersonMode = false;
+ if (this.shaderMode === ShaderMode.FIRST_PERSON) {
+ this.setShaderMode(ShaderMode.SOLID);
+ }
+ }
+
+ updateSceneNode() {
+ // doing it here causes significant jitter - the matrix is instead set in the above setMatrix function
+ // this.sceneNode.setLocalMatrix(this.phone.matrix.elements);
+ }
+
+ /* ---------------- ---------------- */
+
+ enableFrustumCutout() {
+ this.cutoutViewFrustum = true;
+ }
+
+ disableFrustumCutout() {
+ this.cutoutViewFrustum = false;
+ realityEditor.gui.threejsScene.removeMaterialCullingFrustum(this.id);
+ }
+
+ /**
+ * @param {THREE.Color} color
+ */
+ setColor(color) {
+ this.color = color;
+ this.cameraMeshGroupMat.color = color;
+ if (this.material && this.material.uniforms.borderColor) {
+ this.material.uniforms.borderColor.value = color;
+ this.maskMaterial.uniforms.borderColor.value = this.material.uniforms.borderColor.value;
+ }
+ }
+
+ add() {
+ realityEditor.gui.threejsScene.addToScene(this.container);
+ }
+
+ remove() {
+ realityEditor.gui.threejsScene.removeFromScene(this.container);
+ }
+}
diff --git a/content_scripts/CameraVisCoordinator.js b/content_scripts/CameraVisCoordinator.js
new file mode 100644
index 00000000..3541e3ac
--- /dev/null
+++ b/content_scripts/CameraVisCoordinator.js
@@ -0,0 +1,459 @@
+createNameSpace('realityEditor.device.cameraVis');
+
+import * as THREE from '../../thirdPartyCode/three/three.module.js';
+import {CameraVis} from './CameraVis.js';
+import {ShaderMode} from '../../src/spatialCapture/Shaders.js';
+
+(function(exports) {
+ const debug = false;
+ const ENABLE_PICTURE_IN_PICTURE = false;
+ const FIRST_PERSON_CANVAS = false;
+ const DEPTH_WIDTH = 256;
+ const DEPTH_HEIGHT = 144;
+ const CONNECTION_TIMEOUT_MS = 10000;
+
+ const enabledShaderModes = [
+ ShaderMode.SOLID,
+ ShaderMode.DIFF,
+ ShaderMode.POINT,
+ ShaderMode.HOLO,
+ ];
+
+ exports.CameraVisCoordinator = class CameraVisCoordinator {
+ constructor(floorOffset) {
+ this.voxelizer = null;
+ this.webRTCCoordinator = null;
+ this.cameras = {};
+ // this.patches = {}; // NOTE: patches have been moved to userinterface src/spatialCapture/SpatialPatchCoordinator
+ this.visible = true;
+ this.spaghettiVisible = false;
+ this.currentShaderModeIndex = 0;
+ this.floorOffset = floorOffset;
+ this.depthCanvasCache = {};
+ this.colorCanvasCache = {};
+ this.showCanvasTimeout = null;
+ this.callbacks = {
+ onCameraVisCreated: [],
+ onCameraVisRemoved: [],
+ };
+
+ this.onAnimationFrame = this.onAnimationFrame.bind(this);
+ window.requestAnimationFrame(this.onAnimationFrame);
+
+ this.addMenuShortcuts();
+
+ this.onPointerDown = this.onPointerDown.bind(this);
+
+ let threejsCanvas = document.getElementById('mainThreejsCanvas');
+ if (threejsCanvas && ENABLE_PICTURE_IN_PICTURE) {
+ threejsCanvas.addEventListener('pointerdown', this.onPointerDown);
+ }
+
+ this.startWebRTC();
+ }
+
+ addMenuShortcuts() {
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.PointClouds, (toggled) => {
+ this.visible = toggled;
+ for (let camera of Object.values(this.cameras)) {
+ camera.mesh.visible = this.visible;
+ camera.mesh.__hidden = !this.visible;
+ }
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.ResetPaths, () => {
+ for (let camera of Object.values(this.cameras)) {
+ camera.historyPoints = [];
+ camera.historyMesh.setPoints(camera.historyPoints);
+ }
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.SpaghettiMap, (toggled) => {
+ this.spaghettiVisible = toggled;
+ for (let camera of Object.values(this.cameras)) {
+ camera.historyMesh.visible = this.spaghettiVisible;
+ }
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.TakeSpatialSnapshot, () => {
+ realityEditor.spatialCapture.spatialPatchCoordinator.clonePatches(ShaderMode.SOLID, this.cameras);
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.CutoutViewFrustums, (toggled) => {
+ this.cutoutViewFrustums = toggled;
+ for (let camera of Object.values(this.cameras)) {
+ if (toggled) {
+ camera.enableFrustumCutout();
+ } else {
+ camera.disableFrustumCutout();
+ }
+ }
+ });
+ }
+
+ onAnimationFrame() {
+ let now = performance.now();
+ for (let camera of Object.values(this.cameras)) {
+ if (camera.mesh.__hidden) {
+ camera.mesh.visible = false;
+ continue;
+ }
+ if (now - camera.lastUpdate > CONNECTION_TIMEOUT_MS) {
+ camera.remove();
+ delete this.cameras[camera.id];
+ this.callbacks.onCameraVisRemoved.forEach(cb => {
+ cb(camera);
+ });
+ } else if (!camera.mesh.visible) {
+ camera.mesh.visible = true;
+ }
+ }
+ window.requestAnimationFrame(this.onAnimationFrame);
+ }
+
+ updateMatrix(id, mat, delayed, rawMatricesMsg) {
+ if (!this.cameras[id]) {
+ this.createCameraVis(id);
+ }
+ this.cameras[id].update(mat, delayed, rawMatricesMsg);
+ }
+
+ startWebRTC() {
+ const network = 'cam' + Math.floor(Math.random() * 1000);
+
+ const ws = realityEditor.network.realtime.getDesktopSocket();
+ this.webRTCCoordinator = new realityEditor.device.cameraVis.WebRTCCoordinator(this, ws, network);
+ }
+
+ muteMicrophone() {
+ if (!this.webRTCCoordinator) return;
+ this.webRTCCoordinator.mute();
+ }
+
+ unmuteMicrophone() {
+ if (!this.webRTCCoordinator) return;
+ this.webRTCCoordinator.unmute();
+ }
+
+ renderPointCloud(id, textureKey, imageUrl) {
+ if (!this.cameras[id]) {
+ this.createCameraVis(id);
+ }
+ if (this.cameras[id].loading[textureKey]) {
+ return;
+ }
+ this.cameras[id].loading[textureKey] = true;
+ // const pktType = bytes[1];
+ // if (pktType === PKT_MATRIX) {
+ // const text = await msg.data.slice(2, msg.data.length).text();
+ // const mat = JSON.parse(text);
+ // }
+
+ const image = new Image();
+
+ let start = window.performance.now();
+ image.onload = () => {
+ const tex = this.cameras[id][textureKey];
+ tex.dispose();
+ // hmmmmm
+ // most efficient would be if this had a data url for its src
+ // data url = 'data:image/(png|jpeg);' + base64(blob)
+ if (textureKey === 'textureDepth') {
+ if (!this.depthCanvasCache.hasOwnProperty(id)) {
+ let canvas = document.createElement('canvas');
+ this.depthCanvasCache[id] = {
+ canvas,
+ context: canvas.getContext('2d'),
+ };
+ }
+ let {canvas, context} = this.depthCanvasCache[id];
+ if (canvas.width !== image.width || canvas.height !== image.height) {
+ canvas.width = image.width;
+ canvas.height = image.height;
+ }
+ context.drawImage(image, 0, 0, image.width, image.height);
+ } else {
+ if (!this.colorCanvasCache.hasOwnProperty(id)) {
+ let canvas = document.createElement('canvas');
+ this.colorCanvasCache[id] = {
+ canvas,
+ context: canvas.getContext('2d'),
+ };
+ }
+ let {canvas, context} = this.colorCanvasCache[id];
+ if (canvas.width !== image.width || canvas.height !== image.height) {
+ canvas.width = image.width;
+ canvas.height = image.height;
+ }
+ context.drawImage(image, 0, 0, image.width, image.height);
+ }
+ this.finishRenderPointCloudCanvas(id, textureKey, start);
+ URL.revokeObjectURL(imageUrl);
+ };
+ image.onerror = (e) => {
+ console.error(e);
+ };
+ image.src = imageUrl;
+ }
+
+ renderPointCloudRawDepth(id, rawDepth) {
+ const textureKey = 'textureDepth';
+
+ if (!this.cameras[id]) {
+ this.createCameraVis(id);
+ }
+ if (this.cameras[id].loading[textureKey]) {
+ return;
+ }
+ this.cameras[id].loading[textureKey] = true;
+ const tex = this.cameras[id][textureKey];
+ tex.dispose();
+
+ if (!this.depthCanvasCache.hasOwnProperty(id)) {
+ let canvas = document.createElement('canvas');
+ let context = canvas.getContext('2d');
+ let imageData = context.createImageData(DEPTH_WIDTH, DEPTH_HEIGHT);
+ this.depthCanvasCache[id] = {
+ canvas,
+ context,
+ imageData,
+ };
+ }
+
+ let {canvas, context, imageData} = this.depthCanvasCache[id];
+ canvas.width = DEPTH_WIDTH;
+ canvas.height = DEPTH_HEIGHT;
+ let maxDepth14bits = 0;
+ for (let i = 0; i < DEPTH_WIDTH * DEPTH_HEIGHT; i++) {
+ if (rawDepth[i] > maxDepth14bits) {
+ maxDepth14bits = rawDepth[i];
+ }
+ // We get 14 bits of depth information from the RVL-encoded
+ // depth buffer. Note that this means the blue channel is
+ // always zero
+ let depth24Bits = rawDepth[i] << (24 - 14); // * 5 / (1 << 14);
+ if (depth24Bits > 0xffffff) {
+ depth24Bits = 0xffffff;
+ }
+ let b = depth24Bits & 0xff;
+ let g = (depth24Bits >> 8) & 0xff;
+ let r = (depth24Bits >> 16) & 0xff;
+ imageData.data[4 * i + 0] = r;
+ imageData.data[4 * i + 1] = g;
+ imageData.data[4 * i + 2] = b;
+ imageData.data[4 * i + 3] = 255;
+ }
+ this.cameras[id].maxDepthMeters = 5 * (maxDepth14bits / (1 << 14));
+
+ context.putImageData(imageData, 0, 0);
+ this.finishRenderPointCloudCanvas(id, textureKey, -1);
+
+ if (this.voxelizer) {
+ this.voxelizer.raycastDepth(
+ this.cameras[id].phone, {
+ width: DEPTH_WIDTH,
+ height: DEPTH_HEIGHT,
+ },
+ rawDepth
+ );
+ }
+ }
+
+ finishRenderPointCloudCanvas(id, textureKey, start) {
+ const tex = this.cameras[id][textureKey];
+
+ if (textureKey === 'textureDepth') {
+ if (!this.depthCanvasCache.hasOwnProperty(id)) {
+ let canvas = document.createElement('canvas');
+ this.depthCanvasCache[id] = {
+ canvas,
+ context: canvas.getContext('2d'),
+ };
+ }
+ let {canvas} = this.depthCanvasCache[id];
+ tex.image = canvas;
+ } else {
+ if (!this.colorCanvasCache.hasOwnProperty(id)) {
+ let canvas = document.createElement('canvas');
+ this.colorCanvasCache[id] = {
+ canvas,
+ context: canvas.getContext('2d'),
+ };
+ }
+ let {canvas} = this.colorCanvasCache[id];
+ tex.image = canvas;
+ }
+ // tex.needsUpdate = true;
+ // let end = window.performance.now();
+ if (textureKey === 'texture') {
+ // We know that capture takes 30ms
+ // Transmission takes ??s
+ this.cameras[id].setTime(start + 40);
+ }
+ this.cameras[id].loading[textureKey] = false;
+ }
+
+ showFullscreenColorCanvas(id) {
+ let cacheId = id;
+ if (!this.cameras.hasOwnProperty(cacheId)) {
+ cacheId = 'prov' + id;
+ }
+
+ if (FIRST_PERSON_CANVAS) {
+ const doShowCanvas = !document.getElementById('colorCanvas' + cacheId) && !this.showCanvasTimeout;
+ if (this.colorCanvasCache[cacheId] && doShowCanvas) {
+ let canvas = this.colorCanvasCache[cacheId].canvas;
+ canvas.style.position = 'absolute';
+ canvas.style.left = '0';
+ canvas.style.top = '0';
+ canvas.style.width = '100vw';
+ canvas.style.height = '100vh';
+ canvas.style.transform = 'rotate(180deg)';
+ // canvas.style.transition = 'opacity 1.0s ease-in-out';
+ // canvas.style.opacity = '0';
+ canvas.id = 'colorCanvas' + cacheId;
+ this.showCanvasTimeout = setTimeout(() => {
+ document.body.appendChild(canvas);
+ this.showCanvasTimeout = null;
+ }, 300);
+ }
+ } else {
+ const camera = this.cameras[cacheId];
+ if (camera) {
+ camera.enableFirstPersonMode();
+ camera.historyMesh.visible = false;
+ }
+ }
+ }
+
+ hideFullscreenColorCanvas(id) {
+ let cacheId = id;
+ if (!this.cameras.hasOwnProperty(cacheId)) {
+ cacheId = 'prov' + id;
+ }
+
+ if (FIRST_PERSON_CANVAS) {
+ let canvas = document.getElementById('colorCanvas' + cacheId);
+ if (canvas && canvas.parentElement) {
+ canvas.parentElement.removeChild(canvas);
+ }
+ } else {
+ const camera = this.cameras[cacheId];
+ if (this.cameras[cacheId]) {
+ camera.disableFirstPersonMode();
+ camera.historyMesh.visible = this.spaghettiVisible;
+ }
+ }
+ }
+
+ loadPointCloud(id, textureUrl, textureDepthUrl, matrix) {
+ this.renderPointCloud(id, 'texture', textureUrl);
+ this.renderPointCloud(id, 'textureDepth', textureDepthUrl);
+ this.updateMatrix(id, matrix, true, null);
+ }
+
+ hidePointCloud(id) {
+ if (!this.cameras[id]) {
+ console.log('No need to hide camera ' + id + ', it hasn\'t been created yet.');
+ return;
+ }
+ let camera = this.cameras[id];
+ if (camera.mesh) {
+ camera.mesh.visible = false;
+ }
+ }
+
+ onCameraVisCreated(cb) {
+ this.callbacks.onCameraVisCreated.push(cb);
+ }
+
+ onCameraVisRemoved(cb) {
+ this.callbacks.onCameraVisRemoved.push(cb);
+ }
+
+ /**
+ * @param {string} id - id of cameravis to be on the lookout for
+ */
+ startRecheckColorInterval(id) {
+ let recheckColorInterval = setInterval(() => {
+ let colorStr = realityEditor.avatar.getAvatarColorFromProviderId(id);
+ if (!colorStr) {
+ return;
+ }
+ let color = new THREE.Color(colorStr);
+ this.cameras[id].setColor(color);
+ clearInterval(recheckColorInterval);
+ }, 3000);
+ }
+
+ createCameraVis(id) {
+ if (debug) {
+ console.log('new camera', id);
+ }
+ let color;
+ let colorStr = realityEditor.avatar.getAvatarColorFromProviderId(id);
+ if (!colorStr) {
+ console.warn('no color for camera', id);
+ // If it's a webrtc cameravis (id starts with prov) then we
+ // should eventually get this avatar information
+ if (id.startsWith('prov')) {
+ this.startRecheckColorInterval(id);
+ }
+ } else {
+ color = new THREE.Color(colorStr);
+ }
+ this.cameras[id] = new CameraVis(id, this.floorOffset, color);
+ this.cameras[id].add();
+ this.cameras[id].historyMesh.visible = this.spaghettiVisible;
+ this.cameras[id].setShaderMode(enabledShaderModes[this.currentShaderModeIndex]);
+ if (this.cutoutViewFrustums) {
+ this.cameras[id].enableFrustumCutout();
+ } else {
+ this.cameras[id].disableFrustumCutout();
+ }
+ // these menubar shortcuts are disabled by default, enabled when at least one virtualizer connects
+ realityEditor.gui.getMenuBar().setItemEnabled(realityEditor.gui.ITEM.PointClouds, true);
+ realityEditor.gui.getMenuBar().setItemEnabled(realityEditor.gui.ITEM.SpaghettiMap, true);
+
+ realityEditor.gui.getMenuBar().setItemEnabled(realityEditor.gui.ITEM.AdvanceCameraShader, true);
+
+ realityEditor.gui.getMenuBar().setItemEnabled(realityEditor.gui.ITEM.TakeSpatialSnapshot, true);
+
+ this.callbacks.onCameraVisCreated.forEach(cb => {
+ cb(this.cameras[id]);
+ });
+ }
+
+ onPointerDown(e) {
+ let objectsToCheck = Object.values(this.cameras).map(cameraVis => {
+ return cameraVis.cameraMeshGroup;
+ });
+ let intersects = realityEditor.gui.threejsScene.getRaycastIntersects(e.clientX, e.clientY, objectsToCheck);
+
+ intersects.forEach((intersect) => {
+ if (intersect.object.name !== 'cameraVisCamera') {
+ return;
+ }
+
+ let id = intersect.object.cameraVisId;
+ let i = Object.keys(this.cameras).indexOf('' + id);
+ this.cameras[id].toggleColorCube(i);
+
+ // stop propagation if we hit anything, otherwise pass the event on to the rest of the application
+ e.stopPropagation();
+ });
+ }
+
+ advanceShaderMode() {
+ this.currentShaderModeIndex = (this.currentShaderModeIndex + 1) % enabledShaderModes.length;
+ this.setShaderMode(enabledShaderModes[this.currentShaderModeIndex]);
+ }
+
+ setShaderMode(shaderMode) {
+ for (let camera of Object.values(this.cameras)) {
+ camera.setShaderMode(shaderMode);
+ }
+ }
+ };
+
+})(realityEditor.device.cameraVis);
diff --git a/content_scripts/KeyboardListener.js b/content_scripts/KeyboardListener.js
new file mode 100644
index 00000000..9367f96e
--- /dev/null
+++ b/content_scripts/KeyboardListener.js
@@ -0,0 +1,144 @@
+/*
+* Copyright © 2021 PTC
+*/
+
+createNameSpace('realityEditor.device');
+
+(function(exports) {
+ class KeyboardListener {
+ constructor() {
+ /**
+ * Enum mapping readable keyboard names to their keyCode
+ * @type {Readonly<{LEFT: number, UP: number, RIGHT: number, DOWN: number, ONE: number, TWO: number, ESCAPE: number, W: number, A: number, S: number, D: number}>}
+ */
+ this.keyCodes = Object.freeze({
+ BACKSPACE: 8,
+ TAB: 9,
+ ENTER: 13,
+ SHIFT: 16,
+ CTRL: 17,
+ ALT: 18,
+ ESCAPE: 27,
+ SPACE: 32,
+ LEFT: 37,
+ UP: 38,
+ RIGHT: 39,
+ DOWN: 40,
+ _0: 48,
+ _1: 49,
+ _2: 50,
+ _3: 51,
+ _4: 52,
+ _5: 53,
+ _6: 54,
+ _7: 55,
+ _8: 56,
+ _9: 57,
+ A: 65,
+ B: 66,
+ C: 67,
+ D: 68,
+ E: 69,
+ F: 70,
+ G: 71,
+ H: 72,
+ I: 73,
+ J: 74,
+ K: 75,
+ L: 76,
+ M: 77,
+ N: 78,
+ O: 79,
+ P: 80,
+ Q: 81,
+ R: 82,
+ S: 83,
+ T: 84,
+ U: 85,
+ V: 86,
+ W: 87,
+ X: 88,
+ Y: 89,
+ Z: 90,
+ LEFT_WINDOW: 91, // Left Command on Mac
+ RIGHT_WINDOW: 92,
+ SELECT: 93, // Right Command on Mac
+ SEMICOLON: 186,
+ EQUALS: 187,
+ COMMA: 188,
+ DASH: 189,
+ PERIOD: 190,
+ FORWARD_SLASH: 191,
+ OPEN_BRACKET: 219,
+ BACK_SLASH: 220,
+ CLOSE_BRACKET: 221,
+ SINGLE_QUOTE: 222
+ });
+ this.modifiers = [
+ this.keyCodes['SHIFT'],
+ this.keyCodes['CTRL'],
+ this.keyCodes['ALT'],
+ this.keyCodes['LEFT_WINDOW'],
+ this.keyCodes['RIGHT_WINDOW'],
+ this.keyCodes['SELECT']
+ ];
+ this.keyStates = {};
+ this.callbacks = {
+ onKeyDown: [],
+ onKeyUp: []
+ };
+
+ // set up the keyStates map with default value of "up" for each key
+ Object.keys(this.keyCodes).forEach(function(keyName) {
+ this.keyStates[this.keyCodes[keyName]] = 'up';
+ }.bind(this));
+
+ this.initListeners();
+ }
+ initListeners() {
+ // when a key is pressed down, automatically update that entry in keyStates and trigger callbacks
+ document.addEventListener('keydown', function(event) {
+ var code = event.keyCode ? event.keyCode : event.which;
+ if (this.keyStates.hasOwnProperty(code)) {
+ this.keyStates[code] = 'down';
+ this.callbacks.onKeyDown.forEach((cb) => {
+ cb(code, this.getActiveModifiers());
+ });
+ }
+ }.bind(this));
+
+ // when a key is released, automatically update that entry in keyStates and trigger callbacks
+ document.addEventListener('keyup', function(event) {
+ var code = event.keyCode ? event.keyCode : event.which;
+ if (this.keyStates.hasOwnProperty(code)) {
+ this.keyStates[code] = 'up';
+ this.callbacks.onKeyUp.forEach((cb) => {
+ cb(code, this.getActiveModifiers());
+ });
+ }
+ }.bind(this));
+
+ // reset all keys to 'up' if the window loses focus, to prevent cases where a key gets "stuck" down
+ window.addEventListener('blur', () => {
+ for (let code in this.keyStates) {
+ if (this.keyStates.hasOwnProperty(code)) {
+ this.keyStates[code] = 'up';
+ }
+ }
+ });
+ }
+ onKeyDown(callback) {
+ this.callbacks.onKeyDown.push(callback);
+ }
+ onKeyUp(callback) {
+ this.callbacks.onKeyUp.push(callback);
+ }
+ getActiveModifiers() {
+ return this.modifiers.filter((modifier) => {
+ return this.keyStates[modifier] === 'down';
+ });
+ }
+ }
+
+ exports.KeyboardListener = KeyboardListener;
+})(realityEditor.device);
diff --git a/content_scripts/MenuBar.js b/content_scripts/MenuBar.js
new file mode 100644
index 00000000..f25d1b56
--- /dev/null
+++ b/content_scripts/MenuBar.js
@@ -0,0 +1,488 @@
+createNameSpace('realityEditor.gui');
+
+(function(exports) {
+ let _keyboard;
+ function getKeyboard() {
+ if (!_keyboard) {
+ _keyboard = new realityEditor.device.KeyboardListener();
+ }
+ return _keyboard;
+ }
+
+ class MenuBar {
+ constructor() {
+ this.menus = [];
+ this.openMenu = null;
+ this.buildDom();
+ this.setupKeyboard();
+ }
+ buildDom() {
+ this.domElement = document.createElement('div');
+ this.domElement.classList.add('desktopMenuBar');
+ }
+ setupKeyboard() {
+ getKeyboard().onKeyDown((code, modifiers) => {
+ if (realityEditor.device.keyboardEvents.isKeyboardActive()) { return; } // ignore if a tool is using the keyboard
+
+ // check with each of the menu items, whether this triggers anything
+ this.menus.forEach(menu => {
+ menu.items.forEach(item => {
+ if (typeof item.onKeyDown === 'function') {
+ try {
+ item.onKeyDown(code, modifiers);
+ } catch (e) {
+ console.warn('Error in MenuBar item.onKeyDown', e);
+ }
+ }
+ // also add keyboard shortcuts to one-level-deep of submenus
+ if (item.hasSubmenu) {
+ item.submenu.items.forEach(subItem => {
+ if (typeof subItem.onKeyDown === 'function') {
+ try {
+ subItem.onKeyDown(code, modifiers);
+ } catch (e) {
+ console.warn('Error in MenuBar subItem.onKeyDown', e);
+ }
+ }
+ });
+ }
+ });
+ });
+ });
+ }
+ addMenu(menu) {
+ this.menus.push(menu);
+ this.domElement.appendChild(menu.domElement);
+ menu.onMenuTitleClicked = this.onMenuTitleClicked.bind(this);
+ }
+ hideMenu(menu) {
+ if (menu.isHidden) { return; }
+ menu.isHidden = true;
+ this.redraw();
+ }
+ unhideMenu(menu) {
+ if (!menu.isHidden) { return; }
+ menu.isHidden = false;
+ this.redraw();
+ }
+ disableMenu(menu) {
+ if (menu.isDisabled) { return; }
+ menu.isDisabled = true;
+ this.redraw();
+ }
+ enableMenu(menu) {
+ if (!menu.isDisabled) { return; }
+ menu.isDisabled = false;
+ this.redraw();
+ }
+ onMenuTitleClicked(menu) {
+ if (menu.isOpen) {
+ if (this.openMenu && this.openMenu !== menu) {
+ this.openMenu.closeDropdown();
+ }
+ this.openMenu = menu;
+ }
+ }
+ addItemToMenu(menuName, item) {
+ let menu = this.menus.find(menu => {
+ return menu.name === menuName;
+ });
+ if (!menu) {
+ menu = new Menu(menuName);
+ this.menus.push(menu);
+ }
+ menu.addItem(item);
+ this.redraw();
+ }
+ removeItemFromMenu(menuName, itemText) {
+ let menu = this.menus.find(menu => {
+ return menu.name === menuName;
+ });
+ if (!menu) return;
+ menu.removeItem(itemText);
+ this.redraw();
+ }
+ // Note: assumes items in different menus don't have duplicate names
+ addCallbackToItem(itemName, callback) {
+ let item = this.getItemByName(itemName);
+ if (item) {
+ item.addCallback(callback);
+ }
+ }
+ setItemEnabled(itemName, enabled) {
+ let item = this.getItemByName(itemName);
+ if (item) {
+ if (enabled) {
+ item.enable();
+ } else {
+ item.disable();
+ }
+ }
+ }
+ getItemByName(itemName) {
+ let match = null;
+ this.menus.forEach(menu => {
+ if (match) { return; } // only add to the first match
+ // search the menu and one-level-deep of submenus for the matching item
+ let item = menu.items.find(item => {
+ if (item.hasSubmenu) {
+ return item.submenu.items.find(subItem => {
+ return subItem.text === itemName;
+ });
+ }
+ return item.text === itemName;
+ });
+ if (item) {
+ if (item.hasSubmenu) {
+ match = item.submenu.items.find(subItem => {
+ return subItem.text === itemName;
+ });
+ } else {
+ match = item;
+ }
+ }
+ });
+ return match;
+ }
+ redraw() {
+ let numHidden = 0;
+ // tell each menu to redraw
+ this.menus.forEach((menu, index) => {
+ menu.redraw(index - numHidden);
+ if (menu.isHidden) {
+ numHidden++;
+ }
+ });
+ }
+ }
+
+ class Menu {
+ constructor(name) {
+ this.name = name;
+ this.items = [];
+ this.isOpen = false;
+ this.isHidden = false;
+ this.isDisabled = false;
+ this.buildDom();
+ this.menuIndex = 0;
+ this.onMenuTitleClicked = null; // MenuBar can inject callback here to coordinate multiple menus
+ }
+ buildDom() {
+ this.domElement = document.createElement('div');
+ this.domElement.classList.add('desktopMenuBarMenu');
+ const title = document.createElement('div');
+ title.classList.add('desktopMenuBarMenuTitle');
+ title.innerText = this.name;
+ this.domElement.appendChild(title);
+ const dropdown = document.createElement('div');
+ dropdown.classList.add('desktopMenuBarMenuDropdown');
+ dropdown.classList.add('hiddenDropdown');
+ this.domElement.appendChild(dropdown);
+
+ title.addEventListener('pointerdown', () => {
+ this.isOpen = !this.isOpen;
+ this.redraw();
+ if (typeof this.onMenuTitleClicked === 'function') {
+ this.onMenuTitleClicked(this);
+ }
+ });
+ }
+ closeDropdown() {
+ this.isOpen = false;
+ this.redraw();
+ }
+ addItem(menuItem) {
+ let existingIndex = this.items.indexOf(menuItem);
+ if (existingIndex > -1) {
+ this.items.splice(existingIndex, 1); // move item to bottom if already contains it
+ }
+ this.items.push(menuItem);
+ let dropdown = this.domElement.querySelector('.desktopMenuBarMenuDropdown');
+ dropdown.appendChild(menuItem.domElement);
+ menuItem.parent = this;
+ }
+ removeItem(itemText) {
+ let itemIndex = this.items.map(item => item.text).indexOf(itemText);
+ if (itemIndex < 0) return;
+ let menuItem = this.items[itemIndex];
+ let dropdown = this.domElement.querySelector('.desktopMenuBarMenuDropdown');
+ dropdown.removeChild(menuItem.domElement);
+ this.items.splice(itemIndex, 1);
+ }
+ redraw(index) {
+ if (typeof index !== 'undefined') { this.menuIndex = index; }
+ this.domElement.style.left = (100 * this.menuIndex) + 'px';
+
+ let dropdown = this.domElement.querySelector('.desktopMenuBarMenuDropdown');
+ let title = this.domElement.querySelector('.desktopMenuBarMenuTitle');
+ if (this.isOpen) {
+ dropdown.classList.remove('hiddenDropdown');
+ title.classList.add('desktopMenuBarMenuTitleOpen');
+ } else {
+ dropdown.classList.add('hiddenDropdown');
+ title.classList.remove('desktopMenuBarMenuTitleOpen');
+ }
+
+ this.items.forEach((item, itemIndex) => {
+ item.redraw(itemIndex);
+ });
+
+ if (this.isHidden) {
+ this.domElement.style.display = 'none';
+ } else {
+ this.domElement.style.display = '';
+ }
+
+ if (this.isDisabled) {
+ this.domElement.classList.add('desktopMenuBarMenuTitleDisabled');
+ } else {
+ this.domElement.classList.remove('desktopMenuBarMenuTitleDisabled');
+ }
+ }
+ }
+
+ class MenuItem {
+ constructor(text, options, onClick) {
+ this.text = text;
+ this.callbacks = [];
+ if (onClick) {
+ this.addCallback(onClick);
+ }
+ // options include: { shortcutKey: 'M', modifiers: ['SHIFT', 'ALT'], toggle: true, defaultVal: true, disabled: true }
+ // note: shortcutKey should be an entry in the KeyboardListener's keyCodes
+ this.options = options || {};
+ this.buildDom();
+ this.parent = null;
+ }
+ buildDom() {
+ this.domElement = document.createElement('div');
+ this.domElement.classList.add('desktopMenuBarItem');
+
+ let textElement = document.createElement('div');
+ textElement.classList.add('desktopMenuBarItemText');
+ textElement.innerText = this.text;
+
+ if (this.options.isSeparator) {
+ this.domElement.classList.add('desktopMenuBarItemSeparator');
+ textElement.innerHTML = '
';
+ }
+
+ if (this.options.toggle) {
+ let checkmark = document.createElement('div');
+ checkmark.classList.add('desktopMenuBarItemCheckmark');
+ checkmark.innerText = '✓';
+
+ textElement.classList.add('desktopMenuBarItemTextToggle');
+
+ if (!this.options.defaultVal) {
+ checkmark.classList.add('desktopMenuBarItemCheckmarkHidden');
+ }
+ this.domElement.appendChild(checkmark);
+ }
+
+ this.domElement.appendChild(textElement);
+
+ // shortcutKey: 'M', modifiers: ['SHIFT', 'ALT'], toggle: true, defaultVal: true, disabled: true
+ if (this.options.shortcutKey) {
+ const shortcut = document.createElement('div');
+ shortcut.classList.add('desktopMenuBarItemShortcut');
+ shortcut.innerText = getShortcutDisplay(this.options.shortcutKey);
+ this.domElement.appendChild(shortcut);
+
+ const shortcutModifier = document.createElement('div');
+ shortcutModifier.classList.add('desktopMenuBarItemShortcutModifier');
+ shortcutModifier.innerText = this.options.modifiers ? this.options.modifiers.map(modifier => getShortcutDisplay(modifier)).join(' ') : '';
+ this.domElement.appendChild(shortcutModifier);
+
+ const thisKeyCode = getKeyboard().keyCodes[this.options.shortcutKey];
+ const thisModifiers = this.options.modifiers ? this.options.modifiers.map(modifier => getKeyboard().keyCodes[modifier]) : [];
+ const modifierSetsMatch = (modifierSet1, modifierSet2) => {
+ return modifierSet1.length === modifierSet2.length && modifierSet1.every(value => modifierSet2.includes(value));
+ };
+ this.onKeyDown = function(code, activeModifiers) {
+ if (code === thisKeyCode && modifierSetsMatch(thisModifiers, activeModifiers)) {
+ this.triggerItem();
+ }
+ };
+ }
+
+ if (this.options.disabled) {
+ this.disable();
+ }
+
+ this.domElement.addEventListener('pointerup', () => {
+ let succeeded = this.triggerItem();
+ if (succeeded) {
+ this.parent.closeDropdown();
+ }
+ });
+ }
+ triggerItem() {
+ if (this.domElement.classList.contains('desktopMenuBarItemDisabled')) {
+ return false;
+ }
+ let toggled = this.options.toggle ? this.switchToggle() : undefined;
+ this.callbacks.forEach(cb => {
+ cb(toggled);
+ });
+ return true;
+ }
+ switchToggle() {
+ if (!this.options.toggle) { return; }
+ let checkmark = this.domElement.querySelector('.desktopMenuBarItemCheckmark');
+
+ if (checkmark.classList.contains('desktopMenuBarItemCheckmarkHidden')) {
+ checkmark.classList.remove('desktopMenuBarItemCheckmarkHidden');
+ return true;
+ } else {
+ checkmark.classList.add('desktopMenuBarItemCheckmarkHidden');
+ return false;
+ }
+ }
+ disable() {
+ this.domElement.classList.add('desktopMenuBarItemDisabled');
+ let checkmark = this.domElement.querySelector('.desktopMenuBarItemCheckmark');
+ if (checkmark) {
+ checkmark.classList.add('desktopMenuBarItemCheckmarkDisabled');
+ }
+ }
+ enable() {
+ this.domElement.classList.remove('desktopMenuBarItemDisabled');
+ let checkmark = this.domElement.querySelector('.desktopMenuBarItemCheckmark');
+ if (checkmark) {
+ checkmark.classList.remove('desktopMenuBarItemCheckmarkDisabled');
+ }
+ }
+ setText(text) {
+ let textElement = this.domElement.querySelector('.desktopMenuBarItemText');
+ textElement.innerText = text;
+ }
+ redraw() {
+ // currently not used, but can be used to update UI each time menu opens, closes, or contents change
+ }
+ addCallback(callback) {
+ this.callbacks.push(callback);
+ }
+ }
+
+ // when adding a keyboard shortcut, conform to the naming of the keyboard.keyCodes enum
+ // this function maps those names to human-readable shortcut keys to display in the menu
+ const getShortcutDisplay = (keyCodeName) => {
+ if (keyCodeName === 'BACKSPACE') {
+ return '⌫';
+ } else if (keyCodeName === 'TAB') {
+ return '⇥';
+ } else if (keyCodeName === 'ENTER') {
+ return '⏎';
+ } else if (keyCodeName === 'SHIFT') {
+ return '⇪';
+ } else if (keyCodeName === 'CTRL') {
+ return '⌃';
+ } else if (keyCodeName === 'ALT') {
+ return '⎇';
+ } else if (keyCodeName === 'ESCAPE') {
+ return 'Esc';
+ } else if (keyCodeName === 'SPACE') {
+ return '_';
+ } else if (keyCodeName === 'UP') {
+ return '↑';
+ } else if (keyCodeName === 'DOWN') {
+ return '↓';
+ } else if (keyCodeName === 'LEFT') {
+ return '←';
+ } else if (keyCodeName === 'RIGHT') {
+ return '→';
+ } else if (keyCodeName.match(/^_\d$/)) {
+ return keyCodeName[1]; // convert '_0' to '0', '_9' to '9'
+ } else if (keyCodeName === 'SEMICOLON') {
+ return ';';
+ } else if (keyCodeName === 'EQUALS') {
+ return '=';
+ } else if (keyCodeName === 'COMMA') {
+ return ',';
+ } else if (keyCodeName === 'DASH') {
+ return '-';
+ } else if (keyCodeName === 'PERIOD') {
+ return '.';
+ } else if (keyCodeName === 'FORWARD_SLASH') {
+ return '/';
+ } else if (keyCodeName === 'OPEN_BRACKET') {
+ return '[';
+ } else if (keyCodeName === 'BACK_SLASH') {
+ return '\\';
+ } else if (keyCodeName === 'CLOSE_BRACKET') {
+ return ']';
+ } else if (keyCodeName === 'SINGLE_QUOTE') {
+ return '\'';
+ }
+ return keyCodeName;
+ };
+
+ class Submenu extends Menu {
+ constructor(name) {
+ super(name);
+ }
+ redraw(index) {
+ this.isOpen = true; // submenu is always considered open (hidden by its menuItem, not by itself)
+ super.redraw(index);
+
+ this.domElement.style.left = ''; // don't override the css left to be at 0
+
+ // hide the title of the dropdown
+ let title = this.domElement.querySelector('.desktopMenuBarMenuTitle');
+ if (title) {
+ title.style.display = 'none';
+ }
+ }
+ }
+
+ class MenuItemSubmenu extends MenuItem {
+ constructor(text, options, onClick) {
+ super(text, options, onClick);
+
+ this.hasSubmenu = true;
+
+ // add an arrow to signal that this one has a submenu
+ let arrow = document.createElement('div');
+ arrow.classList.add('desktopMenuBarItemArrow');
+ arrow.innerText = '>';
+ this.domElement.appendChild(arrow);
+
+ this.buildSubMenu();
+
+ this.domElement.addEventListener('pointerover', () => {
+ this.showSubMenu();
+ });
+
+ this.domElement.addEventListener('pointerout', () => {
+ this.hideSubMenu();
+ });
+ }
+ addItemToSubmenu(menuItem) {
+ this.submenu.addItem(menuItem);
+ }
+ buildSubMenu() {
+ // the name of the submenu doesn't matter because it isn't rendered
+ this.submenu = new Submenu('Sub Menu');
+ this.submenu.redraw();
+ this.submenu.domElement.classList.add('desktopMenuBarSubmenu');
+ this.domElement.appendChild(this.submenu.domElement);
+ this.hideSubMenu();
+ }
+ showSubMenu() {
+ this.submenu.domElement.classList.remove('hiddenDropdown');
+ this.submenu.domElement.classList.add('desktopMenuBarSubmenu');
+ }
+ hideSubMenu() {
+ if (!this.submenu.domElement) return;
+ if (!this.submenu.domElement.parentElement) return;
+ this.submenu.domElement.classList.add('hiddenDropdown');
+ this.submenu.domElement.classList.remove('desktopMenuBarSubmenu');
+ }
+ }
+
+ exports.MenuBar = MenuBar;
+ exports.Menu = Menu;
+ exports.MenuItem = MenuItem;
+ exports.MenuItemSubmenu = MenuItemSubmenu;
+})(realityEditor.gui);
diff --git a/content_scripts/MotionStudyFollowable.js b/content_scripts/MotionStudyFollowable.js
new file mode 100644
index 00000000..21043660
--- /dev/null
+++ b/content_scripts/MotionStudyFollowable.js
@@ -0,0 +1,56 @@
+import { Followable } from '../../src/gui/ar/Followable.js';
+
+/**
+ * Constructs and updates the sceneNode to follow a humanPoseAnalyzer's pose object
+ */
+export class MotionStudyFollowable extends Followable {
+ static count = 0;
+
+ constructor(frameKey) {
+ MotionStudyFollowable.count++;
+ let parentNode = realityEditor.sceneGraph.getVisualElement('AnalyticsCameraGroupContainer');
+ if (!parentNode) {
+ let gpNode = realityEditor.sceneGraph.getGroundPlaneNode();
+ let motionStudyCameraGroupContainerId = realityEditor.sceneGraph.addVisualElement('AnalyticsCameraGroupContainer', gpNode);
+ parentNode = realityEditor.sceneGraph.getSceneNodeById(motionStudyCameraGroupContainerId);
+ let transformationMatrix = realityEditor.gui.ar.utilities.makeGroundPlaneRotationX(Math.PI / 2);
+ transformationMatrix[13] = -1 * realityEditor.gui.ar.areaCreator.calculateFloorOffset(); // ground plane translation
+ parentNode.setLocalMatrix(transformationMatrix);
+ }
+ let menuItemName = `Analytics ${MotionStudyFollowable.count}`;
+ super(`AnalyticsFollowable_${frameKey}`, menuItemName, parentNode);
+
+ this.frameKey = frameKey;
+ this.floorOffset = realityEditor.gui.ar.areaCreator.calculateFloorOffset();
+ }
+
+ // continuously updates the sceneNode to be positioned a bit behind the
+ // person's chest joint, rotated to match the direction that the person is facing
+ updateSceneNode() {
+ let matchingMotionStudy = realityEditor.motionStudy.getMotionStudyByFrame(this.frameKey);
+ if (!matchingMotionStudy) return;
+ if (matchingMotionStudy.humanPoseAnalyzer.lastDisplayedClones.length === 0) return;
+ // TODO: for now we're following the first clone detected in that timestamp but if we support
+ // tracking multiple people at once then need to implement a way to switch to follow the second person
+ let joints = matchingMotionStudy.humanPoseAnalyzer.lastDisplayedClones[0].pose.joints;
+ let THREE = realityEditor.gui.threejsScene.THREE;
+ // we calculate the direction the person is facing by crossing two vectors:
+ // the neckToHead vector, and the neckToLeftShoulder vector
+ let headPosition = joints.head.position;
+ let neckPosition = joints.neck.position;
+ let leftShoulderPosition = joints.left_shoulder.position;
+ const neckToHeadVector = new THREE.Vector3().subVectors(headPosition, neckPosition).normalize();
+ const neckToShoulderVector = new THREE.Vector3().subVectors(leftShoulderPosition, neckPosition).normalize();
+ const neckRotationAxis = new THREE.Vector3().crossVectors(neckToHeadVector, neckToShoulderVector).normalize();
+ // lookAt gives a convenient way to construct a rotation matrix by looking in the direction of the cross product
+ const neckRotationMatrix = new THREE.Matrix4().lookAt(new THREE.Vector3(0, 0, 0), neckRotationAxis, new THREE.Vector3(0, 1, 0));
+ // calculate the chest position relative to the floor
+ let finalMatrix = new THREE.Matrix4().setPosition(joints.chest.position.x, joints.chest.position.y + this.floorOffset, joints.chest.position.z);
+ // order matters! multiply the rotation after the position so that it doesn't affect the position
+ finalMatrix.multiplyMatrices(finalMatrix, neckRotationMatrix);
+ // move the position of the follow target to be 3 meters behind the person (better centers them in your view)
+ let adjustment = new THREE.Matrix4().setPosition(0, 0, -3000);
+ finalMatrix.multiplyMatrices(finalMatrix, adjustment);
+ this.sceneNode.setLocalMatrix(finalMatrix.elements);
+ }
+}
diff --git a/content_scripts/RealityZoneVoxelizer.js b/content_scripts/RealityZoneVoxelizer.js
new file mode 100644
index 00000000..4f253fb8
--- /dev/null
+++ b/content_scripts/RealityZoneVoxelizer.js
@@ -0,0 +1,537 @@
+createNameSpace('realityEditor.gui.ar.desktopRenderer');
+
+import * as THREE from '../../thirdPartyCode/three/three.module.js';
+import { MeshBVH } from '../../thirdPartyCode/three-mesh-bvh.module.js';
+import { mergeBufferGeometries } from '../../thirdPartyCode/three/BufferGeometryUtils.module.js';
+
+(function(exports) {
+ const MAX_AGE = 100;
+
+ class OctTree {
+ constructor({minX, maxX, minY, maxY, minZ, maxZ}) {
+ this.minX = minX;
+ this.maxX = maxX;
+ this.minY = minY;
+ this.maxY = maxY;
+ this.minZ = minZ;
+ this.maxZ = maxZ;
+ this.tree = [];
+ }
+
+ getOct(x, y, z) {
+ if (typeof x !== 'number' ||
+ typeof y !== 'number' ||
+ typeof z !== 'number') {
+ throw new Error(`incorrect arguments to getOct: ${JSON.stringify({x, y, z})}`);
+ }
+ // TODO
+ // - update displayed voxels
+ // - create entire tree for insertion
+ let {minX, maxX, minY, maxY, minZ, maxZ} = this;
+ let tree = this.tree;
+ while (tree && tree.length > 0) {
+ let midX = (maxX + minX) / 2;
+ let midY = (maxY + minY) / 2;
+ let midZ = (maxZ + minZ) / 2;
+ let idx = 0;
+ // Generate an index into an array of length 8 of the binary form XZY
+ if (x > midX) {
+ idx += 4;
+ }
+ if (z > midZ) {
+ idx += 2;
+ }
+ if (y > midY) {
+ idx += 1;
+ }
+ let cell = tree[idx];
+ if (!Array.isArray(cell)) {
+ return cell;
+ }
+ if (x > midX) {
+ minX = midX;
+ } else {
+ maxX = midX;
+ }
+ if (y > midY) {
+ minY = midY;
+ } else {
+ maxY = midY;
+ }
+ if (z > midZ) {
+ minZ = midZ;
+ } else {
+ maxZ = midZ;
+ }
+ tree = cell;
+ }
+ }
+
+ insert(x, y, z, value) {
+ if (typeof x !== 'number' ||
+ typeof y !== 'number' ||
+ typeof z !== 'number' ||
+ !value) {
+ throw new Error(`incorrect arguments to insert: ${JSON.stringify({x, y, z, value})}`);
+ }
+ // TODO
+ // - update displayed voxels
+ // - create entire tree for insertion
+ let {minX, maxX, minY, maxY, minZ, maxZ} = this;
+ let tree = this.tree;
+ while (tree) {
+ let midX = (maxX + minX) / 2;
+ let midY = (maxY + minY) / 2;
+ let midZ = (maxZ + minZ) / 2;
+ // Generate an index into an array of length 8 of the binary form XZY
+ let idx = 0;
+ if (x > midX) {
+ idx += 4;
+ }
+ if (z > midZ) {
+ idx += 2;
+ }
+ if (y > midY) {
+ idx += 1;
+ }
+ if (typeof tree[idx] === 'undefined') {
+ tree[idx] = value;
+ return;
+ }
+ let cell = tree[idx];
+ if (!Array.isArray(cell)) {
+ let cur = cell;
+ tree[idx] = [];
+ this.insert(cur.pos.x, cur.pos.y, cur.pos.z, cur);
+ }
+ if (x > midX) {
+ minX = midX;
+ } else {
+ maxX = midX;
+ }
+ if (y > midY) {
+ minY = midY;
+ } else {
+ maxY = midY;
+ }
+ if (z > midZ) {
+ minZ = midZ;
+ } else {
+ maxZ = midZ;
+ }
+ tree = tree[idx];
+ }
+ }
+
+ removeOct(x, y, z, isValidDeletion) {
+ if (typeof x !== 'number' ||
+ typeof y !== 'number' ||
+ typeof z !== 'number') {
+ throw new Error(`incorrect arguments to removeOct: ${JSON.stringify({x, y, z})}`);
+ }
+
+ // TODO
+ // - update displayed voxels
+ // - create entire tree for insertion
+ let {minX, maxX, minY, maxY, minZ, maxZ} = this;
+ let tree = this.tree;
+ while (tree && tree.length > 0) {
+ let midX = (maxX + minX) / 2;
+ let midY = (maxY + minY) / 2;
+ let midZ = (maxZ + minZ) / 2;
+ let idx = 0;
+ // Generate an index into an array of length 8 of the binary form XZY
+ if (x > midX) {
+ idx += 4;
+ }
+ if (z > midZ) {
+ idx += 2;
+ }
+ if (y > midY) {
+ idx += 1;
+ }
+ let cell = tree[idx];
+ if (!Array.isArray(cell)) {
+ if (!cell) {
+ return;
+ }
+ if (isValidDeletion && !isValidDeletion(cell, x, y, z)) {
+ return;
+ }
+ delete tree[idx];
+ return cell;
+ }
+ if (x > midX) {
+ minX = midX;
+ } else {
+ maxX = midX;
+ }
+ if (y > midY) {
+ minY = midY;
+ } else {
+ maxY = midY;
+ }
+ if (z > midZ) {
+ minZ = midZ;
+ } else {
+ maxZ = midZ;
+ }
+ tree = cell;
+ }
+ }
+
+ filterOldBoxes() {
+ let queue = [this.tree];
+ while (queue.length > 0) {
+ let tree = queue.pop();
+ for (let i = 0; i < 8; i++) {
+ let cell = tree[i];
+ if (Array.isArray(cell)) {
+ queue.push(cell);
+ continue;
+ }
+ if (!cell) {
+ continue;
+ }
+ cell.box._age -= 1;
+ if (cell.box._age > 0) {
+ continue;
+ }
+
+ cell.box.parent.remove(cell.box);
+ delete tree[i];
+ }
+ }
+ }
+ }
+
+ exports.RealityZoneVoxelizer = class RealityZoneVoxelizer {
+ constructor(floorOffset, gltf, navmesh) {
+ this.floorOffset = floorOffset;
+
+ this.isCloseDeletion = this.isCloseDeletion.bind(this);
+
+ let geometries = [];
+ gltf.traverse(obj => {
+ if (obj.geometry) {
+ let geo = obj.geometry.clone();
+ geo.deleteAttribute('uv'); // Messes with merge if present in some geometries but not others
+ geometries.push(geo);
+ }
+ });
+
+ gltf.children.map(child => {
+ child.geometry.boundsTree = new MeshBVH(child.geometry);
+ });
+
+ this.gltf = gltf;
+
+ let geometry = geometries[0];
+ if (geometries.length > 1) {
+ // We may have to contend with a groundPlaneCollider sneaking
+ // into the geometry list, filter it out
+ geometries = geometries.filter(geo => geo.type === 'BufferGeometry');
+ const mergedGeometry = mergeBufferGeometries(geometries);
+ geometry = mergedGeometry;
+ }
+
+ this.bvh = new MeshBVH(geometry);
+ this.navmesh = navmesh;
+ this.raycaster = new THREE.Raycaster();
+ this.container = new THREE.Group();
+ this.container.position.y = -floorOffset;
+ // this.container.rotation.x = Math.PI / 2;
+ // Can dynamically octree-style subdivide and save a ton of processing
+ this.res = 100 / 1000;
+
+ this.baseMat = new THREE.MeshBasicMaterial({
+ color: 0x777777,
+ transparent: true,
+ opacity: 0.3,
+ wireframe: true,
+ });
+
+ this.addedMat = new THREE.MeshBasicMaterial({
+ color: 0x00ff00,
+ transparent: true,
+ opacity: 0.3,
+ // wireframe: true,
+ });
+
+ this.removedMat = new THREE.MeshBasicMaterial({
+ color: 0xff0000,
+ transparent: true,
+ opacity: 0.3,
+ // wireframe: true,
+ });
+
+ this.baseGeo = new THREE.BoxBufferGeometry(1, 1, 1);
+ const maxBoxes = 1 << 16;
+ this.boxesMesh = new THREE.InstancedMesh(this.baseGeo, this.baseMat, maxBoxes);
+ this.addedRemovedOct = null;
+
+ this.voxOct = null;
+ }
+
+ add() {
+ realityEditor.gui.threejsScene.addToScene(this.container);
+
+ this.boxesMesh.count = 0;
+
+ let startRes = this.res * 8;
+
+ let diffX = Math.ceil((this.navmesh.maxX - this.navmesh.minX) / startRes + 2) * startRes;
+ let diffY = Math.ceil((this.navmesh.maxY - this.navmesh.minY) / startRes + 2) * startRes;
+ let diffZ = Math.ceil((this.navmesh.maxZ - this.navmesh.minZ) / startRes + 2) * startRes;
+ let avgX = (this.navmesh.minX + this.navmesh.maxX) / 2;
+ let avgY = (this.navmesh.minY + this.navmesh.maxY) / 2;
+ let avgZ = (this.navmesh.minZ + this.navmesh.maxZ) / 2;
+ let diff = Math.max(diffX, diffY, diffZ);
+ this.voxOct = new OctTree({
+ minX: avgX - diff / 2,
+ minY: avgY - diff / 2,
+ minZ: avgZ - diff / 2,
+ maxX: avgX + diff / 2,
+ maxY: avgY + diff / 2,
+ maxZ: avgZ + diff / 2,
+ });
+ this.addedRemovedOct = new OctTree({
+ minX: avgX - diff / 2,
+ minY: avgY - diff / 2,
+ minZ: avgZ - diff / 2,
+ maxX: avgX + diff / 2,
+ maxY: avgY + diff / 2,
+ maxZ: avgZ + diff / 2,
+ });
+
+ this.voxOct.tree = this.scanRegion(
+ avgX - diff / 2,
+ avgY - diff / 2,
+ avgZ - diff / 2,
+ avgX + diff / 2,
+ avgY + diff / 2,
+ avgZ + diff / 2,
+ 2);
+ let boxScale = diff;
+ while (boxScale > this.res) {
+ boxScale /= 2;
+ }
+ this.res = boxScale;
+ this.container.add(this.boxesMesh);
+ }
+
+ remove() {
+ realityEditor.gui.threejsScene.removeFromScene(this.container);
+ }
+
+ removeOct(x, y, z) {
+ let n = this.voxOct.remove(x, y, z);
+ if (typeof n !== 'number') {
+ return;
+ }
+ if (n < 0) {
+ return; // hmmmmMMMMMM
+ }
+ let zeros = new THREE.Matrix4(
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 1
+ );
+ this.boxesMesh.setMatrixAt(n, zeros);
+ }
+
+ scanRegion(minX, minY, minZ, maxX, maxY, maxZ, subdivs) {
+ let res = (maxX - minX) / subdivs;
+ let building = res <= this.res;
+
+ let oct = [];
+ for (let xi = 0; xi < subdivs; xi++) {
+ let x = minX + res * (xi + 0.5);
+ for (let zi = 0; zi < subdivs; zi++) {
+ let z = minZ + res * (zi + 0.5);
+ for (let yi = 0; yi < subdivs; yi++) {
+ let y = minY + res * (yi + 0.5);
+ if (this.doRaycastBox(x, y, z, res)) {
+ // let box = new THREE.Mesh(this.baseGeo, this.baseMat);
+ // box.position.set(x * 1000, y * 1000, z * 1000);
+ // box.scale.set(res / this.res, res / this.res, res / this.res);
+ // this.container.add(box);
+ if (building) {
+ let mat = new THREE.Matrix4();
+ mat.makeScale(res * 1000, res * 1000, res * 1000);
+ mat.setPosition(x * 1000, y * 1000, z * 1000);
+ this.boxesMesh.setMatrixAt(this.boxesMesh.count, mat);
+ // let box = new THREE.Mesh(this.baseGeo, this.baseMat);
+ // box.position.set
+ // // this.boxPositions.push(x * 1000, y * 1000, z * 1000);
+ // // box.rotation.y = Math.random() * 0.4;
+ // box.scale.set(res * 1000, res * 1000, res * 1000);
+ // this.container.add(box);
+ oct.push(this.boxesMesh.count);
+ this.boxesMesh.count += 1;
+ } else {
+ oct.push(this.scanRegion(
+ x - res / 2, y - res / 2, z - res / 2,
+ x + res / 2, y + res / 2, z + res / 2,
+ 2));
+ }
+ } else {
+ oct.push([]);
+ }
+ }
+ }
+ }
+ return oct;
+ }
+
+ doRaycastBox(x, y, z, res) {
+ let box = new THREE.Box3();
+ box.min.set(
+ x - res / 2,
+ y - res / 2,
+ z - res / 2
+ );
+ box.max.set(
+ x + res / 2,
+ y + res / 2,
+ z + res / 2
+ );
+ return this.bvh.intersectsBox(box, new THREE.Matrix4());
+ }
+
+ isCloseDeletion(cell, x, y, z) {
+ let otherPos = new THREE.Vector3(x, y, z);
+ return cell.pos.distanceToSquared(otherPos) < this.res * this.res;
+ }
+
+ raycastDepth(mesh, {width, height}, rawDepth) {
+ mesh.updateMatrixWorld();
+ const matrixWorld = mesh.matrixWorld;
+ const matrixWorldInv = mesh.matrix.clone();
+ matrixWorldInv.invert();
+ // Is just 1000x scale and 1/1000x scale, respectively
+ // const gltfMatrixWorldInv = this.gltf.matrixWorld.clone();
+ // gltfMatrixWorldInv.invert();
+ const XtoZ = 1920.0 / 1448.24976; // width over focal length
+ const YtoZ = 1080.0 / 1448.24976;
+ let res = 16;
+ const origin = new THREE.Vector3();
+ origin.setFromMatrixPosition(matrixWorld);
+ origin.x /= 1000;
+ origin.y /= 1000;
+ origin.z /= 1000;
+
+ // Raycast in a square in the center of the screen
+ let xMargin = (width - height) / 2;
+
+ // for (let y = height / 2; y < height; y += 10000)
+ // for (let x = width / 2; x < width; x += 10000)
+ for (let y = 0; y < height; y += res) {
+ for (let x = xMargin; x < width - xMargin; x += res) {
+ const direction = new THREE.Vector3(
+ x / width,
+ y / height,
+ 1
+ );
+ let depth = rawDepth[y * width + x] * 5000 / (1 << 14);
+ const z = depth;
+ direction.x = -(x / width - 0.5) * z * XtoZ;
+ direction.y = -(y / height - 0.5) * z * YtoZ;
+ direction.z = z;
+ const ray = new THREE.Ray(new THREE.Vector3(), direction);
+ // Transforms from a ray relative to the cameravis to a ray
+ // relative to the world
+ ray.applyMatrix4(matrixWorld);
+ ray.origin.x /= 1000;
+ ray.origin.y /= 1000;
+ ray.origin.z /= 1000;
+
+ // TODO this isn't very reliable due to sampling only once
+ // every `res` pixels. Performance could be improved by
+ // doing a true octtree-ray intersection
+
+ // Scan and remove existing boxes that intersect with this
+ // ray of length=measured depth by querying points along
+ // the ray
+ for (let airDepth = 0; airDepth < depth / 1000; airDepth += this.res) {
+ let airPosWorld = ray.origin.clone();
+ let diffVec = ray.direction.clone();
+ diffVec.multiplyScalar(airDepth);
+ airPosWorld.add(diffVec);
+ let cell = this.addedRemovedOct.removeOct(
+ airPosWorld.x, airPosWorld.y, airPosWorld.z,
+ this.isCloseDeletion
+ );
+ if (cell) {
+ cell.box.parent.remove(cell.box);
+ }
+ }
+
+ let hit = this.bvh.raycastFirst(ray, THREE.DoubleSide);
+ if (!hit) {
+ continue;
+ }
+
+ let boxDiffMm = Math.max(100, Math.min(depth * 0.4, 1000));
+ // add green box at ray.origin + ray.direction * depth;
+ let added = depth + boxDiffMm < hit.distance * 1000;
+ // add red box at hit.point
+ let removed = depth - boxDiffMm > hit.distance * 1000;
+
+ if (!added && !removed) {
+ continue;
+ }
+
+ let boxPosWorld = hit.point;
+ if (added) {
+ boxPosWorld.copy(ray.origin);
+ let diffVec = ray.direction.clone();
+ diffVec.multiplyScalar(depth / 1000);
+ boxPosWorld.add(diffVec);
+ }
+
+ let existingBox = this.addedRemovedOct.getOct(boxPosWorld.x, boxPosWorld.y, boxPosWorld.z);
+ if (existingBox && existingBox.pos.distanceToSquared(boxPosWorld) < this.res * this.res) {
+ existingBox.box._age = MAX_AGE;
+ continue;
+ }
+
+
+ let box = new THREE.Mesh(this.baseGeo, added ? this.addedMat : this.removedMat);
+ box.scale.set(this.res * 1000, this.res * 1000, this.res * 1000);
+ box._age = MAX_AGE;
+ this.container.add(box);
+
+ // a point in space aligned to the voxel grid
+ const centerX = (this.navmesh.minX + this.navmesh.maxX) / 2 + this.res / 2;
+ const centerY = (this.navmesh.minY + this.navmesh.maxY) / 2 + this.res / 2;
+ const centerZ = (this.navmesh.minZ + this.navmesh.maxZ) / 2 + this.res / 2;
+ // Use this point in space on the grid to align our box
+ // position to the grid
+ let dx = Math.round((boxPosWorld.x - centerX) / this.res) * this.res;
+ let dy = Math.round((boxPosWorld.y - centerY) / this.res) * this.res;
+ let dz = Math.round((boxPosWorld.z - centerZ) / this.res) * this.res;
+
+ box.position.set(
+ (centerX + dx) * 1000,
+ (centerY + dy) * 1000,
+ (centerZ + dz) * 1000
+ );
+ box.rotation.set(0, 0, 0);
+
+ this.addedRemovedOct.insert(
+ boxPosWorld.x, boxPosWorld.y, boxPosWorld.z,
+ {
+ box,
+ pos: boxPosWorld,
+ }
+ );
+ }
+ }
+ // Cull any boxes that have not been seen for a long time
+ this.addedRemovedOct.filterOldBoxes();
+ }
+ };
+})(realityEditor.gui.ar.desktopRenderer);
+
diff --git a/content_scripts/TimelineController.js b/content_scripts/TimelineController.js
new file mode 100644
index 00000000..05f34d25
--- /dev/null
+++ b/content_scripts/TimelineController.js
@@ -0,0 +1,325 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+
+ // Communicates between the TimelineModel and the TimelineView, and bubbles up events to subscribing modules
+ class TimelineController {
+ constructor() {
+ this.callbacks = {
+ onVideoFrame: [],
+ onDataFrame: [],
+ onSegmentDeselected: []
+ };
+
+ this.model = new realityEditor.videoPlayback.TimelineModel();
+ this.model.onDataLoaded(this.handleDataLoaded.bind(this));
+ this.model.onDataViewUpdated(this.handleDataViewUpdated.bind(this));
+ this.model.onWindowUpdated(this.handleWindowUpdated.bind(this));
+ this.model.onTimestampUpdated(this.handleTimestampUpdated.bind(this));
+ this.model.onPlaybackToggled(this.handlePlaybackToggled.bind(this));
+ this.model.onSpeedUpdated(this.handleSpeedUpdated.bind(this));
+ this.model.onSegmentSelected(this.handleSegmentSelected.bind(this));
+ this.model.onSegmentDeselected(this.handleSegmentDeselected.bind(this));
+ this.model.onSegmentData(this.handleSegmentData.bind(this));
+
+ // set up the View and subscribe to events from the view (buttons pressed, scrollbars moved, etc)
+ this.view = new realityEditor.videoPlayback.TimelineView(document.body);
+ this.setupUserInteractions();
+ }
+ /**
+ * @param {TimelineDatabase} database
+ */
+ setDatabase(database) {
+ this.model.setDatabase(database);
+ }
+ setupCalendarView() {
+ this.calendar = new realityEditor.videoPlayback.Calendar(this.view.timelineContainer, false);
+ this.calendar.onDateSelected(this.handleCalendarDateSelected.bind(this));
+ this.calendar.selectDay(Date.now()); // select today, as an initial default view
+ }
+ setupUserInteractions() {
+ this.view.playButton.addEventListener('pointerup', e => {
+ let isPlayButton = e.currentTarget.src.includes('playButton.svg'); // play vs pause state
+ this.model.togglePlayback(isPlayButton);
+ });
+ this.view.speedButton.addEventListener('pointerup', _e => {
+ this.multiplySpeed(2, true);
+ });
+ this.view.calendarButton.addEventListener('pointerup', _e => {
+ if (this.calendar.dom.classList.contains('timelineCalendarVisible')) {
+ this.calendar.hide();
+ } else {
+ this.calendar.show();
+ }
+ });
+ this.view.onZoomHandleChanged(percentZoom => {
+ let playheadTimePercent = this.model.getPlayheadTimePercent();
+ this.view.onZoomChanged(percentZoom, playheadTimePercent);
+ });
+ this.view.onPlayheadSelected(_ => {
+ this.model.togglePlayback(false);
+ });
+ this.view.onPlayheadChanged(positionInWindow => {
+ this.model.setTimestampFromPositionInWindow(positionInWindow);
+
+ // update the currentTime of each of the selected videos, specifically if UI was touched
+ // if timestamp changes just due to time passing, the video will already play and update
+ this.model.selectedSegments.forEach(segment => {
+ let currentTime = (segment.timeMultiplier || 1) * (this.model.currentTimestamp - segment.start) / 1000;
+
+ let videoElements = this.view.getVideoElementsForTrack(segment.trackId);
+ if (videoElements.color && videoElements.depth) {
+ videoElements.color.currentTime = currentTime;
+ videoElements.depth.currentTime = currentTime;
+ }
+ });
+ });
+ this.view.onScrollbarChanged((zoomPercent, leftPercent, rightPercent) => {
+ this.model.adjustCurrentWindow(leftPercent, rightPercent);
+ });
+ this.view.onVideoElementAdded((videoElement, colorOrDepth) => {
+ if (colorOrDepth !== 'color' && colorOrDepth !== 'depth') {
+ console.warn('unable to parse segment id from video element - not color or depth');
+ return;
+ }
+ let videoSegments = this.model.selectedSegments.filter(segment => segment.type === 'VIDEO_3D');
+ let segmentId = videoElement.querySelector('source').src.split('_session_')[1].split('_start_')[0];
+ let matchingSegment = videoSegments.find(segment => segment.id === segmentId);
+
+ // we correct any temporal warping from the recording process here, by adding a timeMultiplier
+ // e.g. if it was supposed to be 10 fps but frames were added at 7 fps, this will stretch the video back to the correct length
+ videoElement.addEventListener('loadedmetadata', _e => {
+ if (typeof matchingSegment.timeMultiplier === 'undefined') {
+ let videoDuration = videoElement.duration;
+ // this relies on knowing the start and end timestamp of the segment
+ let intendedDuration = (matchingSegment.end - matchingSegment.start) / 1000;
+ matchingSegment.timeMultiplier = videoDuration / intendedDuration;
+ }
+ videoElement.playbackRate = this.model.playbackSpeed * matchingSegment.timeMultiplier;
+ });
+
+ // only need one timeupdate listener per set of color and depth videos - we arbitrarily add it to the color
+ if (colorOrDepth !== 'color') { return; }
+
+ videoElement.addEventListener('timeupdate', _e => {
+ let videoElements = this.view.getVideoElementsForTrack(matchingSegment.trackId);
+ this.callbacks.onVideoFrame.forEach(cb => {
+ cb(videoElements.color, videoElements.depth, matchingSegment);
+ });
+ });
+ });
+ }
+ // when a date is clicked, show those 24 hours on the timeline but zoom in to fit the recorded data from that day
+ handleCalendarDateSelected(dateObject) {
+ this.model.togglePlayback(false);
+ this.calendar.hide();
+
+ this.model.timelineWindow.setWithoutZoomFromDate(dateObject);
+
+ // figure out how much of the day is using any data
+ let minPercent = 1;
+ let maxPercent = 0;
+ let dayMin = this.model.timelineWindow.bounds.withoutZoom.min;
+ let dayMax = this.model.timelineWindow.bounds.withoutZoom.max;
+ let dayLength = dayMax - dayMin;
+ let tracks = this.model.database.getFilteredData(dayMin, dayMax).tracks;
+ for (const [_trackId, track] of Object.entries(tracks)) {
+ let bounds = track.getBounds();
+ minPercent = Math.min(minPercent, (bounds.start - dayMin) / dayLength);
+ maxPercent = Math.max(maxPercent, (bounds.end - dayMin) / dayLength);
+ }
+
+ // skip rescaling window if we don't have any tracks to scale it by
+ if (minPercent !== 1 || maxPercent !== 0) {
+ this.model.timelineWindow.setCurrentFromPercent(Math.max(0, minPercent - 0.01), Math.min(1, maxPercent + 0.01));
+ }
+
+ // move the playhead to the beginning of the recorded data
+ this.model.setTimestamp(Math.max(0, dayMin + minPercent * dayLength));
+ }
+ // set up the calendar and highlight which dates have data, in response to loading a database into the timeline
+ handleDataLoaded() {
+ if (!this.calendar) {
+ try {
+ this.setupCalendarView();
+ } catch (e) {
+ console.warn('error setting up calendar view', e);
+ }
+ }
+ // triggers when data finishes loading
+ this.calendar.highlightDates(this.model.database.getDatesWithData());
+ }
+ /**
+ * @param {DataView} dataView
+ */
+ handleDataViewUpdated(dataView) {
+ // which date is selected, can be used to filter the database
+ let simplifiedTracks = this.generateSimplifiedTracks(dataView);
+
+ this.view.render({
+ tracks: simplifiedTracks,
+ tracksFullUpdate: true
+ });
+ }
+
+ /**
+ * @param {DataView} dataView
+ * @returns {{}}
+ */
+ generateSimplifiedTracks(dataView) {
+ if (!dataView || !dataView.filteredDatabase) { return {}; }
+ // process filtered tracks into just the info needed by the view
+ let simplifiedTracks = {};
+ for (const [trackId, track] of Object.entries(dataView.filteredDatabase.tracks)) {
+ simplifiedTracks[trackId] = {
+ id: trackId,
+ type: track.type,
+ segments: {}
+ };
+ for (const [segmentId, segment] of Object.entries(track.segments)) {
+ simplifiedTracks[trackId].segments[segmentId] = {
+ id: segmentId,
+ type: segment.type,
+ start: this.model.getTimestampAsPercent(segment.start),
+ end: this.model.getTimestampAsPercent(segment.end)
+ };
+ }
+ }
+ return simplifiedTracks;
+ }
+
+ /**
+ * @param {TimelineWindow} window
+ */
+ handleWindowUpdated(window) {
+ this.view.render({
+ zoomPercent: window.getZoomPercent(),
+ scrollLeftPercent: window.getScrollLeftPercent(),
+ tracks: this.generateSimplifiedTracks(this.model.currentDataView),
+ });
+ }
+ handleTimestampUpdated(timestamp) {
+ let percentInWindow = this.model.getPlayheadTimePercent(true);
+ let percentInDay = this.model.getPlayheadTimePercent(false);
+
+ this.view.render({
+ playheadTimePercent: percentInWindow,
+ playheadWithoutZoomPercent: percentInDay,
+ timestamp: timestamp
+ });
+ }
+ onVideoFrame(callback) {
+ this.callbacks.onVideoFrame.push(callback);
+ }
+ onDataFrame(callback) {
+ this.callbacks.onDataFrame.push(callback);
+ }
+ onSegmentDeselected(callback) {
+ this.callbacks.onSegmentDeselected.push(callback);
+ }
+ multiplySpeed(factor = 2, allowLoop = true) {
+ // update the playback speed, which subsequently re-renders the view (button image)
+ this.model.multiplySpeed(factor, allowLoop);
+ }
+ handlePlaybackToggled(isPlaying) {
+ this.view.render({
+ isPlaying: isPlaying
+ });
+
+ // play each of the videos in the view
+ let tracks = this.model.currentDataView.filteredDatabase.tracks;
+ Object.keys(tracks).forEach(trackId => {
+ let videoElements = this.view.getVideoElementsForTrack(trackId);
+ if (videoElements.color && videoElements.depth) {
+ let selectedSegments = this.model.selectedSegments;
+ if (selectedSegments.map(segment => segment.trackId).includes(trackId)) {
+ if (isPlaying) {
+ videoElements.color.play();
+ videoElements.depth.play();
+ } else {
+ videoElements.color.pause();
+ videoElements.depth.pause();
+ }
+ }
+ }
+ });
+ }
+ handleSpeedUpdated(playbackSpeed) {
+ this.view.render({
+ playbackSpeed: playbackSpeed
+ });
+
+ this.model.selectedSegments.forEach(segment => {
+ let colorVideo = this.view.getOrCreateVideoElement(segment.trackId, 'color');
+ let depthVideo = this.view.getOrCreateVideoElement(segment.trackId, 'depth');
+ [colorVideo, depthVideo].forEach(video => {
+ video.playbackRate = playbackSpeed * (segment.timeMultiplier || 1);
+ });
+ });
+ }
+ toggleVisibility(isNowVisible) {
+ if (isNowVisible) {
+ if (!this.model.database) {
+ console.warn('timeline database is null, cant show timeline yet...');
+ return;
+ }
+ this.view.show();
+
+ // zoom to show the most recent date on the timeline
+ let datesWithVideos = this.model.database.getDatesWithData();
+ let mostRecentDate = datesWithVideos.sort((a, b) => {
+ return a.getTime() - b.getTime();
+ })[datesWithVideos.length - 1];
+
+ if (!mostRecentDate) {
+ mostRecentDate = new Date(); // if no data on timeline, default to today
+ }
+ let startOfDay = new Date(mostRecentDate.getFullYear(), mostRecentDate.getMonth(), mostRecentDate.getDate());
+ this.handleCalendarDateSelected(startOfDay);
+
+ } else {
+ this.view.hide();
+ }
+ }
+ handleSegmentSelected(_selectedSegment) {
+ this.renderSelectedSegments();
+ }
+ handleSegmentDeselected(deselectedSegment) {
+ this.renderSelectedSegments();
+ this.callbacks.onSegmentDeselected.forEach(cb => {
+ cb(deselectedSegment); // can use this to hide point clouds when playhead moves away from a segment
+ });
+ }
+ renderSelectedSegments() {
+ let videoSegments = this.model.selectedSegments.filter(segment => segment.type === 'VIDEO_3D');
+ let videoElements = videoSegments.map(segment => {
+ return [
+ { colorOrDepth: 'color', trackId: segment.trackId, src: segment.dataPieces.colorVideo.videoUrl },
+ { colorOrDepth: 'depth', trackId: segment.trackId, src: segment.dataPieces.depthVideo.videoUrl },
+ ];
+ }).flat();
+
+ this.view.render({
+ videoElements: videoElements
+ });
+ }
+ handleSegmentData(segment, _timestamp, _data) {
+ if (segment.type === 'VIDEO_3D') {
+ // TODO: this currently isn't accurate because of a time offset in the recording process
+ // but in theory this can be used for external modules to subscribe to any non-video data on the timeline (such as pose data)
+
+ // let cameraPoseMatrix = segment.dataPieces.poses.getClosestData(timestamp).data;
+ // let colorVideoUrl = segment.dataPieces.colorVideo.videoUrl;
+ // let depthVideoUrl = segment.dataPieces.depthVideo.videoUrl;
+ // let timePercent = segment.getTimestampAsPercent(timestamp);
+ //
+ // this.callbacks.onDataFrame.forEach(cb => {
+ // cb(colorVideoUrl, depthVideoUrl, timePercent, cameraPoseMatrix);
+ // });
+ }
+ // TODO: process other data types (IoT, Human Pose) and trigger other callbacks
+ }
+ }
+
+ exports.TimelineController = TimelineController;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/TimelineDatabase.js b/content_scripts/TimelineDatabase.js
new file mode 100644
index 00000000..8e2e323d
--- /dev/null
+++ b/content_scripts/TimelineDatabase.js
@@ -0,0 +1,232 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+ const TRACK_TYPES = Object.freeze({
+ VIDEO_3D: 'VIDEO_3D',
+ VIDEO_2D: 'VIDEO_2D', // not in use, yet
+ POSE: 'POSE', // not in use, yet
+ IOT: 'IOT' // not in use, yet
+ });
+ const DATA_PIECE_TYPES = Object.freeze({
+ VIDEO_URL: 'VIDEO_URL', // used for VIDEO_3D color and depth
+ TIME_SERIES: 'TIME_SERIES' // used for VIDEO_3D poses
+ });
+
+ // The TimelineDatabase consists of a nested hierarchy of DataTracks -> DataSegments -> DataPieces
+ // These currently correspond to DeviceID -> Recording Session -> (RGB + DEPTH + POSE) data
+ // It is designed to accept a variety of data types, such as pose data or IoT data, which can be added as additional tracks
+ class TimelineDatabase {
+ constructor() {
+ this.tracks = {};
+ }
+ addTrack(track) {
+ this.tracks[track.id] = track;
+ }
+ getBounds() {
+ let minStart = null;
+ let maxEnd = null;
+ for (const [_id, track] of Object.entries(this.tracks)) {
+ let trackBounds = track.getBounds();
+ if (minStart === null || trackBounds.start < minStart) {
+ minStart = trackBounds.start;
+ }
+ if (maxEnd === null || trackBounds.end > maxEnd) {
+ maxEnd = trackBounds.end;
+ }
+ }
+ return {
+ start: minStart,
+ end: maxEnd
+ };
+ }
+ // returns a subset of the tracks, which each contain only the subset of their segments that lie within the specified bounds
+ getFilteredData(minTimestamp, maxTimestamp) {
+ if (typeof minTimestamp !== 'number' || typeof maxTimestamp !== 'number') {
+ return this.tracks;
+ }
+
+ let filteredDatabase = {
+ tracks: {}
+ };
+ for (const [trackId, track] of Object.entries(this.tracks)) {
+ let includeTrack = false;
+ let segmentsToInclude = [];
+ for (const [_segmentId, segment] of Object.entries(track.segments)) {
+ let includeSegment = segment.start >= minTimestamp && segment.end <= maxTimestamp;
+ if (includeSegment) {
+ includeTrack = true;
+ segmentsToInclude.push(segment);
+ }
+ }
+
+ if (includeTrack) {
+ filteredDatabase.tracks[trackId] = new DataTrack(trackId, track.type);
+ segmentsToInclude.forEach(segment => {
+ filteredDatabase.tracks[trackId].addSegment(segment);
+ });
+ }
+ }
+ return filteredDatabase;
+ }
+ getDatesWithData() {
+ let dates = [];
+ for (const [_trackId, track] of Object.entries(this.tracks)) {
+ for (const [_segmentId, segment] of Object.entries(track.segments)) {
+ let date = new Date(segment.start); // TODO: don't assume only lasts one day
+ if (!dates.map(date => JSON.stringify(date)).includes(JSON.stringify(date))) {
+ dates.push(date);
+ }
+ }
+ }
+ return dates;
+ }
+ }
+
+ // A DataTrack contains any number of DataSegments of the same data type (e.g. 3D_VIDEO, 2D_VIDEO, POSES, IOT)
+ // Each DataTrack will be represented by a "layer" on the timeline, for example each unique person or recording device
+ class DataTrack {
+ constructor(id, type) {
+ this.id = id;
+ if (typeof TRACK_TYPES[type] === 'undefined') {
+ console.warn('trying to create an unknown track type');
+ }
+ this.type = type;
+ this.segments = {};
+ }
+ addSegment(segment) {
+ if (segment.type !== this.type) {
+ console.warn('trying to add incompatible segment to track');
+ return;
+ }
+ this.segments[segment.id] = segment;
+ segment.trackId = this.id;
+ }
+ getBounds() {
+ // compute the min/max of segments' starts/ends
+ let minStart = null;
+ let maxEnd = null;
+ for (const [_id, segment] of Object.entries(this.segments)) {
+ if (minStart === null || segment.start < minStart) {
+ minStart = segment.start;
+ }
+ if (maxEnd === null || segment.end > maxEnd) {
+ maxEnd = segment.end;
+ }
+ }
+ return {
+ start: minStart,
+ end: maxEnd
+ };
+ }
+ }
+
+ // A DataSegment is a contiguous data event, such as a 3D video, with a specific start and end time
+ // The actual data payload is contained in one or more DataPieces that the segment contains
+ // DataSegments belong to DataTracks, which will organize them vertically on the timeline.
+ class DataSegment {
+ constructor(id, type, start, end) {
+ this.id = id;
+ if (typeof TRACK_TYPES[type] === 'undefined') {
+ console.warn('trying to create an unknown segment type');
+ }
+ this.type = type;
+ this.start = start;
+ this.end = end;
+ this.dataPieces = {};
+ }
+ addDataPiece(dataPiece) {
+ this.dataPieces[dataPiece.id] = dataPiece;
+ dataPiece.segmentId = this.id;
+ }
+ getTimestampAsPercent(timestamp) {
+ return (timestamp - this.start) / (this.end - this.start);
+ }
+ }
+
+ // A DataPiece is a specific set of data that is attached to a DataSegment
+ // for example, a video or a time-series array of data
+ // A DataSegment can contain multiple DataPieces (e.g. a 3D_VIDEO segment contains 2 videos and an array of poses)
+ class DataPiece {
+ constructor(id, type) {
+ this.id = id;
+ if (typeof DATA_PIECE_TYPES[type] === 'undefined') {
+ console.warn('trying to create an unknown data piece type');
+ }
+ this.type = type;
+ }
+ setVideoUrl(url) {
+ if (this.type !== DATA_PIECE_TYPES.VIDEO_URL) { return; }
+ this.videoUrl = url;
+ }
+ setTimeSeriesData(data) {
+ if (this.type !== DATA_PIECE_TYPES.TIME_SERIES) { return; }
+ if (data.length > 0) {
+ let valid = typeof data[0].data !== 'undefined' && typeof data[0].time !== 'undefined';
+ if (!valid) {
+ console.warn('A TIME_SERIES DataPiece needs the format [{data: _, time: _}, ...]', data[0]);
+ return;
+ }
+ }
+ this.timeSeriesData = data;
+ }
+ getClosestData(timestamp) {
+ if (this.type !== DATA_PIECE_TYPES.TIME_SERIES) { return null; }
+ // TODO: store newer and older so that in future we can have option to interpolate
+ let min_dt = Infinity;
+ let closestEntry = null;
+ this.timeSeriesData.forEach(entry => {
+ let dt = Math.abs(timestamp - entry.time);
+ if (dt < min_dt) {
+ min_dt = dt;
+ closestEntry = entry;
+ }
+ });
+ return closestEntry;
+ }
+ getDataAtIndex(index) {
+ if (this.type !== DATA_PIECE_TYPES.TIME_SERIES) { return null; }
+
+ let clampedIndex = Math.max(0, Math.min(this.timeSeriesData.length - 1, index));
+ return this.timeSeriesData[clampedIndex].data;
+ }
+ }
+
+ // A DataView contains a filteredDatabase, which is a subset of the TimelineDatabase
+ // where all segments are within the start and end timestamps
+ class DataView {
+ constructor(database) {
+ this.start = null;
+ this.end = null;
+ this.database = database;
+ this.filteredDatabase = database;
+ }
+ // updates the filteredDatabase to match the current view
+ setTimeBounds(start, end) {
+ this.start = start;
+ this.end = end;
+ // filter the database, keeping only pointers to segments within the data range
+ this.filteredDatabase = this.database.getFilteredData(start, end);
+ }
+ // given a timestamp, returns all segments that occur at that time (partially overlap/contain that timestamp)
+ processTimestamp(timestamp) {
+ if (!this.filteredDatabase) { return []; }
+ let currentSegments = [];
+ for (const [_trackId, track] of Object.entries(this.filteredDatabase.tracks)) {
+ for (const [_segmentId, segment] of Object.entries(track.segments)) {
+ if (segment.start <= timestamp && segment.end >= timestamp) {
+ currentSegments.push(segment);
+ }
+ }
+ }
+ return currentSegments;
+ }
+ }
+
+ exports.TimelineDatabase = TimelineDatabase;
+ exports.DataTrack = DataTrack;
+ exports.DataSegment = DataSegment;
+ exports.DataPiece = DataPiece;
+ exports.DataView = DataView;
+ exports.TRACK_TYPES = TRACK_TYPES;
+ exports.DATA_PIECE_TYPES = DATA_PIECE_TYPES;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/TimelineModel.js b/content_scripts/TimelineModel.js
new file mode 100644
index 00000000..a600ddb6
--- /dev/null
+++ b/content_scripts/TimelineModel.js
@@ -0,0 +1,209 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+ const MAX_SPEED = 256;
+ const PLAYBACK_FPS = 10; // no need for this to be faster than recording FPS, it'll just waste resources
+
+ // The TimelineModel is the complete data representation needed for the timeline, including the database and the UI state
+ // (such as playhead timestamp, zoom/scroll window, playback speed, etc)
+ class TimelineModel {
+ constructor() {
+ this.database = null;
+ this.currentDataView = null;
+ this.selectedDate = null;
+ this.currentTimestamp = null;
+ this.isPlaying = false;
+ this.playbackSpeed = 1;
+ this.playbackInterval = null;
+ this.lastUpdate = null;
+ this.selectedSegments = [];
+ this.timelineWindow = new realityEditor.videoPlayback.TimelineWindow();
+ this.timelineWindow.onWithoutZoomUpdated(window => {
+ this.handleWindowUpdated(window, true);
+ this.updateDataView(window.bounds.withoutZoom.min, window.bounds.withoutZoom.max);
+ });
+ this.timelineWindow.onCurrentWindowUpdated(window => {
+ this.handleWindowUpdated(window, false);
+ });
+ // controller can subscribe to each of these to update the view in response to data changes
+ this.callbacks = {
+ onDataLoaded: [],
+ onDataViewUpdated: [],
+ onWindowUpdated: [],
+ onTimestampUpdated: [],
+ onPlaybackToggled: [],
+ onSpeedUpdated: [],
+ onSegmentSelected: [],
+ onSegmentDeselected: [],
+ onSegmentData: []
+ };
+ }
+ setDatabase(database) {
+ this.database = database;
+ this.currentDataView = new realityEditor.videoPlayback.DataView(database);
+ this.callbacks.onDataLoaded.forEach(cb => {
+ cb();
+ });
+ }
+ handleWindowUpdated(window, resetPlayhead) {
+ let playheadTime = resetPlayhead ? window.bounds.current.min : this.currentTimestamp;
+ this.setTimestamp(playheadTime);
+
+ this.callbacks.onWindowUpdated.forEach(cb => {
+ cb(window);
+ });
+ }
+ updateDataView(minTimestamp, maxTimestamp) {
+ if (!this.database) { return; }
+
+ // updates the currentDataView.filteredDatabase
+ this.currentDataView.setTimeBounds(minTimestamp, maxTimestamp);
+
+ this.callbacks.onDataViewUpdated.forEach(cb => {
+ cb(this.currentDataView);
+ });
+ }
+ setTimestamp(newTimestamp) {
+ this.currentTimestamp = newTimestamp;
+ this.callbacks.onTimestampUpdated.forEach(cb => {
+ cb(this.currentTimestamp);
+ });
+
+ // determine if there are any overlapping segments
+ if (!this.currentDataView) { return; }
+
+ // trigger onSegmentSelected and onSegmentDeselected callbacks
+ let previousSegments = JSON.parse(JSON.stringify(this.selectedSegments));
+ let currentSegments = this.currentDataView.processTimestamp(newTimestamp);
+ this.selectedSegments = currentSegments;
+ let selectedIds = currentSegments.map(segment => segment.id);
+ let previousIds = previousSegments.map(segment => segment.id);
+ // trigger events based on difference between currentSegments and previous selectedSegments
+ currentSegments.forEach(segment => {
+ if (!previousIds.includes(segment.id)) {
+ this.callbacks.onSegmentSelected.forEach(cb => {
+ cb(segment);
+ });
+ }
+ });
+ previousSegments.forEach(segment => {
+ if (!selectedIds.includes(segment.id)) {
+ this.callbacks.onSegmentDeselected.forEach(cb => {
+ cb(segment);
+ });
+ }
+ });
+
+ // trigger onSegmentData callbacks to process the dataPieces that occur at this timestamp
+ currentSegments.forEach(segment => {
+ this.callbacks.onSegmentData.forEach(cb => {
+ cb(segment, newTimestamp, segment.dataPieces);
+ });
+ });
+ }
+ getPlayheadTimePercent(inWindow) {
+ let min, max;
+ if (inWindow) {
+ min = this.timelineWindow.bounds.current.min;
+ max = this.timelineWindow.bounds.current.max;
+ } else {
+ min = this.timelineWindow.bounds.withoutZoom.min;
+ max = this.timelineWindow.bounds.withoutZoom.max;
+ }
+ return (this.currentTimestamp - min) / (max - min);
+ }
+ getTimestampAsPercent(timestamp) {
+ let bounds = this.timelineWindow.bounds;
+ return {
+ withoutZoom: (timestamp - bounds.withoutZoom.min) / (bounds.withoutZoom.max - bounds.withoutZoom.min),
+ currentWindow: (timestamp - bounds.current.min) / (bounds.current.max - bounds.current.min)
+ };
+ }
+ setTimestampFromPositionInWindow(percentInWindow) {
+ let min = this.timelineWindow.bounds.current.min;
+ let max = this.timelineWindow.bounds.current.max;
+ let current = min + (max - min) * percentInWindow;
+ this.setTimestamp(current);
+ }
+ adjustCurrentWindow(leftPercent, rightPercent) {
+ this.timelineWindow.setCurrentFromPercent(leftPercent, rightPercent);
+ }
+ togglePlayback(toggled) {
+ if (this.isPlaying === toggled) { return; }
+ this.isPlaying = toggled;
+
+ // create a loop that will increment the currentTimestamp as time passes
+ if (this.isPlaying) {
+ if (!this.playbackInterval) {
+ this.lastUpdate = Date.now();
+ this.playbackInterval = setInterval(() => {
+ let now = Date.now();
+ let dt = now - this.lastUpdate;
+ this.lastUpdate = now;
+ this.playbackLoop(dt);
+ }, 1000 / PLAYBACK_FPS);
+ }
+ } else if (this.playbackInterval) {
+ clearInterval(this.playbackInterval);
+ this.playbackInterval = null;
+ }
+
+ // play videos, begin data processing, etc
+ this.callbacks.onPlaybackToggled.forEach(cb => {
+ cb(this.isPlaying);
+ });
+ }
+ playbackLoop(dt) {
+ // update the timestamp based on time passed (but stop if reached end of timeline)
+ let newTime = this.currentTimestamp + dt * this.playbackSpeed;
+ if (newTime > this.timelineWindow.bounds.withoutZoom.max) {
+ newTime = this.timelineWindow.bounds.withoutZoom.max;
+ this.togglePlayback(false);
+ }
+ this.setTimestamp(newTime); // this will process any data segments
+ }
+ multiplySpeed(factor, allowLoop) {
+ this.playbackSpeed *= factor;
+ if (this.playbackSpeed > MAX_SPEED) {
+ this.playbackSpeed = allowLoop ? 1 : MAX_SPEED;
+ } else if (this.playbackSpeed < 1) {
+ this.playbackSpeed = allowLoop ? MAX_SPEED : 1;
+ }
+ this.callbacks.onSpeedUpdated.forEach(cb => {
+ cb(this.playbackSpeed);
+ });
+ }
+ /*
+ Callback Subscription Methods
+ */
+ onDataLoaded(callback) {
+ this.callbacks.onDataLoaded.push(callback);
+ }
+ onDataViewUpdated(callback) {
+ this.callbacks.onDataViewUpdated.push(callback);
+ }
+ onWindowUpdated(callback) {
+ this.callbacks.onWindowUpdated.push(callback);
+ }
+ onTimestampUpdated(callback) {
+ this.callbacks.onTimestampUpdated.push(callback);
+ }
+ onPlaybackToggled(callback) {
+ this.callbacks.onPlaybackToggled.push(callback);
+ }
+ onSpeedUpdated(callback) {
+ this.callbacks.onSpeedUpdated.push(callback);
+ }
+ onSegmentSelected(callback) {
+ this.callbacks.onSegmentSelected.push(callback);
+ }
+ onSegmentDeselected(callback) {
+ this.callbacks.onSegmentDeselected.push(callback);
+ }
+ onSegmentData(callback) {
+ this.callbacks.onSegmentData.push(callback);
+ }
+ }
+
+ exports.TimelineModel = TimelineModel;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/TimelineView.js b/content_scripts/TimelineView.js
new file mode 100644
index 00000000..9fc0bf45
--- /dev/null
+++ b/content_scripts/TimelineView.js
@@ -0,0 +1,694 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+ const ZOOM_EXPONENT = 0.5; // the zoom bar doesn't zoom linearly with position
+ const MAX_ZOOM_FACTOR = 96; // 96 means maximum zoom narrows down 24 hours to 15 minutes
+ const SUPPORTED_SPEEDS = [1, 2, 4, 8, 16, 32, 64, 128, 256]; // only have these SVGs for now
+ const TRACK_HEIGHT_PERCENT = 80.0; // other 20% of container is split among margins between each track
+ // constants that need to be updated if SVG size or CSS is updated:
+ const PLAYHEAD_WIDTH = 20;
+ const TRACK_CONTAINER_MARGIN = 20;
+ const VIDEO_PREVIEW_CONTAINER_OFFSET = 160;
+ const PLAYHEAD_DOT_WIDTH = 10;
+ const ZOOM_BAR_MARGIN = 20;
+
+ // The TimelineView is responsible for creating and updating the DOM elements to match the TimelineModel
+ // It's main input is the render function, which can be triggered to update the UI with the state passed in
+ // It contains a few callbacks that the Controller can subscribe to in order to update the model in response to interactions
+ class TimelineView {
+ constructor(parent) {
+ this.playButton = null;
+ this.speedButton = null;
+ this.calendarButton = null;
+ this.callbacks = {
+ onZoomHandleChanged: [],
+ onPlayheadSelected: [],
+ onPlayheadChanged: [],
+ onScrollbarChanged: [],
+ onVideoElementAdded: []
+ };
+ this.videoElements = {};
+ this.buildDomElement(parent);
+ }
+ onZoomHandleChanged(callback) {
+ this.callbacks.onZoomHandleChanged.push(callback);
+ }
+ onPlayheadSelected(callback) {
+ this.callbacks.onPlayheadSelected.push(callback);
+ }
+ onPlayheadChanged(callback) {
+ this.callbacks.onPlayheadChanged.push(callback);
+ }
+ onScrollbarChanged(callback) {
+ this.callbacks.onScrollbarChanged.push(callback);
+ }
+ buildDomElement(parent) {
+ // create a timeline, a playhead on the timeline for scrolling, and play/pause/controls
+ this.timelineContainer = this.createTimelineElement();
+ parent.appendChild(this.timelineContainer);
+
+ // create containers for two preview videos
+ let videoPreviewContainer = document.getElementById('timelineVideoPreviewContainer');
+
+ let colorPreviewContainer = document.createElement('div');
+ colorPreviewContainer.classList.add('videoPreviewContainer');
+ colorPreviewContainer.id = 'timelineColorPreviewContainer';
+
+ let depthPreviewContainer = document.createElement('div');
+ depthPreviewContainer.classList.add('videoPreviewContainer');
+ depthPreviewContainer.id = 'timelineDepthPreviewContainer';
+ depthPreviewContainer.style.left = 256 + 'px';
+
+ videoPreviewContainer.appendChild(colorPreviewContainer);
+ // videoPreviewContainer.appendChild(depthPreviewContainer);
+ }
+ createTimelineElement() {
+ let container = document.createElement('div');
+ container.id = 'timelineContainer';
+ // container has a left box to hold date/time, a center box for the timeline, and a right box for playback controls
+ let leftBox = document.createElement('div');
+ let centerBox = document.createElement('div');
+ let rightBox = document.createElement('div');
+ [leftBox, centerBox, rightBox].forEach(elt => {
+ elt.classList.add('timelineBox');
+ container.appendChild(elt);
+ });
+ leftBox.id = 'timelineVisibilityBox';
+ centerBox.id = 'timelineTrackBox';
+ rightBox.id = 'timelineControlsBox';
+
+ // extra div added to box with slightly different dimensions, to hide the horizontal overflow
+ let centerScrollBox = document.createElement('div');
+ centerScrollBox.id = 'timelineTrackScrollBox';
+ centerBox.appendChild(centerScrollBox);
+
+ let innerScrollBox = document.createElement('div');
+ innerScrollBox.id = 'timelineTrackScrollBoxInner';
+ centerScrollBox.appendChild(innerScrollBox);
+
+ // the element that will actually hold the data tracks and segments
+ let timelineTracksContainer = document.createElement('div');
+ timelineTracksContainer.id = 'timelineTracksContainer';
+ innerScrollBox.appendChild(timelineTracksContainer);
+
+ let timestampDisplay = document.createElement('div');
+ timestampDisplay.id = 'timelineTimestampDisplay';
+ timestampDisplay.innerText = this.getFormattedTime(new Date(0));
+ leftBox.appendChild(timestampDisplay);
+
+ let dateDisplay = document.createElement('div');
+ dateDisplay.id = 'timelineDateDisplay';
+ dateDisplay.innerText = this.getFormattedDate(Date.now());
+ leftBox.appendChild(dateDisplay);
+
+ let calendarButton = document.createElement('img');
+ calendarButton.id = 'timelineCalendarButton';
+ calendarButton.src = 'addons/vuforia-spatial-remote-operator-addon/calendarButton.svg';
+ leftBox.appendChild(calendarButton);
+ this.calendarButton = calendarButton;
+
+ let zoomBar = this.createZoomBar();
+ zoomBar.id = 'timelineZoomBar';
+ leftBox.appendChild(zoomBar);
+
+ let playhead = document.createElement('img');
+ playhead.id = 'timelinePlayhead';
+ playhead.src = 'addons/vuforia-spatial-remote-operator-addon/timelinePlayhead.svg';
+ innerScrollBox.appendChild(playhead);
+ this.playhead = playhead;
+
+ let playheadDot = document.createElement('div');
+ playheadDot.id = 'timelinePlayheadDot';
+ innerScrollBox.appendChild(playheadDot);
+
+ let videoPreviewContainer = document.createElement('div');
+ videoPreviewContainer.id = 'timelineVideoPreviewContainer';
+ videoPreviewContainer.classList.add('timelineBox');
+ videoPreviewContainer.classList.add('timelineVideoPreviewNoTrack'); // need to click on timeline to select
+ innerScrollBox.appendChild(videoPreviewContainer);
+ // left = -68px is most left as possible
+ // width = 480px for now, to show both, but should change to 240px eventually
+
+ let scrollBar = this.createScrollBar();
+ scrollBar.id = 'timelineScrollBar';
+ innerScrollBox.appendChild(scrollBar);
+
+ let playButton = document.createElement('img');
+ playButton.id = 'timelinePlayButton';
+ playButton.src = 'addons/vuforia-spatial-remote-operator-addon/playButton.svg';
+ this.playButton = playButton;
+
+ // let seekButton = document.createElement('img');
+ // seekButton.id = 'timelineSeekButton';
+ // seekButton.src = 'addons/vuforia-spatial-remote-operator-addon/seekButton.svg';
+
+ let speedButton = document.createElement('img');
+ speedButton.id = 'timelineSpeedButton';
+ speedButton.src = 'addons/vuforia-spatial-remote-operator-addon/speedButton_1x.svg';
+ this.speedButton = speedButton;
+
+ [playButton, speedButton].forEach(elt => {
+ elt.classList.add('timelineControlButton');
+ rightBox.appendChild(elt);
+ });
+
+ this.setupPlayhead();
+
+ return container;
+ }
+ createZoomBar() {
+ let container = document.createElement('div');
+ let slider = document.createElement('img');
+ slider.id = 'zoomSliderBackground';
+ slider.src = 'addons/vuforia-spatial-remote-operator-addon/zoomSliderBackground.svg';
+ container.appendChild(slider);
+ let handle = document.createElement('img');
+ handle.id = 'zoomSliderHandle';
+ handle.src = 'addons/vuforia-spatial-remote-operator-addon/zoomSliderHandle.svg';
+ container.appendChild(handle);
+ let isDown = false;
+ handle.addEventListener('pointerdown', _e => {
+ isDown = true;
+ });
+ document.addEventListener('pointerup', _e => {
+ isDown = false;
+ });
+ document.addEventListener('pointercancel', _e => {
+ isDown = false;
+ });
+ document.addEventListener('pointermove', e => {
+ if (!isDown) { return; }
+ let pointerX = e.pageX;
+ let leftMargin = ZOOM_BAR_MARGIN;
+ let rightMargin = ZOOM_BAR_MARGIN;
+ let sliderRect = slider.getBoundingClientRect();
+ let handleWidth = handle.getBoundingClientRect().width;
+
+ if (pointerX < (sliderRect.left + leftMargin)) {
+ handle.style.left = leftMargin - (handleWidth / 2) + 'px';
+ } else if (pointerX > (sliderRect.right - rightMargin)) {
+ handle.style.left = (sliderRect.width - rightMargin - handleWidth / 2) + 'px';
+ } else {
+ handle.style.left = pointerX - sliderRect.left - (handleWidth / 2) + 'px';
+ }
+
+ // we scale from linear to sqrt so that it zooms in faster when it is further zoomed out than when it is already zoomed in a lot
+ let linearZoom = (parseFloat(handle.style.left) - handleWidth / 2) / ((sliderRect.right - rightMargin) - (sliderRect.left + leftMargin));
+ let percentZoom = Math.pow(Math.max(0, linearZoom), ZOOM_EXPONENT);
+ let MAX_ZOOM = 1.0 - (1.0 / MAX_ZOOM_FACTOR); // max zoom level is 96x (15 minutes vs 1 day)
+
+ // trigger callbacks to respond to the updated GUI
+ this.callbacks.onZoomHandleChanged.forEach(cb => {
+ cb(Math.max(0, Math.min(MAX_ZOOM, percentZoom)));
+ });
+
+ // update the scrollbar, which in return will update the model dataview
+ });
+ return container;
+ }
+ onZoomChanged(zoomPercent, playheadTimePercent, scrollbarLeftPercent) { // TODO: make use of render functions to simplify
+ // make the zoom bar handle fill 1.0 - zoomPercent of the overall bar
+ let scrollBar = document.getElementById('timelineScrollBar');
+ let handle = scrollBar.querySelector('.timelineScrollBarHandle');
+ let playheadDot = document.getElementById('timelinePlayheadDot');
+ let trackBox = document.getElementById('timelineTrackBox');
+ handle.style.width = (1.0 - zoomPercent) * 100 + '%';
+
+ if (zoomPercent < 0.01) {
+ scrollBar.classList.add('timelineScrollBarFadeout');
+ playheadDot.classList.add('timelineScrollBarFadeout');
+ } else {
+ scrollBar.classList.remove('timelineScrollBarFadeout');
+ playheadDot.classList.remove('timelineScrollBarFadeout');
+ }
+
+ let handleRect = handle.getBoundingClientRect();
+ let scrollBarRect = scrollBar.getBoundingClientRect();
+ let trackBoxRect = trackBox.getBoundingClientRect();
+ let newWidth = handleRect.width;
+ let maxLeft = scrollBarRect.width - newWidth;
+
+ if (typeof scrollbarLeftPercent === 'undefined') {
+ // keep the timeline playhead at the same timestamp and zoom around that
+ let containerWidth = trackBoxRect.width;
+ let playheadElement = document.getElementById('timelinePlayhead');
+ let leftMargin = 20;
+ let rightMargin = 20;
+ let halfPlayheadWidth = 10;
+ let playheadLeft = parseInt(playheadElement.style.left) || halfPlayheadWidth;
+ let playheadTimePercentWindow = (playheadLeft + halfPlayheadWidth - leftMargin) / (containerWidth - halfPlayheadWidth - leftMargin - rightMargin);
+
+ // let absoluteTime = this.dayBounds.min + playheadTimePercentDay * this.DAY_LENGTH;
+
+ // TODO: separate metadata time from window time from day length so we can perform these calculations
+ // let playheadTimePercent = (this.playheadTimestamp - this.trackInfo.metadata.minTime) / (this.trackInfo.metadata.maxTime - this.trackInfo.metadata.minTime);
+ // console.log('timepercent = ' + playheadTimePercent);
+
+ // reposition the scrollbar handle left so that it would keep the playhead at the same spot.
+
+ // if previous leftPercent is 0, new leftPercent is 0
+ // move scrollbar handle to playheadTimePercent, then move it so playhead is playheadTimePercent within the handle width
+ let newLeft = (playheadTimePercent * scrollBarRect.width) - (playheadTimePercentWindow * handleRect.width);
+ // TODO: this is off if you scroll in halfway, move playhead sideways, then continue scrolling
+ handle.style.left = Math.max(0, Math.min(maxLeft, newLeft)) + 'px';
+
+ } else {
+ let newLeft = scrollbarLeftPercent * scrollBar.getBoundingClientRect().width;
+ handle.style.left = Math.max(0, Math.min(maxLeft, newLeft)) + 'px';
+ }
+ handleRect = handle.getBoundingClientRect(); // recompute after moving handle
+
+ let startPercent = (handleRect.left - scrollBarRect.left) / scrollBarRect.width;
+ let endPercent = (handleRect.right - scrollBarRect.left) / scrollBarRect.width;
+ // this.onTimelineWindowChanged(zoomPercent, startPercent, endPercent);
+
+ this.callbacks.onScrollbarChanged.forEach(cb => {
+ cb(zoomPercent, startPercent, endPercent);
+ });
+ }
+ createScrollBar() {
+ let container = document.createElement('div');
+ let handle = document.createElement('div');
+ container.appendChild(handle);
+ // container.classList.add('hiddenScrollBar'); // TODO: add this after a few seconds of not interacting or hovering over the scrollable panel
+ container.classList.add('timelineScrollBarContainer');
+ handle.classList.add('timelineScrollBarHandle');
+ let isDown = false;
+ let pointerOffset = 0;
+ handle.addEventListener('pointerdown', e => {
+ isDown = true;
+ let handleRect = handle.getBoundingClientRect();
+ pointerOffset = e.pageX - (handleRect.left + handleRect.width / 2);
+ });
+ document.addEventListener('pointerup', _e => {
+ isDown = false;
+ });
+ document.addEventListener('pointercancel', _e => {
+ isDown = false;
+ });
+ document.addEventListener('pointermove', e => {
+ if (!isDown) { return; }
+ let pointerX = e.pageX;
+ let containerRect = container.getBoundingClientRect();
+ let handleRect = handle.getBoundingClientRect();
+
+ if (pointerX < containerRect.left + handleRect.width / 2) {
+ handle.style.left = '0px';
+ } else if (pointerX > containerRect.right - handleRect.width / 2) {
+ handle.style.left = (containerRect.width - handleRect.width) + 'px';
+ } else {
+ handle.style.left = (pointerX - pointerOffset) - (containerRect.left + handleRect.width / 2) + 'px';
+ }
+
+ handleRect = handle.getBoundingClientRect(); // recompute handleRect after moving handle
+ let startPercent = (handleRect.left - containerRect.left) / containerRect.width;
+ let endPercent = (handleRect.right - containerRect.left) / containerRect.width;
+ let zoomPercent = 1.0 - handleRect.width / containerRect.width;
+
+ this.callbacks.onScrollbarChanged.forEach(cb => {
+ cb(zoomPercent, startPercent, endPercent);
+ });
+ });
+ return container;
+ }
+ setupPlayhead() {
+ let playheadElement = this.playhead;
+ document.addEventListener('pointermove', e => {
+ this.onDocumentPointerMove(e);
+ });
+ playheadElement.addEventListener('pointerdown', _e => {
+ this.playheadClickedDown = true;
+ playheadElement.classList.add('timelinePlayheadSelected');
+
+ let playheadDot = document.getElementById('timelinePlayheadDot');
+ playheadDot.classList.add('timelinePlayheadSelected');
+
+ let videoPreview = document.getElementById('timelineVideoPreviewContainer');
+ videoPreview.classList.add('timelineVideoPreviewSelected');
+
+ this.callbacks.onPlayheadSelected.forEach(cb => {
+ cb();
+ });
+ });
+ document.addEventListener('pointerup', e => {
+ this.onDocumentPointerUp(e);
+ });
+ document.addEventListener('pointercancel', e => {
+ this.onDocumentPointerUp(e);
+ });
+ }
+ onDocumentPointerUp(_e) {
+ // reset playhead selection
+ this.playheadClickedDown = false;
+ let playheadElement = document.getElementById('timelinePlayhead');
+ playheadElement.classList.remove('timelinePlayheadSelected');
+
+ let playheadDot = document.getElementById('timelinePlayheadDot');
+ playheadDot.classList.remove('timelinePlayheadSelected');
+
+ let videoPreview = document.getElementById('timelineVideoPreviewContainer');
+ videoPreview.classList.remove('timelineVideoPreviewSelected');
+ }
+ onDocumentPointerMove(e) {
+ if (this.playheadClickedDown) {
+ this.onPointerMovePlayhead(e);
+ }
+ }
+ onPointerMovePlayhead(e) {
+ let playheadElement = document.getElementById('timelinePlayhead');
+
+ // calculate new X position to follow mouse, constrained to trackBox element
+ let pointerX = e.pageX;
+
+ let trackBox = document.getElementById('timelineTrackBox');
+ let trackBoxRect = trackBox.getBoundingClientRect();
+
+ let relativeX = pointerX - trackBoxRect.left;
+ let leftMargin = 20;
+ let rightMargin = 20;
+ let halfPlayheadWidth = 10;
+ playheadElement.style.left = Math.min(trackBoxRect.width - halfPlayheadWidth - rightMargin, Math.max(leftMargin, relativeX)) - halfPlayheadWidth + 'px';
+ let playheadLeft = parseInt(playheadElement.style.left) || halfPlayheadWidth;
+ // move timelineVideoPreviewContainer to correct spot (constrained to -68px < left < (innerWidth - 588)
+ this.displayPlayheadVideoPreview(playheadLeft, halfPlayheadWidth);
+
+ let playheadTimePercentWindow = (playheadLeft + halfPlayheadWidth - leftMargin) / (trackBoxRect.width - halfPlayheadWidth - leftMargin - rightMargin);
+
+ this.callbacks.onPlayheadChanged.forEach(cb => {
+ cb(playheadTimePercentWindow);
+ });
+ }
+ /**
+ * Update the GUI in response to new data from the model/controller
+ * @param {{playheadTimePercent: number, timestamp: number, zoomPercent: number, scrollLeftPercent: number,
+ * isPlaying: boolean, playbackSpeed: number, tracks: {}}} props
+ */
+ render(props) {
+ // don't render if the timeline view is hidden
+ if (this.timelineContainer.classList.contains('hiddenTimeline')) {
+ return;
+ }
+
+ if (typeof props.playheadTimePercent !== 'undefined') {
+ this.displayPlayhead(props.playheadTimePercent);
+ }
+ if (typeof props.playheadWithoutZoomPercent !== 'undefined') {
+ this.displayPlayheadDot(props.playheadWithoutZoomPercent);
+ }
+ if (typeof props.timestamp !== 'undefined') {
+ this.displayTime(props.timestamp);
+ }
+ if (typeof props.zoomPercent !== 'undefined') {
+ this.displayZoom(props.zoomPercent);
+ if (typeof props.scrollLeftPercent !== 'undefined') {
+ this.displayScroll(props.scrollLeftPercent, props.zoomPercent);
+ }
+ }
+ if (typeof props.isPlaying !== 'undefined') {
+ this.displayIsPlaying(props.isPlaying);
+ }
+ if (typeof props.playbackSpeed !== 'undefined') {
+ this.displayPlaybackSpeed(props.playbackSpeed);
+ }
+ // TODO: add a more optimized pathway if we know the filtered database hasn't changed
+ // e.g. (no new tracks/segments, just repositioning them within the changing window)
+ if (typeof props.tracks !== 'undefined') {
+ let fullUpdate = props.tracksFullUpdate;
+ if (fullUpdate) {
+ this.fullUpdateTracks(props.tracks);
+ } else {
+ this.updateTracks(props.tracks);
+ }
+ // this.displayTracks(props.tracks, fullUpdate);
+ }
+ if (typeof props.videoElements !== 'undefined') {
+ this.displayVideoElements(props.videoElements);
+ }
+ }
+ /**
+ * @param {{colorOrDepth: string, trackId: string, src: string}[]} videoElements
+ */
+ displayVideoElements(videoElements) {
+ // hide video preview if no elements in array
+ if (videoElements.length === 0) {
+ let videoPreviewContainer = document.getElementById('timelineVideoPreviewContainer');
+ videoPreviewContainer.classList.add('timelineVideoPreviewNoTrack');
+ let colorContainer = document.getElementById('timelineColorPreviewContainer');
+ while (colorContainer.firstChild) {
+ colorContainer.removeChild(colorContainer.firstChild);
+ }
+ }
+
+ videoElements.forEach(info => {
+ // it is necessary to update the color and depth video elements with the correct src for this segment
+ // since the point cloud rendering depends on reading pixel data from these video elements
+ let videoElement = this.getOrCreateVideoElement(info.trackId, info.colorOrDepth, info.src);
+
+ if (info.colorOrDepth !== 'color') { return; }
+
+ // add color video to preview container
+ let colorContainer = document.getElementById('timelineColorPreviewContainer');
+ while (colorContainer.firstChild) {
+ colorContainer.removeChild(colorContainer.firstChild);
+ }
+ colorContainer.appendChild(videoElement);
+ let videoPreviewContainer = document.getElementById('timelineVideoPreviewContainer');
+ videoPreviewContainer.classList.remove('timelineVideoPreviewNoTrack');
+ console.log('videoElement added to video preview container');
+ });
+ }
+ getOrCreateVideoElement(trackId, colorOrDepth, src) {
+ if (colorOrDepth !== 'color' && colorOrDepth !== 'depth') { console.warn('colorOrDepth is invalid in getVideoElement'); }
+
+ if (typeof this.videoElements[trackId] === 'undefined') {
+ this.videoElements[trackId] = { color: null, depth: null };
+ }
+
+ let videoElement = this.videoElements[trackId][colorOrDepth];
+ if (!videoElement) {
+ videoElement = this.createVideoElement(trackId, colorOrDepth);
+ }
+
+ // updates the src of this videoElement. there is one videoElement per track, but each time the segment changes this will update
+ if (typeof src !== 'undefined') {
+ let filename = src.replace(/^.*[\\\/]/, '');
+ if (!videoElement.querySelector('source').src.includes(filename)) {
+ videoElement.querySelector('source').src = '/virtualizer_recording/' + trackId + '/' + colorOrDepth + '/' + filename;
+ videoElement.load();
+ this.callbacks.onVideoElementAdded.forEach(cb => {
+ cb(videoElement, colorOrDepth);
+ });
+ }
+ }
+ return videoElement;
+ }
+ getVideoElementsForTrack(trackId) {
+ if (!this.videoElements[trackId]) { return { color: null, depth: null }; }
+ return {
+ color: this.videoElements[trackId].color,
+ depth: this.videoElements[trackId].depth
+ };
+ }
+ createVideoElement(trackId, colorOrDepth) {
+ const id = colorOrDepth + '_video_' + trackId;
+ let video = document.createElement('video');
+ video.id = id;
+ video.classList.add('videoPreview');
+ video.setAttribute('width', '256');
+ // video.setAttribute('controls', 'controls');
+ video.setAttribute('muted', 'muted');
+ let source = document.createElement('source');
+ video.appendChild(source);
+
+ if (colorOrDepth === 'color') {
+ this.videoElements[trackId].color = video;
+ } else if (colorOrDepth === 'depth') {
+ this.videoElements[trackId].depth = video;
+ }
+ return video;
+ }
+ onVideoElementAdded(callback) {
+ this.callbacks.onVideoElementAdded.push(callback);
+ }
+ displayPlayhead(percentInWindow) {
+ let trackBox = document.getElementById('timelineTrackBox');
+ let containerWidth = trackBox.getBoundingClientRect().width;
+ let halfPlayheadWidth = PLAYHEAD_WIDTH / 2;
+ let leftMargin = TRACK_CONTAINER_MARGIN;
+ let rightMargin = TRACK_CONTAINER_MARGIN;
+ let playheadLeft = (percentInWindow * (containerWidth - halfPlayheadWidth - leftMargin - rightMargin)) - (halfPlayheadWidth - leftMargin);
+ let playheadElement = document.getElementById('timelinePlayhead');
+ playheadElement.style.left = playheadLeft + 'px';
+
+ this.displayPlayheadVideoPreview(playheadLeft, halfPlayheadWidth);
+ }
+ displayPlayheadVideoPreview(playheadLeft, halfPlayheadWidth) {
+ let videoPreviewContainer = document.getElementById('timelineVideoPreviewContainer');
+ let videoPreviewContainerRect = videoPreviewContainer.getBoundingClientRect();
+ if (videoPreviewContainer && videoPreviewContainerRect) {
+ let previewWidth = videoPreviewContainerRect.width;
+ let previewRelativeX = playheadLeft + halfPlayheadWidth - previewWidth / 2;
+ let timelineTracksRight = document.getElementById('timelineTracksContainer').getBoundingClientRect().right;
+ let clampMin = 0;
+ let clampMax = (timelineTracksRight - previewWidth) - VIDEO_PREVIEW_CONTAINER_OFFSET;
+ videoPreviewContainer.style.left = Math.min(clampMax, Math.max(clampMin, previewRelativeX)) + 'px';
+ }
+ }
+ displayPlayheadDot(percentInDay) {
+ // put a little dot on the scrollbar showing the currentWindow-agnostic position of the playhead
+ let playheadDot = document.getElementById('timelinePlayheadDot');
+ let trackBox = document.getElementById('timelineTrackBox');
+ let containerWidth = trackBox.getBoundingClientRect().width;
+ let leftMargin = TRACK_CONTAINER_MARGIN;
+ let rightMargin = TRACK_CONTAINER_MARGIN;
+ let halfPlayheadWidth = PLAYHEAD_WIDTH / 2;
+ let halfDotWidth = PLAYHEAD_DOT_WIDTH / 2;
+ playheadDot.style.left = (leftMargin - halfDotWidth) + percentInDay * (containerWidth - halfPlayheadWidth - leftMargin - rightMargin) + 'px';
+ }
+ displayTime(timestamp) {
+ let textfield = document.getElementById('timelineTimestampDisplay');
+ textfield.innerText = this.getFormattedTime(timestamp);
+ let dateTextfield = document.getElementById('timelineDateDisplay');
+ dateTextfield.innerText = this.getFormattedDate(timestamp);
+ }
+ displayZoom(zoomPercent) {
+ let slider = document.getElementById('zoomSliderBackground');
+ let handle = document.getElementById('zoomSliderHandle');
+ let leftMargin = ZOOM_BAR_MARGIN;
+ let sliderRect = slider.getBoundingClientRect();
+ let handleWidth = handle.getBoundingClientRect().width;
+
+ // percentZoom = Math.pow(Math.max(0, linearZoom), 0.25)
+ let linearZoom = Math.pow(zoomPercent, 1.0 / ZOOM_EXPONENT);
+ let handleLeft = linearZoom * ((sliderRect.right - leftMargin) - (sliderRect.left + leftMargin)) + (handleWidth / 2);
+ handle.style.left = handleLeft + 'px';
+ }
+ displayScroll(scrollLeftPercent, zoomPercent) {
+ // make the zoom bar handle fill 1.0 - zoomPercent of the overall bar
+ let scrollBar = document.getElementById('timelineScrollBar');
+ let handle = scrollBar.querySelector('.timelineScrollBarHandle');
+ let trackBox = document.getElementById('timelineTrackBox');
+ let containerWidth = trackBox.getBoundingClientRect().width;
+ let leftMargin = TRACK_CONTAINER_MARGIN;
+ let rightMargin = TRACK_CONTAINER_MARGIN;
+ let halfPlayheadWidth = PLAYHEAD_WIDTH / 2;
+
+ handle.style.width = (1.0 - zoomPercent) * 100 + '%';
+ handle.style.left = scrollLeftPercent * (containerWidth - halfPlayheadWidth - leftMargin - rightMargin) + 'px';
+
+ if (zoomPercent < 0.01) {
+ scrollBar.classList.add('timelineScrollBarFadeout');
+ } else {
+ scrollBar.classList.remove('timelineScrollBarFadeout');
+ }
+ }
+ displayIsPlaying(isPlaying) {
+ let playButton = document.getElementById('timelinePlayButton');
+ let playheadElement = document.getElementById('timelinePlayhead');
+ let playheadDot = document.getElementById('timelinePlayheadDot');
+ if (isPlaying) {
+ playButton.src = 'addons/vuforia-spatial-remote-operator-addon/pauseButton.svg';
+ playheadElement.classList.add('timelinePlayheadPlaying');
+ playheadDot.classList.add('timelinePlayheadPlaying');
+ } else {
+ playButton.src = 'addons/vuforia-spatial-remote-operator-addon/playButton.svg';
+ playheadElement.classList.remove('timelinePlayheadPlaying');
+ playheadDot.classList.remove('timelinePlayheadPlaying');
+ }
+ }
+ displayPlaybackSpeed(playbackSpeed) {
+ if (!SUPPORTED_SPEEDS.includes(playbackSpeed)) {
+ console.warn('no SVG button for playback speed ' + playbackSpeed);
+ }
+ let speedButton = document.getElementById('timelineSpeedButton');
+ speedButton.src = 'addons/vuforia-spatial-remote-operator-addon/speedButton_' + playbackSpeed + 'x.svg';
+ }
+ getFormattedTime(relativeTimestamp) {
+ return new Date(relativeTimestamp).toLocaleTimeString();
+ }
+ getFormattedDate(timestamp) { // Format: 'Mon, Apr 18, 2022'
+ return new Date(timestamp).toLocaleDateString('en-us', {
+ weekday: 'short', year: 'numeric', month: 'short', day: 'numeric'
+ });
+ }
+ // completely deletes and re-creates tracks and segments
+ fullUpdateTracks(tracks) {
+ console.log('fullUpdate tracks');
+ let numTracks = Object.keys(tracks).length;
+ let container = document.getElementById('timelineTracksContainer');
+ while (container.firstChild) {
+ container.removeChild(container.firstChild);
+ }
+ Object.entries(tracks).forEach(([trackId, track]) => {
+ let index = Object.keys(tracks).indexOf(trackId); // get a consistent index across both for-each loops
+ console.log('creating elements for track: ' + trackId);
+ let trackElement = document.createElement('div');
+ trackElement.classList.add('timelineTrack');
+ trackElement.id = this.getTrackElementId(trackId);
+ container.appendChild(trackElement);
+ this.positionAndScaleTrack(trackElement, track, index, numTracks);
+
+ Object.entries(track.segments).forEach(([segmentId, segment]) => {
+ let segmentElement = document.createElement('div');
+ segmentElement.classList.add('timelineSegment');
+ segmentElement.id = this.getSegmentElementId(trackId, segmentId);
+ trackElement.appendChild(segmentElement);
+ this.positionAndScaleSegment(segmentElement, segment);
+ });
+ });
+ }
+ // doesn't delete tracks/segments, just moves them around (use this if scrolling/zooming, use fullUpdate if changing dataset)
+ updateTracks(tracks) {
+ let numTracks = Object.keys(tracks).length;
+ let container = document.getElementById('timelineTracksContainer');
+
+ // compute a quick checksum to ensure we have the right data to update
+ let childrenChecksum = Array.from(container.children).map(elt => elt.id).join('');
+ let tracksChecksum = Object.keys(tracks).map(id => this.getTrackElementId(id)).join('');
+ if (childrenChecksum !== tracksChecksum) {
+ console.warn('tracks needs a full update instead... performing one now');
+ this.fullUpdateTracks(tracks);
+ return;
+ }
+
+ Object.entries(tracks).forEach(([trackId, track]) => {
+ let index = Object.keys(tracks).indexOf(trackId); // get a consistent index across both for-each loops
+ let elementId = this.getTrackElementId(trackId);
+ let trackElement = document.getElementById(elementId);
+ this.positionAndScaleTrack(trackElement, track, index, numTracks);
+
+ Object.entries(track.segments).forEach(([segmentId, segment]) => {
+ // let index = Object.keys(tracks).indexOf(trackId); // get a consistent index across both for-each loops
+ let elementId = this.getSegmentElementId(trackId, segmentId);
+ let segmentElement = document.getElementById(elementId);
+ this.positionAndScaleSegment(segmentElement, segment);
+ });
+ });
+ }
+ positionAndScaleTrack(trackElement, track, index, numTracks) {
+ let heightPercent = (TRACK_HEIGHT_PERCENT / numTracks);
+ let marginPercent = ((100.0 - TRACK_HEIGHT_PERCENT) / (numTracks + 1)); // there are one more margins than tracks
+ trackElement.style.top = ((marginPercent * (index + 1)) + (heightPercent * index)) + '%';
+ trackElement.style.height = heightPercent + '%';
+ }
+ positionAndScaleSegment(segmentElement, segment) {
+ let durationPercentCurrentWindow = segment.end.currentWindow - segment.start.currentWindow;
+ segmentElement.style.width = Math.max(0.1, (durationPercentCurrentWindow * 100)) + '%';
+ segmentElement.style.left = (segment.start.currentWindow * 100) + '%';
+ }
+ getTrackElementId(trackId) {
+ return 'timelineTrack_' + trackId;
+ }
+ getSegmentElementId(trackId, segmentId) {
+ return 'timelineSegment_' + trackId + '_' + segmentId;
+ }
+ show() {
+ this.timelineContainer.classList.remove('hiddenTimeline');
+ }
+ hide() {
+ this.timelineContainer.classList.add('hiddenTimeline');
+ }
+ }
+
+ exports.TimelineView = TimelineView;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/TimelineWindow.js b/content_scripts/TimelineWindow.js
new file mode 100644
index 00000000..68fbb0a0
--- /dev/null
+++ b/content_scripts/TimelineWindow.js
@@ -0,0 +1,55 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+ const DAY_LENGTH_MS = 1000 * 60 * 60 * 24;
+
+ // Helper class to manage the viewport of the timeline
+ // this includes its bounds (as timestamps) when fully zoomed out,
+ // as well as the timestamps defining the current zoom/scroll of the window
+ class TimelineWindow {
+ constructor() {
+ this.bounds = {
+ withoutZoom: { min: 0, max: Date.now() },
+ current: { min: 0, max: Date.now() }
+ };
+ this.callbacks = {
+ onWithoutZoomUpdated: [],
+ onCurrentWindowUpdated: []
+ };
+ }
+ setWithoutZoomFromDate(dateObject) {
+ this.bounds.withoutZoom.min = dateObject.getTime();
+ this.bounds.withoutZoom.max = dateObject.getTime() + DAY_LENGTH_MS - 1; // remove 1ms so that day ends at 11:59:59.99
+
+ // by default, also adjusts the current view to be the entire withoutZoom bounds
+ this.bounds.current.min = this.bounds.withoutZoom.min;
+ this.bounds.current.max = this.bounds.withoutZoom.max;
+
+ this.callbacks.onWithoutZoomUpdated.forEach(cb => {
+ cb(this);
+ });
+ }
+ setCurrentFromPercent(minPercent, maxPercent) {
+ let fullLength = this.bounds.withoutZoom.max - this.bounds.withoutZoom.min;
+ this.bounds.current.min = this.bounds.withoutZoom.min + minPercent * fullLength;
+ this.bounds.current.max = this.bounds.withoutZoom.min + maxPercent * fullLength;
+
+ this.callbacks.onCurrentWindowUpdated.forEach(cb => {
+ cb(this);
+ });
+ }
+ getZoomPercent() { // 0 if not zoomed at all (see 100%), 1 if current window is 0% of the withoutZoom window
+ return 1.0 - (this.bounds.current.max - this.bounds.current.min) / (this.bounds.withoutZoom.max - this.bounds.withoutZoom.min);
+ }
+ getScrollLeftPercent() { // helper to get relative location of current.min within the withoutZoom window
+ return (this.bounds.current.min - this.bounds.withoutZoom.min) / (this.bounds.withoutZoom.max - this.bounds.withoutZoom.min);
+ }
+ onWithoutZoomUpdated(callback) {
+ this.callbacks.onWithoutZoomUpdated.push(callback);
+ }
+ onCurrentWindowUpdated(callback) {
+ this.callbacks.onCurrentWindowUpdated.push(callback);
+ }
+ }
+ exports.TimelineWindow = TimelineWindow;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/TouchControlButtons.js b/content_scripts/TouchControlButtons.js
new file mode 100644
index 00000000..d8b48fd2
--- /dev/null
+++ b/content_scripts/TouchControlButtons.js
@@ -0,0 +1,83 @@
+export class TouchControlButtons {
+ constructor() {
+ if (realityEditor.device.environment.isDesktop()) {
+ console.warn('Are you sure you want to create the TouchControlButtons on desktop?');
+ }
+
+ this.buttons = [];
+ this.MODES = Object.freeze({
+ pointer: 'pointer',
+ pan: 'pan',
+ rotate: 'rotate',
+ zoom: 'zoom'
+ });
+ this.callbacks = {
+ onModeSelected: [],
+ };
+
+ let iconSrc = {};
+ iconSrc[this.MODES.pointer] = 'addons/vuforia-spatial-remote-operator-addon/touch-controls-white-pointer.svg';
+ iconSrc[this.MODES.pan] = 'addons/vuforia-spatial-remote-operator-addon/touch-controls-white-pan.svg';
+ iconSrc[this.MODES.rotate] = 'addons/vuforia-spatial-remote-operator-addon/touch-controls-white-rotate.svg';
+ iconSrc[this.MODES.zoom] = 'addons/vuforia-spatial-remote-operator-addon/touch-controls-white-zoom.svg';
+
+ // create the elements
+ let container = document.createElement('div');
+ this.container = container;
+
+ Object.values(this.MODES).forEach(mode => {
+ let button = this.createButton(mode, iconSrc[mode]);
+ container.appendChild(button);
+ this.buttons.push(button);
+ });
+ }
+ createButton(mode, src) {
+ let div = document.createElement('div');
+ div.classList.add('touchControlButtonContainer');
+ div.id = `touchControlButton_${mode}`;
+ let icon = document.createElement('img');
+ icon.classList.add('touchControlButtonIcon');
+ icon.src = src;
+ div.appendChild(icon);
+
+ icon.addEventListener('pointerdown', () => {
+ this.buttonPressedDown = mode;
+ });
+
+ icon.addEventListener('pointerup', () => {
+ if (mode === this.buttonPressedDown) { // make sure you press down and up on the same button
+ this.selectMode(mode);
+ }
+ this.buttonPressedDown = null;
+ });
+
+ return div;
+ }
+ selectMode(mode) {
+ this.deselectButtons();
+ let button = document.querySelector(`#touchControlButton_${mode}`);
+ if (button) {
+ button.classList.add('selected');
+ }
+
+ this.callbacks.onModeSelected.forEach(cb => {
+ cb(mode);
+ });
+ }
+ deselectButtons() {
+ this.buttons.forEach(button => {
+ button.classList.remove('selected');
+ });
+ }
+ onModeSelected(callback) {
+ this.callbacks.onModeSelected.push(callback);
+ }
+ activate() {
+ this.container.style.display = '';
+ this.container.style.pointerEvents = '';
+ }
+ deactivate() {
+ this.container.style.display = 'none';
+ this.container.style.pointerEvents = 'none';
+ }
+}
diff --git a/content_scripts/VideoPlaybackCoordinator.js b/content_scripts/VideoPlaybackCoordinator.js
new file mode 100644
index 00000000..42e019f9
--- /dev/null
+++ b/content_scripts/VideoPlaybackCoordinator.js
@@ -0,0 +1,166 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+ const DEVICE_ID_PREFIX = 'device'; // should match DEVICE_ID_PREFIX in backend recording system
+ let menuItemsAdded = false;
+
+ // The Video Playback Coordinator creates a Timeline and loads all VideoSources into it via a TimelineDatabase
+ // When the Timeline plays or is scrolled, responds to the RGB+Depth+Pose data and tells the CameraVisCoordinator to render point clouds
+ class VideoPlaybackCoordinator {
+ constructor() {
+ this.canvasElements = {};
+ this.timelineVisibile = true;
+ this.POSE_FPS = 10; // if the recording FPS changes, this const needs to be updated to synchronize the playback
+ }
+ load() {
+ let playback = realityEditor.videoPlayback;
+ this.timelineController = new playback.TimelineController();
+ this.database = new playback.TimelineDatabase();
+ let _videoSources = new playback.VideoSources((videoInfo, trackInfo) => {
+ console.debug('VideoPlaybackCoordinator got trackInfo', trackInfo);
+ for (const [trackId, trackData] of Object.entries(trackInfo.tracks)) {
+ let track = new playback.DataTrack(trackId, playback.TRACK_TYPES.VIDEO_3D);
+ for (const [segmentId, segmentData] of Object.entries(trackData.segments)) {
+ let segment = new playback.DataSegment(segmentId, playback.TRACK_TYPES.VIDEO_3D, segmentData.start, segmentData.end);
+ let colorVideo = new playback.DataPiece('colorVideo', playback.DATA_PIECE_TYPES.VIDEO_URL);
+ colorVideo.setVideoUrl(segmentData.colorVideo);
+ let depthVideo = new playback.DataPiece('depthVideo', playback.DATA_PIECE_TYPES.VIDEO_URL);
+ depthVideo.setVideoUrl(segmentData.depthVideo);
+ let poses = new playback.DataPiece('poses', playback.DATA_PIECE_TYPES.TIME_SERIES);
+ poses.setTimeSeriesData(segmentData.poses.map(elt => {
+ return {data: elt.pose, time: elt.time};
+ }));
+ segment.addDataPiece(colorVideo);
+ segment.addDataPiece(depthVideo);
+ segment.addDataPiece(poses);
+ track.addSegment(segment);
+ }
+ this.database.addTrack(track);
+ }
+ console.debug('VideoPlaybackCoordinator database', this.database);
+
+ // this.timeline.loadTracks(trackInfo);
+ this.timelineController.setDatabase(this.database);
+
+ // TODO: make the VideoSources listen for newly uploaded videos, and when loaded, append to timeline
+ });
+ this.timelineController.onVideoFrame((colorVideo, depthVideo, segment) => {
+ if (!this.timelineController.model.selectedSegments.map(segment => segment.id).includes(segment.id)) {
+ console.log('dont process video frame for deselected segment');
+ return;
+ }
+ let deviceId = segment.trackId;
+ let colorVideoCanvas = this.getCanvasElement(deviceId, 'color');
+ let depthVideoCanvas = this.getCanvasElement(deviceId, 'depth');
+
+ let colorCtx = colorVideoCanvas.getContext('2d');
+ let depthCtx = depthVideoCanvas.getContext('2d');
+ colorCtx.drawImage(colorVideo, 0, 0, 960, 540);
+ depthCtx.drawImage(depthVideo, 0, 0, 256, 144);
+
+ // get pose that accurately matches actual video playback currentTime. relies on knowing FPS of the recording.
+ // a 179.5-second video has 1795 poses, so use the (video timestamp * 10) as index to retrieve pose (if video is 10fps)
+ let closestPoseBase64 = segment.dataPieces.poses.getDataAtIndex(Math.floor(colorVideo.currentTime * this.POSE_FPS));
+ let closestPoseMatrix = this.getPoseMatrixFromData(closestPoseBase64);
+ let colorImageUrl = colorVideoCanvas.toDataURL('image/jpeg');
+ let depthImageUrl = depthVideoCanvas.toDataURL('image/png');
+
+ if (closestPoseMatrix) {
+ if (typeof this.loadPointCloud !== 'undefined') {
+ this.loadPointCloud(this.getCameraId(deviceId), colorImageUrl, depthImageUrl, closestPoseMatrix);
+ }
+ }
+ });
+ this.timelineController.onSegmentDeselected(segment => {
+ if (typeof this.hidePointCloud === 'function') {
+ this.hidePointCloud(this.getCameraId(segment.trackId));
+ }
+ });
+
+ this.toggleVisibility(false); // default to hidden
+
+ // Note: onDataFrame not working because of an error in recorded pose timestamps, so we calculate pose in onVideoFrame instead
+ // this.timelineController.onDataFrame((colorVideoUrl, depthVideoUrl, timePercent, cameraPoseMatrixBase64) => {
+ // // console.log('onDataFrame', colorVideoUrl, depthVideoUrl, timePercent, cameraPoseMatrix);
+ // this.mostRecentPose = this.getPoseMatrixFromData(cameraPoseMatrixBase64);
+ // this.mostRecentPoseTimePercent = timePercent;
+ // });
+ }
+ getCameraId(deviceId) {
+ // add 255 to go outside the range of camera ids, so playback cameras are independent of realtime cameras
+ return parseInt(deviceId.replace(DEVICE_ID_PREFIX, '')) + 255;
+ }
+ getPoseMatrixFromData(poseBase64) {
+ if (!poseBase64) { return null; }
+
+ let byteCharacters = window.atob(poseBase64);
+ const byteNumbers = new Array(byteCharacters.length);
+ for (let i = 0; i < byteCharacters.length; i++) {
+ byteNumbers[i] = byteCharacters.charCodeAt(i);
+ }
+ const byteArray = new Uint8Array(byteNumbers);
+ return new Float32Array(byteArray.buffer);
+ }
+ getCanvasElement(trackId, colorOrDepth) {
+ if (colorOrDepth !== 'color' && colorOrDepth !== 'depth') { console.warn('passing invalid colorOrDepth to getCanvasElement', colorOrDepth); }
+
+ if (typeof this.canvasElements[trackId] === 'undefined') {
+ this.canvasElements[trackId] = {};
+ }
+ if (typeof this.canvasElements[trackId].depth === 'undefined') {
+ this.canvasElements[trackId].depth = this.createCanvasElement('depth_canvas_' + trackId, 256, 144);
+ }
+ if (typeof this.canvasElements[trackId].color === 'undefined') {
+ this.canvasElements[trackId].color = this.createCanvasElement('color_canvas_' + trackId, 960, 540);
+ }
+
+ return this.canvasElements[trackId][colorOrDepth];
+ }
+ createCanvasElement(id, width, height) {
+ let canvas = document.createElement('canvas');
+ canvas.id = id;
+ canvas.width = width;
+ canvas.height = height;
+ canvas.style.display = 'none';
+ document.body.appendChild(canvas);
+ return canvas;
+ }
+ setPointCloudCallback(callback) {
+ this.loadPointCloud = callback;
+ }
+ setHidePointCloudCallback(callback) {
+ this.hidePointCloud = callback;
+ }
+ toggleVisibility(toggled) {
+ if (this.timelineVisibile || (typeof toggled !== 'undefined' && !toggled)) {
+ this.timelineVisibile = false;
+ } else {
+ this.timelineVisibile = true;
+ this.addMenuItems();
+ }
+
+ this.timelineController.toggleVisibility(this.timelineVisibile);
+ }
+ addMenuItems() {
+ if (!menuItemsAdded) {
+ menuItemsAdded = true;
+ // set up keyboard shortcuts
+ let togglePlayback = new realityEditor.gui.MenuItem('Toggle Playback', { shortcutKey: 'SPACE', toggle: true, defaultVal: false}, (toggled) => {
+ this.timelineController.model.togglePlayback(toggled);
+ });
+ realityEditor.gui.getMenuBar().addItemToMenu(realityEditor.gui.MENU.History, togglePlayback);
+
+ let slower = new realityEditor.gui.MenuItem('Play Slower', { shortcutKey: 'COMMA' }, () => {
+ this.timelineController.multiplySpeed(0.5, false);
+ });
+ realityEditor.gui.getMenuBar().addItemToMenu(realityEditor.gui.MENU.History, slower);
+
+ let faster = new realityEditor.gui.MenuItem('Play Faster', { shortcutKey: 'PERIOD' }, () => {
+ this.timelineController.multiplySpeed(2.0, false);
+ });
+ realityEditor.gui.getMenuBar().addItemToMenu(realityEditor.gui.MENU.History, faster);
+ }
+ }
+ }
+ exports.VideoPlaybackCoordinator = VideoPlaybackCoordinator;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/VideoSources.js b/content_scripts/VideoSources.js
new file mode 100644
index 00000000..b244a20e
--- /dev/null
+++ b/content_scripts/VideoSources.js
@@ -0,0 +1,158 @@
+createNameSpace('realityEditor.videoPlayback');
+
+(function (exports) {
+ // VideoSources is set up to discover all 3D video paths from the server and put them into a usable data structure
+ // It also adds the pose data to each recording. The videoInfo it returns can be loaded into a TimelineDatabase.
+ class VideoSources {
+ constructor(onDataLoaded) {
+ this.onDataLoaded = onDataLoaded;
+ this.loadAvailableVideos('/virtualizer_recordings').then(info => {
+ this.videoInfo = info;
+ if (this.videoInfo) {
+ // videoInfo is a json blob from the server with less structure
+ // create trackInfo, which adds metadata and pose data to the videoInfo
+ this.createTrackInfo(this.videoInfo); // triggers onDataLoaded when it's done
+ }
+ }).catch(error => {
+ console.warn('error loading /virtualizer_recordings', error);
+ });
+ }
+ createTrackInfo(videoInfo) {
+ this.trackInfo = {
+ tracks: {}, // each device gets its own track. more than one segment can be on that track
+ metadata: { minTime: 0, maxTime: 1 }
+ };
+
+ let earliestTime = Date.now();
+ let latestTime = 0;
+ let trackIndex = 0;
+
+ Object.keys(videoInfo).forEach(deviceId => {
+ // console.log('loading track for device: ' + deviceId);
+ Object.keys(videoInfo[deviceId]).forEach(sessionId => {
+ // console.log('loading ' + deviceId + ' session ' + sessionId);
+ let sessionInfo = videoInfo[deviceId][sessionId];
+ if (typeof sessionInfo.color === 'undefined' || typeof sessionInfo.depth === 'undefined') {
+ return; // skip entries that don't have both videos
+ }
+ if (typeof this.trackInfo.tracks[deviceId] === 'undefined') {
+ this.trackInfo.tracks[deviceId] = {
+ segments: {},
+ index: trackIndex
+ };
+ trackIndex++;
+ }
+ let timeInfo = this.parseTimeInfo(sessionInfo.color);
+ this.trackInfo.tracks[deviceId].segments[sessionId] = {
+ colorVideo: sessionInfo.color,
+ depthVideo: sessionInfo.depth,
+ start: parseInt(timeInfo.start),
+ end: parseInt(timeInfo.end),
+ visible: true,
+ };
+ earliestTime = Math.min(earliestTime, timeInfo.start);
+ latestTime = Math.max(latestTime, timeInfo.end);
+ });
+ });
+
+ this.trackInfo.metadata.minTime = earliestTime;
+ this.trackInfo.metadata.maxTime = latestTime > 0 ? latestTime : Date.now();
+ console.debug('trackInfo', this.trackInfo);
+
+ this.addPoseInfoToTracks().then(response => {
+ console.debug('addPoseInfoToTracks', response);
+ this.onDataLoaded(this.videoInfo, this.trackInfo);
+ }).catch(error => {
+ console.warn('error in addPoseInfoToTracks', error);
+ });
+ }
+ parseTimeInfo(filename) {
+ let re_start = new RegExp('start_[0-9]{13,}');
+ let re_end = new RegExp('end_[0-9]{13,}');
+ let startMatches = filename.match(re_start);
+ let endMatches = filename.match(re_end);
+ if (!startMatches || !endMatches || startMatches.length === 0 || endMatches.length === 0) { return null; }
+ return {
+ start: startMatches[0].replace('start_', ''),
+ end: endMatches[0].replace('end_', '')
+ };
+ }
+ loadAvailableVideos(url) {
+ return new Promise((resolve, reject) => {
+ // this.downloadVideoInfo().then(info => console.log(info));
+ // httpGet('http://' + this.ip + ':31337/videoInfo').then(info => {
+ this.httpGet(url).then(info => {
+ console.debug('loadAvailableVideos httpGet', info);
+ resolve(info);
+ }).catch(reason => {
+ console.warn('loadAvailableVideos error', reason);
+ reject(reason);
+ });
+ });
+ }
+ async addPoseInfoToTracks() {
+ return new Promise((resolve, reject) => {
+ // add pose info to tracks
+ // http://localhost:8081/virtualizer_recording/device_21/pose/device_device_21_session_wE1fcfcd.json
+ let promises = [];
+ Object.keys(this.trackInfo.tracks).forEach(deviceId => {
+ Object.keys(this.trackInfo.tracks[deviceId].segments).forEach(segmentId => {
+ promises.push(this.loadPoseInfo(deviceId, segmentId));
+ });
+ });
+ if (promises.length === 0) {
+ resolve();
+ return;
+ }
+ Promise.all(promises).then((poses) => {
+ poses.forEach(response => {
+ let segment = this.trackInfo.tracks[response.deviceId].segments[response.segmentId];
+ segment.poses = response.poseInfo;
+ });
+ resolve();
+ }).catch(error => {
+ console.warn('addPoseInfoToTracks failed', error);
+ reject();
+ });
+ });
+ }
+ loadPoseInfo(deviceId, segmentId) {
+ return new Promise((resolve, reject) => {
+ this.httpGet(this.getPoseUrl(deviceId, segmentId)).then(poseInfo => {
+ resolve({
+ deviceId: deviceId,
+ segmentId: segmentId,
+ poseInfo: poseInfo
+ });
+ }).catch(reason => {
+ console.warn('loadPoseInfo failed', reason);
+ reject(reason);
+ });
+ });
+ }
+ getPoseUrl(deviceId, segmentId) {
+ // http://localhost:8081/virtualizer_recording/device21/pose/device_device21_session_wE1fcfcd.json
+ return '/virtualizer_recording/' + deviceId + '/pose/device_' + deviceId + '_session_' + segmentId + '.json';
+ }
+ httpGet(url) {
+ return new Promise((resolve, reject) => {
+ let req = new XMLHttpRequest();
+ req.open('GET', url, true);
+ req.onreadystatechange = function () {
+ if (req.readyState === 4) {
+ if (req.status === 0) {
+ return;
+ }
+ if (req.status !== 200) {
+ reject('Invalid status code <' + req.status + '>');
+ }
+ resolve(JSON.parse(req.responseText));
+ }
+ };
+ req.send();
+ });
+ }
+ }
+
+ exports.VideoSources = VideoSources;
+})(realityEditor.videoPlayback);
diff --git a/content_scripts/VirtualCamera.js b/content_scripts/VirtualCamera.js
new file mode 100644
index 00000000..e68e88ca
--- /dev/null
+++ b/content_scripts/VirtualCamera.js
@@ -0,0 +1,1405 @@
+/*
+* Copyright © 2021 PTC
+*/
+
+createNameSpace('realityEditor.device');
+
+import * as THREE from '../../thirdPartyCode/three/three.module.js';
+import Splatting from '../../src/splatting/Splatting.js';
+
+(function (exports) {
+
+ const DISPLAY_PERSPECTIVE_CUBES = false;
+ const FOCUS_DISTANCE_MM_IN_FRONT_OF_VIRTUALIZER = 1000; // what point to focus on when we rotate/pan away from following
+ const MIN_FIRST_PERSON_DISTANCE = 0;
+
+ class VirtualCamera {
+ constructor(cameraNode, kTranslation, kRotation, kScale, initialPosition, floorOffset) {
+ if (!cameraNode) { console.warn('cameraNode is undefined!'); }
+
+ this.cameraNode = cameraNode;
+ this.projectionMatrix = [];
+ this.idleOrbitting = false;
+ this.isFlying = false;
+
+ this.initialPosition = [0, 0, 0];
+ this.position = [1, 1, 1];
+ if (typeof initialPosition !== 'undefined') {
+ this.initialPosition = [initialPosition[0], initialPosition[1], initialPosition[2]];
+ this.position = [initialPosition[0], initialPosition[1], initialPosition[2]];
+ }
+ // focusDistance ~ distance between position and targetPosition
+ // needs to be smaller than scrollOperation targetToFocus threshold, b/c otherwise target position is so far away from
+ // camera position that when zooming in, camera position reaches focus point before target position reaches focus point
+ this.focusDistance = 150;
+ this.targetPosition = [0, 0, 0];
+ this.velocity = [0, 0, 0];
+ this.targetVelocity = [0, 0, 0];
+ this.distanceToTarget = 1;
+ this.preRotateDistanceToTarget = null;
+ this.preStopFollowingDistanceToTarget = null;
+ this.afterNFrames = []; // can be used to trigger delayed actions that gives the camera precise time to update
+ this.speedFactors = {
+ translation: kTranslation || 1,
+ rotation: kRotation || 1,
+ scale: kScale || 1
+ };
+ this.mouseInput = {
+ unprocessedDX: 0,
+ unprocessedDY: 0,
+ unprocessedScroll: 0,
+ isPointerDown: false,
+ isRightClick: false,
+ isRotateRequested: false,
+ isStrafeRequested: false,
+ first: { x: 0, y: 0 },
+ last: { x: 0, y: 0 },
+ lastWorldPos: [0, 0, 0],
+ latest: {x: 0, y: 0},
+ };
+ this.mouseFlyInput = {
+ justSwitched: true,
+ last: {x: 0, y: 0},
+ unprocessedDX: 0,
+ unprocessedDY: 0,
+ };
+ this.pauseTouchGestures = false;
+ this.zoomOutTransition = false;
+ this.keyboard = new realityEditor.device.KeyboardListener();
+ this.followerName = 'cameraFollower' + cameraNode.id;
+ this.followingState = {
+ active: false,
+ selectedId: null,
+ currentFollowingDistance: 0,
+ // three.js objects used to calculate the following trajectory
+ unstabilizedContainer: null,
+ stabilizedContainer: null,
+ forwardTargetObject: null, // this is unstabilized
+ levelTargetObject: null, // this is fully stabilized, has height = virtualizer height
+ partiallyStabilizedTargetObject: null // this is actually what we lookAt, in between forwardObj and levelObj
+ };
+ this.lockOnMode = null; // directly lock-on to the perspective of another user's virtual camera
+
+ this.callbacks = {
+ onPanToggled: [],
+ onRotateToggled: [],
+ onScaleToggled: [],
+ onStopFollowing: [], // other modules can discover when pan/rotate forced this camera out of follow mode
+ onFirstPersonDistanceToggled: [],
+ onStopLockOnMode: [],
+ };
+
+ this.promptContainer = this.createPromptContainer();
+ document.body.appendChild(this.promptContainer);
+ this.addNormalModePrompt();
+ this.addFlyModePrompt();
+
+ this.focusTargetCube = null;
+ this.addFocusTargetCube();
+ this.addEventListeners();
+
+ this.threeJsContainer = new THREE.Group();
+ this.threeJsContainer.name = 'VirtualCamera_' + cameraNode.id + '_threeJsContainer';
+ this.threeJsContainer.position.y = -floorOffset;
+ this.threeJsContainer.rotation.x = Math.PI / 2;
+ realityEditor.gui.threejsScene.addToScene(this.threeJsContainer);
+ }
+
+ /**
+ * Updates the vertical position of the three.js container used to offset the following mechanism
+ * This needs to be called anytime the navmesh floorOffset updates, e.g. if you drop in a new gltf model
+ * @param {number} floorOffset
+ */
+ updateFloorOffset(floorOffset) {
+ this.threeJsContainer.position.y = -floorOffset;
+ }
+
+ /**
+ * Start or stop lockOnMode - which snaps your camera view to another user's
+ * @param {string|null} objectId
+ * @returns {boolean}
+ */
+ toggleLockOnMode(objectId) {
+ if (this.lockOnMode) {
+ this.lockOnMode = null;
+ return false;
+ } else if (objectId) {
+ this.lockOnMode = objectId;
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Register a new callback to be triggered when you stop lockOnMode
+ * @param {function} cb
+ */
+ onStopLockOnMode(cb) {
+ this.callbacks.onStopLockOnMode.push(cb);
+ }
+ addFocusTargetCube() {
+ if (this.focusTargetCube === null) {
+ this.focusTargetCube = new THREE.Mesh(
+ new THREE.BoxGeometry(20, 20, 20),
+ new THREE.MeshBasicMaterial({color: 0x00ffff})
+ );
+ this.focusTargetCube.position.copy(new THREE.Vector3().fromArray(this.mouseInput.lastWorldPos));
+ // realityEditor.gui.threejsScene.addToScene(this.focusTargetCube);
+ }
+ }
+ // div/container where all prompts get appended to
+ createPromptContainer () {
+ let promptContainer = document.createElement('div');
+ promptContainer.classList.add('mode-prompt-container');
+
+ return promptContainer;
+ }
+ // function for creating prompts
+ createPrompt(titleText, bodyText) {
+ // don't show keyboard controls when remote operator loaded into AR app
+ if (realityEditor.device.environment.isWithinToolboxApp()) return;
+
+ // add normal mode prompt
+ let prompt = document.createElement('div');
+ prompt.classList.add('mode-prompt');
+
+ let modeText = document.createElement('div');
+ modeText.classList.add('mode-prompt-big-font');
+ modeText.innerText = titleText;
+ prompt.appendChild(modeText);
+
+ let modeControls = document.createElement('ul');
+ if (bodyText.length && bodyText.length > 0) {
+ bodyText.forEach(item => {
+ let listEl = document.createElement('li');
+ listEl.innerText = item;
+ modeControls.appendChild(listEl);
+ });
+ } else if (bodyText.typeof('string')) {
+ let promptText = document.createElement('li');
+ promptText.innerText = bodyText;
+ modeControls.appendChild(promptText);
+ }
+ prompt.appendChild(modeControls);
+
+ prompt.addEventListener('animationend', () => {
+ this.promptContainer.removeChild(prompt);
+ });
+ return prompt;
+ }
+ addNormalModePrompt() {
+ if (realityEditor.device.environment.isWithinToolboxApp()) return;
+ if (this.isFlying) return;
+
+ // add normal mode prompt
+ let promptTitle = 'Entered normal mode';
+ let promptText = ['F - switch mode', 'G - focus', 'RMB - rotate', 'MMB/RMB+Alt - pan', 'scroll wheel - zoom'];
+
+ let normalModePrompt = this.createPrompt(promptTitle, promptText);
+ this.promptContainer.appendChild(normalModePrompt);
+ }
+ addFlyModePrompt() {
+ if (realityEditor.device.environment.isWithinToolboxApp()) return;
+ if (!this.isFlying) return;
+
+ // add fly mode prompt
+ let promptTitle = 'Entered fly mode';
+ let promptText = ['F - switch mode', 'G - focus', 'Q/E - down/up', 'W/A/S/D - move', 'SHIFT - speed up'];
+
+ let flyModePrompt = this.createPrompt(promptTitle, promptText);
+ this.promptContainer.appendChild(flyModePrompt);
+ }
+ switchMode() {
+ if (this.isFlying) {
+ this.addFlyModePrompt();
+ } else if (!this.isFlying) {
+ this.addNormalModePrompt();
+ }
+ }
+ // when in normal mode, right click to add a green focus cube to the scene
+ setFocusTargetCube(event, forceSet = false) {
+ if (this.isFlying) return;
+ // conform to spatial cursor mousemove event pageX and pageY
+ // if (event.button === 2 || !realityEditor.device.environment.variables.requiresMouseEvents) {
+ if (forceSet || event.button === 2 || !realityEditor.device.environment.variables.requiresMouseEvents) {
+ let worldIntersectPoint = realityEditor.spatialCursor.getRaycastCoordinates(event.pageX, event.pageY, true).point;
+ if (worldIntersectPoint === undefined) return;
+ // record pointerdown world intersect point, for off-center camera rotation
+ this.mouseInput.lastWorldPos = [worldIntersectPoint.x, worldIntersectPoint.y, worldIntersectPoint.z];
+ if (this.focusTargetCube === null) {
+ this.focusTargetCube = new THREE.Mesh(
+ new THREE.BoxGeometry(20, 20, 20),
+ new THREE.MeshBasicMaterial({color: 0x00ffff})
+ );
+ this.focusTargetCube.position.copy(worldIntersectPoint);
+ realityEditor.gui.threejsScene.addToScene(this.focusTargetCube);
+ } else {
+ this.focusTargetCube.position.copy(worldIntersectPoint);
+ }
+ }
+ }
+ addEventListeners() {
+ if (!realityEditor.device.environment.variables.requiresMouseEvents) {
+ this.addTouchEvents();
+ return;
+ }
+
+ let scrollTimeout = null;
+ window.addEventListener('wheel', function (event) {
+ // restrict deltaY between [-100, 100], to prevent mouse wheel deltaY so large that camera cannot focus on focus point when zooming in
+ let wheelAmt = Math.max(-40, Math.min(40, event.deltaY));
+ this.mouseInput.unprocessedScroll += wheelAmt;
+ event.preventDefault();
+
+ // update scale callbacks based on whether you've scrolled in this 150ms time period
+ this.triggerScaleCallbacks(true);
+ this.preRotateDistanceToTarget = null; // if we rotate and scroll, don't lock zoom to pre-rotate level
+
+ if (scrollTimeout !== null) {
+ Splatting.toggleGSRaycast(false);
+ clearTimeout(scrollTimeout);
+ }
+ scrollTimeout = setTimeout(function () {
+ this.triggerScaleCallbacks(false);
+ this.preRotateDistanceToTarget = null;
+ Splatting.toggleGSRaycast(false);
+
+ }.bind(this), 150);
+
+ }.bind(this), { passive: false }); // in order to call preventDefault, wheel needs to be active not passive
+
+ document.addEventListener('pointerdown', function (event) {
+ if (event.button === 2 || event.button === 1) { // 2 is right click, 0 is left, 1 is middle button
+ this.mouseInput.isPointerDown = true;
+ this.mouseInput.isRightClick = false;
+ this.mouseInput.isRotateRequested = false;
+ this.mouseInput.isStrafeRequested = false;
+ if (event.button === 1 || this.keyboard.keyStates[this.keyboard.keyCodes.ALT] === 'down') {
+ this.mouseInput.isStrafeRequested = true;
+ this.triggerPanCallbacks(true);
+ } else if (event.button === 2) {
+ this.setFocusTargetCube(event);
+ this.mouseInput.isRightClick = true;
+ this.mouseInput.isRotateRequested = true;
+ Splatting.toggleGSRaycast(true);
+ this.triggerRotateCallbacks(true);
+ if (!this.followingState.active) { // we preserve distance to virtualizer if following, not distance to target
+ this.preRotateDistanceToTarget = this.distanceToTarget;
+ }
+ }
+ this.mouseInput.first.x = event.pageX;
+ this.mouseInput.first.y = event.pageY;
+ this.mouseInput.last.x = event.pageX;
+ this.mouseInput.last.y = event.pageY;
+ // follow a tool if you click it with shift held down
+ }
+ }.bind(this));
+
+ let pointermoveTimeout = null;
+ document.addEventListener('pointermove', function (event) {
+ this.mouseInput.latest.x = event.pageX;
+ this.mouseInput.latest.y = event.pageY;
+ Splatting.toggleGSRaycast(true);
+ if (pointermoveTimeout !== null) {
+ clearTimeout(pointermoveTimeout);
+ }
+ pointermoveTimeout = setTimeout(function () {
+ Splatting.toggleGSRaycast(false);
+ }.bind(this), 150);
+ if (this.idleOrbitting || this.mouseInput.isRotateRequested || this.mouseInput.isStrafeRequested) {
+ Splatting.toggleGSRaycast(false);
+ return;
+ }
+ this.setFocusTargetCube(event, true);
+ }.bind(this));
+
+ const pointerReset = () => {
+ this.mouseInput.isPointerDown = false;
+ this.mouseInput.isRightClick = false;
+ this.mouseInput.isRotateRequested = false;
+ this.mouseInput.isStrafeRequested = false;
+ Splatting.toggleGSRaycast(false);
+
+ this.mouseInput.first.x = 0;
+ this.mouseInput.first.y = 0;
+ this.mouseInput.last.x = 0;
+ this.mouseInput.last.y = 0;
+
+ if (this.preRotateDistanceToTarget !== null) {
+ this.preRotateDistanceToTarget = null;
+ }
+
+ this.triggerPanCallbacks(false);
+ this.triggerRotateCallbacks(false);
+ this.triggerScaleCallbacks(false);
+ };
+
+ document.addEventListener('pointerup', pointerReset);
+ document.addEventListener('pointercancel', pointerReset);
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.FocusCamera, () => {
+ this.focus(this.focusTargetCube.position.clone());
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.ToggleFlyMode, (toggled) => {
+ this.isFlying = toggled;
+ if (this.isFlying) {
+ document.body.requestPointerLock();
+ } else {
+ document.exitPointerLock();
+ }
+ });
+
+ document.addEventListener('pointerlockchange', () => {
+ if (document.pointerLockElement === document.body) {
+ if (!this.isFlying) {
+ realityEditor.gui.getMenuBar().getItemByName(realityEditor.gui.ITEM.ToggleFlyMode).switchToggle();
+ this.isFlying = true;
+ Splatting.toggleGSRaycast(false);
+ }
+ } else if (document.pointerLockElement === null) {
+ if (this.isFlying) {
+ // make sure the menu item toggle state updates in response to escape key, etc
+ realityEditor.gui.getMenuBar().getItemByName(realityEditor.gui.ITEM.ToggleFlyMode).switchToggle();
+ this.isFlying = false;
+ Splatting.toggleGSRaycast(false);
+ }
+ }
+ this.switchMode();
+ });
+
+ document.addEventListener('pointermove', function (event) {
+ if ( document.pointerLockElement === document.body ) {
+ this.mouseFlyInput.unprocessedDX = event.movementX;
+ this.mouseFlyInput.unprocessedDY = event.movementY;
+ } else {
+ if (this.mouseInput.isPointerDown) {
+
+ let xOffset = event.pageX - this.mouseInput.last.x;
+ let yOffset = event.pageY - this.mouseInput.last.y;
+
+ this.mouseInput.unprocessedDX += xOffset;
+ this.mouseInput.unprocessedDY += yOffset;
+
+ if (this.followingState.active) {
+ this.callbacks.onFirstPersonDistanceToggled.forEach(cb => cb(false, this.followingState.currentFollowingDistance));
+ }
+
+ this.mouseInput.last.x = event.pageX;
+ this.mouseInput.last.y = event.pageY;
+ }
+ }
+ }.bind(this));
+ }
+ /**
+ * by default, uses multitouch gestures to pan/rotate/zoom, but a specific mode can be passed in
+ * and only that mode will be able to be controlled (and will be controlled by a 1 finger drag)
+ * @param {string} mode
+ */
+ setTouchControlMode(mode) {
+ this.touchControlMode = mode;
+ }
+
+ /**
+ * True if touchControlMode is a camera control mode (not 'pointer' mode or null)
+ */
+ isTouchControlModeActive() {
+ return this.touchControlMode && (this.touchControlMode === 'pan' ||
+ this.touchControlMode === 'rotate' || this.touchControlMode === 'zoom');
+ }
+ /**
+ * Sets up the multitouch gesture controls, as well as the touch controls for specifically-activated modes
+ */
+ addTouchEvents() {
+ // on mobile browsers, we add touch controls instead of mouse controls, to move the camera. additional
+ // code is added to avoid iOS's pesky safari gestures, such as pull-to-refresh and swiping between tabs
+
+ let isMultitouchGestureActive = false;
+ let didMoveAtAll = false;
+ let initialPosition = null;
+ let initialDistance = 0;
+ let lastDistance = 0;
+
+ // Prevent the default pinch gesture response (zooming) on mobile browsers
+ document.addEventListener('gesturestart', (event) => {
+ event.preventDefault();
+ });
+
+ // convert touch movement into rotation, pan, or zoom amount (unprocessedDX, unprocessedDY, unprocessedScroll)
+ const analyzeTouchMovement = (event) => {
+ if (this.mouseInput.last.x && this.mouseInput.last.y) {
+ let xOffset = event.pageX - this.mouseInput.last.x;
+ let yOffset = event.pageY - this.mouseInput.last.y;
+
+ if (!this.touchControlMode || this.touchControlMode === 'pan' || this.touchControlMode === 'rotate') {
+ this.mouseInput.unprocessedDX += xOffset;
+ this.mouseInput.unprocessedDY += yOffset;
+ } else if (this.touchControlMode === 'zoom') {
+ this.mouseInput.unprocessedScroll += (yOffset * 2);
+ }
+ }
+
+ this.mouseInput.last.x = event.pageX;
+ this.mouseInput.last.y = event.pageY;
+ };
+
+ // Handle one-finger drag to rotate (labeled "multitouch" here to group it with the pinch and pan gestures)
+ const handleMultitouchRotate = (event) => {
+ event.preventDefault();
+ if (event.touches.length === 1) {
+ analyzeTouchMovement(event); // rotates because isRotateRequested is true
+
+ if (!initialPosition) {
+ initialPosition = { x: event.pageX, y: event.pageY };
+ } else {
+ const distance = Math.hypot(event.pageX - initialPosition.x, event.pageY - initialPosition.y);
+ if (distance > 10) {
+ didMoveAtAll = true; // only add focus cube if touch moved less than this threshold
+ }
+ }
+ }
+ };
+
+ // Handle two-finger drag to pan
+ const handleMultitouchPan = (event) => {
+ event.preventDefault();
+
+ // don't allow pan within the AR app, because two-finger-gesture is used to transition between AR<>VR
+ if (realityEditor.device.environment.isWithinToolboxApp()) return;
+
+ if (event.touches.length === 2) {
+ analyzeTouchMovement(event); // pans because isStrafeRequested is true
+
+ // pan faster on touchscreens by scaling up proportional to distance to focus cube
+ const focusCubePosition = [this.focusTargetCube.position.x, this.focusTargetCube.position.y, this.focusTargetCube.position.z];
+ const distanceToFocusCube = magnitude(add(this.position, negate(focusCubePosition)));
+ const distancePanFactor = 1.8 + Math.max(0.2, distanceToFocusCube / 5000); // speed when 1 meter units away, scales up w/ distance
+ this.mouseInput.unprocessedDX *= distancePanFactor;
+ this.mouseInput.unprocessedDY *= distancePanFactor;
+ }
+ };
+
+ // Handle pinch to zoom
+ const handleMultitouchPinch = (event) => {
+ event.preventDefault();
+
+ // don't allow pinch within the AR app, because pinch is used to transition between AR<>VR
+ if (realityEditor.device.environment.isWithinToolboxApp()) return;
+
+ if (event.touches.length === 2) {
+ const touch1 = event.touches[0];
+ const touch2 = event.touches[1];
+ const currentDistance = Math.hypot(touch2.clientX - touch1.clientX, touch2.clientY - touch1.clientY);
+
+ if (initialDistance === 0) { // indicates the start of the pinch gesture
+ initialDistance = currentDistance;
+ lastDistance = initialDistance;
+ } else {
+ // Calculate the pinch scale based on the change in distance over time.
+ // 5 is empirically determined to feel natural. -= so bigger distance leads to closer zoom
+ this.mouseInput.unprocessedScroll -= 5 * (currentDistance - lastDistance);
+ lastDistance = currentDistance;
+ }
+ }
+ };
+
+ // Handles touchstart events when a specific touchControlMode has been selected
+ const handleTouchControlDown = (event) => {
+ initialPosition = null;
+ this.mouseInput.last.x = 0;
+ this.mouseInput.last.y = 0;
+
+ this.setFocusTargetCube(event);
+
+ if (this.touchControlMode === 'pan') {
+ this.mouseInput.isStrafeRequested = true;
+ this.mouseInput.isRotateRequested = false;
+ this.triggerPanCallbacks(true);
+ } else if (this.touchControlMode === 'rotate') {
+ this.mouseInput.isRotateRequested = true;
+ this.mouseInput.isStrafeRequested = false;
+ this.triggerRotateCallbacks(true);
+ } else if (this.touchControlMode === 'zoom') {
+ this.triggerScaleCallbacks(true);
+ }
+
+ // the touch visual feedback circle stops working automatically when we're in this mode, so we need to manually update it here
+ if (overlayDiv) {
+ overlayDiv.style.display = 'inline';
+ overlayDiv.style.transform = `translate3d(${event.touches[0].clientX}px, ${event.touches[0].clientY}px, 1200px)`;
+ }
+ };
+
+ // Handles touchmove events when a specific touchControlMode has been selected
+ const handleTouchControlMove = (event) => {
+ // pans, rotates, or zooms based on isStrafeRequested, isRotateRequested, and touchControlMode
+ analyzeTouchMovement(event);
+
+ // the touch visual feedback circle stops working automatically when we're in this mode, so we need to manually update it here
+ if (overlayDiv) {
+ overlayDiv.style.transform = `translate3d(${event.touches[0].clientX}px, ${event.touches[0].clientY}px, 1200px)`;
+ }
+ };
+
+ // Handles touchend events when a specific touchControlMode has been selected
+ const handleTouchControlUp = (_event) => {
+ // stops the visual feedback of the camera control event
+ if (this.touchControlMode === 'pan') {
+ this.triggerPanCallbacks(false);
+ } else if (this.touchControlMode === 'rotate') {
+ this.triggerRotateCallbacks(false);
+ } else if (this.touchControlMode === 'zoom') {
+ this.triggerScaleCallbacks(false);
+ }
+
+ // the touch visual feedback circle stops working automatically when we're in this mode, so we need to manually update it here
+ if (overlayDiv) {
+ overlayDiv.style.display = 'none';
+ }
+ };
+
+ // Add touch event listeners to the document, which trigger the "TouchControl" functions
+ // if a specific touchControlMode has been selected, or the "multitouch" functions if not
+ document.addEventListener('touchstart', (event) => {
+ if (!realityEditor.device.utilities.isEventHittingBackground(event)) return;
+ // while pinching to enter remote operator in AR app, don't trigger additional camera gestures
+ if (this.pauseTouchGestures) return;
+
+ isMultitouchGestureActive = true;
+
+ if (this.isTouchControlModeActive()) {
+ handleTouchControlDown(event);
+ return;
+ }
+
+ if (event.touches.length === 1) {
+ initialPosition = null;
+ didMoveAtAll = false;
+ this.mouseInput.isRotateRequested = true; // rotate
+ this.mouseInput.isStrafeRequested = false;
+ this.mouseInput.last.x = 0;
+ this.mouseInput.last.y = 0;
+ // this.setFocusTargetCube(event);
+
+ } else if (event.touches.length === 2) {
+ initialDistance = 0; // Reset pinch distance
+ this.mouseInput.isRotateRequested = false;
+ this.mouseInput.isStrafeRequested = true; // pan
+ this.mouseInput.last.x = 0;
+ this.mouseInput.last.y = 0;
+ }
+ });
+ document.addEventListener('touchmove', (event) => {
+ if (!isMultitouchGestureActive) return;
+ if (this.pauseTouchGestures) return;
+ event.preventDefault();
+
+ // Ensure regular zoom level
+ document.documentElement.style.zoom = '1';
+ // Ensure no page offset
+ window.scrollTo(0, 0);
+
+ if (this.isTouchControlModeActive()) {
+ handleTouchControlMove(event);
+ return;
+ }
+
+ if (event.touches.length === 1) {
+ handleMultitouchRotate(event);
+
+ } else if (event.touches.length === 2) {
+ // pans based on overall translation of both fingers
+ handleMultitouchPan(event);
+ // zooms based on changing distance between fingers
+ handleMultitouchPinch(event);
+ didMoveAtAll = true;
+ }
+ });
+ document.addEventListener('touchend', (event) => {
+ initialDistance = 0;
+ this.mouseInput.isRotateRequested = false;
+ this.mouseInput.isStrafeRequested = false; // do we add this, or only if zero touches left?
+ isMultitouchGestureActive = false;
+
+ if (this.pauseTouchGestures) return;
+
+ // tapping without dragging moves the focus cube to the tapped location
+ if (!didMoveAtAll) {
+ this.setFocusTargetCube(event);
+ }
+
+ if (this.isTouchControlModeActive()) {
+ handleTouchControlUp(event);
+ }
+ });
+ }
+ reset() {
+ this.stopFollowing();
+ this.position = [this.initialPosition[0], this.initialPosition[1], this.initialPosition[2]];
+ this.targetPosition = [0, 0, 0];
+ this.mouseInput.lastWorldPos = [0, 0, 0];
+ this.focusTargetCube.position.copy(new THREE.Vector3().fromArray(this.mouseInput.lastWorldPos));
+ }
+ adjustEnvVars(distanceToTarget) {
+ // places new tools at the camera's targetPosition...
+ // relies on the fact that new tools are dropped 400mm in front of camera by default
+ realityEditor.device.environment.variables.newFrameDistanceMultiplier = distanceToTarget / 400;
+ }
+ getTargetMatrix() {
+ return [
+ 1, 0, 0, 0,
+ 0, 1, 0, 0,
+ 0, 0, 1, 0,
+ this.targetPosition[0], this.targetPosition[1], this.targetPosition[2], 1
+ ];
+ }
+ getFocusTargetCubeMatrix() {
+ return [
+ 1, 0, 0, 0,
+ 0, 1, 0, 0,
+ 0, 0, 1, 0,
+ this.focusTargetCube.position.x, this.focusTargetCube.position.y, this.focusTargetCube.position.z, 1
+ ];
+ }
+ onPanToggled(callback) {
+ this.callbacks.onPanToggled.push(callback);
+ }
+ onRotateToggled(callback) {
+ this.callbacks.onRotateToggled.push(callback);
+ }
+ onScaleToggled(callback) {
+ this.callbacks.onScaleToggled.push(callback);
+ }
+ triggerPanCallbacks(newValue) {
+ this.callbacks.onPanToggled.forEach(function(cb) { cb(newValue); });
+ }
+ triggerRotateCallbacks(newValue) {
+ this.callbacks.onRotateToggled.forEach(function(cb) { cb(newValue); });
+ }
+ triggerScaleCallbacks(newValue) {
+ this.callbacks.onScaleToggled.forEach(function(cb) { cb(newValue); });
+ }
+ zoomBackToPreStopFollowLevel() {
+ if (this.preStopFollowingDistanceToTarget === null) { return; }
+ let cameraNormalizedVector = normalize(add(this.position, negate(this.targetPosition)));
+ this.position = add(this.targetPosition, scalarMultiply(cameraNormalizedVector, this.preStopFollowingDistanceToTarget));
+ }
+ onFirstPersonDistanceToggled(callback) {
+ this.callbacks.onFirstPersonDistanceToggled.push(callback);
+ }
+ onStopFollowing(callback) {
+ this.callbacks.onStopFollowing.push(callback);
+ }
+ // get the camera direction as an array
+ getCameraDirection() {
+ return normalize(sub(this.targetPosition, this.position));
+ }
+ // set the target position based on the camera direction
+ setCameraDirection(cameraDirection) {
+ this.targetPosition = add(this.position, cameraDirection);
+ }
+ // if specify a focus direction, the camera will look into that direction. Note that dir is expected to be a unit vector
+ // if not, move the camera while keeping its lookAt direction
+ focus(pos, dir, zoomDistanceMm = 3000) {
+ let zoomFactor = zoomDistanceMm;
+ this.targetPosition[0] = pos.x;
+ this.targetPosition[1] = pos.y;
+ this.targetPosition[2] = pos.z;
+ if (dir !== undefined) {
+ this.position[0] = this.targetPosition[0] + dir.x * zoomFactor;
+ this.position[1] = this.targetPosition[1] + dir.y * zoomFactor;
+ this.position[2] = this.targetPosition[2] + dir.z * zoomFactor;
+ } else {
+ let camDir = this.getCameraDirection();
+ this.position[0] = this.targetPosition[0] - camDir[0] * zoomFactor;
+ this.position[1] = this.targetPosition[1] - camDir[1] * zoomFactor;
+ this.position[2] = this.targetPosition[2] - camDir[2] * zoomFactor;
+ }
+ }
+ orbit(xRot, yRot, camPos, camLookAt, target) {
+ const yaxis = new THREE.Vector3(0, 1, 0);
+ const newXAxis = new THREE.Vector3();
+ // basically set newXAxis x and z to direction tangent to camera lookAt direction, and y to 0
+ newXAxis.x = -camLookAt.z;
+ newXAxis.z = camLookAt.x;
+ newXAxis.y = 0;
+
+ newXAxis.normalize();
+
+ // step 1: first change the camera position
+ // method 1:
+ const newCamPos = camPos
+ .sub(target)
+ .applyAxisAngle(newXAxis, xRot)
+ .applyAxisAngle(yaxis, yRot)
+ .add(target);
+ this.position = newCamPos.toArray();
+
+ // step 2: change the camera lookAt/target position
+ const relLookAt = camLookAt
+ .multiplyScalar(this.focusDistance)
+ .applyAxisAngle(newXAxis, xRot)
+ .applyAxisAngle(yaxis, yRot)
+ .add(newCamPos);
+ this.targetPosition = relLookAt.toArray();
+ }
+ // this needs to be called externally each frame that you want it to update
+ update(options = { skipApplying: false }) {
+ this.velocity = [0, 0, 0];
+ this.targetVelocity = [0, 0, 0];
+
+ // following lets you choose a target that you can zoom in/out to "drift" behind / follow over the shoulder
+ if (this.followingState.active) {
+ this.updateFollowing();
+ }
+
+ // lockOnMode exactly snaps your viewpoint to match the lockOn target's sceneNode matrix
+ if (this.lockOnMode) {
+ this.updateLockOnMode();
+ return; // stop executing - cannot zoom or pan while in lock-on mode
+ }
+
+ // let previousTargetPosition = [this.targetPosition[0], this.targetPosition[1], this.targetPosition[2]];
+ // move camera to cameraPosition and look at cameraTargetPosition
+ let newCameraLookAtMatrix = lookAt(this.position[0], this.position[1], this.position[2], this.targetPosition[0], this.targetPosition[1], this.targetPosition[2], 0, 1, 0);
+
+ let ev = this.position;
+ let cv = this.targetPosition;
+ let uv = [0, 1, 0];
+
+ this.distanceToTarget = magnitude(add(ev, negate(cv)));
+ this.adjustEnvVars(this.distanceToTarget);
+
+ let mCamera = newCameraLookAtMatrix; // translation is based on what direction you're facing,
+ let vCamX = normalize([mCamera[0], mCamera[4], mCamera[8]]);
+ let vCamY = normalize([mCamera[1], mCamera[5], mCamera[9]]);
+ let _vCamZ = normalize([mCamera[2], mCamera[6], mCamera[10]]);
+
+ let forwardVector = normalize(add(ev, negate(cv))); // vector from the camera to the center point
+ let horizontalVector = normalize(crossProduct(uv, forwardVector)); // a "right" vector, orthogonal to n and the lookup vector
+ let verticalVector = crossProduct(forwardVector, horizontalVector); // resulting orthogonal vector to n and u, as the up vector isn't necessarily one anymore
+
+ let distanceToFocus = magnitude(sub(this.position, this.mouseInput.lastWorldPos));
+ let distancePanFactor = Math.max(2, distanceToFocus / 1000); // speed when 1 meter units away, scales up w/ distance
+
+ if (this.idleOrbitting) {
+ this.mouseInput.unprocessedDX = 0.15;
+ this.mouseInput.isRotateRequested = true;
+ this.mouseInput.isStrafeRequested = false;
+ }
+
+ // rotate
+ if (this.mouseInput.isRotateRequested && (this.mouseInput.unprocessedDX !== 0 || this.mouseInput.unprocessedDY !== 0)) {
+ let camLookAt = new THREE.Vector3().fromArray(this.getCameraDirection());
+ let angle = camLookAt.clone().angleTo(new THREE.Vector3(camLookAt.x, 0, camLookAt.z));
+ // rotateFactor is a quadratic function that goes through (+-PI/2, 0) and (0, 1), so that when camera gets closer to 2 poles, the slower it rotates
+ let rotateFactor = -Math.pow(angle / Math.PI, 2) * 4 + 1;
+ let xRot = -this.mouseInput.unprocessedDY * 0.01 * rotateFactor;
+ let yRot = -this.mouseInput.unprocessedDX * 0.01 * rotateFactor;
+ let camPos = new THREE.Vector3().fromArray(this.position);
+ let target = new THREE.Vector3().fromArray(this.mouseInput.lastWorldPos);
+ this.orbit(xRot, yRot, camPos, camLookAt, target);
+
+ this.deselectTarget();
+
+ this.mouseInput.unprocessedDX = 0;
+ this.mouseInput.unprocessedDY = 0;
+ }
+
+ // strafe
+ if (this.mouseInput.isStrafeRequested) {
+ if (this.mouseInput.unprocessedDX !== 0) { // strafe left-right
+ let vector = scalarMultiply(negate(horizontalVector), distancePanFactor * this.speedFactors.translation * this.mouseInput.unprocessedDX * getCameraPanSensitivity());
+ this.targetVelocity = add(this.targetVelocity, vector);
+ this.velocity = add(this.velocity, vector);
+ this.deselectTarget();
+
+ this.mouseInput.unprocessedDX = 0;
+ }
+ if (this.mouseInput.unprocessedDY !== 0) { // strafe up-down
+ let vector = scalarMultiply(verticalVector, distancePanFactor * this.speedFactors.translation * this.mouseInput.unprocessedDY * getCameraPanSensitivity());
+ this.targetVelocity = add(this.targetVelocity, vector);
+ this.velocity = add(this.velocity, vector);
+ this.deselectTarget();
+
+ this.mouseInput.unprocessedDY = 0;
+ }
+ }
+
+ // scroll
+ scrollOperation: if (this.mouseInput.unprocessedScroll !== 0) {
+
+ // prevent from scrolling while rotating
+ if (this.mouseInput.isRotateRequested) {
+ this.mouseInput.unprocessedScroll = 0;
+ this.deselectTarget();
+ break scrollOperation;
+ }
+
+ if (this.followingState.active) {
+ // while following, zooming in-out moves the camera along a parametric curve up and behind the virtualizer
+ let dDist = this.speedFactors.scale * getCameraZoomSensitivity() * this.mouseInput.unprocessedScroll;
+ this.followingState.currentFollowingDistance = Math.min(10000, Math.max(0, this.followingState.currentFollowingDistance + dDist));
+
+ this.updateParametricTargetAndPosition(this.followingState.currentFollowingDistance);
+
+ } else {
+ let camToFocusCube = magnitude(sub(this.mouseInput.lastWorldPos, this.position));
+ // increase speed as distance increases
+ let nonLinearFactor = 1.05; // closer to 1 = less intense log (bigger as distance bigger)
+ let isZoomingIn = this.mouseInput.unprocessedScroll < 0;
+ let baseLog = getBaseLog(nonLinearFactor, camToFocusCube) / 100;
+
+ // move camera & camera target along the camera <---> focus cube line
+ {
+ let distanceMultiplier = Math.max(1, baseLog);
+ let cameraToFocusUnit = normalize(sub(this.position, this.mouseInput.lastWorldPos));
+ let vector = scalarMultiply(cameraToFocusUnit, distanceMultiplier * this.speedFactors.scale * getCameraZoomSensitivity() * this.mouseInput.unprocessedScroll);
+ // if distanceToTarget <= 200, slow down zooming speed quadratically to prevent from zooming too close / beyond the target
+ if (isZoomingIn && camToFocusCube <= 200) {
+ let scrollFactor = null;
+ if (camToFocusCube <= 100) {
+ scrollFactor = 0;
+ } else {
+ scrollFactor = Math.pow(camToFocusCube / 200, 2);
+ }
+ vector = scalarMultiply(vector, scrollFactor);
+ }
+ this.velocity = add(this.velocity, vector);
+
+ this.targetVelocity = add(this.targetVelocity, vector);
+ }
+
+ this.deselectTarget();
+ }
+
+ this.mouseInput.unprocessedScroll = 0; // reset now that data is processed
+ }
+
+ let flyingSpeed = 30;
+ if (this.isFlying) {
+ // handle mouse movements
+ let mouseVector = [0, 0, 0];
+ mouseVector = add(mouseVector, scalarMultiply(vCamX, 0.0005 * (2 * Math.PI * this.distanceToTarget) * this.mouseFlyInput.unprocessedDX));
+ this.mouseFlyInput.unprocessedDX = 0;
+ mouseVector = add(mouseVector, scalarMultiply(negate(vCamY), 0.0005 * (2 * Math.PI * this.distanceToTarget) * this.mouseFlyInput.unprocessedDY));
+ this.mouseFlyInput.unprocessedDY = 0;
+ this.targetVelocity = add(this.targetVelocity, mouseVector);
+ // handle WASDQE movements, shift to speed up
+ let transformKeys = {
+ W: this.keyboard.keyStates[this.keyboard.keyCodes.W] === 'down',
+ A: this.keyboard.keyStates[this.keyboard.keyCodes.A] === 'down',
+ S: this.keyboard.keyStates[this.keyboard.keyCodes.S] === 'down',
+ D: this.keyboard.keyStates[this.keyboard.keyCodes.D] === 'down',
+ Q: this.keyboard.keyStates[this.keyboard.keyCodes.Q] === 'down',
+ E: this.keyboard.keyStates[this.keyboard.keyCodes.E] === 'down',
+ };
+ let keyDirection = [transformKeys.S - transformKeys.W, transformKeys.D - transformKeys.A, transformKeys.E - transformKeys.Q];
+ if (magnitude(keyDirection) !== 0) {
+ let vector = [0, 0, 0];
+ flyingSpeed = this.keyboard.keyStates[this.keyboard.keyCodes.SHIFT] === 'down' ? 40 : 20;
+ let forwardValue = scalarMultiply(forwardVector, keyDirection[0]);
+ let horizontalValue = scalarMultiply(horizontalVector, keyDirection[1]);
+ let verticalValue = scalarMultiply([0, 1, 0], keyDirection[2]);
+ vector = add(vector, add(add(forwardValue, horizontalValue), verticalValue));
+ vector = scalarMultiply(normalize(vector), flyingSpeed);
+ this.targetVelocity = add(this.targetVelocity, vector);
+ this.velocity = add(this.velocity, vector);
+ }
+ }
+
+ // TODO: add back keyboard controls
+ // TODO: add back 6D mouse controls
+
+ // this is where the velocity gets added to the position...
+ // anything that modifies the camera movement should be above this line in the update function
+ if (!this.mouseInput.isRotateRequested || this.isFlying || this.zoomOutTransition) {
+ // let camLookAt = new THREE.Vector3().fromArray(this.getCameraDirection());
+ // let angle = camLookAt.clone().angleTo(new THREE.Vector3(camLookAt.x, 0, camLookAt.z));
+ // rotateFactor is a quadratic function that goes through (+-PI/2, 0) and (0, 1), so that when camera gets closer to 2 poles, the slower it rotates
+ this.position = add(this.position, this.velocity);
+ this.targetPosition = add(this.targetPosition, this.targetVelocity);
+ }
+
+ // tween the matrix every frame to animate it to the new position
+ // let cameraNode = realityEditor.sceneGraph.getSceneNodeById('CAMERA');
+ let currentCameraMatrix = realityEditor.gui.ar.utilities.copyMatrix(this.cameraNode.localMatrix);
+ let destinationCameraMatrix = realityEditor.gui.ar.utilities.invertMatrix(newCameraLookAtMatrix);
+ let totalDifference = sumOfElementDifferences(destinationCameraMatrix, currentCameraMatrix);
+ if (totalDifference < 0.00001) {
+ return; // don't animate the matrix with an infinite level of precision, stop when it gets very close to destination
+ }
+
+ // disables smoothing while following, to provide a tighter sync with the followed element
+ let shouldSmoothCamera = !this.isFollowingFirstPerson() && !this.zoomOutTransition;
+ let animationSpeed = shouldSmoothCamera ? (realityEditor.spatialCursor.isGSActive() ? 0.6 : 0.3) : 1.0;
+ let newCameraMatrix = tweenMatrix(currentCameraMatrix, destinationCameraMatrix, animationSpeed);
+
+ if (!options.skipApplying) {
+ if (this.cameraNode.id === 'CAMERA') {
+ realityEditor.sceneGraph.setCameraPosition(newCameraMatrix);
+ let cameraNode = realityEditor.sceneGraph.getCameraNode();
+ cameraNode.needsRerender = true;
+ } else {
+ this.cameraNode.setLocalMatrix(newCameraMatrix);
+ }
+ }
+
+ // allows us to schedule code to trigger exactly after the camera has updated its position N times
+ // useful for some calculations that require an up-to-date camera. can also be used for animations
+ let callbacksToTrigger = [];
+ this.afterNFrames.forEach((info) => {
+ info.n -= 1;
+ if (info.n <= 0) {
+ callbacksToTrigger.push(info.callback);
+ }
+ });
+ this.afterNFrames = this.afterNFrames.filter(entry => entry.n > 0);
+ callbacksToTrigger.forEach(cb => cb());
+ }
+
+ // returns the position [xDist,yDist,zDist] within the coordinate system defined by startPosition and startTargetPosition
+ getRelativePosition(startPosition, startTargetPosition, xDist = 0, yDist = 0, zDist = 1000) {
+ let ev = startPosition;
+ let cv = startTargetPosition;
+ let uv = [0, 1, 0];
+
+ let forwardVector = normalize(add(ev, negate(cv))); // vector from the camera to the center point
+ let horizontalVector = normalize(crossProduct(uv, forwardVector)); // a "right" vector, orthogonal to n and the lookup vector
+ let verticalVector = crossProduct(forwardVector, horizontalVector); // resulting orthogonal vector to n and u, as the up vector isn't necessarily one anymore
+
+ let endPosition = [...startPosition];
+ endPosition = add(endPosition, scalarMultiply(horizontalVector, xDist));
+ endPosition = add(endPosition, scalarMultiply(verticalVector, yDist));
+ endPosition = add(startPosition, scalarMultiply(forwardVector, zDist));
+ return endPosition;
+ }
+
+ isFollowingFirstPerson() {
+ return this.followingState.active &&
+ this.followingState.currentFollowingDistance <= MIN_FIRST_PERSON_DISTANCE;
+ }
+
+ /////////////////////////////
+ // FOLLOWING THE VIRTUALIZER
+ /////////////////////////////
+ follow(sceneNodeToFollow, initialFollowDistance) {
+ this.followingState.active = true;
+ this.followingState.selectedId = sceneNodeToFollow.id;
+ if (typeof initialFollowDistance !== 'undefined') {
+ this.followingState.currentFollowingDistance = initialFollowDistance; // can adjust with scroll wheel
+
+ let isFirstPerson = initialFollowDistance <= MIN_FIRST_PERSON_DISTANCE;
+ this.callbacks.onFirstPersonDistanceToggled.forEach(cb => cb(isFirstPerson, this.followingState.currentFollowingDistance));
+ }
+
+ this.updateParametricTargetAndPosition(this.followingState.currentFollowingDistance);
+ }
+ deselectTarget() {
+ // set the target position to the mouse cursor position, and then stop following
+ let selectedNode = realityEditor.sceneGraph.getSceneNodeById(this.followingState.selectedId);
+ if (!selectedNode) { return; }
+ let virtualizerMatrixThree = new THREE.Matrix4();
+ let relativeMatrix = selectedNode.getMatrixRelativeTo(realityEditor.sceneGraph.getSceneNodeById(realityEditor.sceneGraph.getWorldId()));
+ realityEditor.gui.threejsScene.setMatrixFromArray(virtualizerMatrixThree, relativeMatrix);
+
+ // the new focus of the camera should be the point 1 meter in front of the virtualizer
+ // let virtualizerForwardPosition = this.followingState.partiallyStabilizedTargetObject.getWorldPosition(new THREE.Vector3());
+ // this.targetPosition = [virtualizerForwardPosition.x, virtualizerForwardPosition.y, virtualizerForwardPosition.z];
+
+ if (this.preStopFollowingDistanceToTarget === null) {
+ // calculate distance from this.position to virtualizerForwardPosition, so that we can zoom back to this
+ this.preStopFollowingDistanceToTarget = magnitude(add(this.position, negate(this.targetPosition)));
+ }
+
+ // need to allow the camera position to update once to actually lookAt the targetPosition before we stop
+ this.afterOneFrame(() => {
+ if (this.followingState.active) {
+ this.stopFollowing();
+ }
+ this.cancelLockOnMode();
+ });
+ }
+ afterOneFrame(callback) {
+ this.afterNFrames.push({
+ n: 1,
+ callback: callback
+ });
+ }
+ stopFollowing() {
+ this.followingState.active = false;
+ this.followingState.selectedId = null;
+
+ if (this.preStopFollowingDistanceToTarget !== null) {
+ this.zoomBackToPreStopFollowLevel();
+ this.preRotateDistanceToTarget = this.preStopFollowingDistanceToTarget; // todo Steve: this.preRotateDistanceToTarget seems not to be used anywhere in the code. Is it still necessary?
+ this.preStopFollowingDistanceToTarget = null;
+ }
+
+ this.cancelLockOnMode();
+
+ this.callbacks.onStopFollowing.forEach(cb => cb());
+ }
+
+ /**
+ * Stops lockOnMode (if currently locked on), and triggers callbacks
+ */
+ cancelLockOnMode() {
+ if (this.lockOnMode) {
+ this.lockOnMode = null;
+ this.callbacks.onStopLockOnMode.forEach((cb) => {
+ cb();
+ });
+ }
+ }
+ /**
+ * Update the camera position to exactly match the sceneNode of the id stored in this.lockOnMode
+ */
+ updateLockOnMode() {
+ // move the camera position to the exact position of the avatar they're set to track
+ if (!this.lockOnMode) return;
+ let objectSceneNode = realityEditor.sceneGraph.getSceneNodeById(this.lockOnMode);
+ if (!objectSceneNode) return;
+ let lockedOnWorldMatrix = objectSceneNode.worldMatrix;
+
+ const APPLY_SMOOTHING_TO_LOCK_ON_MODE = false; // this didn't look too good when I turned it on, but we could further experiment with it in future
+
+ let newCameraMatrix;
+ if (APPLY_SMOOTHING_TO_LOCK_ON_MODE) {
+ let totalDifference = sumOfElementDifferences(lockedOnWorldMatrix, this.cameraNode.localMatrix);
+ if (totalDifference < 0.00001) {
+ return; // don't animate the matrix with an infinite level of precision, stop when it gets very close to destination
+ }
+ let animationSpeed = 0.3;
+ newCameraMatrix = tweenMatrix(this.cameraNode.localMatrix, lockedOnWorldMatrix, animationSpeed);
+ } else {
+ newCameraMatrix = realityEditor.gui.ar.utilities.copyMatrix(lockedOnWorldMatrix);
+ }
+
+ // update the three.js camera position
+ if (this.cameraNode.id === 'CAMERA') {
+ realityEditor.sceneGraph.setCameraPosition(newCameraMatrix);
+ let cameraNode = realityEditor.sceneGraph.getCameraNode();
+ cameraNode.needsRerender = true;
+ }
+ }
+ // trigger this in the main update loop each frame while we are following, to perform the following camera motion
+ updateFollowing() {
+ // don't update parametric position while we are disengaging from a follow, otherwise the target velocity goes crazy
+ if (this.preStopFollowingDistanceToTarget !== null) { return; }
+
+ // check that the sceneNode exists with its worldMatrix positioned at the virtualizer
+ let targetPosition = realityEditor.sceneGraph.getWorldPosition(this.followingState.selectedId);
+ if (!targetPosition) { this.stopFollowing(); return; }
+
+ // to work in portrait or landscape, the curve that the camera follows (which is nested inside a threejs group)
+ // needs to not have the exact transform of the virtualizer – ignore the twist and tilt of the phone
+ this.stabilizeCameraFollowPathUpVector();
+
+ // updates the camera position to equal the parametricPositionObject, and its direction to lookAt the parametricTargetObject
+ // these invisible threejs objects are nested inside the stabilizedContainer and their positions move as you scroll in/out
+ this.moveCameraToParametricFollowPosition();
+ }
+ stabilizeCameraFollowPathUpVector() {
+ // before beginning, ensure the right group hierarchies exist to compute the lookAt vector
+ this.createMissingThreejsFollowingGroups();
+
+ // 1. directly set the matrix of the UN-stabilized container to that of the virtualizer
+ let selectedNode = realityEditor.sceneGraph.getSceneNodeById(this.followingState.selectedId);
+ realityEditor.gui.threejsScene.setMatrixFromArray(this.followingState.unstabilizedContainer.matrix, selectedNode.worldMatrix);
+
+ // 2. set the position of the stabilized container = the UN-stabilized container
+ let virtualizerPosition = new THREE.Vector3().setFromMatrixPosition(this.followingState.unstabilizedContainer.matrix);
+ this.followingState.stabilizedContainer.position.set(virtualizerPosition.x, virtualizerPosition.y, virtualizerPosition.z);
+
+ // 3. get the "world position" of the forwardTargetObject, which is a child of the UN-stabilized container (in the coordinate system of the threeJsContainer)
+ let tiltedForwardPosition = this.followingState.forwardTargetObject.position.clone().applyMatrix4(this.followingState.unstabilizedContainer.matrix);
+
+ // 4. set the "world position" of the levelTargetObject to that of the forwardTargetObject, but with its height = the height of the virtualizer
+ this.followingState.levelTargetObject.position.set(tiltedForwardPosition.x, tiltedForwardPosition.y, virtualizerPosition.z);
+
+ // 5. stabilize the camera's up-down tilt more and more as you zoom out, such that when you are fully zoomed out, it doesn't tilt up and down at all,
+ // but when you are fully zoomed in it is completely un-stabilized, so you tilt up and down exactly with the virtualizer perspective
+ // (this step is optional but leads to stable birds-eye and seamless first-person transition)
+ let parametricTargetObject = realityEditor.gui.threejsScene.getObjectByName('parametricTargetObject');
+ let MIN_Z = 1250; // these can be calculated from passing min and max follow distance into updateParametricTargetAndPosition
+ let MAX_Z = 2500;
+ let unclampedPercent = 1.0 - (Math.abs(parametricTargetObject.position.z) - MIN_Z) / (MAX_Z - MIN_Z);
+ let stabilizationPercent = clamp(unclampedPercent, 0, 1);
+ let stabilizedHeight = virtualizerPosition.z * stabilizationPercent + tiltedForwardPosition.z * (1 - stabilizationPercent);
+ let stabilizedTargetPosition = new THREE.Vector3(this.followingState.levelTargetObject.position.x, this.followingState.levelTargetObject.position.y, stabilizedHeight);
+
+ if (this.followingState.partiallyStabilizedTargetObject) {
+ // set before doing localToWorld, since both are children of the threeJsContainer
+ this.followingState.partiallyStabilizedTargetObject.position.set(stabilizedTargetPosition.x, stabilizedTargetPosition.y, stabilizedTargetPosition.z);
+ }
+
+ // 6. rotate the stabilized container to lookAt the (partially stabilized depending on distance) levelTargetObject
+ this.threeJsContainer.localToWorld(stabilizedTargetPosition); // THREE.lookAt takes in world coordinates, so requires a transform
+ this.followingState.stabilizedContainer.lookAt(stabilizedTargetPosition.x, stabilizedTargetPosition.y, stabilizedTargetPosition.z);
+ }
+ createMissingThreejsFollowingGroups() {
+ if (!this.followingState.unstabilizedContainer) {
+ this.followingState.unstabilizedContainer = new THREE.Group();
+ this.followingState.unstabilizedContainer.name = 'followingElementGroup';
+ this.followingState.unstabilizedContainer.matrixAutoUpdate = false;
+ this.followingState.unstabilizedContainer.visible = DISPLAY_PERSPECTIVE_CUBES;
+ this.threeJsContainer.add(this.followingState.unstabilizedContainer);
+
+ let debugCube = new THREE.Mesh(new THREE.BoxGeometry(100, 100, 100), new THREE.MeshBasicMaterial({ color: '#ffffff' }));
+ this.followingState.unstabilizedContainer.add(debugCube);
+
+ // These boxes could be groups / empty objects rather than meshes, but we have them to help debug
+ let forwardTarget = new THREE.Mesh(new THREE.BoxGeometry(40, 40, 40), new THREE.MeshBasicMaterial({ color: '#0000ff' }));
+ forwardTarget.position.set(0, 0, FOCUS_DISTANCE_MM_IN_FRONT_OF_VIRTUALIZER);
+ forwardTarget.name = 'forwardFollowTargetObject';
+ forwardTarget.visible = DISPLAY_PERSPECTIVE_CUBES;
+ this.followingState.unstabilizedContainer.add(forwardTarget);
+ this.followingState.forwardTargetObject = forwardTarget;
+
+ let level = new THREE.Mesh(new THREE.BoxGeometry(40, 40, 40), new THREE.MeshBasicMaterial({ color: '#00ffff' }));
+ level.name = 'levelFollowTargetObject';
+ level.visible = DISPLAY_PERSPECTIVE_CUBES;
+ this.threeJsContainer.add(level);
+ this.followingState.levelTargetObject = level;
+ }
+
+ if (!this.followingState.stabilizedContainer) {
+ let container = new THREE.Group();
+ container.name = 'followStabilizedContainer';
+ container.visible = DISPLAY_PERSPECTIVE_CUBES;
+ this.followingState.stabilizedContainer = container;
+ this.threeJsContainer.add(container);
+
+ let obj = new THREE.Mesh(new THREE.BoxGeometry(20, 20, 20), new THREE.MeshBasicMaterial({ color: '#ff0000' }));
+ obj.name = 'parametricPositionObject';
+ obj.visible = DISPLAY_PERSPECTIVE_CUBES;
+ container.add(obj);
+
+ let target = new THREE.Mesh(new THREE.BoxGeometry(20, 20, 20), new THREE.MeshBasicMaterial({ color: '#ff0000' }));
+ target.name = 'parametricTargetObject';
+ target.visible = DISPLAY_PERSPECTIVE_CUBES;
+ container.add(target);
+
+ // set initial positions of objects otherwise camera following will break
+ this.updateParametricTargetAndPosition(this.followingState.currentFollowingDistance);
+ }
+
+ if (!this.followingState.partiallyStabilizedTargetObject) {
+ let obj = new THREE.Mesh(new THREE.BoxGeometry(40, 40, 40), new THREE.MeshBasicMaterial({ color: '#00ffff' }));
+ obj.name = 'partiallyStabilizedFollowTargetObject';
+ obj.visible = DISPLAY_PERSPECTIVE_CUBES;
+ this.threeJsContainer.add(obj);
+ this.followingState.partiallyStabilizedTargetObject = obj;
+ }
+ }
+ // actually moves the camera to be located at the positionObject and looking at the targetObject
+ // (which are nested inside the stabilized following container)
+ moveCameraToParametricFollowPosition() {
+ let positionObject = realityEditor.gui.threejsScene.getObjectByName('parametricPositionObject');
+ let targetObject = realityEditor.gui.threejsScene.getObjectByName('parametricTargetObject');
+
+ let groundPlaneMatrixArray = realityEditor.sceneGraph.getGroundPlaneNode().worldMatrix;
+ let invGroundPlaneMatrix = new THREE.Matrix4();
+ realityEditor.gui.threejsScene.setMatrixFromArray(invGroundPlaneMatrix, groundPlaneMatrixArray);
+ invGroundPlaneMatrix.invert();
+
+ let vec1 = new THREE.Vector3();
+ let vec2 = new THREE.Vector3();
+ targetObject.getWorldPosition(vec1);
+ positionObject.getWorldPosition(vec2);
+ vec1.applyMatrix4(invGroundPlaneMatrix);
+ vec2.applyMatrix4(invGroundPlaneMatrix);
+ let newPosVec = vec2.toArray();
+ let newTargetPosVec = vec1.toArray();
+
+ let movement = add(newPosVec, negate(this.position));
+ if (movement[0] !== 0 || movement[1] !== 0 || movement[2] !== 0) {
+ this.velocity = add(this.velocity, movement);
+ }
+
+ let targetMovement = add(newTargetPosVec, negate(this.targetPosition));
+ if (targetMovement[0] !== 0 || targetMovement[1] !== 0 || targetMovement[2] !== 0) {
+ this.targetVelocity = add(this.targetVelocity, targetMovement);
+ }
+ }
+ // moves the parametricPositionObject and parametricTargetObject along curves,
+ // based on distance between VirtualCamera and the virtualizer we are following
+ updateParametricTargetAndPosition(distanceToCamera) {
+ let positionObject = realityEditor.gui.threejsScene.getObjectByName('parametricPositionObject');
+ let targetObject = realityEditor.gui.threejsScene.getObjectByName('parametricTargetObject');
+
+ if (!positionObject || !targetObject) {
+ return;
+ }
+
+ // Diagram showing the path that the curve follows relative to the virtualizer (from a side view)
+ // A is where that object will be when the camera is zoomed in all the way
+ // B is where that object will be when the camera is zoomed out all the way
+ //
+ // [POSITION OBJECT]
+ // -B>|
+ // --
+ // ---
+ // ----
+ // || [VIRTUALIZER] | cb(this.isFollowingFirstPerson(), this.followingState.currentFollowingDistance));
+ }
+ }
+
+ //************************************************ Utilities *************************************************//
+
+ // since groundPlaneMatrix is applied to threejsContainerObj in threejsScene.js
+ // here we apply the same groundPlaneMatrix to VirtualCamera, so that it's in the same coord space as threejsContainerObj
+ function convertToThreejsContainerObjSpace(eyeX, eyeY, eyeZ, centerX, centerY, centerZ) {
+ let groundPlaneMatrixArray = realityEditor.sceneGraph.getGroundPlaneNode().worldMatrix;
+ let groundPlaneMatrix = new THREE.Matrix4();
+ realityEditor.gui.threejsScene.setMatrixFromArray(groundPlaneMatrix, groundPlaneMatrixArray);
+ let cameraPos = new THREE.Vector3(eyeX, eyeY, eyeZ);
+ cameraPos.applyMatrix4(groundPlaneMatrix);
+ eyeX = cameraPos.x;
+ eyeY = cameraPos.y;
+ eyeZ = cameraPos.z;
+ let targetPos = new THREE.Vector3(centerX, centerY, centerZ);
+ targetPos.applyMatrix4(groundPlaneMatrix);
+ centerX = targetPos.x;
+ centerY = targetPos.y;
+ centerZ = targetPos.z;
+ return {a: eyeX, b: eyeY, c: eyeZ, d: centerX, e: centerY, f: centerZ};
+ }
+
+ // Working look-at matrix generator (with a set of vector3 math functions)
+ function lookAt( eyeX, eyeY, eyeZ, centerX, centerY, centerZ, upX, upY, upZ ) {
+ let {a, b, c, d, e, f} = convertToThreejsContainerObjSpace(eyeX, eyeY, eyeZ, centerX, centerY, centerZ);
+ eyeX = a; eyeY = b; eyeZ = c; centerX = d; centerY = e; centerZ = f;
+ var ev = [eyeX, eyeY, eyeZ];
+ var cv = [centerX, centerY, centerZ];
+ var uv = [upX, upY, upZ];
+
+ var n = normalize(add(ev, negate(cv))); // vector from the camera to the center point
+ var u = normalize(crossProduct(uv, n)); // a "right" vector, orthogonal to n and the lookup vector
+ var v = crossProduct(n, u); // resulting orthogonal vector to n and u, as the up vector isn't necessarily one anymore
+
+ return [u[0], v[0], n[0], 0,
+ u[1], v[1], n[1], 0,
+ u[2], v[2], n[2], 0,
+ dotProduct(negate(u), ev), dotProduct(negate(v), ev), dotProduct(negate(n), ev), 1];
+ }
+
+ function scalarMultiply(A, x) {
+ return [A[0] * x, A[1] * x, A[2] * x];
+ }
+
+ function negate(A) {
+ return [-A[0], -A[1], -A[2]];
+ }
+
+ function add(A, B) {
+ return [A[0] + B[0], A[1] + B[1], A[2] + B[2]];
+ }
+
+ function sub(A, B) {
+ return add(A, negate(B));
+ }
+
+ // function hadamardProduct(A, B) {
+ // return [A[0] * B[0], A[1] * B[1], A[2] * B[2]];
+ // }
+
+ function magnitude(A) {
+ return Math.sqrt(A[0] * A[0] + A[1] * A[1] + A[2] * A[2]);
+ }
+
+ function normalize(A) {
+ // include the edge case where A === [0, 0, 0]
+ if (A[0] === 0 && A[1] === 0 && A[2] === 0) return A;
+ var mag = magnitude(A);
+ return [A[0] / mag, A[1] / mag, A[2] / mag];
+ }
+
+ function crossProduct(A, B) {
+ var a = A[1] * B[2] - A[2] * B[1];
+ var b = A[2] * B[0] - A[0] * B[2];
+ var c = A[0] * B[1] - A[1] * B[0];
+ return [a, b, c];
+ }
+
+ function dotProduct(A, B) {
+ return A[0] * B[0] + A[1] * B[1] + A[2] * B[2];
+ }
+
+ // function multiplyMatrixVector(M, v) {
+ // return [M[0] * v[0] + M[1] * v[1] + M[2] * v[2],
+ // M[3] * v[0] + M[4] * v[1] + M[5] * v[2],
+ // M[6] * v[0] + M[7] * v[1] + M[8] * v[2]];
+ // }
+
+ function sumOfElementDifferences(M1, M2) {
+ // assumes M1 and M2 are of equal length
+ let sum = 0;
+ for (let i = 0; i < M1.length; i++) {
+ sum += Math.abs(M1[i] - M2[i]);
+ }
+ return sum;
+ }
+
+ function getBaseLog(x, y) {
+ return Math.log(y) / Math.log(x);
+ }
+
+ function clamp(num, min, max) {
+ return Math.min(Math.max(num, min), max);
+ }
+
+ // function prettyPrint(matrix, precision) {
+ // return '[ ' + matrix[0].toFixed(precision) + ', ' + matrix[1].toFixed(precision) + ', ' + matrix[2].toFixed(precision) + ']';
+ // }
+
+ function tweenMatrix(currentMatrix, destination, tweenSpeed) {
+ if (typeof tweenSpeed === 'undefined') { tweenSpeed = 0.5; } // default value
+
+ if (currentMatrix.length !== destination.length) {
+ console.warn('matrices are inequal lengths. cannot be tweened so just assigning current=destination');
+ return realityEditor.gui.ar.utilities.copyMatrix(destination);
+ }
+ if (tweenSpeed <= 0 || tweenSpeed >= 1) {
+ return realityEditor.gui.ar.utilities.copyMatrix(destination);
+ }
+
+ var m = [];
+ for (var i = 0; i < currentMatrix.length; i++) {
+ m[i] = destination[i] * tweenSpeed + currentMatrix[i] * (1.0 - tweenSpeed);
+ }
+ return m;
+ }
+
+ function getCameraZoomSensitivity() {
+ return Math.max(0.01, realityEditor.gui.settings.toggleStates.cameraZoomSensitivity || 0.5);
+ }
+
+ function getCameraPanSensitivity() {
+ return Math.max(0.01, realityEditor.gui.settings.toggleStates.cameraPanSensitivity || 0.5);
+ }
+
+ // function getCameraRotateSensitivity() {
+ // return Math.max(0.01, realityEditor.gui.settings.toggleStates.cameraRotateSensitivity || 0.5);
+ // }
+
+ exports.VirtualCamera = VirtualCamera;
+})(realityEditor.device);
diff --git a/content_scripts/WebRTCCoordinator.js b/content_scripts/WebRTCCoordinator.js
new file mode 100644
index 00000000..3212bb14
--- /dev/null
+++ b/content_scripts/WebRTCCoordinator.js
@@ -0,0 +1,605 @@
+createNameSpace('realityEditor.device.cameraVis');
+
+import RVLParser from '../../thirdPartyCode/rvl/RVLParser.js';
+
+(function(exports) {
+ const DEPTH_REPR_FORCE_PNG = false;
+ const DEBUG = false;
+ // Coordinator re-sends joinNetwork message at this interval of ms until
+ // discoverPeers acknowledgement is received from remote peer
+ const JOIN_NETWORK_INTERVAL = 5000;
+
+ const decoder = new TextDecoder();
+
+ const ErrorMessage = {
+ autoplayBlocked: 'Autoplay blocked. Interact with page or grant permission in browser settings.',
+ noMicrophonePermissions: 'No microphone permission. Grant permission from browser and refresh page.',
+ webrtcIssue: 'Internal WebRTC issue.',
+ };
+
+ /**
+ * @param {ErrorMessage} message - human readable error text
+ * @param {Error} error - error responsible for causing this
+ * @param errorId - html element id for parent div
+ * @param errorTextId - html element for error text
+ * @param {number} duration - ms duration of notification popup
+ */
+ function showError(message, error, errorId, errorTextId, duration) {
+ console.error('webrtc error', error);
+ // showBannerNotification removes notification after set time so no additional function is needed
+ realityEditor.gui.modal.showBannerNotification(message, errorId, errorTextId, duration);
+ }
+
+ class WebRTCCoordinator {
+ constructor(cameraVisCoordinator, ws, consumerId) {
+ this.cameraVisCoordinator = cameraVisCoordinator;
+ /** @type ToolSocket */
+ this.ws = ws;
+ this.audioStream = null;
+ this.consumerId = consumerId;
+ this.muted = true;
+
+ // setInterval result used for repeatedly sending joinNetwork
+ this.joinNetworkInterval = null;
+
+ this.webrtcConnections = {};
+
+ this.subscribedObjects = {};
+
+ this.onToolsocketMessage = this.onToolsocketMessage.bind(this);
+ this.sendSignallingMessage = this.sendSignallingMessage.bind(this);
+
+ this.ws.on('/signalling', this.onToolsocketMessage);
+ const joinNetwork = () => {
+ this.sendSignallingMessage({
+ command: 'joinNetwork',
+ src: this.consumerId,
+ role: 'consumer',
+ });
+ };
+ joinNetwork();
+ this.joinNetworkInterval = setInterval(joinNetwork, JOIN_NETWORK_INTERVAL);
+
+ this.audioStreamPromise = navigator.mediaDevices.getUserMedia({
+ video: false,
+ audio: {
+ noiseSuppression: true,
+ },
+ }).then((stream) => {
+ this.audioStream = this.improveAudioStream(stream);
+ for (let conn of Object.values(this.webrtcConnections)) {
+ conn.audioStream = this.audioStream;
+ conn.localConnection.addStream(conn.audioStream);
+ }
+ this.updateMutedState();
+ }).catch(err => {
+ showError(ErrorMessage.noMicrophonePermissions, err, 'audioErrorUI', 'audioErrorText', 10000);
+ });
+ }
+
+ sendSignallingMessage(message) {
+ let identifier = 'unused';
+ const worldObject = realityEditor.worldObjects.getBestWorldObject();
+ if (worldObject) {
+ identifier = worldObject.port;
+ if (!this.subscribedObjects[identifier]) {
+ this.subscribedObjects[identifier] = true;
+ let serverSocket = realityEditor.network.realtime.getServerSocketForObject(worldObject.objectId);
+ serverSocket.on('/signalling', this.onToolsocketMessage);
+ }
+ }
+ this.ws.emit(realityEditor.network.getIoTitle(identifier, '/signalling'), message);
+ }
+
+ improveAudioStream(stream) {
+ const context = new AudioContext();
+ const src = context.createMediaStreamSource(stream);
+ const dst = context.createMediaStreamDestination();
+ const gainNode = context.createGain();
+ gainNode.gain.value = 6;
+ src.connect(gainNode);
+ gainNode.connect(dst);
+ return dst.stream;
+ }
+
+ updateMutedState() {
+ if (!this.audioStream) return;
+ for (let track of this.audioStream.getTracks()) {
+ track.enabled = !this.muted;
+ }
+ }
+
+ mute() {
+ this.muted = true;
+ this.updateMutedState();
+ }
+
+ unmute() {
+ this.muted = false;
+ this.updateMutedState();
+ }
+
+ async onToolsocketMessage(msgRaw) {
+ let msg;
+ try {
+ msg = typeof msgRaw === 'string' ? JSON.parse(msgRaw) : msgRaw;
+ } catch (e) {
+ console.warn('ws parse error', e, event);
+ return;
+ }
+ if (DEBUG) {
+ console.log('webrtc msg', msg);
+ }
+
+ if (msg.command === 'joinNetwork') {
+ if (msg.role === 'provider') {
+ await this.initConnection(msg.src);
+ }
+ return;
+ }
+
+ if (msg.command === 'discoverPeers' && msg.dest === this.consumerId) {
+ if (this.joinNetworkInterval) {
+ clearInterval(this.joinNetworkInterval);
+ this.joinNetworkInterval = null;
+ }
+ for (let provider of msg.providers) {
+ await this.initConnection(provider);
+ }
+ for (let consumer of msg.consumers) {
+ if (consumer !== this.consumerId) {
+ await this.initConnection(consumer);
+ }
+ }
+ return;
+ }
+
+ if (msg.dest !== this.consumerId) {
+ if (DEBUG) {
+ console.warn('discarding not mine', this.consumerId, msg);
+ }
+ return;
+ }
+
+ if (!this.webrtcConnections[msg.src]) {
+ if (!this.audioStream) {
+ await this.audioStreamPromise;
+ }
+
+ this.webrtcConnections[msg.src] = new WebRTCConnection(
+ this.sendSignallingMessage,
+ this.cameraVisCoordinator,
+ this.ws,
+ this.audioStream,
+ this.consumerId,
+ msg.src
+ );
+ this.webrtcConnections[msg.src].initLocalConnection();
+ }
+ this.webrtcConnections[msg.src].onSignallingMessage(msg);
+ }
+
+ async initConnection(otherId) {
+ const conn = this.webrtcConnections[otherId];
+ const goodChannelStates = ['connecting', 'open'];
+
+ if (conn) {
+ // connection already as good as it gets
+ if (conn.receiveChannel &&
+ goodChannelStates.includes(conn.receiveChannel.readyState)) {
+ return;
+ }
+
+ // This was initiated by the provider side, don't mess with it
+ if (!conn.offered) {
+ return;
+ }
+ }
+
+ if (!this.audioStream) {
+ await this.audioStreamPromise;
+ }
+
+ let newConn = new WebRTCConnection(
+ this.sendSignallingMessage,
+ this.cameraVisCoordinator,
+ this.ws,
+ this.audioStream,
+ this.consumerId,
+ otherId,
+ );
+
+ this.webrtcConnections[otherId] = newConn;
+ newConn.connect();
+ }
+ }
+
+ class WebRTCConnection {
+ constructor(sendSignallingMessageImpl, cameraVisCoordinator, ws, audioStream, consumerId, providerId) {
+ this.sendSignallingMessageImpl = sendSignallingMessageImpl;
+ this.cameraVisCoordinator = cameraVisCoordinator;
+ this.ws = ws;
+ this.consumerId = consumerId;
+ this.providerId = providerId;
+ this.audioStream = audioStream;
+ this.offered = false;
+
+ this.receiveChannel = null;
+ this.localConnection = null;
+
+ this.onSignallingMessage = this.onSignallingMessage.bind(this);
+
+ this.onReceiveChannelStatusChange =
+ this.onReceiveChannelStatusChange.bind(this);
+ this.onReceiveChannelMessage =
+ this.onReceiveChannelMessage.bind(this);
+ this.onSendChannelStatusChange =
+ this.onSendChannelStatusChange.bind(this);
+ this.onWebRTCError =
+ this.onWebRTCError.bind(this);
+ }
+
+ async onSignallingMessage(msg) {
+ if (msg.command === 'newIceCandidate') {
+ if (DEBUG) {
+ console.log('webrtc remote candidate', msg);
+ }
+ this.localConnection.addIceCandidate(msg.candidate)
+ .catch(this.onWebRTCError);
+ return;
+ }
+
+ if (msg.command === 'newDescription') {
+ try {
+ await this.localConnection.setRemoteDescription(msg.description);
+ if (!this.offered) {
+ let answer = await this.localConnection.createAnswer();
+ await this.localConnection.setLocalDescription(answer);
+ this.sendSignallingMessage({
+ src: this.consumerId,
+ dest: this.providerId,
+ command: 'newDescription',
+ description: this.localConnection.localDescription,
+ });
+ }
+ } catch (e) {
+ // This error only occurs as a result of older WebRTC implementations
+ if (this.localConnection.signalingState === 'stable' && e.name === 'InvalidStateError') {
+ console.warn('setRemoteDescription error', e);
+ return;
+ }
+ this.onWebRTCError(e);
+ }
+ }
+ }
+
+ initLocalConnection() {
+ this.localConnection = new RTCPeerConnection({
+ iceServers: [{
+ urls: [
+ 'stun:stun.l.google.com:19302',
+ 'stun:stun4.l.google.com:19305',
+ ],
+ }, {
+ urls: [
+ 'stun:stun.relay.metered.ca:80',
+ ],
+ }, {
+ urls: [
+ 'turn:turn.meta.ptc.io:443'
+ ],
+ username: 'test',
+ credential: 'uWmkoS44agy7GTN',
+ }, {
+ urls: [
+ 'turn:a.relay.metered.ca:443',
+ ],
+ username: 'c35c5795da892aeead553ae7',
+ credential: 'QNFn8q+yPVb1XG6k',
+ }, {
+ urls: [
+ 'turn:a.relay.metered.ca:80?transport=tcp',
+ ],
+ username: 'c35c5795da892aeead553ae7',
+ credential: 'QNFn8q+yPVb1XG6k',
+ }, {
+ urls: [
+ 'turn:spatial.ptc.io:3478'
+ ],
+ username: 'ptc',
+ credential: 'pgnhOCNXxwH2rz1qiV2hOuckkOtuu6Tx',
+ }],
+ });
+
+ this.localConnection.addEventListener('icecandidate', (e) => {
+ if (DEBUG) {
+ console.log('webrtc local candidate', e);
+ }
+
+ if (!e.candidate) {
+ return;
+ }
+
+ this.sendSignallingMessage({
+ src: this.consumerId,
+ dest: this.providerId,
+ command: 'newIceCandidate',
+ candidate: e.candidate,
+ });
+ });
+
+ this.localConnection.addEventListener('datachannel', (e) => {
+ if (DEBUG) {
+ console.log('webrtc datachannel', e);
+ }
+
+ this.sendChannel = e.channel;
+ this.sendChannel.addEventListener('open', this.onSendChannelStatusChange);
+ this.sendChannel.addEventListener('close', this.onSendChannelStatusChange);
+ });
+
+ this.localConnection.addEventListener('track', (e) => {
+ if (DEBUG) {
+ console.log('webrtc track event', e);
+ }
+
+ if (e.streams.length === 0) {
+ return;
+ }
+ const elt = document.createElement('video');
+ // elt.style.position = 'absolute';
+ // elt.style.top = 0;
+ // elt.style.left = 0;
+ // elt.style.zIndex = 10000;
+ // elt.style.transform = 'translateZ(10000px)';
+ // elt.controls = true;
+
+ elt.autoplay = true;
+ elt.srcObject = e.streams[0];
+ let timesFailed = 0;
+ let autoplayWhenAvailableInterval = setInterval(() => {
+ try {
+ elt.play();
+ } catch (err) {
+ if (DEBUG) {
+ console.log('autoplay failed', err);
+ }
+ timesFailed += 1;
+ // this is a delay of 3000 ms = 250 ms * 12 so that
+ // notifications don't overlap but stay on screen for a
+ // decent amount of time
+ if (timesFailed > 12) {
+ showError(ErrorMessage.autoplayBlocked, err, 'autoplayErrorUI', 'autoplayErrorText', 12 * 250);
+ timesFailed = 0;
+ }
+ }
+ }, 250);
+ elt.addEventListener('play', function clearAutoplayInterval() {
+ clearInterval(autoplayWhenAvailableInterval);
+ elt.removeEventListener('play', clearAutoplayInterval);
+ });
+ document.body.appendChild(elt);
+ });
+
+ this.receiveChannel = this.localConnection.createDataChannel(
+ 'sendChannel',
+ {
+ ordered: false,
+ maxRetransmits: 0,
+ },
+ );
+ this.receiveChannel.binaryType = 'arraybuffer';
+ this.receiveChannel.onopen = this.onReceiveChannelStatusChange;
+ this.receiveChannel.onclose = this.onReceiveChannelStatusChange;
+ this.receiveChannel.addEventListener('message', this.onReceiveChannelMessage);
+
+ if (this.audioStream) {
+ this.localConnection.addStream(this.audioStream);
+ } else {
+ console.warn('missing audiostream');
+ }
+ }
+
+ async connect() {
+ if (!this.localConnection) {
+ this.initLocalConnection();
+ }
+
+ this.offered = true;
+ const offer = await this.localConnection.createOffer({
+ offerToReceiveAudio: true,
+ offerToReceiveVideo: true,
+ });
+ await this.localConnection.setLocalDescription(offer);
+
+ this.sendSignallingMessage({
+ src: this.consumerId,
+ dest: this.providerId,
+ command: 'newDescription',
+ description: this.localConnection.localDescription,
+ });
+ }
+
+ sendSignallingMessage(message) {
+ this.sendSignallingMessageImpl(message);
+ }
+
+ onSendChannelStatusChange() {
+ if (!this.sendChannel) {
+ return;
+ }
+
+ const state = this.sendChannel.readyState;
+ if (DEBUG) {
+ console.log('webrtc onSendChannelStatusChange', state);
+ }
+ }
+
+ onReceiveChannelStatusChange() {
+ if (!this.receiveChannel) {
+ return;
+ }
+
+ const state = this.receiveChannel.readyState;
+ if (DEBUG) {
+ console.log('webrtc onReceiveChannelStatusChange', state);
+ }
+
+ if (state === 'open') {
+ // create cameravis with receiveChannel
+ }
+ }
+
+ async onReceiveChannelMessage(event) {
+ const id = this.providerId;
+ let bytes = event.data;
+ if (bytes instanceof ArrayBuffer) {
+ bytes = new Uint8Array(event.data);
+ }
+ if (bytes.length === 0) {
+ return;
+ }
+
+ if (bytes.length < 1000) {
+ // const decoder = new TextDecoder();
+ const matricesMsg = decoder.decode(bytes);
+ // blah blah it's matrix
+ const matrices = JSON.parse(matricesMsg);
+ this.onMatrices(id, matrices);
+ return;
+ }
+
+ if (DEPTH_REPR_FORCE_PNG) {
+ switch (bytes[0]) {
+ case 0xff: {
+ const imageUrl = URL.createObjectURL(new Blob([event.data], {type: 'image/jpeg'}));
+ // Color is always JPEG which has first byte 0xff
+ this.cameraVisCoordinator.renderPointCloud(id, 'texture', imageUrl);
+ }
+ break;
+
+ case 0x89: {
+ const imageUrl = URL.createObjectURL(new Blob([event.data], {type: 'image/png'}));
+ // Depth is always PNG which has first byte 0x89
+ this.cameraVisCoordinator.renderPointCloud(id, 'textureDepth', imageUrl);
+ }
+ break;
+ }
+ } else {
+ // jpeg start of image, chance of this happening from rvl is probably 0 but at most 1/(1 << 16)
+ if (bytes[0] === 0xff && bytes[1] === 0xd8) {
+ const imageUrl = URL.createObjectURL(new Blob([event.data], {type: 'image/jpeg'}));
+ // Color is always JPEG which has first byte 0xff
+ this.cameraVisCoordinator.renderPointCloud(id, 'texture', imageUrl);
+ // PNG header for depth just in case
+ } else if (bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) {
+ const imageUrl = URL.createObjectURL(new Blob([event.data], {type: 'image/png'}));
+ this.cameraVisCoordinator.renderPointCloud(id, 'textureDepth', imageUrl);
+ } else {
+ // if (!window.timings) {
+ // window.timings = {
+ // parseFrame: [],
+ // parseDepth: [],
+ // parseMats: [],
+ // doMats: [],
+ // doDepth: [],
+ // };
+ // }
+ // let start = window.performance.now();
+ const parser = new RVLParser(bytes.buffer);
+ // let parseFrame = window.performance.now();
+ const rawDepth = parser.getFrameRawDepth(parser.currentFrame);
+ // let parseDepth = window.performance.now();
+ const matricesMsg = decoder.decode(parser.currentFrame.payload);
+ const matrices = JSON.parse(matricesMsg);
+ // let parseMats = window.performance.now();
+ this.onMatrices(id, matrices);
+ // let doMats = window.performance.now();
+ if (!rawDepth) {
+ console.warn('RVL depth unparsed');
+ return;
+ }
+ this.cameraVisCoordinator.renderPointCloudRawDepth(id, rawDepth);
+ // let doDepth = window.performance.now();
+ // window.timings.parseFrame.push(parseFrame - start);
+ // window.timings.parseDepth.push(parseDepth - parseFrame);
+ // window.timings.parseMats.push(parseMats - parseDepth);
+ // window.timings.doMats.push(doMats - parseMats);
+ // window.timings.doDepth.push(doDepth - doMats);
+ }
+ }
+ }
+
+ onMatrices(id, matrices) {
+ let cameraNode = new realityEditor.sceneGraph.SceneNode(id + '-camera');
+ cameraNode.setLocalMatrix(matrices.camera);
+ cameraNode.updateWorldMatrix();
+
+ let gpNode = new realityEditor.sceneGraph.SceneNode(id + '-gp');
+ gpNode.needsRotateX = true;
+ let gpRxNode = new realityEditor.sceneGraph.SceneNode(id + '-gp' + 'rotateX');
+ gpRxNode.addTag('rotateX');
+ gpRxNode.setParent(gpNode);
+
+ const c = Math.cos(-Math.PI / 2);
+ const s = Math.sin(-Math.PI / 2);
+ let rxMat = [
+ 1, 0, 0, 0,
+ 0, c, -s, 0,
+ 0, s, c, 0,
+ 0, 0, 0, 1
+ ];
+ gpRxNode.setLocalMatrix(rxMat);
+
+ // let gpNode = realityEditor.sceneGraph.getSceneNodeById(
+ // realityEditor.sceneGraph.NAMES.GROUNDPLANE + realityEditor.sceneGraph.TAGS.ROTATE_X);
+ // if (!gpNode) {
+ // gpNode = realityEditor.sceneGraph.getSceneNodeById(realityEditor.sceneGraph.NAMES.GROUNDPLANE);
+ // }
+ gpNode.setLocalMatrix(matrices.groundplane);
+ gpNode.updateWorldMatrix();
+ // gpRxNode.updateWorldMatrix();
+
+ let sceneNode = new realityEditor.sceneGraph.SceneNode(id);
+ sceneNode.setParent(realityEditor.sceneGraph.getSceneNodeById('ROOT'));
+
+ let initialVehicleMatrix = [
+ -1, 0, 0, 0,
+ 0, 1, 0, 0,
+ 0, 0, -1, 0,
+ 0, 0, 0, 1,
+ ];
+
+ sceneNode.setPositionRelativeTo(cameraNode, initialVehicleMatrix);
+ sceneNode.updateWorldMatrix();
+
+ let cameraMat = sceneNode.getMatrixRelativeTo(gpRxNode);
+ this.cameraVisCoordinator.updateMatrix(id, new Float32Array(cameraMat), false, matrices);
+ }
+
+ onWebRTCError(e) {
+ console.error('webrtc error', e);
+ showError(ErrorMessage.webrtcIssue, e, 'webRTCErrorUI', 'webRTCErrorText', 5000);
+ }
+
+ disconnect() {
+ this.sendSignallingMessage({
+ src: this.consumerId,
+ dest: this.providerId,
+ command: 'leaveNetwork',
+ });
+
+ this.sendChannel.close();
+ this.receiveChannel.close();
+
+ this.localConnection.close();
+
+ this.sendChannel = null;
+ this.receiveChannel = null;
+ this.localConnection = null;
+ }
+ }
+
+ exports.WebRTCCoordinator = WebRTCCoordinator;
+})(realityEditor.device.cameraVis);
+
diff --git a/content_scripts/desktopAdapter.js b/content_scripts/desktopAdapter.js
new file mode 100644
index 00000000..1652f128
--- /dev/null
+++ b/content_scripts/desktopAdapter.js
@@ -0,0 +1,521 @@
+/*
+* Copyright © 2018 PTC
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+/* globals globalCanvas */
+
+createNameSpace('realityEditor.device.desktopAdapter');
+
+/**
+ * @fileOverview realityEditor.device.desktopAdapter.js
+ * If the editor frontend is loaded on a desktop browser, re-maps some native functions, adjusts some CSS, and
+ * waits for a connection from a mobile editor that will stream matrices here
+ */
+
+(function(exports) {
+
+ const PROXY = !window.location.port || window.location.port === "443";
+
+ // holds the most recent set of objectId/matrix pairs so that they can be rendered on the next frame
+ let visibleObjects = {};
+
+ let didAddModeTransitionListeners = false;
+
+ let env = realityEditor.device.environment.variables;
+
+ /**
+ * initialize the desktop adapter only if we are running on a desktop environment
+ */
+ function initService() {
+ // add these so that we can activate the addon later if we enable AR mode
+ addModeTransitionListeners();
+
+ // by including this check, we can tolerate compiling this add-on into the app without breaking everything
+ // (ideally this add-on should only be added to a "desktop" server but this should effectively ignore it on mobile)
+ if (realityEditor.device.environment.isARMode()) { return; }
+
+ if (!env) {
+ env = realityEditor.device.environment.variables; // ensure that this alias is set correctly if loaded too fast
+ }
+
+ // Set the correct environment variables so that this add-on changes the app to run in desktop mode
+ env.requiresMouseEvents = realityEditor.device.environment.isDesktop(); // this fixes touch events to become mouse events
+ env.supportsDistanceFading = false; // this prevents things from disappearing when the camera zooms out
+ env.ignoresFreezeButton = true; // no need to "freeze the camera" on desktop
+ env.shouldDisplayLogicMenuModally = true; // affects appearance of crafting board
+ env.lineWidthMultiplier = 5; // makes links thicker (more visible)
+ env.distanceScaleFactor = 30; // makes distance-based interactions work at further distances than mobile
+ env.newFrameDistanceMultiplier = 6; // makes new tools spawn further away from camera position
+ // globalStates.defaultScale *= 3; // make new tools bigger
+ env.localServerPort = PROXY ? 443 : 8080; // this would let it find world_local if it exists (but it probably doesn't exist)
+ env.shouldCreateDesktopSocket = true; // this lets UDP messages get sent over socket instead
+ env.isCameraOrientationFlipped = true; // otherwise new tools and anchors get placed upside-down
+ env.waitForARTracking = false; // don't show loading UI waiting for vuforia to give us camera matrices
+ env.supportsAreaTargetCapture = false; // don't show Create Area Target UI when app loads
+ env.hideOriginCube = true; // don't show a set of cubes at the world origin
+ env.addOcclusionGltf = false; // don't add transparent world gltf, because we're already adding the visible mesh
+ env.transformControlsSize = 0.3; // gizmos for ground plane anchors are smaller
+ env.defaultShowGroundPlane = true;
+ env.groundWireframeColor = 'rgb(255, 240, 0)'; // make the ground holo-deck styled
+
+ globalStates.groundPlaneOffset = 0.77;
+ if (PROXY) {
+ realityEditor.app.callbacks.acceptUDPBeats = false;
+ globalStates.a = 0.77;
+ realityEditor.network.state.isCloudInterface = true;
+ }
+ // default values that I may or may not need to invert:
+ // shouldBroadcastUpdateObjectMatrix: false,
+
+ restyleForDesktop();
+ modifyGlobalNamespace();
+
+ let worldIdQueryItem = getPrimaryWorldId();
+ if (worldIdQueryItem) {
+ realityEditor.network.discovery.setPrimaryWorld(null, worldIdQueryItem);
+ }
+
+ function setupMenuBarWhenReady() {
+ if (realityEditor.gui.setupMenuBar) {
+ realityEditor.gui.setupMenuBar();
+ setupMenuBarItems();
+ return;
+ }
+ setTimeout(setupMenuBarWhenReady, 50);
+ }
+
+ setupMenuBarWhenReady();
+
+ // TODO realtime interactions between remote operator and AR clients need to be re-tested and possibly fixed
+ setTimeout(function() {
+ addSocketListeners(); // HACK. this needs to happen after realtime module finishes loading
+ }, 100);
+
+ calculateProjectionMatrices(window.innerWidth, window.innerHeight);
+
+ function setupKeyboardWhenReady() {
+ if (realityEditor.device.KeyboardListener) {
+ setupKeyboard();
+ return;
+ }
+ setTimeout(setupKeyboardWhenReady, 50);
+ }
+
+ setupKeyboardWhenReady();
+
+ setTimeout(() => {
+ realityEditor.gui.threejsScene.getInternals().setAnimationLoop(update);
+ }, 100);
+ }
+
+ function calculateProjectionMatrices(viewportWidth, viewportHeight) {
+ const iPhoneVerticalFOV = 41.22673; // https://discussions.apple.com/thread/250970597
+ const desktopProjectionMatrix = projectionMatrixFrom(iPhoneVerticalFOV, viewportWidth / viewportHeight, 10, 300000);
+
+ realityEditor.gui.ar.setProjectionMatrix(desktopProjectionMatrix);
+
+ let cameraNode = realityEditor.sceneGraph.getCameraNode();
+ if (cameraNode) {
+ cameraNode.needsRerender = true; // make sure the sceneGraph is rendered with the right projection matrix
+ }
+ }
+
+ function setupMenuBarItems() {
+ const menuBar = realityEditor.gui.getMenuBar();
+
+ menuBar.addCallbackToItem(realityEditor.gui.ITEM.DarkMode, (value) => {
+ if (value) {
+ menuBar.domElement.classList.remove('desktopMenuBarLight');
+ Array.from(document.querySelectorAll('.desktopMenuBarMenuDropdown')).forEach(dropdown => {
+ dropdown.classList.remove('desktopMenuBarLight');
+ });
+ document.body.style.backgroundColor = 'rgb(50, 50, 50)';
+ env.groundWireframeColor = 'rgb(255, 240, 0)'; // make the ground holo-deck styled yellow
+ realityEditor.gui.ar.groundPlaneRenderer.updateGridStyle({
+ color: env.groundWireframeColor,
+ thickness: 0.075 // relatively thick
+ });
+ // todo Steve: make ai chatbox turn on dark mode
+ let aiContainer = document.getElementById('ai-chat-tool-container');
+ let searchTextArea = [...aiContainer.querySelectorAll('#searchTextArea')][0];
+ searchTextArea.classList.remove('searchTextArea-light');
+ let myAiDialogues = [...aiContainer.querySelectorAll('.ai-chat-tool-dialogue-my')];
+ myAiDialogues.forEach(dialogue => dialogue.classList.remove('ai-chat-tool-dialogue-my-light'));
+ let aiAiDialogues = [...aiContainer.querySelectorAll('.ai-chat-tool-dialogue-ai')];
+ aiAiDialogues.forEach(dialogue => dialogue.classList.remove('ai-chat-tool-dialogue-ai-light'));
+ } else {
+ menuBar.domElement.classList.add('desktopMenuBarLight');
+ Array.from(document.querySelectorAll('.desktopMenuBarMenuDropdown')).forEach(dropdown => {
+ dropdown.classList.add('desktopMenuBarLight');
+ });
+ document.body.style.backgroundColor = 'rgb(225, 225, 225)';
+ env.groundWireframeColor = 'rgb(150, 150, 150)'; // make the ground plane subtle grey
+ realityEditor.gui.ar.groundPlaneRenderer.updateGridStyle({
+ color: env.groundWireframeColor,
+ thickness: 0.025 // relatively thin
+ });
+ // todo Steve: make ai chatbox turn off dark mode
+ let aiContainer = document.getElementById('ai-chat-tool-container');
+ let searchTextArea = aiContainer.querySelector('#searchTextArea');
+ searchTextArea.classList.add('searchTextArea-light');
+ let myAiDialogues = [...aiContainer.querySelectorAll('.ai-chat-tool-dialogue-my')];
+ myAiDialogues.forEach(dialogue => dialogue.classList.add('ai-chat-tool-dialogue-my-light'));
+ let aiAiDialogues = [...aiContainer.querySelectorAll('.ai-chat-tool-dialogue-ai')];
+ aiAiDialogues.forEach(dialogue => dialogue.classList.add('ai-chat-tool-dialogue-ai-light'));
+ }
+ });
+
+ menuBar.addCallbackToItem(realityEditor.gui.ITEM.SurfaceAnchors, (value) => {
+ realityEditor.gui.ar.groundPlaneAnchors.togglePositioningMode(value);
+ });
+ }
+
+ // add a keyboard listener to toggle visibility of the zone/phone discovery buttons
+ function setupKeyboard() {
+ let keyboard = new realityEditor.device.KeyboardListener();
+ keyboard.onKeyDown(function(code) {
+ if (realityEditor.device.keyboardEvents.isKeyboardActive()) { return; } // ignore if a tool is using the keyboard
+
+ // if hold press S while dragging an element, scales it
+ if (code === keyboard.keyCodes.S) {
+ let touchPosition = realityEditor.gui.ar.positioning.getMostRecentTouchPosition();
+
+ if (!realityEditor.device.editingState.syntheticPinchInfo) {
+ realityEditor.device.editingState.syntheticPinchInfo = {
+ startX: touchPosition.x,
+ startY: touchPosition.y
+ };
+ }
+ } else if (code === keyboard.keyCodes.R) {
+ // rotate tool towards camera a single time when you press the R key while dragging a tool
+ let tool = realityEditor.device.getEditingVehicle();
+ if (!tool) return;
+ let toolSceneNode = realityEditor.sceneGraph.getSceneNodeById(tool.uuid);
+ if (!toolSceneNode) return;
+ // we don't include scale in new matrix otherwise it can shrink/grow
+ let modelMatrix = realityEditor.sceneGraph.getModelMatrixLookingAt(tool.uuid, 'CAMERA', {flipX: true, flipY: true, includeScale: false});
+ let rootNode = realityEditor.sceneGraph.getSceneNodeById('ROOT');
+ toolSceneNode.setPositionRelativeTo(rootNode, modelMatrix);
+ }
+ });
+ keyboard.onKeyUp(function(code) {
+ if (realityEditor.device.keyboardEvents.isKeyboardActive()) { return; } // ignore if a tool is using the keyboard
+
+ if (code === keyboard.keyCodes.S) {
+ realityEditor.device.editingState.syntheticPinchInfo = null;
+ globalCanvas.hasContent = true; // force the canvas to be cleared
+ }
+ });
+ }
+
+ /**
+ * Builds a projection matrix from field of view, aspect ratio, and near and far planes
+ */
+ function projectionMatrixFrom(vFOV, aspect, near, far) {
+ var top = near * Math.tan((Math.PI / 180) * 0.5 * vFOV );
+ var height = 2 * top;
+ var width = aspect * height;
+ var left = -0.5 * width;
+ return makePerspective( left, left + width, top, top - height, near, far );
+ }
+
+ /**
+ * Helper function for creating a projection matrix
+ */
+ function makePerspective ( left, right, top, bottom, near, far ) {
+
+ var te = [];
+ var x = 2 * near / ( right - left );
+ var y = 2 * near / ( top - bottom );
+
+ var a = ( right + left ) / ( right - left );
+ var b = ( top + bottom ) / ( top - bottom );
+ var c = - ( far + near ) / ( far - near );
+ var d = - 2 * far * near / ( far - near );
+
+ te[ 0 ] = x; te[ 4 ] = 0; te[ 8 ] = a; te[ 12 ] = 0;
+ te[ 1 ] = 0; te[ 5 ] = y; te[ 9 ] = b; te[ 13] = 0;
+ te[ 2 ] = 0; te[ 6 ] = 0; te[ 10 ] = c; te[ 14 ] = d;
+ te[ 3 ] = 0; te[ 7 ] = 0; te[ 11 ] = - 1; te[ 15 ] = 0;
+
+ return te;
+
+ }
+
+ /**
+ * Adjust visuals for desktop rendering -> set background color and add "Waiting for Connection..." indicator
+ */
+ function restyleForDesktop() {
+
+ document.getElementById('groundPlaneResetButton').classList.add('hiddenDesktopButton');
+
+ realityEditor.device.layout.onWindowResized(({width, height}) => {
+ calculateProjectionMatrices(width, height);
+ });
+
+ const DISABLE_SAFE_MODE = true;
+ if (!DISABLE_SAFE_MODE) {
+ if (window.outerWidth !== document.body.offsetWidth) {
+ alert('Reset browser zoom level to get accurate calculations');
+ }
+ }
+
+ realityEditor.gui.ar.injectClosestObjectFilter(function(objectKey) {
+ let object = realityEditor.getObject(objectKey);
+ if (!object) { return false; }
+ let isWorld = object.isWorldObject || object.type === 'world';
+ if (!isWorld && realityEditor.sceneGraph.getDistanceToCamera(objectKey) > 2000) {
+ return false;
+ }
+ return true;
+ });
+ }
+
+ /**
+ * Re-maps native app calls to functions within this file.
+ * E.g. calling realityEditor.app.setPause() will be rerouted to this file's setPause() function.
+ * @todo Needs to be manually modified as more native calls are added. Add one switch case per native app call.
+ */
+ function modifyGlobalNamespace() {
+ if (realityEditor.device.environment.isWithinToolboxApp()) {
+ console.warn('Preventing modifyGlobalNamespace - we are within the toolbox app');
+ return;
+ }
+
+ // mark that we've manipulated the webkit reference, so that we
+ // can still detect isWithinToolboxApp vs running in mobile browser
+ window.webkitWasTamperedWith = true;
+
+ // set up object structure if it doesn't exist yet
+ window.webkit = {
+ messageHandlers: {
+ realityEditor: {}
+ }
+ };
+
+ // intercept postMessage calls to the messageHandlers and manually handle each case by functionName
+ window.webkit.messageHandlers.realityEditor.postMessage = function(messageBody) {
+ switch (messageBody.functionName) {
+ // case 'setPause':
+ // setPause();
+ // break;
+ // case 'setResume':
+ // setResume();
+ // break;
+ case 'getVuforiaReady':
+ getVuforiaReady(messageBody.arguments);
+ break;
+ case 'sendUDPMessage':
+ sendUDPMessage(messageBody.arguments);
+ break;
+ // case 'getUDPMessages':
+ // getUDPMessages(messageBody.callback);
+ case 'muteMicrophone':
+ realityEditor.gui.ar.desktopRenderer.muteMicrophoneForCameraVis();
+ break;
+ case 'unmuteMicrophone':
+ realityEditor.gui.ar.desktopRenderer.unmuteMicrophoneForCameraVis();
+ break;
+ default:
+ return;
+ }
+ };
+
+ // we also manually overwrite some of the promise wrappers for certain webkit messageHandlers
+ // else they'll never resolve (e.g. realityEditor.app.promises.setPause().then(success => {...})
+ realityEditor.app.promises.setPause = async () => {
+ return new Promise((resolve, _reject) => {
+ resolve(true); // setPause resolves immediately on desktop
+ });
+ };
+ realityEditor.app.promises.setResume = async () => {
+ return new Promise((resolve, _reject) => {
+ resolve(true); // setResume resolves immediately on desktop
+ });
+ };
+
+ // don't need to polyfill webkit functions for Chrome here because it is already polyfilled in the userinterface
+
+ // TODO: unsure if env.isCameraOrientationFlipped was only necessary because rotateX needs to be different on desktop..
+ // investigate this further to potentially simplify calculations
+ // rotateX = [
+ // 1, 0, 0, 0,
+ // 0, -1, 0, 0,
+ // 0, 0, -1, 0,
+ // 0, 0, 0, 1
+ // ];
+ // realityEditor.gui.ar.draw.rotateX = rotateX;
+
+ window.DEBUG_CLIENT_NAME = 'Remote Operator';
+ }
+
+ /**
+ * Adds socket.io listeners for UDP messages necessary to setup editor without mobile environment
+ * (e.g. object discovery)
+ */
+ function addSocketListeners() {
+
+ realityEditor.network.addObjectDiscoveredCallback(function(object, objectKey) {
+ // make objects show up by default at the origin
+ if (object.matrix.length === 0) {
+ object.matrix = realityEditor.gui.ar.utilities.newIdentityMatrix();
+ visibleObjects[objectKey] = realityEditor.gui.ar.utilities.newIdentityMatrix();
+ }
+
+ // subscribe to new object matrices to update where the object is in the world
+ realityEditor.network.realtime.subscribeToObjectMatrices(objectKey, function(data) {
+ if (globalStates.freezeButtonState) { return; }
+
+ var msgData = JSON.parse(data);
+ if (msgData.objectKey === objectKey && msgData.propertyPath === 'matrix') {
+ // TODO: set sceneGraph localMatrix to msgData.newValue?
+ // var rotatedObjectMatrix = realityEditor.gui.ar.utilities.copyMatrix(msgData.newValue);
+ // object.matrix = rotatedObjectMatrix;
+ // visibleObjects[msgData.objectKey] = rotatedObjectMatrix;
+
+ visibleObjects[msgData.objectKey] = realityEditor.gui.ar.utilities.newIdentityMatrix();
+ }
+ });
+ });
+
+ realityEditor.network.realtime.addDesktopSocketMessageListener('reloadScreen', function(_msgContent) {
+ // window.location.reload(); // reload screen when server restarts
+ });
+
+ realityEditor.network.realtime.addDesktopSocketMessageListener('udpMessage', function(msgContent) {
+
+ if (typeof msgContent.id !== 'undefined' &&
+ typeof msgContent.ip !== 'undefined') {
+
+ if (typeof realityEditor.network.discovery !== 'undefined') {
+ realityEditor.network.discovery.processHeartbeat(msgContent);
+ }
+ }
+
+ if (typeof msgContent.action !== 'undefined') {
+ if (typeof msgContent.action === 'string') {
+ try {
+ msgContent.action = JSON.parse(msgContent.action);
+ } catch (e) {
+ // console.log('dont need to parse');
+ }
+ }
+ realityEditor.network.onAction(msgContent.action);
+ }
+
+ // forward the message to a generic message handler that various modules use to subscribe to different messages
+ realityEditor.network.onUDPMessage(msgContent);
+
+ });
+
+ }
+
+ function addModeTransitionListeners() {
+ if (didAddModeTransitionListeners) return;
+ didAddModeTransitionListeners = true;
+
+ // start the update loop when the remote operator is shown
+ realityEditor.device.modeTransition.onRemoteOperatorShown(() => {
+ realityEditor.gui.threejsScene.getInternals().setAnimationLoop(update); // start update loop
+ calculateProjectionMatrices(window.innerWidth, window.innerHeight); // update proj matrices
+ });
+ }
+
+ /**
+ * The 60 FPS render loop. Smoothly calls realityEditor.gui.ar.draw.update to render the most recent visibleObjects
+ * Also smoothly updates camera postion when paused
+ */
+ function update() {
+ if (realityEditor.device.environment.isARMode()) { return; } // stop the update loop if we enter AR mode
+
+ // TODO: ensure that visibleObjects that aren't known objects get filtered out
+
+ realityEditor.gui.ar.draw.update(getVisibleObjects());
+ }
+
+ function getVisibleObjects() {
+ // render everything that has been localized
+ let tempVisibleObjects = {};
+
+ // first process the world objects
+ let visibleWorlds = [];
+ Object.keys(objects).forEach(function(objectKey) {
+ let object = objects[objectKey];
+
+ // always add world object to scene unless we set a primaryWorldId in the URLSearchParams
+ if (object.isWorldObject || object.type === 'world') {
+ let primaryWorld = getPrimaryWorldId();
+
+ if (!primaryWorld || objectKey === primaryWorld) {
+ tempVisibleObjects[objectKey] = object.matrix; // actual matrix doesn't matter, just that it's visible
+ visibleWorlds.push(objectKey);
+ }
+ }
+ });
+
+ // now process the other objects
+ Object.keys(objects).forEach(function(objectKey) {
+ let object = objects[objectKey];
+
+ // we already added world objects. also ignore the avatar objects
+ if (object.isWorldObject || object.type === 'world' || realityEditor.avatar.utils.isAvatarObject(object)) {
+ return;
+ }
+
+ // if there isn't a world object, it's ok to load objects without a world (e.g. as a debugger)
+ if (visibleWorlds.length === 0) {
+ if (!object.worldId) {
+ tempVisibleObjects[objectKey] = object.matrix; // actual matrix doesn't matter, just that it's visible
+ }
+ } else {
+ // if there is a world loaded, only show objects localized within that world, not at the identity matrix
+ if (object.worldId && visibleWorlds.includes(object.worldId)) {
+ if (!realityEditor.gui.ar.utilities.isIdentityMatrix(object.matrix)) {
+ tempVisibleObjects[objectKey] = object.matrix; // actual matrix doesn't matter , just that it's visible
+ }
+ }
+ }
+ });
+
+ return tempVisibleObjects;
+ }
+
+ function createNativeAPISocket() {
+ // lazily instantiate the socket to the server if it doesn't exist yet
+ var socketsIps = realityEditor.network.realtime.getSocketIPsForSet('nativeAPI');
+ // var hostedServerIP = 'http://127.0.0.1:' + window.location.port;
+ var hostedServerIP = window.location.protocol + '//' + window.location.host; //'http://127.0.0.1:' + window.location.port;
+
+ if (socketsIps.indexOf(hostedServerIP) < 0) {
+ realityEditor.network.realtime.createSocketInSet('nativeAPI', hostedServerIP);
+ realityEditor.network.realtime.addDesktopSocketMessageListener('test', function(message) {
+ console.log('received message from socketMessageListener', message);
+ });
+ }
+ }
+
+ function sendUDPMessage(args) {
+ createNativeAPISocket();
+ realityEditor.network.realtime.sendMessageToSocketSet('nativeAPI', '/nativeAPI/sendUDPMessage', args.message);
+ }
+
+ function getVuforiaReady(_args) {
+ // just immediately call the callback because vuforia will never load on the desktop
+
+ // this is the only functionality we still need from the original callback (realityEditor.app.callbacks.vuforiaIsReady)
+ realityEditor.app.getUDPMessages('realityEditor.app.callbacks.receivedUDPMessage');
+ }
+
+ function getPrimaryWorldId() {
+ return (new URLSearchParams(window.location.search)).get('world');
+ }
+
+ exports.getPrimaryWorldId = getPrimaryWorldId;
+
+ // this happens only for desktop editors
+ realityEditor.addons.addCallback('init', initService);
+}(realityEditor.device.desktopAdapter));
diff --git a/content_scripts/desktopCamera.js b/content_scripts/desktopCamera.js
new file mode 100644
index 00000000..0e94967c
--- /dev/null
+++ b/content_scripts/desktopCamera.js
@@ -0,0 +1,782 @@
+/*
+* Copyright © 2018 PTC
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+createNameSpace('realityEditor.device.desktopCamera');
+
+import { Vector3 } from '../../thirdPartyCode/three/three.module.js';
+import { CameraFollowCoordinator } from './CameraFollowCoordinator.js';
+import { MotionStudyFollowable } from './MotionStudyFollowable.js';
+import { TouchControlButtons } from './TouchControlButtons.js';
+
+/**
+ * @fileOverview realityEditor.device.desktopCamera.js
+ * Responsible for manipulating the camera position and resulting view matrix, on remote desktop clients
+ */
+
+(function(exports) {
+ const DEBUG = false;
+
+ // arbitrary birds-eye view to start the camera with. it will look towards the world object origin
+ let INITIAL_CAMERA_POSITION = [-1499.9648912671637, 8275.552791086136, 5140.3791620707225];
+
+ // used to render an icon at the target position to help you navigate the scene
+ let rotateCenterElementId = null;
+
+ // polyfill for requestAnimationFrame to provide a smooth update loop
+ let requestAnimationFrame = window.requestAnimationFrame ||
+ window.webkitRequestAnimationFrame || function(cb) {setTimeout(cb, 17);};
+ let virtualCamera;
+
+ let knownInteractionStates = {
+ pan: false,
+ rotate: false,
+ scale: false
+ };
+
+ let staticInteractionCursor = null;
+ let interactionCursor = null;
+ let pointerPosition = { x: 0, y: 0 };
+ let cameraTargetIcon = null;
+
+ let followCoordinator = null;
+
+ // used for transitioning from AR view to remote operator virtual camera
+ let didAddModeTransitionListeners = false;
+ let virtualCameraEnabled = false;
+ let cameraTransitionPosition_AR = null;
+ let cameraTransitionTarget_AR = null;
+ let cameraTransitionPosition_VR = null;
+ let cameraTransitionTarget_VR = null;
+
+ // on touchscreen VR mode devices, these let us toggle between camera controls and pointer
+ let touchControlButtons = null;
+
+ let motionStudyFollowables = {};
+
+ /**
+ * Public init method to enable rendering if isDesktop
+ */
+ function initService(floorOffset) {
+ if (!realityEditor.device.desktopAdapter) {
+ setTimeout(function() {
+ initService(floorOffset);
+ }, 100);
+ return;
+ }
+
+ addModeTransitionListeners();
+
+ if (realityEditor.device.environment.isARMode()) { return; }
+
+ if (!realityEditor.sceneGraph.getSceneNodeById('CAMERA')) { // reload after camera has been created
+ setTimeout(function() {
+ initService(floorOffset);
+ }, 100);
+ return;
+ }
+
+ let parentNode = realityEditor.sceneGraph.getGroundPlaneNode();
+
+ let cameraNode = realityEditor.sceneGraph.getSceneNodeById('CAMERA');
+ virtualCamera = new realityEditor.device.VirtualCamera(cameraNode, 1, 0.001, 10, INITIAL_CAMERA_POSITION, floorOffset);
+ virtualCameraEnabled = true;
+
+ followCoordinator = new CameraFollowCoordinator(virtualCamera);
+ window.followCoordinator = followCoordinator;
+ followCoordinator.addMenuItems();
+
+ // ---- Add and remove follow targets when virtualizers connect ---- //
+
+ function addCameraVisCallbacks() {
+ let cameraVisCoordinator = realityEditor.gui.ar.desktopRenderer.getCameraVisCoordinator();
+ if (!cameraVisCoordinator) {
+ setTimeout(addCameraVisCallbacks, 100);
+ return;
+ }
+ cameraVisCoordinator.onCameraVisCreated(cameraVis => {
+ followCoordinator.addFollowTarget(cameraVis);
+ });
+ cameraVisCoordinator.onCameraVisRemoved(cameraVis => {
+ followCoordinator.removeFollowTarget(cameraVis.id);
+ });
+ }
+ addCameraVisCallbacks();
+
+ // set rotateCenterElementId parent as groundPlaneNode to make the coord space of rotateCenterElementId the same as virtual camera and threejsContainerObj
+ rotateCenterElementId = realityEditor.sceneGraph.addVisualElement('rotateCenter', parentNode, undefined, virtualCamera.getFocusTargetCubeMatrix());
+
+
+ virtualCamera.onPanToggled(function(isPanning) {
+ if (virtualCamera.lockOnMode) {
+ isPanning = false; // can't pan while locked onto another user's perspective
+ }
+ if (isPanning && !knownInteractionStates.pan) {
+ knownInteractionStates.pan = true;
+ panToggled();
+ } else if (!isPanning && knownInteractionStates.pan) {
+ knownInteractionStates.pan = false;
+ panToggled();
+ }
+ realityEditor.gui.ar.positioning.coverFull2DTools(isPanning);
+ });
+ virtualCamera.onRotateToggled(function(isRotating) {
+ if (virtualCamera.lockOnMode) {
+ isRotating = false; // can't rotate while locked onto another user's perspective
+ }
+ if (isRotating && !knownInteractionStates.rotate) {
+ knownInteractionStates.rotate = true;
+ knownInteractionStates.pan = false; // stop panning if you start rotating
+ rotateToggled();
+ } else if (!isRotating && knownInteractionStates.rotate) {
+ knownInteractionStates.rotate = false;
+ rotateToggled();
+ }
+ realityEditor.gui.ar.positioning.coverFull2DTools(isRotating);
+ });
+ virtualCamera.onScaleToggled(function(isScaling) {
+ if (virtualCamera.lockOnMode) {
+ isScaling = false;
+ }
+ if (isScaling && !knownInteractionStates.scale) {
+ knownInteractionStates.scale = true;
+ scaleToggled();
+ } else if (!isScaling && knownInteractionStates.scale) {
+ knownInteractionStates.scale = false;
+ scaleToggled();
+ }
+ });
+
+ // virtualCamera.onStopFollowing(() => {
+ // currentlyFollowingId = null;
+ // });
+
+ interactionCursor = document.createElement('img');
+ interactionCursor.id = 'interactionCursor';
+ document.body.appendChild(interactionCursor);
+
+ staticInteractionCursor = document.createElement('img');
+ staticInteractionCursor.id = 'staticInteractionCursor';
+ document.body.appendChild(staticInteractionCursor);
+
+ // necessary to make the interactionCursor start at the right position on touchscreens
+ document.addEventListener('pointerdown', (e) => {
+ pointerPosition.x = e.clientX;
+ pointerPosition.y = e.clientY;
+ });
+
+ document.addEventListener('pointermove', function(e) {
+ pointerPosition.x = e.clientX;
+ pointerPosition.y = e.clientY;
+
+ let interactionRect = getRectSafe(interactionCursor);
+ if (interactionRect) {
+ interactionCursor.style.left = (pointerPosition.x - interactionRect.width / 2) + 'px';
+ interactionCursor.style.top = (pointerPosition.y - interactionRect.height / 2) + 'px';
+ }
+ });
+
+ onFrame();
+
+ // disable right-click context menu so we can use right-click to rotate camera
+ document.addEventListener('contextmenu', event => event.preventDefault());
+
+ try {
+ addSensitivitySlidersToMenu();
+ } catch (e) {
+ console.warn('Slider components for settings menu not available, skipping', e);
+ }
+
+ setupTouchControlButtons();
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.ResetCameraPosition, () => {
+ console.log('reset camera position');
+ virtualCamera.reset();
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.OrbitCamera, (value) => {
+ virtualCamera.idleOrbitting = value;
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.StopFollowing, () => {
+ virtualCamera.stopFollowing();
+ });
+
+ // ---- Add and remove follow targets when video players are created ---- //
+
+ realityEditor.network.addPostMessageHandler('followCameraOnPlayback', (msgData) => {
+ const cameraTargets = followCoordinator.followTargets;
+ for (let cameraTarget in cameraTargets) {
+ if (cameraTargets[cameraTarget].followable.frameKey === msgData.frame) {
+ followCoordinator.follow(cameraTargets[cameraTarget].id, msgData.distance);
+ }
+ }
+ });
+
+ realityEditor.network.addPostMessageHandler('stopFollowingCamera', (msgData) => {
+ const cameraTargets = followCoordinator.followTargets;
+ for (let cameraTarget in cameraTargets) {
+ if (cameraTargets[cameraTarget].followable.frameKey === msgData.frame) {
+ followCoordinator.unfollow();
+ }
+ }
+ });
+
+ let videoPlayback = realityEditor.gui.ar.videoPlayback;
+ videoPlayback.onVideoCreated(player => {
+ followCoordinator.addFollowTarget(player);
+ });
+ videoPlayback.onVideoDisposed(id => {
+ followCoordinator.removeFollowTarget(id);
+ });
+ // TODO: should we do anything when videos pause/resume?
+ // videoPlayback.onVideoPlayed(_player => {
+ // console.log('onVideoPlayed', player.id, player);
+ // });
+ // videoPlayback.onVideoPaused(_player => {
+ // console.log('onVideoPaused', player.id, player);
+ // });
+
+ // ---- Add and remove follow targets when motion studies are opened ---- //
+
+ realityEditor.network.addPostMessageHandler('analyticsOpen', (msgData) => {
+ if (typeof motionStudyFollowables[msgData.frame] === 'undefined') {
+ motionStudyFollowables[msgData.frame] = new MotionStudyFollowable(msgData.frame);
+ }
+ followCoordinator.addFollowTarget(motionStudyFollowables[msgData.frame]);
+ });
+
+ realityEditor.network.addPostMessageHandler('analyticsClose', (msgData) => {
+ if (!motionStudyFollowables[msgData.frame]) return;
+ followCoordinator.removeFollowTarget(motionStudyFollowables[msgData.frame].id);
+ });
+
+ const keyboard = new realityEditor.device.KeyboardListener();
+
+ // Setup Save/Load Camera Position System
+ // Allows for quickly jumping between different camera positions
+ let getSavedCameraDataLocalStorageKey = (index) => `savedCameraData${index}-${realityEditor.sceneGraph.getWorldId()}`;
+
+ const saveCameraData = (index) => {
+ const cameraPosition = [...virtualCamera.position];
+ const cameraDirection = virtualCamera.getCameraDirection();
+ const cameraData = { cameraPosition, cameraDirection };
+ const cameraDataJsonString = JSON.stringify(cameraData);
+ localStorage.setItem(getSavedCameraDataLocalStorageKey(index), cameraDataJsonString);
+ };
+
+ const loadCameraData = (index) => {
+ if (virtualCamera.lockOnMode) return;
+ const cameraDataJsonString = localStorage.getItem(getSavedCameraDataLocalStorageKey(index));
+ if (!cameraDataJsonString) {
+ return;
+ }
+ try {
+ const cameraData = JSON.parse(cameraDataJsonString);
+ virtualCamera.position = [...cameraData.cameraPosition];
+ virtualCamera.setCameraDirection(cameraData.cameraDirection);
+ return cameraData;
+ } catch (e) {
+ console.warn('Error parsing saved camera position data', e);
+ }
+ };
+
+ // Only one gets a menu item to avoid crowding, but they all get a shortcut key
+ const saveCameraPositionMenuItem = new realityEditor.gui.MenuItem('Save Camera Position', { shortcutKey: '_1', modifiers: ['ALT'], toggle: false, disabled: false }, () => {
+ saveCameraData(0);
+ });
+ const loadCameraPositionMenuItem = new realityEditor.gui.MenuItem('Load Camera Position', { shortcutKey: '_1', modifiers: ['SHIFT'], toggle: false, disabled: false }, () => {
+ loadCameraData(0);
+ });
+ realityEditor.gui.getMenuBar().addItemToMenu(realityEditor.gui.MENU.Camera, saveCameraPositionMenuItem);
+ realityEditor.gui.getMenuBar().addItemToMenu(realityEditor.gui.MENU.Camera, loadCameraPositionMenuItem);
+ [2, 3, 4, 5, 6, 7, 8, 9, 0].forEach(key => {
+ // Would be nice to deduplicate some of this logic, shared with MenuBar and MenuItem
+ keyboard.onKeyDown((code, activeModifiers) => {
+ if (realityEditor.device.keyboardEvents.isKeyboardActive()) { return; } // ignore if a tool is using the keyboard
+ const modifierSetsMatch = (modifierSet1, modifierSet2) => {
+ return modifierSet1.length === modifierSet2.length && modifierSet1.every(value => modifierSet2.includes(value));
+ };
+ if (code === keyboard.keyCodes[`_${key}`] && modifierSetsMatch([keyboard.keyCodes['ALT']], activeModifiers)) {
+ saveCameraData(key - 1);
+ }
+ if (code === keyboard.keyCodes[`_${key}`] && modifierSetsMatch([keyboard.keyCodes['SHIFT']], activeModifiers)) {
+ loadCameraData(key - 1);
+ }
+ });
+ });
+
+ /**
+ * Stops following the previous target, tells the virtualCamera lock on to the new target id,
+ * shows visual feedback (a colored screen border), and notifies other clients via the avatar publicData
+ * @param {string} avatarToLockOntoId
+ */
+ const lockOnToTarget = (avatarToLockOntoId) => {
+ if (virtualCamera.lockOnMode && virtualCamera.lockOnMode !== avatarToLockOntoId) {
+ // stop following previous and start following new
+ virtualCamera.toggleLockOnMode(null);
+ }
+ let newLockOnMode = virtualCamera.toggleLockOnMode(avatarToLockOntoId);
+ try {
+ let avatarObject = realityEditor.getObject(avatarToLockOntoId);
+ let color = realityEditor.avatar.utils.getColor(avatarObject);
+ let avatarProfile = realityEditor.avatar.getConnectedAvatarList()[avatarToLockOntoId];
+
+ if (newLockOnMode) {
+ let avatarDescription = avatarProfile.name ? `${avatarProfile.name}'s` : `Anonymous User's`;
+ let description = `Press to stop viewing ${avatarDescription} perspective`;
+ addScreenBorder(color, description);
+ realityEditor.avatar.writeMyLockOnMode(avatarToLockOntoId);
+ } else {
+ removeScreenBorder();
+ realityEditor.avatar.writeMyLockOnMode(null);
+ }
+ } catch (e) {
+ console.warn('error locking onto target', e);
+ }
+ };
+
+ /**
+ * Tells another user to lock onto my view, by sending a message via that avatar's publicData
+ * @param {string} otherAvatarId
+ */
+ const lockOnToMe = (otherAvatarId) => {
+ realityEditor.avatar.writeLockOnToMe(otherAvatarId);
+ };
+
+ // Depending on which avatar menu item is clicked, perform the right LockOn action
+ realityEditor.avatar.iconMenu.onAvatarIconMenuItemSelected((params) => {
+ let {avatarObjectId, buttonText } = params;
+
+ if (buttonText === realityEditor.avatar.iconMenu.MENU_ITEMS.FollowThem) {
+ lockOnToTarget(avatarObjectId);
+ } else if (buttonText === realityEditor.avatar.iconMenu.MENU_ITEMS.FollowMe) {
+ lockOnToMe(avatarObjectId);
+ } else if (buttonText === realityEditor.avatar.iconMenu.MENU_ITEMS.AllFollowMe) {
+ // for each other avatar in the same world... tell them to lock on to me
+ let myId = realityEditor.avatar.getMyAvatarId();
+ realityEditor.forEachObject((object, objectId) => {
+ if (!realityEditor.avatar.utils.isAvatarObject(object)) return; // only works with avatars
+ if (objectId === myId) return; // don't lock self onto self
+ lockOnToMe(objectId);
+ });
+ // if I'm locked onto someone else, stop following them when I ask everyone to follow me
+ if (virtualCamera.lockOnMode) {
+ virtualCamera.toggleLockOnMode(null);
+ realityEditor.avatar.writeMyLockOnMode(null);
+ removeScreenBorder();
+ }
+ }
+ });
+
+ // If virtual camera stops lockOnMode (e.g. using escape key), remove the border and notify others
+ virtualCamera.onStopLockOnMode(() => {
+ removeScreenBorder();
+ realityEditor.avatar.writeMyLockOnMode(null);
+ });
+
+ // detect when other users started/stopped following me, by subscribing to my avatar's userProfile
+ realityEditor.avatar.registerOnMyAvatarInitializedCallback((myAvatarObject) => {
+ const subscriptionCallbacks = {};
+ subscriptionCallbacks[realityEditor.avatar.utils.PUBLIC_DATA_KEYS.userProfile] = (msgContent) => {
+ const userProfile = msgContent.publicData.userProfile;
+ let avatarToLockOntoId = userProfile.lockOnMode;
+ lockOnToTarget(avatarToLockOntoId);
+ };
+ realityEditor.avatar.network.subscribeToAvatarPublicData(myAvatarObject, subscriptionCallbacks);
+ });
+
+ realityEditor.ai.registerCallback('shouldFocusVirtualCamera', function (params) {
+ focusVirtualCamera(new Vector3(params.pos.x, params.pos.y, params.pos.z), new Vector3(params.dir.x, params.dir.y, params.dir.z), params.zoomDistanceMm);
+ });
+ }
+
+ /**
+ * Update the floorOffset of the camera system - useful if new gltf loads with new navmesh/floorOffset
+ * @param {number} floorOffset
+ */
+ function updateCameraFloorOffset(floorOffset) {
+ if (!virtualCamera) {
+ console.warn('cant update camera with floorOffset because no camera yet');
+ return;
+ }
+ virtualCamera.updateFloorOffset(floorOffset);
+ }
+
+ /**
+ * For lockOnMode: add "screen share"-style border to edge of screen to indicate that you are following another user
+ * @param {string} color - hsl/rgb/hex string
+ * @param {string} descriptionText - what text to display on the screen while following
+ */
+ function addScreenBorder(color, descriptionText) {
+ let existingBorder = document.getElementById('avatar-follow-border');
+ if (existingBorder) {
+ changeBorderColor(color);
+ return;
+ }
+
+ let border = document.createElement('div');
+ border.style.border = '8px solid ' + color;
+ border.id = 'avatar-follow-border';
+
+ if (descriptionText) {
+ let textDiv = document.createElement('div');
+ textDiv.classList.add('fullscreenSubtitle');
+ textDiv.textContent = descriptionText;
+ border.appendChild(textDiv);
+ }
+
+ document.body.appendChild(border);
+ }
+
+ /**
+ * Remove the colored border when you stop lockOnMode
+ */
+ function removeScreenBorder() {
+ let border = document.getElementById('avatar-follow-border');
+ if (border) {
+ border.parentNode.removeChild(border);
+ }
+ }
+
+ /**
+ * Change the lockOnMode screen border color
+ * @param {string} color - hsl/rgb/hex string
+ */
+ function changeBorderColor(color) {
+ let border = document.getElementById('avatar-follow-border');
+ if (border) {
+ border.style.border = '8px solid ' + color;
+ }
+ }
+
+ /**
+ * More accessible controls for touchscreen devices:
+ * Adds buttons to the screen where the user can toggle between pan/rotate/zoom/pointer
+ * The selected mode will be used by the primary pointer (left click, first tap, etc)
+ */
+ function setupTouchControlButtons() {
+ if (realityEditor.device.environment.isDesktop()) return; // only show them on iPhone/iPad remote operators
+ if (touchControlButtons) return; // only initialize one time - but remember to show/hide if we go from AR<>VR
+
+ // add the buttons to the screen
+ touchControlButtons = new TouchControlButtons();
+ document.body.appendChild(touchControlButtons.container);
+ touchControlButtons.container.id = 'touchControlsContainer';
+
+ const FLAG_NAME = 'touchCameraControlButtons';
+
+ touchControlButtons.onModeSelected((mode) => {
+ virtualCamera.setTouchControlMode(mode);
+
+ // prevent avatar pointer from activating when we are in pan/rotate/zoom mode
+ if (mode === 'pan' || mode === 'rotate' || mode === 'zoom') {
+ realityEditor.device.setFlagForPointerOccupiedByCamera(FLAG_NAME);
+ } else { // if 'pointer', or null
+ realityEditor.device.clearFlagForPointerOccupiedByCamera(FLAG_NAME);
+ }
+ });
+
+ touchControlButtons.selectMode('pointer'); // select this mode by default
+ }
+
+ function addSensitivitySlidersToMenu() {
+ // add sliders for strafe, rotate, and zoom sensitivity
+ realityEditor.gui.settings.addSlider('Zoom Sensitivity', 'how fast scroll wheel zooms camera', 'cameraZoomSensitivity', '../../../svg/cameraZoom.svg', 0.5, function(newValue) {
+ if (DEBUG) {
+ console.log('zoom value = ' + newValue);
+ }
+ });
+
+ realityEditor.gui.settings.addSlider('Pan Sensitivity', 'how fast keyboard pans camera', 'cameraPanSensitivity', '../../../svg/cameraPan.svg', 0.5, function(newValue) {
+ if (DEBUG) {
+ console.log('pan value = ' + newValue);
+ }
+ });
+
+ realityEditor.gui.settings.addSlider('Rotate Sensitivity', 'how fast right-click dragging rotates camera', 'cameraRotateSensitivity', '../../../svg/cameraRotate.svg', 0.5, function(newValue) {
+ if (DEBUG) {
+ console.log('rotate value = ' + newValue);
+ }
+ });
+ }
+
+ function panToggled() {
+ if (!cameraTargetIcon) return;
+ cameraTargetIcon.visible = knownInteractionStates.pan || knownInteractionStates.rotate || knownInteractionStates.scale;
+ updateInteractionCursor(cameraTargetIcon.visible, 'addons/vuforia-spatial-remote-operator-addon/cameraPan.svg');
+ }
+ function rotateToggled() {
+ if (!cameraTargetIcon) return;
+ cameraTargetIcon.visible = knownInteractionStates.rotate || knownInteractionStates.pan || knownInteractionStates.scale;
+ updateInteractionCursor(cameraTargetIcon.visible, 'addons/vuforia-spatial-remote-operator-addon/cameraRotate.svg');
+ }
+ function scaleToggled() {
+ if (!cameraTargetIcon) return;
+ cameraTargetIcon.visible = knownInteractionStates.scale || knownInteractionStates.pan || knownInteractionStates.rotate;
+
+ if (virtualCamera.touchControlMode !== 'zoom') {
+ // most of the time, this is the zooming cursor UI that we use
+ updateInteractionCursor(cameraTargetIcon.visible, 'addons/vuforia-spatial-remote-operator-addon/cameraZoom.svg');
+ } else {
+ // show different UI if you're zooming with touchscreen, which indicates that +Y is zoom in, -Y is out
+ let dynamicImageSrc = 'addons/vuforia-spatial-remote-operator-addon/touch-control-zoom-dynamic.svg';
+ let staticImageInfo = {
+ src: 'addons/vuforia-spatial-remote-operator-addon/touch-control-zoom-static.svg',
+ width: 30,
+ height: 90
+ };
+ updateInteractionCursor(cameraTargetIcon.visible, dynamicImageSrc, staticImageInfo);
+ }
+ }
+
+ /**
+ * This updates the state of the two cursor images that appear while you're manipulating the camera. The first
+ * image follows your cursor. The second "static" one appears faintly at the position where you originally clicked
+ * down. The static image defaults to be the same as the moving cursor image, but a different image can be provided.
+ * @param {boolean} visible
+ * @param {string} imageSrc
+ * @param {*|null} staticImageInfo
+ */
+ function updateInteractionCursor(visible, imageSrc, staticImageInfo = {src: null, width: null, height: null}) {
+ interactionCursor.style.display = visible ? 'inline' : 'none';
+ if (imageSrc) {
+ interactionCursor.src = imageSrc;
+ }
+ let interactionRect = getRectSafe(interactionCursor);
+ if (interactionRect) {
+ interactionCursor.style.left = (pointerPosition.x - interactionRect.width / 2) + 'px';
+ interactionCursor.style.top = (pointerPosition.y - interactionRect.height / 2) + 'px';
+ }
+
+ staticInteractionCursor.style.display = visible ? 'inline' : 'none';
+ if (staticImageInfo && staticImageInfo.src) {
+ staticInteractionCursor.src = staticImageInfo.src;
+ staticInteractionCursor.style.width = `${staticImageInfo.width}px` || '30px';
+ staticInteractionCursor.style.height = `${staticImageInfo.height}px` || '30px';
+ } else if (imageSrc) {
+ staticInteractionCursor.src = imageSrc;
+ staticInteractionCursor.style.width = '30px';
+ staticInteractionCursor.style.height = '30px';
+ }
+ let staticInteractionRect = getRectSafe(staticInteractionCursor);
+ if (staticInteractionRect) {
+ staticInteractionCursor.style.left = (pointerPosition.x - staticInteractionRect.width / 2) + 'px';
+ staticInteractionCursor.style.top = (pointerPosition.y - staticInteractionRect.height / 2) + 'px';
+ }
+ }
+ function getRectSafe(div) {
+ if (!div || div.style.display === 'none') { return null; }
+ let rects = div.getClientRects();
+ if (!rects || rects.length === 0) { return null; }
+ return rects[0];
+ }
+
+ /**
+ * Update loop governed by requestAnimationFrame
+ */
+ function onFrame() {
+ update(false);
+ requestAnimationFrame(onFrame);
+ }
+
+ /**
+ * Main update function
+ * @param forceCameraUpdate - Whether this update forces virtualCamera to
+ * update even if it's in 2d (locked follow) mode
+ */
+ function update(forceCameraUpdate) {
+ if (virtualCamera && virtualCameraEnabled) {
+ try {
+ if (followCoordinator) {
+ followCoordinator.update();
+ }
+
+ let skipUpdate = followCoordinator.currentFollowTarget &&
+ followCoordinator.currentFollowTarget.followable &&
+ followCoordinator.currentFollowTarget.isFollowing2D &&
+ followCoordinator.currentFollowTarget.followable.doesOverrideCameraUpdatesInFirstPerson();
+
+ let skipApplying = skipUpdate && !forceCameraUpdate;
+ virtualCamera.update({ skipApplying: skipApplying });
+
+ let worldObject = realityEditor.worldObjects.getBestWorldObject();
+ if (worldObject) {
+ let worldId = worldObject.objectId;
+
+ // render a cube at the virtual camera's target position
+ let sceneNode = realityEditor.sceneGraph.getSceneNodeById(rotateCenterElementId);
+ sceneNode.setLocalMatrix(virtualCamera.getFocusTargetCubeMatrix());
+
+ if (!cameraTargetIcon && worldId !== realityEditor.worldObjects.getLocalWorldId()) {
+ cameraTargetIcon = {};
+ cameraTargetIcon.visible = false;
+ }
+
+ let cameraNode = realityEditor.sceneGraph.getSceneNodeById('CAMERA');
+ let gpNode = realityEditor.sceneGraph.getSceneNodeById(realityEditor.sceneGraph.NAMES.GROUNDPLANE + realityEditor.sceneGraph.TAGS.ROTATE_X);
+ if (!gpNode) {
+ gpNode = realityEditor.sceneGraph.getSceneNodeById(realityEditor.sceneGraph.NAMES.GROUNDPLANE);
+ }
+ realityEditor.network.realtime.sendCameraMatrix(worldId, cameraNode.getMatrixRelativeTo(gpNode));
+ }
+ } catch (e) {
+ if (DEBUG) {
+ console.warn('error updating Virtual Camera', e);
+ }
+ }
+ }
+ }
+
+ let transitionPercent = -1;
+
+ // these only affect the camera when you load the remote operator view in the AR app, not in the browser
+ function addModeTransitionListeners() {
+ if (didAddModeTransitionListeners) return;
+ didAddModeTransitionListeners = true;
+
+ // move the camera based on the combination of the transitionPercent and the transition endpoint positions
+ const processDevicePosition = () => {
+ if (transitionPercent <= 0 || transitionPercent === 1) {
+ virtualCamera.zoomOutTransition = false;
+ return;
+ }
+ if (!cameraTransitionPosition_AR || !cameraTransitionTarget_AR ||
+ !cameraTransitionPosition_VR || !cameraTransitionTarget_VR) return;
+
+ // only starts moving after the first 5% of the pinch gesture / slider
+ let percent = Math.max(0, Math.min(1, (transitionPercent - 0.1) / 0.9));
+
+ let groundPlaneNode = realityEditor.sceneGraph.getGroundPlaneNode();
+
+ virtualCamera.position = [
+ (1.0 - percent) * cameraTransitionPosition_AR[0] + percent * cameraTransitionPosition_VR[0],
+ (1.0 - percent) * cameraTransitionPosition_AR[1] + percent * cameraTransitionPosition_VR[1],
+ (1.0 - percent) * cameraTransitionPosition_AR[2] + percent * cameraTransitionPosition_VR[2]
+ ];
+ virtualCamera.position[1] -= groundPlaneNode.worldMatrix[13]; // TODO: this works but spatial cursor ends up in weird positions
+
+ virtualCamera.targetPosition = [
+ (1.0 - percent) * cameraTransitionTarget_AR[0] + percent * cameraTransitionTarget_VR[0],
+ (1.0 - percent) * cameraTransitionTarget_AR[1] + percent * cameraTransitionTarget_VR[1],
+ (1.0 - percent) * cameraTransitionTarget_AR[2] + percent * cameraTransitionTarget_VR[2]
+ ];
+ virtualCamera.targetPosition[1] -= groundPlaneNode.worldMatrix[13]; // TODO: this works but spatial cursor ends up in weird positions
+
+ virtualCamera.zoomOutTransition = percent !== 0 && percent !== 1;
+ };
+
+ // when the slider or pinch gesture updates, move the virtual camera based on the transition endpoints
+ realityEditor.device.modeTransition.onTransitionPercent((percent) => {
+ transitionPercent = percent;
+ if (!virtualCamera) return; // wait for virtual camera to initialize
+ virtualCamera.pauseTouchGestures = percent < 1;
+ processDevicePosition();
+ });
+
+ // when the device itself moves, update the transition endpoints
+ realityEditor.device.modeTransition.onDeviceCameraPosition((_cameraMatrix) => {
+ let deviceNode = realityEditor.sceneGraph.getDeviceNode();
+ let worldNode = realityEditor.sceneGraph.getSceneNodeById(realityEditor.sceneGraph.getWorldId());
+ let position = realityEditor.sceneGraph.convertToNewCoordSystem([0, 0, 0], deviceNode, worldNode);
+
+ // get the current camera target position, so we maintain the same perspective when we turn on the scene
+ // defaults the target position to 1 meter in front of the camera
+ let targetPositionObj = realityEditor.sceneGraph.getPointAtDistanceFromCamera(window.innerWidth / 2, window.innerHeight / 2, 1000, worldNode, deviceNode);
+ let targetPosition = [targetPositionObj.x, targetPositionObj.y, targetPositionObj.z];
+
+ if (position) {
+ cameraTransitionPosition_AR = [...position];
+ }
+ if (targetPosition) {
+ cameraTransitionTarget_AR = [...targetPosition];
+ cameraTransitionTarget_VR = [...targetPosition];
+ }
+ cameraTransitionPosition_VR = virtualCamera.getRelativePosition(cameraTransitionPosition_AR, cameraTransitionTarget_AR, 0, 3000, 8000);
+
+ if (transitionPercent === 1) {
+ cameraTransitionTarget_VR = [...virtualCamera.targetPosition];
+ cameraTransitionPosition_VR = [...virtualCamera.position];
+ }
+
+ processDevicePosition();
+ });
+
+ // move the virtual camera to a good starting position when the remote operator first loads in the AR app
+ // TODO: there's some redundant code in here that should be removed and rely on onDeviceCameraPosition instead
+ realityEditor.device.modeTransition.onRemoteOperatorShown(() => {
+ if (virtualCameraEnabled) return; // don't do this multiple times per transition
+ virtualCameraEnabled = true;
+ if (!virtualCamera) return;
+
+ // get the current camera position
+ let cameraNode = realityEditor.sceneGraph.getCameraNode();
+ let deviceNode = realityEditor.sceneGraph.getDeviceNode();
+ let groundPlaneNode = realityEditor.sceneGraph.getGroundPlaneNode();
+ let position = realityEditor.sceneGraph.convertToNewCoordSystem([0, 0, 0], cameraNode, groundPlaneNode);
+
+ // get the current camera target position, so we maintain the same perspective when we turn on the scene
+ // defaults the target position to 1 meter in front of the camera
+ let targetPositionObj = realityEditor.sceneGraph.getPointAtDistanceFromCamera(window.innerWidth / 2, window.innerHeight / 2, 1000, groundPlaneNode, deviceNode);
+ let targetPosition = [targetPositionObj.x, targetPositionObj.y, targetPositionObj.z];
+
+ if (position) {
+ cameraTransitionPosition_AR = [...position];
+ virtualCamera.position = [...position];
+ }
+ if (targetPosition) {
+ cameraTransitionTarget_AR = [...targetPosition];
+ virtualCamera.targetPosition = [...targetPosition];
+ }
+
+ // calculate the end position of the transition, and assign to the _VR variables
+ cameraTransitionPosition_VR = virtualCamera.getRelativePosition(cameraTransitionPosition_AR, cameraTransitionTarget_AR, 0, 3000, 8000);
+ cameraTransitionTarget_VR = [...targetPosition]; // where you're looking doesn't change
+
+ if (virtualCamera.focusTargetCube) {
+ virtualCamera.focusTargetCube.position.copy({
+ x: targetPosition[0],
+ y: targetPosition[1],
+ z: targetPosition[2]
+ });
+ virtualCamera.mouseInput.lastWorldPos = [...targetPosition];
+ }
+
+ virtualCamera.zoomOutTransition = true;
+
+ // force it to update
+ virtualCamera.update();
+
+ if (touchControlButtons) {
+ touchControlButtons.activate();
+ touchControlButtons.selectMode('pointer'); // select pointer mode by default
+ }
+ });
+ realityEditor.device.modeTransition.onRemoteOperatorHidden(() => {
+ virtualCameraEnabled = false;
+ virtualCamera.zoomOutTransition = false;
+ cameraTransitionPosition_AR = null;
+ cameraTransitionTarget_AR = null;
+ cameraTransitionPosition_VR = null;
+ cameraTransitionTarget_VR = null;
+
+ if (touchControlButtons) {
+ touchControlButtons.deactivate();
+ touchControlButtons.selectMode(null); // deselect all modes and propagate state to virtual camera
+ }
+ });
+ }
+
+ function focusVirtualCamera(pos, dir, zoomDistanceMm = 3000) {
+ if (!virtualCamera || !virtualCameraEnabled) return;
+ virtualCamera.focus(pos, dir, zoomDistanceMm);
+ }
+
+ exports.update = update;
+ exports.initService = initService;
+ exports.updateCameraFloorOffset = updateCameraFloorOffset;
+})(realityEditor.device.desktopCamera);
diff --git a/content_scripts/desktopRenderer.js b/content_scripts/desktopRenderer.js
new file mode 100644
index 00000000..dac63324
--- /dev/null
+++ b/content_scripts/desktopRenderer.js
@@ -0,0 +1,664 @@
+/*
+* Copyright © 2018 PTC
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+createNameSpace('realityEditor.gui.ar.desktopRenderer');
+
+import * as THREE from '../../thirdPartyCode/three/three.module.js';
+import {UNIFORMS, MAX_VIEW_FRUSTUMS} from '../../src/gui/ViewFrustum.js';
+import {ShaderMode} from '../../src/spatialCapture/Shaders.js';
+
+/**
+ * @fileOverview realityEditor.device.desktopRenderer.js
+ * For remote desktop operation: renders background graphics simulating the context streamed from a connected phone.
+ * e.g. a point or plane for each target, or an entire point cloud of the background contents
+ */
+
+(function(exports) {
+ const PROXY = !window.location.port || window.location.port === "443";
+
+ /**
+ * @type {Canvas} - the DOM element where the images streamed from a reality zone are rendered
+ */
+ var backgroundCanvas;
+ /**
+ * @type {Canvas}
+ * Scratch space to draw and chroma-key the image from the RZ which is
+ * drawing the point cloud and background
+ */
+ var primaryBackgroundCanvas;
+ // Whether the primary canvas is ready for use in bg rendering
+ var primaryDrawn = false;
+
+ /**
+ * @type {Canvas}
+ * Scratch space to draw and chroma-key the image from the RZ which is
+ * drawing only its point cloud
+ */
+ var secondaryBackgroundCanvas;
+ // Whether the secondary canvas is ready for use in bg rendering
+ var secondaryDrawn = false;
+
+ var ONLY_REQUIRE_PRIMARY = true;
+
+ // let gltfPath = null; //'./svg/office.glb'; //null; // './svg/BenApt1_authoring.glb';
+ let isGlbLoaded = false;
+
+ let gltf = null;
+ let staticModelMode = false;
+ let videoPlayback = null;
+ let cameraVisCoordinator = null;
+ let cameraVisSceneNodes = [];
+
+ let cameraVisFrustums = [];
+
+ let didInit = false; // true as soon as initService is called
+ let cameraSystemInitialized = false; // true when a valid world has been discovered
+ let sceneInitialized = false; // true when the world mesh has loaded
+
+ let didAddModeTransitionListeners = false;
+ let gltfUpdateCallbacks = [];
+
+ /**
+ * Public init method to enable rendering if isDesktop
+ */
+ function initService() {
+ if (!realityEditor.device.desktopAdapter) {
+ setTimeout(initService, 100);
+ return;
+ }
+
+ // always trigger this even on an AR device, since these listeners can trigger initService
+ addModeTransitionListeners();
+
+ // only continue initializing the remote renderer if we're not in AR mode (happens on desktop, or by pinching on AR device)
+ if (realityEditor.device.environment.isARMode()) { return; }
+ if (didInit) return;
+
+ didInit = true;
+
+ const renderingFlagName = 'loadingWorldMesh';
+ realityEditor.device.environment.addSuppressedObjectRenderingFlag(renderingFlagName); // hide tools until the model is loaded
+
+ // when a new object is detected, check if we need to create a socket connection with its server
+ realityEditor.network.addObjectDiscoveredCallback(function(object, objectKey) {
+ if (isGlbLoaded) { return; } // only do this for the first world object detected
+
+ // let primaryWorldId = realityEditor.device.desktopAdapter.getPrimaryWorldId();
+ let primaryWorldId = realityEditor.network.discovery.getPrimaryWorldInfo() ?
+ realityEditor.network.discovery.getPrimaryWorldInfo().id : null;
+ let isConnectedViaIp = window.location.hostname.split('').every(char => '0123456789.'.includes(char)); // Already know hostname is valid, this is enough to check for IP
+ let isSameIp = object.ip === window.location.hostname;
+ let isWorldObject = object.isWorldObject || object.type === 'world';
+
+ let allCriteriaMet;
+ if (primaryWorldId) {
+ allCriteriaMet = objectKey === primaryWorldId; // Connecting to specific world object via search param
+ } else {
+ if (isConnectedViaIp) {
+ allCriteriaMet = isSameIp && isWorldObject; // Connecting to same world object running on remote operator (excluding when connecting via domain name)
+ } else {
+ allCriteriaMet = isWorldObject; // Otherwise, connect to first available world object
+ }
+ }
+
+ if (!allCriteriaMet) {
+ return;
+ }
+
+ if (objectKey.includes('_local')) {
+ console.warn('Rejected local world object');
+ return;
+ }
+
+ // initialize the camera system first with a 0 floor offset - proper offset gets set after gltf loaded
+ // this lets the camera work even when there's an empty world object (just seeing the holodeck)
+ initializeCameraSystem(0);
+
+ realityEditor.device.meshLine.inject();
+
+ // try loading area target GLB file into the threejs scene
+ isGlbLoaded = true;
+ let gltfPath = realityEditor.network.getURL(object.ip, realityEditor.network.getPort(object), '/obj/' + object.name + '/target/target.glb');
+ let checkGltfPath = realityEditor.network.getURL(object.ip, realityEditor.network.getPort(object), '/object/' + objectKey + '/checkFileExists/target/target.glb');
+ function checkExist() {
+ let start = Date.now(); // prevent cached responses
+ fetch(`${checkGltfPath}?t=${start}`).then((res) => {
+ return res.json();
+ }).then((body) => {
+ if (!body.exists) {
+ setTimeout(checkExist, 500);
+ } else {
+ realityEditor.app.targetDownloader.createNavmesh(gltfPath, objectKey, createNavmeshCallback);
+ }
+ }).catch(_ => {
+ setTimeout(checkExist, 500);
+ });
+ }
+
+ function createNavmeshCallback(navmesh) {
+ let floorOffset = navmesh.floorOffset * 1000;
+ let buffer = 0; // can be used to offset the mesh from the groundplane, but other alignment issues throughout the system may arise if it isn't 0
+ floorOffset += buffer;
+ let groundPlaneMatrix = [
+ 1, 0, 0, 0,
+ 0, 1, 0, 0,
+ 0, 0, 1, 0,
+ 0, floorOffset, 0, 1
+ ];
+ realityEditor.sceneGraph.setGroundPlanePosition(groundPlaneMatrix);
+
+ // update the camera system with the correct floor offset from the navmesh
+ initializeCameraSystem(floorOffset);
+
+ let ceilingHeight = Math.max(
+ navmesh.maxY - navmesh.minY,
+ navmesh.maxX - navmesh.minX,
+ navmesh.maxZ - navmesh.minZ
+ );
+ let ceilingAndFloor = {
+ ceiling: navmesh.maxY,
+ floor: navmesh.minY
+ };
+ let center = {
+ x: (navmesh.maxX + navmesh.minX) / 2,
+ y: navmesh.minY,
+ z: (navmesh.maxZ + navmesh.minZ) / 2,
+ };
+ let map = navmesh.map;
+ let steepnessMap = navmesh.steepnessMap;
+ let heightMap = navmesh.heightMap;
+ realityEditor.gui.threejsScene.addGltfToScene(gltfPath, map, steepnessMap, heightMap, {x: 0, y: -floorOffset, z: 0}, {x: 0, y: 0, z: 0}, ceilingHeight, ceilingAndFloor, center, function(createdMesh) {
+
+ initializeSceneAfterMeshLoaded();
+
+ gltf = createdMesh;
+ gltf.name = 'areaTargetMesh';
+
+ const greyMaterial = new THREE.MeshBasicMaterial({
+ color: 0x777777,
+ wireframe: true,
+ });
+
+ gltf.traverse(obj => {
+ if (obj.type === 'Mesh' && obj.material) {
+ obj.oldMaterial = greyMaterial;
+
+ // to improve performance on mobile devices, switch to simpler material (has very large effect)
+ if (!realityEditor.device.environment.isDesktop() && typeof obj.originalMaterial !== 'undefined') {
+ obj.material.dispose(); // free resources from the advanced material
+ obj.material = obj.originalMaterial;
+
+ gltfUpdateCallbacks.push((percent) => {
+ let scaledPercent = percent * 20; // fully fades in when slider is 5% activated
+ // TODO: figure out best-looking transition. for now, just make it appear all at once
+ if (percent < 0.05) {
+ scaledPercent = 0;
+ } else {
+ scaledPercent = 1;
+ }
+ if (scaledPercent < 1) {
+ obj.material.transparent = true;
+ } else {
+ obj.material.transparent = false;
+ }
+ obj.material.opacity = Math.max(0, Math.min(1.0, scaledPercent));
+ });
+ }
+ }
+ });
+
+ // this will trigger any onLocalizedWithinWorld callbacks in the userinterface, such as creating the Avatar
+ let identity = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
+ realityEditor.worldObjects.setOrigin(objectKey, identity);
+
+ let realityZoneVoxelizer;
+ function enableVoxelizer() {
+ if (realityZoneVoxelizer) {
+ realityZoneVoxelizer.remove();
+ }
+ realityZoneVoxelizer = new realityEditor.gui.ar.desktopRenderer.RealityZoneVoxelizer(floorOffset, createdMesh, navmesh);
+ realityZoneVoxelizer.add();
+ cameraVisCoordinator.voxelizer = realityZoneVoxelizer;
+ }
+ function disableVoxelizer() {
+ if (!realityZoneVoxelizer) {
+ return;
+ }
+
+ realityZoneVoxelizer.remove();
+ realityZoneVoxelizer = null;
+ cameraVisCoordinator.voxelizer = null;
+ }
+
+ function setupMenuBar() {
+ if (realityEditor.device.environment.isWithinToolboxApp()) return;
+
+ if (!realityEditor.gui.getMenuBar) {
+ setTimeout(setupMenuBar, 100);
+ return;
+ }
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.Voxelizer, (toggled) => {
+ if (toggled) {
+ enableVoxelizer();
+ } else {
+ disableVoxelizer();
+ }
+ });
+
+ cameraVisCoordinator = new realityEditor.device.cameraVis.CameraVisCoordinator(floorOffset);
+ cameraVisCoordinator.onCameraVisCreated(cameraVis => {
+ console.log('onCameraVisCreated', cameraVis);
+ cameraVisSceneNodes.push(cameraVis.sceneNode);
+
+ // add to cameraVisFrustums so that material uniforms can be updated
+ cameraVisFrustums.push(cameraVis.id);
+ });
+
+ cameraVisCoordinator.onCameraVisRemoved(cameraVis => {
+ console.log('onCameraVisRemoved', cameraVis);
+ cameraVisSceneNodes = cameraVisSceneNodes.filter(sceneNode => {
+ return sceneNode !== cameraVis.sceneNode;
+ });
+
+ // remove from cameraVisFrustums so that material uniforms can be updated
+ cameraVisFrustums = cameraVisSceneNodes.filter(id => {
+ return id !== cameraVis.id;
+ });
+ realityEditor.gui.threejsScene.removeMaterialCullingFrustum(cameraVis.id);
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.AdvanceCameraShader, () => {
+ cameraVisCoordinator.advanceShaderMode();
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.DownloadScan, () => {
+ window.open(gltfPath, '_blank');
+ });
+ realityEditor.gui.getMenuBar().setItemEnabled(realityEditor.gui.ITEM.DownloadScan, true);
+
+ const ENABLE_VIDEO_TIMELINE_COMPONENT = false;
+ if (ENABLE_VIDEO_TIMELINE_COMPONENT && !PROXY) {
+ const createVideoPlayback = () => {
+ videoPlayback = new realityEditor.videoPlayback.VideoPlaybackCoordinator();
+ videoPlayback.setPointCloudCallback(cameraVisCoordinator.loadPointCloud.bind(cameraVisCoordinator));
+ videoPlayback.setHidePointCloudCallback(cameraVisCoordinator.hidePointCloud.bind(cameraVisCoordinator));
+ videoPlayback.load();
+ // window.videoPlayback = videoPlayback;
+ };
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.VideoPlayback, (toggled) => {
+ if (!videoPlayback) {
+ createVideoPlayback(); // only create it if the user decides to use it
+ }
+ videoPlayback.toggleVisibility(toggled);
+ });
+ }
+ }
+
+ setupMenuBar();
+ });
+ }
+
+ checkExist();
+ });
+
+ if (!realityEditor.device.environment.isWithinToolboxApp()) {
+ document.body.style.backgroundColor = 'rgb(50,50,50)';
+ }
+
+ // create background canvas and supporting canvasses
+
+ backgroundCanvas = document.createElement('canvas');
+ backgroundCanvas.id = 'desktopBackgroundRenderer';
+ backgroundCanvas.classList.add('desktopBackgroundRenderer');
+ backgroundCanvas.style.transform = 'matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1)'; // render behind three.js
+ backgroundCanvas.style.transformOrigin = 'top left';
+ backgroundCanvas.style.position = 'absolute';
+ backgroundCanvas.style.visibility = 'hidden';
+ primaryBackgroundCanvas = document.createElement('canvas');
+ secondaryBackgroundCanvas = document.createElement('canvas');
+
+ updateCanvasSize();
+ window.addEventListener('resize', updateCanvasSize);
+
+ // backgroundRenderer.src = "https://www.youtube.com/embed/XOacA3RYrXk?enablejsapi=1&rel=0&controls=0&playsinline=1&vq=large";
+
+ // add the Reality Zone background behind everything else
+ document.body.insertBefore(backgroundCanvas, document.body.childNodes[0]);
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.ModelVisibility, (value) => {
+ if (!gltf) { return; }
+ staticModelMode = value;
+ if (staticModelMode) {
+ gltf.visible = true;
+ console.log('show gtlf');
+ } else {
+ gltf.visible = false;
+ console.log('hide gltf');
+ }
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.ModelTexture, () => {
+ if (!gltf) {
+ return;
+ }
+ gltf.traverse(obj => {
+ if (obj.type === 'Mesh' && obj.material) {
+ let tmp = obj.material;
+ obj.material = obj.oldMaterial;
+ obj.oldMaterial = tmp;
+ }
+ });
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.ToggleMotionStudySettings, () => {
+ if (!realityEditor.humanPose.draw) { return; }
+ realityEditor.humanPose.draw.toggleAnalyzerSettingsUI();
+ });
+
+ realityEditor.gui.buttons.registerCallbackForButton(
+ 'logic',
+ function onLogicMode() {
+ const logicCanvas = document.getElementById('canvas');
+ logicCanvas.style.pointerEvents = 'auto';
+ }
+ );
+ realityEditor.gui.buttons.registerCallbackForButton(
+ 'gui',
+ function onGuiMode() {
+ const logicCanvas = document.getElementById('canvas');
+ logicCanvas.style.pointerEvents = 'none';
+ }
+ );
+ }
+
+ /**
+ * Take care of any initialization steps that should happen as soon as a valid world object has been discovered
+ * @param floorOffset
+ */
+ function initializeCameraSystem(floorOffset = 0) {
+ if (cameraSystemInitialized) {
+ realityEditor.device.desktopCamera.updateCameraFloorOffset(floorOffset);
+ return;
+ }
+ // make sure this only happens once
+ cameraSystemInitialized = true;
+ realityEditor.device.desktopCamera.initService(floorOffset);
+ }
+
+ /**
+ * Take care of any initialization steps that should happen after the mesh has been loaded
+ */
+ function initializeSceneAfterMeshLoaded() {
+ if (sceneInitialized) return; // make sure this only happens once
+ sceneInitialized = true;
+ realityEditor.device.environment.clearSuppressedObjectRenderingFlag('loadingWorldMesh'); // stop hiding tools
+
+ let endMarker = document.createElement('div');
+ endMarker.style.display = 'none';
+ endMarker.id = 'gltf-added';
+ document.body.appendChild(endMarker);
+ }
+
+ /**
+ * @param {ShaderMode} shaderMode - initial shader mode to set on the patches
+ * @return {{[key: string]: THREE.Object3D}} map from key to patch
+ */
+ function cloneCameraVisPatches(shaderMode = ShaderMode.SOLID) {
+ let spatialPatchCoordinator = realityEditor.spatialCapture.spatialPatchCoordinator;
+ if (!spatialPatchCoordinator) {
+ return null;
+ }
+ return spatialPatchCoordinator.clonePatches(shaderMode);
+ }
+ exports.cloneCameraVisPatches = cloneCameraVisPatches;
+
+ /**
+ * @return {{[key: string]: THREE.Object3D}} map from key to patch
+ */
+ function getCameraVisPatches() {
+ let spatialPatchCoordinator = realityEditor.spatialCapture.spatialPatchCoordinator;
+ if (!spatialPatchCoordinator) {
+ return null;
+ }
+ return spatialPatchCoordinator.patches;
+ }
+ exports.getCameraVisPatches = getCameraVisPatches;
+
+ function showCameraCanvas(id) {
+ if (cameraVisCoordinator) {
+ cameraVisCoordinator.showFullscreenColorCanvas(id);
+ isVirtualizerRenderingIn2D[id] = true;
+ }
+ }
+ exports.showCameraCanvas = showCameraCanvas;
+
+ function hideCameraCanvas(id) {
+ if (cameraVisCoordinator) {
+ cameraVisCoordinator.hideFullscreenColorCanvas(id);
+ isVirtualizerRenderingIn2D[id] = false;
+ }
+ }
+ exports.hideCameraCanvas = hideCameraCanvas;
+
+ // can use this to preserve 2D rendering if we switch from one camera target to another
+ let isVirtualizerRenderingIn2D = {};
+ exports.getVirtualizers2DRenderingState = function() {
+ return isVirtualizerRenderingIn2D;
+ };
+
+ /**
+ * Updates canvas size for resize events
+ */
+ function updateCanvasSize() {
+ backgroundCanvas.width = window.innerWidth;
+ backgroundCanvas.height = window.innerHeight;
+ primaryBackgroundCanvas.width = window.innerWidth;
+ primaryBackgroundCanvas.height = window.innerHeight;
+ secondaryBackgroundCanvas.width = window.innerWidth;
+ secondaryBackgroundCanvas.height = window.innerHeight;
+ primaryDrawn = false;
+ secondaryDrawn = false;
+ }
+
+ /**
+ * Takes a message containing an encoded image, and chroma keys it for use as the fullscreen background on the desktop
+ * @param {string} source - either primary or secondary
+ * @param {string} msgContent - contains the image data encoded as a base64 string
+ */
+ function processImageFromSource(source, msgContent) {
+ // if (typeof msgContent.base64String !== 'undefined') {
+ // var imageBlobUrl = realityEditor.device.utilities.decodeBase64JpgToBlobUrl(msgContent.base64String);
+ // backgroundRenderer.src = imageBlobUrl;
+ // }
+ let parts = msgContent.split(';_;');
+ let rgbImage = parts[0];
+ let alphaImage = parts[1];
+ let editorId = parts[2];
+ let rescaleFactor = parts[3];
+
+ if (editorId !== globalStates.tempUuid) {
+ // console.log('ignoring image from other editorId');
+ return;
+ }
+
+ let prom;
+ if (source === 'primary') {
+ prom = renderImageAndChromaKey(primaryBackgroundCanvas, rgbImage, alphaImage).then(function() {
+ primaryDrawn = true;
+ });
+ } else if (source === 'secondary') {
+ prom = renderImageAndChromaKey(secondaryBackgroundCanvas, rgbImage, alphaImage).then(function() {
+ secondaryDrawn = true;
+ });
+ }
+ if (!prom) {
+ return;
+ }
+ prom.then(function() {
+ if (primaryDrawn && (secondaryDrawn || ONLY_REQUIRE_PRIMARY)) {
+ renderBackground();
+ backgroundCanvas.style.transform = 'matrix3d(' + rescaleFactor + ', 0, 0, 0, 0, ' + rescaleFactor + ', 0, 0, 0, 0, 1, 0, 0, 0, 1, 1)';
+ }
+ });
+ }
+
+ function renderBackground() {
+ let gfx = backgroundCanvas.getContext('2d');
+ gfx.clearRect(0, 0, backgroundCanvas.width, backgroundCanvas.height);
+ gfx.drawImage(primaryBackgroundCanvas, 0, 0);
+ gfx.drawImage(secondaryBackgroundCanvas, 0, 0);
+ realityEditor.device.desktopStats.imageRendered();
+
+ if (staticModelMode) {
+ // desktopBackgroundRenderer
+ backgroundCanvas.style.visibility = 'hidden';
+ } else {
+ backgroundCanvas.style.visibility = '';
+ }
+ }
+
+ function loadImage(width, height, imageStr) {
+ if (!imageStr) {
+ return Promise.resolve(null);
+ }
+ return new Promise(function(res) {
+ let img = new Image(width, height);
+ img.onload = function() {
+ img.onload = null;
+ res(img);
+ };
+ img.src = imageStr;
+ });
+ }
+
+ function renderImageAndChromaKey(canvas, rgbImageStr, alphaImageStr) {
+ return Promise.all([
+ loadImage(canvas.width, canvas.height, rgbImageStr),
+ loadImage(canvas.width, canvas.height, alphaImageStr),
+ ]).then(function([rgbImage, alphaImage]) {
+ let gfx = canvas.getContext('2d');
+
+ if (!alphaImage) {
+ gfx.drawImage(rgbImage, 0, 0);
+ return;
+ }
+
+ gfx.drawImage(alphaImage, 0, 0);
+ let alphaId = gfx.getImageData(0, 0, canvas.width, canvas.height);
+ gfx.drawImage(rgbImage, 0, 0);
+ let id = gfx.getImageData(0, 0, canvas.width, canvas.height);
+ let nPixels = canvas.width * canvas.height;
+ for (let i = 0; i < nPixels; i++) {
+ id.data[4 * i + 3] = alphaId.data[4 * i + 0];
+ }
+ gfx.putImageData(id, 0, 0);
+ });
+ }
+
+ exports.processImageFromSource = processImageFromSource;
+
+ exports.getCameraVisSceneNodes = () => {
+ return cameraVisSceneNodes;
+ };
+
+ exports.updateAreaGltfForCamera = function(cameraId, cameraWorldMatrix, maxDepthMeters) {
+ if (!gltf || typeof gltf.traverse === 'undefined') return;
+ const utils = realityEditor.gui.ar.utilities;
+
+ let cameraPosition = new THREE.Vector3(
+ cameraWorldMatrix.elements[12] / 1000,
+ cameraWorldMatrix.elements[13] / 1000,
+ cameraWorldMatrix.elements[14] / 1000
+ );
+ let cameraPos = [cameraPosition.x, cameraPosition.y, cameraPosition.z];
+ let cameraDirection = utils.normalize(utils.getForwardVector(cameraWorldMatrix.elements));
+ let cameraLookAtPosition = utils.add(cameraPos, cameraDirection);
+ let cameraUp = utils.normalize(utils.getUpVector(cameraWorldMatrix.elements));
+
+ let thisFrustumPlanes = realityEditor.gui.threejsScene.updateMaterialCullingFrustum(cameraId, cameraPos, cameraLookAtPosition, cameraUp, maxDepthMeters);
+
+ gltf.traverse(child => {
+ updateFrustumUniforms(child, cameraId, thisFrustumPlanes);
+ });
+ };
+
+ function updateFrustumUniforms(mesh, cameraId, frustumPlanes) {
+ if (!mesh.material || !mesh.material.uniforms) return;
+
+ let cameraFrustumIndex = cameraVisFrustums.indexOf(cameraId);
+ if (cameraFrustumIndex >= MAX_VIEW_FRUSTUMS || cameraFrustumIndex === -1) {
+ return;
+ }
+
+ mesh.material.uniforms[UNIFORMS.numFrustums].value = Math.min(cameraVisFrustums.length, MAX_VIEW_FRUSTUMS);
+
+ if (typeof mesh.material.uniforms[UNIFORMS.frustums] !== 'undefined') {
+ // update this frustum with all of the normals and constants
+ let existingFrustums = mesh.material.uniforms[UNIFORMS.frustums].value;
+ existingFrustums[cameraFrustumIndex] = frustumPlanes;
+ mesh.material.uniforms[UNIFORMS.frustums].value = existingFrustums;
+ mesh.material.needsUpdate = true;
+ }
+ }
+
+ function muteMicrophoneForCameraVis() {
+ if (!cameraVisCoordinator) return;
+ cameraVisCoordinator.muteMicrophone();
+ }
+
+ function unmuteMicrophoneForCameraVis() {
+ if (!cameraVisCoordinator) return;
+ cameraVisCoordinator.unmuteMicrophone();
+ }
+
+ exports.muteMicrophoneForCameraVis = muteMicrophoneForCameraVis;
+ exports.unmuteMicrophoneForCameraVis = unmuteMicrophoneForCameraVis;
+
+ // when transitioning from AR to VR, add or remove the gltf from the three.js scene
+ function addModeTransitionListeners() {
+ if (didAddModeTransitionListeners) return;
+ didAddModeTransitionListeners = true;
+
+ realityEditor.device.modeTransition.onRemoteOperatorShown(() => {
+ initService(); // init if needed
+ showScene();
+ });
+
+ realityEditor.device.modeTransition.onRemoteOperatorHidden(() => {
+ hideScene();
+ });
+
+ realityEditor.device.modeTransition.onTransitionPercent((percent) => {
+ gltfUpdateCallbacks.forEach(callback => {
+ callback(percent);
+ });
+ });
+ }
+
+ function showScene() {
+ if (!gltf) return;
+ gltf.visible = true;
+ realityEditor.gui.threejsScene.addToScene(gltf);
+ }
+
+ function hideScene() {
+ if (!gltf) return;
+ gltf.visible = false;
+ realityEditor.gui.threejsScene.removeFromScene(gltf);
+ }
+
+ exports.initService = initService;
+
+ exports.getCameraVisCoordinator = () => {
+ return cameraVisCoordinator;
+ };
+
+ realityEditor.addons.addCallback('init', initService);
+})(realityEditor.gui.ar.desktopRenderer);
diff --git a/content_scripts/desktopStats.js b/content_scripts/desktopStats.js
new file mode 100644
index 00000000..26f8d397
--- /dev/null
+++ b/content_scripts/desktopStats.js
@@ -0,0 +1,126 @@
+/*
+* Copyright © 2018 PTC
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+createNameSpace('realityEditor.device.desktopStats');
+
+(function(exports) {
+
+ let stats = new Stats();
+
+ let imagesPerSecond = 0;
+ let numImages = 0;
+ let imageStartTime = null;
+ let currentImageTime = null;
+ let imagesPerSecondElement = null;
+
+ let isVisible = false;
+
+ function initService() {
+ if (!realityEditor.device.desktopAdapter) {
+ setTimeout(initService, 50);
+ return;
+ }
+
+ if (realityEditor.device.environment.isARMode()) { return; }
+
+ // wait until the menubar is initialized
+ if (typeof realityEditor.gui.setupMenuBar !== 'function') {
+ setTimeout(initService, 100);
+ return;
+ }
+ realityEditor.gui.setupMenuBar(); // this can be safely called multiple times to ensure it is created
+
+ stats.dom.position = 'absolute';
+ stats.dom.style.top = realityEditor.device.environment.variables.screenTopOffset + 'px';
+ document.body.appendChild(stats.dom);
+ stats.dom.style.left = (window.innerWidth - stats.dom.getBoundingClientRect().width) + 'px';
+
+ imagesPerSecondElement = document.createElement('div');
+ imagesPerSecondElement.style.color = 'white';
+ imagesPerSecondElement.style.fontSize = '30px';
+ imagesPerSecondElement.style.position = 'absolute';
+ imagesPerSecondElement.style.right = '100px';
+ imagesPerSecondElement.style.top = realityEditor.device.environment.variables.screenTopOffset + 'px';
+ document.body.appendChild(imagesPerSecondElement);
+
+ isVisible = true;
+
+ update(); // start update loop
+ hide(); // default hidden
+ }
+
+ function update() {
+ if (!isVisible) {
+ return;
+ }
+
+ if (isVisible) {
+ stats.update();
+ }
+ requestAnimationFrame(update);
+
+ if (imageStartTime !== null) {
+ updateImagesPerSecond();
+ }
+ }
+
+ function startImageTimer() {
+ imageStartTime = (new Date()).getTime();
+ }
+
+ function resetImageTimer() {
+ numImages = 0;
+ imageStartTime = null;
+ currentImageTime = null;
+ }
+
+ function imageRendered() {
+ if (!isVisible) {
+ return;
+ }
+
+ if (currentImageTime > 10000) {
+ resetImageTimer(); // reset every 10 seconds to maintain accurate temporal averages
+ }
+ if (imageStartTime === null) {
+ startImageTimer();
+ }
+ numImages += 1;
+ }
+
+ function updateImagesPerSecond() {
+ currentImageTime = (new Date()).getTime() - imageStartTime;
+ imagesPerSecond = numImages / (currentImageTime / 1000);
+ imagesPerSecondElement.innerText = imagesPerSecond.toFixed(2);
+ }
+
+ function show() {
+ stats.dom.style.visibility = 'visible';
+ imagesPerSecondElement.style.visibility = 'visible';
+ isVisible = true;
+ resetImageTimer();
+ update();
+ }
+
+ function hide() {
+ if (stats && stats.dom) {
+ stats.dom.style.visibility = 'hidden';
+ }
+ if (imagesPerSecond) {
+ imagesPerSecondElement.style.visibility = 'hidden';
+ }
+ isVisible = false;
+ }
+
+ exports.imageRendered = imageRendered;
+ exports.resetImageTimer = resetImageTimer;
+ exports.show = show;
+ exports.hide = hide;
+
+ realityEditor.addons.addCallback('init', initService);
+})(realityEditor.device.desktopStats);
diff --git a/content_scripts/meshLine.js b/content_scripts/meshLine.js
new file mode 100644
index 00000000..dc9ea43b
--- /dev/null
+++ b/content_scripts/meshLine.js
@@ -0,0 +1,707 @@
+createNameSpace('realityEditor.device.meshLine');
+
+import * as THREE from '../../thirdPartyCode/three/three.module.js';
+
+realityEditor.device.meshLine.inject = function() {
+ 'use strict';
+
+ class MeshLine extends THREE.BufferGeometry {
+ constructor() {
+ super();
+
+ this.type = 'MeshLine';
+ this.isMeshLine = true;
+
+ this.positions = [];
+
+ this.previous = [];
+ this.next = [];
+ this.side = [];
+ this.width = [];
+ this.indices_array = [];
+ this.uvs = [];
+ this.counters = [];
+ this._points = [];
+ this._geom = null;
+ this.raycast = MeshLineRaycast;
+
+ this.widthCallback = null;
+
+ // Used to raycast
+ this.matrixWorld = new THREE.Matrix4();
+
+ Object.defineProperties(this, {
+ // this is now a bufferGeometry
+ // add getter to support previous api
+ geometry: {
+ enumerable: true,
+ get: function() {
+ return this;
+ },
+ },
+ geom: {
+ enumerable: true,
+ get: function() {
+ return this._geom;
+ },
+ set: function(value) {
+ this.setGeometry(value, this.widthCallback);
+ },
+ },
+ // for declaritive architectures
+ // to return the same value that sets the points
+ // eg. this.points = points
+ // console.log(this.points) -> points
+ points: {
+ enumerable: true,
+ get: function() {
+ return this._points;
+ },
+ set: function(value) {
+ this.setPoints(value, this.widthCallback);
+ },
+ },
+ });
+ }
+
+ setMatrixWorld(matrixWorld) {
+ this.matrixWorld = matrixWorld;
+ }
+
+ // setting via a geometry is rather superfluous
+ // as you're creating a unecessary geometry just to throw away
+ // but exists to support previous api
+ setGeometry(g, c) {
+ // as the input geometry are mutated we store them
+ // for later retreival when necessary (declaritive architectures)
+ this._geometry = g;
+ if (g instanceof THREE.Geometry) {
+ this.setPoints(g.vertices, c);
+ } else if (g instanceof THREE.BufferGeometry) {
+ this.setPoints(g.getAttribute('position').array, c);
+ } else {
+ this.setPoints(g, c);
+ }
+ }
+
+ setPoints(points, wcb) {
+ if (!(points instanceof Float32Array) && !(points instanceof Array)) {
+ console.error(
+ 'ERROR: The BufferArray of points is not instancied correctly.'
+ );
+ return;
+ }
+ // as the points are mutated we store them
+ // for later retreival when necessary (declaritive architectures)
+ this._points = points;
+ this.widthCallback = wcb;
+ this.positions = [];
+ this.counters = [];
+ if (points.length && points[0] instanceof THREE.Vector3) {
+ // could transform Vector3 array into the array used below
+ // but this approach will only loop through the array once
+ // and is more performant
+ for (let j = 0; j < points.length; j++) {
+ let p = points[j];
+ let c = j / points.length;
+ this.positions.push(p.x, p.y, p.z);
+ this.positions.push(p.x, p.y, p.z);
+ this.counters.push(c);
+ this.counters.push(c);
+ }
+ } else {
+ for (let j = 0; j < points.length; j += 3) {
+ let c = j / points.length;
+ this.positions.push(points[j], points[j + 1], points[j + 2]);
+ this.positions.push(points[j], points[j + 1], points[j + 2]);
+ this.counters.push(c);
+ this.counters.push(c);
+ }
+ }
+ this.process();
+ }
+
+ compareV3(a, b) {
+ var aa = a * 6;
+ var ab = b * 6;
+ return (
+ this.positions[aa] === this.positions[ab] &&
+ this.positions[aa + 1] === this.positions[ab + 1] &&
+ this.positions[aa + 2] === this.positions[ab + 2]
+ );
+ }
+
+ copyV3(a) {
+ var aa = a * 6;
+ return [this.positions[aa], this.positions[aa + 1], this.positions[aa + 2]];
+ }
+
+ process() {
+ var l = this.positions.length / 6;
+
+ this.previous = [];
+ this.next = [];
+ this.side = [];
+ this.width = [];
+ this.indices_array = [];
+ this.uvs = [];
+
+ var w;
+
+ var v;
+ // initial previous points
+ if (this.compareV3(0, l - 1)) {
+ v = this.copyV3(l - 2);
+ } else {
+ v = this.copyV3(0);
+ }
+ this.previous.push(v[0], v[1], v[2]);
+ this.previous.push(v[0], v[1], v[2]);
+
+ for (var j = 0; j < l; j++) {
+ // sides
+ this.side.push(1);
+ this.side.push(-1);
+
+ // widths
+ if (this.widthCallback) w = this.widthCallback(j / (l - 1));
+ else w = 1;
+ this.width.push(w);
+ this.width.push(w);
+
+ // uvs
+ this.uvs.push(j / (l - 1), 0);
+ this.uvs.push(j / (l - 1), 1);
+
+ if (j < l - 1) {
+ // points previous to poisitions
+ v = this.copyV3(j);
+ this.previous.push(v[0], v[1], v[2]);
+ this.previous.push(v[0], v[1], v[2]);
+
+ // indices
+ var n = j * 2;
+ this.indices_array.push(n, n + 1, n + 2);
+ this.indices_array.push(n + 2, n + 1, n + 3);
+ }
+ if (j > 0) {
+ // points after poisitions
+ v = this.copyV3(j);
+ this.next.push(v[0], v[1], v[2]);
+ this.next.push(v[0], v[1], v[2]);
+ }
+ }
+
+ // last next point
+ if (this.compareV3(l - 1, 0)) {
+ v = this.copyV3(1);
+ } else {
+ v = this.copyV3(l - 1);
+ }
+ this.next.push(v[0], v[1], v[2]);
+ this.next.push(v[0], v[1], v[2]);
+
+ // redefining the attribute seems to prevent range errors
+ // if the user sets a differing number of vertices
+ if (!this._attributes || this._attributes.position.count !== this.positions.length) {
+ this._attributes = {
+ position: new THREE.BufferAttribute(new Float32Array(this.positions), 3),
+ previous: new THREE.BufferAttribute(new Float32Array(this.previous), 3),
+ next: new THREE.BufferAttribute(new Float32Array(this.next), 3),
+ side: new THREE.BufferAttribute(new Float32Array(this.side), 1),
+ width: new THREE.BufferAttribute(new Float32Array(this.width), 1),
+ uv: new THREE.BufferAttribute(new Float32Array(this.uvs), 2),
+ index: new THREE.BufferAttribute(new Uint16Array(this.indices_array), 1),
+ counters: new THREE.BufferAttribute(new Float32Array(this.counters), 1),
+ };
+ } else {
+ this._attributes.position.copyArray(new Float32Array(this.positions));
+ this._attributes.position.needsUpdate = true;
+ this._attributes.previous.copyArray(new Float32Array(this.previous));
+ this._attributes.previous.needsUpdate = true;
+ this._attributes.next.copyArray(new Float32Array(this.next));
+ this._attributes.next.needsUpdate = true;
+ this._attributes.side.copyArray(new Float32Array(this.side));
+ this._attributes.side.needsUpdate = true;
+ this._attributes.width.copyArray(new Float32Array(this.width));
+ this._attributes.width.needsUpdate = true;
+ this._attributes.uv.copyArray(new Float32Array(this.uvs));
+ this._attributes.uv.needsUpdate = true;
+ this._attributes.index.copyArray(new Uint16Array(this.indices_array));
+ this._attributes.index.needsUpdate = true;
+ }
+
+ this.setAttribute('position', this._attributes.position);
+ this.setAttribute('previous', this._attributes.previous);
+ this.setAttribute('next', this._attributes.next);
+ this.setAttribute('side', this._attributes.side);
+ this.setAttribute('width', this._attributes.width);
+ this.setAttribute('uv', this._attributes.uv);
+ this.setAttribute('counters', this._attributes.counters);
+
+ this.setIndex(this._attributes.index);
+
+ this.computeBoundingSphere();
+ this.computeBoundingBox();
+ }
+
+ /**
+ * Fast method to advance the line by one position. The oldest position is removed.
+ * @param position
+ */
+ advance(position) {
+ var positions = this._attributes.position.array;
+ var previous = this._attributes.previous.array;
+ var next = this._attributes.next.array;
+ var l = positions.length;
+
+ // PREVIOUS
+ memcpy(positions, 0, previous, 0, l);
+
+ // POSITIONS
+ memcpy(positions, 6, positions, 0, l - 6);
+
+ positions[l - 6] = position.x;
+ positions[l - 5] = position.y;
+ positions[l - 4] = position.z;
+ positions[l - 3] = position.x;
+ positions[l - 2] = position.y;
+ positions[l - 1] = position.z;
+
+ // NEXT
+ memcpy(positions, 6, next, 0, l - 6);
+
+ next[l - 6] = position.x;
+ next[l - 5] = position.y;
+ next[l - 4] = position.z;
+ next[l - 3] = position.x;
+ next[l - 2] = position.y;
+ next[l - 1] = position.z;
+
+ this._attributes.position.needsUpdate = true;
+ this._attributes.previous.needsUpdate = true;
+ this._attributes.next.needsUpdate = true;
+ }
+
+
+ }
+
+ function MeshLineRaycast(raycaster, intersects) {
+ var inverseMatrix = new THREE.Matrix4();
+ var ray = new THREE.Ray();
+ var sphere = new THREE.Sphere();
+ var interRay = new THREE.Vector3();
+ var geometry = this.geometry;
+ // Checking boundingSphere distance to ray
+
+ sphere.copy(geometry.boundingSphere);
+ sphere.applyMatrix4(this.matrixWorld);
+
+ if (raycaster.ray.intersectSphere(sphere, interRay) === false) {
+ return;
+ }
+
+ inverseMatrix.getInverse(this.matrixWorld);
+ ray.copy(raycaster.ray).applyMatrix4(inverseMatrix);
+
+ var vStart = new THREE.Vector3();
+ var vEnd = new THREE.Vector3();
+ var interSegment = new THREE.Vector3();
+ var step = this instanceof THREE.LineSegments ? 2 : 1;
+ var index = geometry.index;
+ var attributes = geometry.attributes;
+
+ if (index !== null) {
+ var indices = index.array;
+ var positions = attributes.position.array;
+ var widths = attributes.width.array;
+
+ for (var i = 0, l = indices.length - 1; i < l; i += step) {
+ var a = indices[i];
+ var b = indices[i + 1];
+
+ vStart.fromArray(positions, a * 3);
+ vEnd.fromArray(positions, b * 3);
+ var width = widths[Math.floor(i / 3)] != undefined ? widths[Math.floor(i / 3)] : 1;
+ var precision = raycaster.params.Line.threshold + (this.material.lineWidth * width) / 2;
+ var precisionSq = precision * precision;
+
+ var distSq = ray.distanceSqToSegment(vStart, vEnd, interRay, interSegment);
+
+ if (distSq > precisionSq) continue;
+
+ interRay.applyMatrix4(this.matrixWorld); //Move back to world space for distance calculation
+
+ var distance = raycaster.ray.origin.distanceTo(interRay);
+
+ if (distance < raycaster.near || distance > raycaster.far) continue;
+
+ intersects.push({
+ distance: distance,
+ // What do we want? intersection point on the ray or on the segment??
+ // point: raycaster.ray.at( distance ),
+ point: interSegment.clone().applyMatrix4(this.matrixWorld),
+ index: i,
+ face: null,
+ faceIndex: null,
+ object: this,
+ });
+ // make event only fire once
+ i = l;
+ }
+ }
+ }
+
+
+ function memcpy(src, srcOffset, dst, dstOffset, length) {
+ var i;
+
+ src = src.subarray || src.slice ? src : src.buffer;
+ dst = dst.subarray || dst.slice ? dst : dst.buffer;
+
+ src = srcOffset
+ ? src.subarray
+ ? src.subarray(srcOffset, length && srcOffset + length)
+ : src.slice(srcOffset, length && srcOffset + length)
+ : src;
+
+ if (dst.set) {
+ dst.set(src, dstOffset);
+ } else {
+ for (i = 0; i < src.length; i++) {
+ dst[i + dstOffset] = src[i];
+ }
+ }
+
+ return dst;
+ }
+
+ THREE.ShaderChunk['meshline_vert'] = [
+ '',
+ THREE.ShaderChunk.logdepthbuf_pars_vertex,
+ THREE.ShaderChunk.fog_pars_vertex,
+ '',
+ 'attribute vec3 previous;',
+ 'attribute vec3 next;',
+ 'attribute float side;',
+ 'attribute float width;',
+ 'attribute float counters;',
+ '',
+ 'uniform vec2 resolution;',
+ 'uniform float lineWidth;',
+ 'uniform vec3 color;',
+ 'uniform float opacity;',
+ 'uniform float sizeAttenuation;',
+ '',
+ 'varying vec2 vUV;',
+ 'varying vec4 vColor;',
+ 'varying float vCounters;',
+ '',
+ 'vec2 fix( vec4 i, float aspect ) {',
+ '',
+ ' vec2 res = i.xy / i.w;',
+ ' res.x *= aspect;',
+ ' vCounters = counters;',
+ ' return res;',
+ '',
+ '}',
+ '',
+ 'void main() {',
+ '',
+ ' float aspect = resolution.x / resolution.y;',
+ '',
+ ' vColor = vec4( color, opacity );',
+ ' vUV = uv;',
+ '',
+ ' mat4 m = projectionMatrix * modelViewMatrix;',
+ ' vec4 finalPosition = m * vec4( position, 1.0 );',
+ ' vec4 prevPos = m * vec4( previous, 1.0 );',
+ ' vec4 nextPos = m * vec4( next, 1.0 );',
+ '',
+ ' vec2 currentP = fix( finalPosition, aspect );',
+ ' vec2 prevP = fix( prevPos, aspect );',
+ ' vec2 nextP = fix( nextPos, aspect );',
+ '',
+ ' float w = lineWidth * width;',
+ '',
+ ' vec2 dir;',
+ ' if( nextP == currentP ) dir = normalize( currentP - prevP );',
+ ' else if( prevP == currentP ) dir = normalize( nextP - currentP );',
+ ' else {',
+ ' vec2 dir1 = normalize( currentP - prevP );',
+ ' vec2 dir2 = normalize( nextP - currentP );',
+ ' dir = normalize( dir1 + dir2 );',
+ '',
+ ' vec2 perp = vec2( -dir1.y, dir1.x );',
+ ' vec2 miter = vec2( -dir.y, dir.x );',
+ ' //w = clamp( w / dot( miter, perp ), 0., 4. * lineWidth * width );',
+ '',
+ ' }',
+ '',
+ ' //vec2 normal = ( cross( vec3( dir, 0. ), vec3( 0., 0., 1. ) ) ).xy;',
+ ' vec4 normal = vec4( -dir.y, dir.x, 0., 1. );',
+ ' normal.xy *= .5 * w;',
+ ' normal *= projectionMatrix;',
+ ' if( sizeAttenuation == 0. ) {',
+ ' normal.xy *= finalPosition.w;',
+ ' normal.xy /= ( vec4( resolution, 0., 1. ) * projectionMatrix ).xy;',
+ ' }',
+ '',
+ ' finalPosition.xy += normal.xy * side;',
+ '',
+ ' gl_Position = finalPosition;',
+ '',
+ THREE.ShaderChunk.logdepthbuf_vertex,
+ THREE.ShaderChunk.fog_vertex && ' vec4 mvPosition = modelViewMatrix * vec4( position, 1.0 );',
+ THREE.ShaderChunk.fog_vertex,
+ '}',
+ ].join('\n');
+
+ THREE.ShaderChunk['meshline_frag'] = [
+ '',
+ THREE.ShaderChunk.fog_pars_fragment,
+ THREE.ShaderChunk.logdepthbuf_pars_fragment,
+ '',
+ 'uniform sampler2D map;',
+ 'uniform sampler2D alphaMap;',
+ 'uniform float useMap;',
+ 'uniform float useAlphaMap;',
+ 'uniform float useDash;',
+ 'uniform float dashArray;',
+ 'uniform float dashOffset;',
+ 'uniform float dashRatio;',
+ 'uniform float visibility;',
+ 'uniform float alphaTest;',
+ 'uniform vec2 repeat;',
+ '',
+ 'varying vec2 vUV;',
+ 'varying vec4 vColor;',
+ 'varying float vCounters;',
+ '',
+ 'void main() {',
+ '',
+ THREE.ShaderChunk.logdepthbuf_fragment,
+ '',
+ ' vec4 c = vColor;',
+ ' if( useMap == 1. ) c *= texture2D( map, vUV * repeat );',
+ ' if( useAlphaMap == 1. ) c.a *= texture2D( alphaMap, vUV * repeat ).a;',
+ ' if( c.a < alphaTest ) discard;',
+ ' if( useDash == 1. ){',
+ ' c.a *= ceil(mod(vCounters + dashOffset, dashArray) - (dashArray * dashRatio));',
+ ' }',
+ ' gl_FragColor = c;',
+ ' gl_FragColor.a *= step(vCounters, visibility);',
+ '',
+ THREE.ShaderChunk.fog_fragment,
+ '}',
+ ].join('\n');
+
+ class MeshLineMaterial extends THREE.ShaderMaterial {
+ constructor(parameters) {
+ super({
+ uniforms: Object.assign({}, THREE.UniformsLib.fog, {
+ lineWidth: { value: 1 },
+ map: { value: null },
+ useMap: { value: 0 },
+ alphaMap: { value: null },
+ useAlphaMap: { value: 0 },
+ color: { value: new THREE.Color(0xffffff) },
+ opacity: { value: 1 },
+ resolution: { value: new THREE.Vector2(1, 1) },
+ sizeAttenuation: { value: 1 },
+ dashArray: { value: 0 },
+ dashOffset: { value: 0 },
+ dashRatio: { value: 0.5 },
+ useDash: { value: 0 },
+ visibility: { value: 1 },
+ alphaTest: { value: 0 },
+ repeat: { value: new THREE.Vector2(1, 1) },
+ }),
+
+ vertexShader: THREE.ShaderChunk.meshline_vert,
+
+ fragmentShader: THREE.ShaderChunk.meshline_frag,
+ });
+
+ this.type = 'MeshLineMaterial';
+ this.isMeshLineMaterial = true;
+
+ Object.defineProperties(this, {
+ lineWidth: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.lineWidth.value;
+ },
+ set: function(value) {
+ this.uniforms.lineWidth.value = value;
+ },
+ },
+ map: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.map.value;
+ },
+ set: function(value) {
+ this.uniforms.map.value = value;
+ },
+ },
+ useMap: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.useMap.value;
+ },
+ set: function(value) {
+ this.uniforms.useMap.value = value;
+ },
+ },
+ alphaMap: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.alphaMap.value;
+ },
+ set: function(value) {
+ this.uniforms.alphaMap.value = value;
+ },
+ },
+ useAlphaMap: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.useAlphaMap.value;
+ },
+ set: function(value) {
+ this.uniforms.useAlphaMap.value = value;
+ },
+ },
+ color: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.color.value;
+ },
+ set: function(value) {
+ this.uniforms.color.value = value;
+ },
+ },
+ opacity: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.opacity.value;
+ },
+ set: function(value) {
+ this.uniforms.opacity.value = value;
+ },
+ },
+ resolution: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.resolution.value;
+ },
+ set: function(value) {
+ this.uniforms.resolution.value.copy(value);
+ },
+ },
+ sizeAttenuation: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.sizeAttenuation.value;
+ },
+ set: function(value) {
+ this.uniforms.sizeAttenuation.value = value;
+ },
+ },
+ dashArray: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.dashArray.value;
+ },
+ set: function(value) {
+ this.uniforms.dashArray.value = value;
+ this.useDash = value !== 0 ? 1 : 0;
+ },
+ },
+ dashOffset: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.dashOffset.value;
+ },
+ set: function(value) {
+ this.uniforms.dashOffset.value = value;
+ },
+ },
+ dashRatio: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.dashRatio.value;
+ },
+ set: function(value) {
+ this.uniforms.dashRatio.value = value;
+ },
+ },
+ useDash: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.useDash.value;
+ },
+ set: function(value) {
+ this.uniforms.useDash.value = value;
+ },
+ },
+ visibility: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.visibility.value;
+ },
+ set: function(value) {
+ this.uniforms.visibility.value = value;
+ },
+ },
+ alphaTest: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.alphaTest.value;
+ },
+ set: function(value) {
+ this.uniforms.alphaTest.value = value;
+ },
+ },
+ repeat: {
+ enumerable: true,
+ get: function() {
+ return this.uniforms.repeat.value;
+ },
+ set: function(value) {
+ this.uniforms.repeat.value.copy(value);
+ },
+ },
+ });
+
+ this.setValues(parameters);
+ }
+
+ copy(source) {
+ super.copy(source);
+
+ this.lineWidth = source.lineWidth;
+ this.map = source.map;
+ this.useMap = source.useMap;
+ this.alphaMap = source.alphaMap;
+ this.useAlphaMap = source.useAlphaMap;
+ this.color.copy(source.color);
+ this.opacity = source.opacity;
+ this.resolution.copy(source.resolution);
+ this.sizeAttenuation = source.sizeAttenuation;
+ this.dashArray.copy(source.dashArray);
+ this.dashOffset.copy(source.dashOffset);
+ this.dashRatio.copy(source.dashRatio);
+ this.useDash = source.useDash;
+ this.visibility = source.visibility;
+ this.alphaTest = source.alphaTest;
+ this.repeat.copy(source.repeat);
+
+ return this;
+ }
+ }
+
+ realityEditor.device.meshLine.MeshLine = MeshLine;
+ realityEditor.device.meshLine.MeshLineMaterial = MeshLineMaterial;
+ realityEditor.device.meshLine.MeshLineRaycast = MeshLineRaycast;
+};
diff --git a/content_scripts/multiclientUI.js b/content_scripts/multiclientUI.js
new file mode 100644
index 00000000..6871b13c
--- /dev/null
+++ b/content_scripts/multiclientUI.js
@@ -0,0 +1,316 @@
+/*
+* Copyright © 2018 PTC
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+createNameSpace('realityEditor.device.multiclientUI');
+
+import * as THREE from '../../thirdPartyCode/three/three.module.js';
+
+(function(_exports) {
+ let allConnectedCameras = {};
+ let isCameraSubscriptionActiveForObject = {};
+
+ const USE_ICOSAHEDRON = false;
+ let showViewCones = false;
+
+ const wireVertex = `
+ attribute vec3 center;
+ varying vec3 vCenter;
+ void main() {
+ vCenter = center;
+ gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
+ }
+ `;
+
+ const wireFragment = `
+ uniform float thickness;
+ uniform vec3 color;
+ varying vec3 vCenter;
+
+ void main() {
+ vec3 afwidth = fwidth(vCenter.xyz);
+ vec3 edge3 = smoothstep((thickness - 1.0) * afwidth, thickness * afwidth, vCenter.xyz);
+ float edge = 1.0 - min(min(edge3.x, edge3.y), edge3.z);
+ gl_FragColor.rgb = gl_FrontFacing ? color : (color * 0.5);
+ gl_FragColor.a = edge;
+ }
+ `;
+
+ const wireMat = new THREE.ShaderMaterial({
+ uniforms: {
+ thickness: {
+ value: 5.0,
+ },
+ color: {
+ value: new THREE.Color(0.9, 0.9, 1.0),
+ },
+ },
+ vertexShader: wireVertex,
+ fragmentShader: wireFragment,
+ side: THREE.DoubleSide,
+ alphaToCoverage: true,
+ });
+ wireMat.extensions.derivatives = true;
+ window.wireMat = wireMat;
+
+ function initService() {
+ if (!realityEditor.device.desktopAdapter || !realityEditor.device.KeyboardListener || !realityEditor.gui.getMenuBar) {
+ setTimeout(initService, 100);
+ return;
+ }
+
+ if (realityEditor.device.environment.isARMode()) { return; }
+
+ realityEditor.network.addObjectDiscoveredCallback(function(object, objectKey) {
+ setTimeout(function() {
+ setupWorldSocketSubscriptionsIfNeeded(objectKey);
+ }, 100); // give time for bestWorldObject to update before checking
+ });
+
+ update();
+
+ let keyboard = new realityEditor.device.KeyboardListener();
+ keyboard.onKeyDown(function(code) {
+ if (realityEditor.device.keyboardEvents.isKeyboardActive()) { return; } // ignore if a tool is using the keyboard
+
+ // while shift is down, turn on the laser beam
+ if (code === keyboard.keyCodes.SHIFT) {
+ let touchPosition = realityEditor.gui.ar.positioning.getMostRecentTouchPosition();
+ realityEditor.avatar.setBeamOn(touchPosition.x, touchPosition.y);
+ }
+ });
+ keyboard.onKeyUp(function(code) {
+ if (realityEditor.device.keyboardEvents.isKeyboardActive()) { return; } // ignore if a tool is using the keyboard
+
+ // when shift is released, turn off the laser beam
+ if (code === keyboard.keyCodes.SHIFT) {
+ let touchPosition = realityEditor.gui.ar.positioning.getMostRecentTouchPosition();
+ realityEditor.avatar.setBeamOff(touchPosition.x, touchPosition.y);
+ }
+ });
+
+ realityEditor.gui.getMenuBar().addCallbackToItem(realityEditor.gui.ITEM.ViewCones, (toggled) => {
+ showViewCones = toggled;
+
+ Object.keys(allConnectedCameras).forEach(function(editorId) {
+ // let cameraMatrix = allConnectedCameras[editorId];
+ let coneMesh = realityEditor.gui.threejsScene.getObjectByName('camera_' + editorId + '_coneMesh');
+ let coneMesh2 = realityEditor.gui.threejsScene.getObjectByName('camera_' + editorId + '_coneMesh2');
+ if (coneMesh && coneMesh2) {
+ coneMesh.visible = showViewCones;
+ coneMesh2.visible = showViewCones;
+ }
+ });
+ });
+ }
+
+ function setupWorldSocketSubscriptionsIfNeeded(objectKey) {
+ if (isCameraSubscriptionActiveForObject[objectKey]) {
+ return;
+ }
+
+ // subscribe to remote operator camera positions
+ // right now this assumes there will only be one world object in the network
+ let object = realityEditor.getObject(objectKey);
+ if (object && (object.isWorldObject || object.type === 'world')) {
+ realityEditor.network.realtime.subscribeToCameraMatrices(objectKey, onCameraMatrix);
+ isCameraSubscriptionActiveForObject[objectKey] = true;
+ }
+ }
+
+ function onCameraMatrix(data) {
+ let msgData = JSON.parse(data);
+ if (typeof msgData.cameraMatrix !== 'undefined' && typeof msgData.editorId !== 'undefined') {
+ allConnectedCameras[msgData.editorId] = msgData.cameraMatrix;
+ }
+ }
+
+ // helper function to generate an integer hash from a string (https://stackoverflow.com/a/15710692)
+ function hashCode(s) {
+ return s.split("").reduce(function(a, b) {
+ a = ((a << 5) - a) + b.charCodeAt(0);
+ return a & a;
+ }, 0);
+ }
+
+ function update() {
+ // this remote operator's camera position already gets sent in desktopCamera.js
+ // here we render boxes at the location of each other camera...
+
+ try {
+ Object.keys(allConnectedCameras).forEach(function(editorId) {
+ let cameraMatrix = allConnectedCameras[editorId];
+ let existingMesh = realityEditor.gui.threejsScene.getObjectByName('camera_' + editorId);
+ if (!existingMesh) {
+ // each client gets a random but consistent color based on their editorId
+ let id = Math.abs(hashCode(editorId));
+ const color = `hsl(${(id % Math.PI) * 360 / Math.PI}, 100%, 50%)`;
+
+ // render either a simple box, or a more complicated icosahedron, located at the remote camera position
+ let mesh;
+ if (USE_ICOSAHEDRON) {
+ const geo = new THREE.IcosahedronBufferGeometry(100);
+ geo.deleteAttribute('normal');
+ geo.deleteAttribute('uv');
+
+ const vectors = [
+ new THREE.Vector3(1, 0, 0),
+ new THREE.Vector3(0, 1, 0),
+ new THREE.Vector3(0, 0, 1)
+ ];
+
+ const position = geo.attributes.position;
+ const centers = new Float32Array(position.count * 3);
+
+ for (let i = 0, l = position.count; i < l; i ++) {
+ vectors[i % 3].toArray(centers, i * 3);
+ }
+
+ geo.setAttribute('center', new THREE.BufferAttribute(centers, 3));
+
+ const mat = wireMat.clone();
+ mat.uniforms.color.value = new THREE.Color(color);
+ mesh = new THREE.Mesh(geo, mat);
+ } else {
+ let cubeSize = 50;
+ const geo = new THREE.BoxGeometry(cubeSize, cubeSize, cubeSize);
+ const mat = new THREE.MeshBasicMaterial({color: color});
+ mesh = new THREE.Mesh(geo, mat);
+ }
+
+ const fov = 0.1 * Math.PI;
+ const points = [
+ // new THREE.Vector2(100 * Math.sin(fov), 100 * Math.cos(fov)),
+ new THREE.Vector2(0, 0),
+ new THREE.Vector2(15 * 1000 * Math.sin(fov), 15 * 1000 * Math.cos(fov)),
+ ];
+ const coneGeo = new THREE.LatheGeometry(points, 4);
+ const coneMesh = new THREE.Mesh(
+ coneGeo,
+ new THREE.MeshBasicMaterial({
+ color: new THREE.Color(color),
+ transparent: true,
+ depthWrite: false,
+ opacity: 0.05,
+ })
+ );
+ coneMesh.name = 'camera_' + editorId + '_coneMesh';
+ coneMesh.rotation.x = -Math.PI / 2;
+ coneMesh.rotation.y = Math.PI / 4;
+ coneMesh.position.z = 0; // 7.5 * 1000;
+
+ const coneMesh2 = new THREE.Mesh(
+ coneGeo,
+ new THREE.MeshBasicMaterial({
+ color: new THREE.Color(color),
+ wireframe: true,
+ })
+ );
+ coneMesh2.name = 'camera_' + editorId + '_coneMesh2';
+ coneMesh2.rotation.x = -Math.PI / 2;
+ coneMesh2.rotation.y = Math.PI / 4;
+ coneMesh2.position.z = 0; // 7.5 * 1000;
+
+ if (!showViewCones) {
+ coneMesh.visible = false;
+ coneMesh2.visible = false;
+ }
+
+ existingMesh = new THREE.Group();
+ existingMesh.add(coneMesh);
+ existingMesh.add(coneMesh2);
+ existingMesh.add(mesh);
+
+ existingMesh.name = 'camera_' + editorId;
+ existingMesh.matrixAutoUpdate = false;
+ realityEditor.gui.threejsScene.addToScene(existingMesh);
+ }
+
+ const ANIMATE = false;
+ if (ANIMATE) {
+ // let animatedMatrix = realityEditor.gui.ar.utilities.tweenMatrix(existingMesh.matrix.elements, cameraMatrix, 0.05);
+ let animatedMatrix = realityEditor.gui.ar.utilities.animationVectorLinear(existingMesh.matrix.elements, cameraMatrix, 30);
+ realityEditor.gui.threejsScene.setMatrixFromArray(existingMesh.matrix, animatedMatrix);
+ } else {
+ realityEditor.gui.threejsScene.setMatrixFromArray(existingMesh.matrix, cameraMatrix);
+ }
+
+ updateCubeVisibility(existingMesh, editorId);
+ });
+ } catch (e) {
+ console.warn(e);
+ }
+
+ requestAnimationFrame(update);
+ }
+
+ /**
+ * Hide cubes part of my following group (who I'm locked onto, and others locked onto that same target)
+ * Show all other cubes
+ * @param {THREE.Object3D} cubeMesh
+ * @param {string} editorId
+ */
+ function updateCubeVisibility(cubeMesh, editorId) {
+ // get the avatarId that corresponds with this cubeMesh
+ let cubeAvatarId = Object.keys(objects).find(objectId => {
+ return objectId.includes(editorId); // This relies on the fact that the avatar names include the temp_uuid
+ });
+
+ let isPartOfMyGroup = isAvatarPartOfMyFollowingGroup(cubeAvatarId);
+ if (cubeMesh.visible) {
+ // hide if part of my group
+ if (isPartOfMyGroup) {
+ cubeMesh.visible = false;
+ }
+ } else {
+ // show if not part of my group
+ if (!isPartOfMyGroup) {
+ cubeMesh.visible = true;
+ }
+ }
+ }
+
+ /**
+ * Returns true for:
+ * 1. Any avatars following me
+ * 2. The avatar that I am following
+ * 3. Any avatars following the same avatar as I am
+ */
+ function isAvatarPartOfMyFollowingGroup(cubeAvatarId) {
+ let myAvatarId = realityEditor.avatar.getMyAvatarId();
+ if (!myAvatarId || !cubeAvatarId) return false;
+ if (myAvatarId && cubeAvatarId === myAvatarId) return true; // I am always part of my group
+ let cubeAvatarObject = realityEditor.getObject(cubeAvatarId);
+ if (!cubeAvatarObject) return false; // defaults to false if anything is wrong
+ let myAvatarObject = realityEditor.getObject(myAvatarId);
+ if (!myAvatarId) return false;
+
+ let cubeInfo = realityEditor.avatar.utils.getAvatarNodeInfo(cubeAvatarObject);
+ let cubeNode = realityEditor.getNode(cubeInfo.objectKey, cubeInfo.frameKey, cubeInfo.nodeKey);
+ let cubeUserProfile = cubeNode.publicData.userProfile;
+
+ // 1. any avatars following me
+ if (cubeUserProfile && cubeUserProfile.lockOnMode && cubeUserProfile.lockOnMode === myAvatarId) return true;
+
+ let myInfo = realityEditor.avatar.utils.getAvatarNodeInfo(myAvatarObject);
+ let myNode = realityEditor.getNode(myInfo.objectKey, myInfo.frameKey, myInfo.nodeKey);
+ let myUserProfile = myNode.publicData.userProfile;
+
+ // 2. the avatar that I am following
+ if (myUserProfile && myUserProfile.lockOnMode && myUserProfile.lockOnMode === cubeAvatarId) return true;
+
+ // 3. avatars following the same avatar that I am following
+ let avatarMyAvatarIsFollowing = realityEditor.getObject(myUserProfile.lockOnMode);
+ if (avatarMyAvatarIsFollowing) {
+ if (cubeUserProfile && cubeUserProfile.lockOnMode && cubeUserProfile.lockOnMode === avatarMyAvatarIsFollowing.objectId) return true;
+ }
+
+ return false;
+ }
+
+ realityEditor.addons.addCallback('init', initService);
+})(realityEditor.device.multiclientUI);
diff --git a/content_scripts/setupMenuBar.js b/content_scripts/setupMenuBar.js
new file mode 100644
index 00000000..c29edf61
--- /dev/null
+++ b/content_scripts/setupMenuBar.js
@@ -0,0 +1,231 @@
+createNameSpace('realityEditor.gui');
+
+import Splatting from '../../src/splatting/Splatting.js';
+
+(function(exports) {
+ let menuBar = null;
+
+ const MENU = Object.freeze({
+ View: 'View',
+ Camera: 'Camera',
+ Follow: 'Follow',
+ History: 'History',
+ Help: 'Help',
+ Develop: 'Develop'
+ });
+ exports.MENU = MENU;
+
+ const ITEM = Object.freeze({
+ PointClouds: '3D Videos',
+ SpaghettiMap: 'Spaghetti Map',
+ ModelVisibility: 'Model Visibility',
+ ModelTexture: 'Model Texture',
+ SurfaceAnchors: 'Surface Anchors',
+ VideoPlayback: 'Video Timeline',
+ Voxelizer: 'Model Voxelizer',
+ Follow1stPerson: 'Follow 1st-Person',
+ Follow3rdPerson: 'Follow 3rd-Person',
+ StopFollowing: 'Stop Following',
+ TakeSpatialSnapshot: 'Take Spatial Snapshot',
+ OrbitCamera: 'Orbit Camera',
+ ResetCameraPosition: 'Reset Camera Position',
+ GettingStarted: 'Getting Started',
+ ShowDeveloperMenu: 'Show Developer Menu',
+ DebugAvatarConnections: 'Debug Avatar Connections',
+ DeleteAllTools: 'Delete All Tools',
+ DownloadScan: 'Download Scan',
+ ViewCones: 'Show View Cones',
+ AdvanceCameraShader: 'Next Camera Lens',
+ ToggleMotionStudySettings: 'Toggle Analytics Settings',
+ DarkMode: 'Dark Mode',
+ CutoutViewFrustums: 'Cut Out 3D Videos',
+ ShowFPS: 'Show FPS',
+ ActivateProfiler: 'Activate Profiler',
+ ToggleFlyMode: 'Fly Mode',
+ FocusCamera: 'Focus Camera',
+ ShowAIChatbot: 'AI Assist',
+ ReloadPage: 'Reload Page',
+ GSSettingsPanel: 'GS Settings Panel'
+ });
+ exports.ITEM = ITEM;
+
+ // sets up the initial contents of the menuBar
+ // other modules can add more to it by calling getMenuBar().addItemToMenu(menuName, menuItem)
+ const setupMenuBar = () => {
+ if (menuBar) { return; }
+
+ const MenuBar = realityEditor.gui.MenuBar;
+ const Menu = realityEditor.gui.Menu;
+ const MenuItem = realityEditor.gui.MenuItem;
+
+ menuBar = new MenuBar();
+ // menuBar.addMenu(new Menu('File'));
+ // menuBar.addMenu(new Menu('Edit'));
+ menuBar.addMenu(new Menu(MENU.View));
+ menuBar.addMenu(new Menu(MENU.Camera));
+ let followMenu = new Menu(MENU.Follow); // keep a reference, so we can show/hide it on demand
+ exports.followMenu = followMenu;
+ menuBar.addMenu(followMenu);
+ menuBar.disableMenu(followMenu);
+ menuBar.addMenu(new Menu(MENU.History));
+ let developMenu = new Menu(MENU.Develop); // keep a reference, so we can show/hide it on demand
+ menuBar.addMenu(developMenu);
+ menuBar.hideMenu(developMenu);
+ menuBar.addMenu(new Menu(MENU.Help));
+
+ const togglePointClouds = new MenuItem(ITEM.PointClouds, { shortcutKey: 'M', toggle: true, defaultVal: true, disabled: true }, (value) => {
+ console.log('toggle point clouds', value);
+ });
+ menuBar.addItemToMenu(MENU.View, togglePointClouds);
+
+ const toggleSpaghetti = new MenuItem(ITEM.SpaghettiMap, { shortcutKey: 'N', toggle: true, defaultVal: false, disabled: true }, null);
+ menuBar.addItemToMenu(MENU.View, toggleSpaghetti);
+
+ const toggleModelVisibility = new MenuItem(ITEM.ModelVisibility, { shortcutKey: 'T', toggle: true, defaultVal: true }, null); // other module can attach a callback later
+ menuBar.addItemToMenu(MENU.View, toggleModelVisibility);
+
+ const toggleModelTexture = new MenuItem(ITEM.ModelTexture, { shortcutKey: 'Y', toggle: true, defaultVal: true }, null);
+ menuBar.addItemToMenu(MENU.View, toggleModelTexture);
+
+ const toggleViewCones = new MenuItem(ITEM.ViewCones, { shortcutKey: 'K', toggle: true, defaultVal: false }, null);
+ menuBar.addItemToMenu(MENU.View, toggleViewCones);
+
+ const toggleCutoutViewFrustums = new MenuItem(ITEM.CutoutViewFrustums, { toggle: true, defaultVal: false }, null);
+ menuBar.addItemToMenu(MENU.View, toggleCutoutViewFrustums);
+
+ // Note: these features still exist in the codebase, but have been removed from the menu for now
+ // const toggleSurfaceAnchors = new MenuItem(ITEM.SurfaceAnchors, { shortcutKey: 'SEMICOLON', toggle: true, defaultVal: false }, null); // other module can attach a callback later
+ // menuBar.addItemToMenu(MENU.View, toggleSurfaceAnchors);
+ // const toggleVideoPlayback = new MenuItem(ITEM.VideoPlayback, { shortcutKey: 'OPEN_BRACKET', toggle: true, defaultVal: false }, null); // other module can attach a callback later
+ // menuBar.addItemToMenu(MENU.View, toggleVideoPlayback);
+
+ const toggleDarkMode = new MenuItem(ITEM.DarkMode, { toggle: true, defaultVal: true }, null);
+ menuBar.addItemToMenu(MENU.View, toggleDarkMode);
+
+ const toggleFlyMode = new MenuItem(ITEM.ToggleFlyMode, { toggle: true, shortcutKey: 'F', defaultVal: false }, null);
+ menuBar.addItemToMenu(MENU.Camera, toggleFlyMode);
+
+ const focusCamera = new MenuItem(ITEM.FocusCamera, { shortcutKey: 'G' }, null);
+ menuBar.addItemToMenu(MENU.Camera, focusCamera);
+
+ const rzvAdvanceCameraShader = new MenuItem(ITEM.AdvanceCameraShader, { disabled: true }, null);
+ menuBar.addItemToMenu(MENU.Camera, rzvAdvanceCameraShader);
+
+ const toggleMotionStudySettings = new MenuItem(ITEM.ToggleMotionStudySettings, { toggle: true, defaultVal: false }, null);
+ menuBar.addItemToMenu(MENU.History, toggleMotionStudySettings);
+
+ const takeSpatialSnapshot = new MenuItem(ITEM.TakeSpatialSnapshot, { shortcutKey: 'P', disabled: true }, null);
+ menuBar.addItemToMenu(MENU.History, takeSpatialSnapshot);
+
+ const toggleVoxelizer = new MenuItem(ITEM.Voxelizer, { shortcutKey: '', toggle: true, defaultVal: false }, null); // other module can attach a callback later
+ menuBar.addItemToMenu(MENU.History, toggleVoxelizer);
+
+ const stopFollowing = new MenuItem(ITEM.StopFollowing, { shortcutKey: '_0', toggle: false, disabled: true }, null);
+ exports.stopFollowingItem = stopFollowing;
+ menuBar.addItemToMenu(MENU.Follow, stopFollowing);
+
+ const orbitCamera = new MenuItem(ITEM.OrbitCamera, { shortcutKey: 'O', toggle: true, defaultVal: false }, null);
+ menuBar.addItemToMenu(MENU.Camera, orbitCamera);
+
+ const resetCamera = new MenuItem(ITEM.ResetCameraPosition, { shortcutKey: 'ESCAPE' }, null);
+ menuBar.addItemToMenu(MENU.Camera, resetCamera);
+
+ // TODO: build a better Getting Started / Help experience
+ // const gettingStarted = new MenuItem(ITEM.GettingStarted, null, () => {
+ // window.open('https://spatialtoolbox.vuforia.com/', '_blank');
+ // });
+ // menuBar.addItemToMenu(MENU.Help, gettingStarted);
+
+ // useful in Teams or other iframe-embedded versions of the app, where you are otherwise unable to refresh the page
+ const reloadPage = new MenuItem(ITEM.ReloadPage, null, () => {
+ // reload and bypass the cache (https://stackoverflow.com/questions/2099201/javascript-hard-refresh-of-current-page)
+ window.location.reload(true);
+ });
+ menuBar.addItemToMenu(MENU.Help, reloadPage);
+
+ const activateProfiler = new MenuItem(ITEM.ActivateProfiler, { toggle: true, defaultVal: false }, (checked) => {
+ if (checked) {
+ if (realityEditor.device.profiling) realityEditor.device.profiling.show();
+ } else {
+ if (realityEditor.device.profiling) realityEditor.device.profiling.hide();
+ }
+ });
+ menuBar.addItemToMenu(MENU.Develop, activateProfiler);
+
+ const debugAvatars = new MenuItem(ITEM.DebugAvatarConnections, { toggle: true }, (checked) => {
+ realityEditor.avatar.toggleDebugMode(checked);
+ });
+ menuBar.addItemToMenu(MENU.Develop, debugAvatars);
+
+ const showFPS = new MenuItem(ITEM.ShowFPS, { toggle: true }, (checked) => {
+ if (checked) {
+ realityEditor.device.desktopStats.show();
+ } else {
+ realityEditor.device.desktopStats.hide();
+ }
+ });
+ menuBar.addItemToMenu(MENU.Develop, showFPS);
+
+ const deleteAllTools = new MenuItem(ITEM.DeleteAllTools, { toggle: false }, () => {
+ realityEditor.forEachFrameInAllObjects((objectKey, frameKey) => {
+ let object = realityEditor.getObject(objectKey);
+ if (!object) return;
+ // only delete for regular objects, world objects, and anchor objects - don't delete avatar or human pose frames
+ if (object.type !== 'object' && object.type !== 'world' && object.type !== 'anchor') return;
+ let frameToDelete = realityEditor.getFrame(objectKey, frameKey);
+ realityEditor.device.tryToDeleteSelectedVehicle(frameToDelete);
+ });
+ });
+ menuBar.addItemToMenu(MENU.Develop, deleteAllTools);
+
+ const downloadScan = new MenuItem(ITEM.DownloadScan, { disabled: true });
+ menuBar.addItemToMenu(MENU.Develop, downloadScan);
+
+ const showDeveloper = new MenuItem(ITEM.ShowDeveloperMenu, { toggle: true }, (checked) => {
+ if (checked) {
+ menuBar.unhideMenu(developMenu);
+ } else {
+ menuBar.hideMenu(developMenu);
+ }
+ });
+ menuBar.addItemToMenu(MENU.Help, showDeveloper);
+
+ const showAIChat = new MenuItem(ITEM.ShowAIChatbot, { toggle: true, shortcutKey: 'BACK_SLASH' }, (checked) => {
+ if (checked) {
+ realityEditor.ai.showDialogue();
+ } else {
+ realityEditor.ai.hideDialogue();
+ }
+ });
+ menuBar.addItemToMenu(MENU.Help, showAIChat);
+
+ const gsSettingsPanel = new MenuItem(ITEM.GSSettingsPanel, { toggle: true, defaultVal: false }, (checked) => {
+ if (checked) {
+ Splatting.showGSSettingsPanel()
+ } else {
+ Splatting.hideGSSettingsPanel();
+ }
+ })
+ menuBar.addItemToMenu(MENU.Develop, gsSettingsPanel);
+
+ document.body.appendChild(menuBar.domElement);
+
+ // Offset certain UI elements that align to the top of the screen, such as the envelope X button
+ realityEditor.device.environment.variables.screenTopOffset = menuBar.domElement.getBoundingClientRect().height;
+ };
+
+ const getMenuBar = () => { // use this to access the shared MenuBar instance
+ if (!menuBar) {
+ try {
+ setupMenuBar();
+ } catch (e) {
+ console.warn(e);
+ }
+ }
+ return menuBar;
+ };
+
+ exports.setupMenuBar = setupMenuBar;
+ exports.getMenuBar = getMenuBar;
+
+})(realityEditor.gui);
diff --git a/content_scripts/webrtc-adapter.js b/content_scripts/webrtc-adapter.js
new file mode 100644
index 00000000..9571ac3d
--- /dev/null
+++ b/content_scripts/webrtc-adapter.js
@@ -0,0 +1,3250 @@
+(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.adapter = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i 0 && arguments[0] !== undefined ? arguments[0] : {},
+ window = _ref.window;
+ var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {
+ shimChrome: true,
+ shimFirefox: true,
+ shimSafari: true
+ };
+ // Utils.
+ var logging = utils.log;
+ var browserDetails = utils.detectBrowser(window);
+ var adapter = {
+ browserDetails: browserDetails,
+ commonShim: commonShim,
+ extractVersion: utils.extractVersion,
+ disableLog: utils.disableLog,
+ disableWarnings: utils.disableWarnings,
+ // Expose sdp as a convenience. For production apps include directly.
+ sdp: sdp
+ };
+
+ // Shim browser if found.
+ switch (browserDetails.browser) {
+ case 'chrome':
+ if (!chromeShim || !chromeShim.shimPeerConnection || !options.shimChrome) {
+ logging('Chrome shim is not included in this adapter release.');
+ return adapter;
+ }
+ if (browserDetails.version === null) {
+ logging('Chrome shim can not determine version, not shimming.');
+ return adapter;
+ }
+ logging('adapter.js shimming chrome.');
+ // Export to the adapter global object visible in the browser.
+ adapter.browserShim = chromeShim;
+
+ // Must be called before shimPeerConnection.
+ commonShim.shimAddIceCandidateNullOrEmpty(window, browserDetails);
+ commonShim.shimParameterlessSetLocalDescription(window, browserDetails);
+ chromeShim.shimGetUserMedia(window, browserDetails);
+ chromeShim.shimMediaStream(window, browserDetails);
+ chromeShim.shimPeerConnection(window, browserDetails);
+ chromeShim.shimOnTrack(window, browserDetails);
+ chromeShim.shimAddTrackRemoveTrack(window, browserDetails);
+ chromeShim.shimGetSendersWithDtmf(window, browserDetails);
+ chromeShim.shimSenderReceiverGetStats(window, browserDetails);
+ chromeShim.fixNegotiationNeeded(window, browserDetails);
+ commonShim.shimRTCIceCandidate(window, browserDetails);
+ commonShim.shimRTCIceCandidateRelayProtocol(window, browserDetails);
+ commonShim.shimConnectionState(window, browserDetails);
+ commonShim.shimMaxMessageSize(window, browserDetails);
+ commonShim.shimSendThrowTypeError(window, browserDetails);
+ commonShim.removeExtmapAllowMixed(window, browserDetails);
+ break;
+ case 'firefox':
+ if (!firefoxShim || !firefoxShim.shimPeerConnection || !options.shimFirefox) {
+ logging('Firefox shim is not included in this adapter release.');
+ return adapter;
+ }
+ logging('adapter.js shimming firefox.');
+ // Export to the adapter global object visible in the browser.
+ adapter.browserShim = firefoxShim;
+
+ // Must be called before shimPeerConnection.
+ commonShim.shimAddIceCandidateNullOrEmpty(window, browserDetails);
+ commonShim.shimParameterlessSetLocalDescription(window, browserDetails);
+ firefoxShim.shimGetUserMedia(window, browserDetails);
+ firefoxShim.shimPeerConnection(window, browserDetails);
+ firefoxShim.shimOnTrack(window, browserDetails);
+ firefoxShim.shimRemoveStream(window, browserDetails);
+ firefoxShim.shimSenderGetStats(window, browserDetails);
+ firefoxShim.shimReceiverGetStats(window, browserDetails);
+ firefoxShim.shimRTCDataChannel(window, browserDetails);
+ firefoxShim.shimAddTransceiver(window, browserDetails);
+ firefoxShim.shimGetParameters(window, browserDetails);
+ firefoxShim.shimCreateOffer(window, browserDetails);
+ firefoxShim.shimCreateAnswer(window, browserDetails);
+ commonShim.shimRTCIceCandidate(window, browserDetails);
+ commonShim.shimConnectionState(window, browserDetails);
+ commonShim.shimMaxMessageSize(window, browserDetails);
+ commonShim.shimSendThrowTypeError(window, browserDetails);
+ break;
+ case 'safari':
+ if (!safariShim || !options.shimSafari) {
+ logging('Safari shim is not included in this adapter release.');
+ return adapter;
+ }
+ logging('adapter.js shimming safari.');
+ // Export to the adapter global object visible in the browser.
+ adapter.browserShim = safariShim;
+
+ // Must be called before shimCallbackAPI.
+ commonShim.shimAddIceCandidateNullOrEmpty(window, browserDetails);
+ commonShim.shimParameterlessSetLocalDescription(window, browserDetails);
+ safariShim.shimRTCIceServerUrls(window, browserDetails);
+ safariShim.shimCreateOfferLegacy(window, browserDetails);
+ safariShim.shimCallbacksAPI(window, browserDetails);
+ safariShim.shimLocalStreamsAPI(window, browserDetails);
+ safariShim.shimRemoteStreamsAPI(window, browserDetails);
+ safariShim.shimTrackEventTransceiver(window, browserDetails);
+ safariShim.shimGetUserMedia(window, browserDetails);
+ safariShim.shimAudioContext(window, browserDetails);
+ commonShim.shimRTCIceCandidate(window, browserDetails);
+ commonShim.shimRTCIceCandidateRelayProtocol(window, browserDetails);
+ commonShim.shimMaxMessageSize(window, browserDetails);
+ commonShim.shimSendThrowTypeError(window, browserDetails);
+ commonShim.removeExtmapAllowMixed(window, browserDetails);
+ break;
+ default:
+ logging('Unsupported browser!');
+ break;
+ }
+ return adapter;
+}
+
+},{"./chrome/chrome_shim":3,"./common_shim":5,"./firefox/firefox_shim":6,"./safari/safari_shim":9,"./utils":10,"sdp":11}],3:[function(require,module,exports){
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+/* eslint-env node */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.fixNegotiationNeeded = fixNegotiationNeeded;
+exports.shimAddTrackRemoveTrack = shimAddTrackRemoveTrack;
+exports.shimAddTrackRemoveTrackWithNative = shimAddTrackRemoveTrackWithNative;
+exports.shimGetSendersWithDtmf = shimGetSendersWithDtmf;
+Object.defineProperty(exports, "shimGetUserMedia", {
+ enumerable: true,
+ get: function get() {
+ return _getusermedia.shimGetUserMedia;
+ }
+});
+exports.shimMediaStream = shimMediaStream;
+exports.shimOnTrack = shimOnTrack;
+exports.shimPeerConnection = shimPeerConnection;
+exports.shimSenderReceiverGetStats = shimSenderReceiverGetStats;
+var utils = _interopRequireWildcard(require("../utils.js"));
+var _getusermedia = require("./getusermedia");
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
+function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
+function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
+function shimMediaStream(window) {
+ window.MediaStream = window.MediaStream || window.webkitMediaStream;
+}
+function shimOnTrack(window) {
+ if (_typeof(window) === 'object' && window.RTCPeerConnection && !('ontrack' in window.RTCPeerConnection.prototype)) {
+ Object.defineProperty(window.RTCPeerConnection.prototype, 'ontrack', {
+ get: function get() {
+ return this._ontrack;
+ },
+ set: function set(f) {
+ if (this._ontrack) {
+ this.removeEventListener('track', this._ontrack);
+ }
+ this.addEventListener('track', this._ontrack = f);
+ },
+ enumerable: true,
+ configurable: true
+ });
+ var origSetRemoteDescription = window.RTCPeerConnection.prototype.setRemoteDescription;
+ window.RTCPeerConnection.prototype.setRemoteDescription = function setRemoteDescription() {
+ var _this = this;
+ if (!this._ontrackpoly) {
+ this._ontrackpoly = function (e) {
+ // onaddstream does not fire when a track is added to an existing
+ // stream. But stream.onaddtrack is implemented so we use that.
+ e.stream.addEventListener('addtrack', function (te) {
+ var receiver;
+ if (window.RTCPeerConnection.prototype.getReceivers) {
+ receiver = _this.getReceivers().find(function (r) {
+ return r.track && r.track.id === te.track.id;
+ });
+ } else {
+ receiver = {
+ track: te.track
+ };
+ }
+ var event = new Event('track');
+ event.track = te.track;
+ event.receiver = receiver;
+ event.transceiver = {
+ receiver: receiver
+ };
+ event.streams = [e.stream];
+ _this.dispatchEvent(event);
+ });
+ e.stream.getTracks().forEach(function (track) {
+ var receiver;
+ if (window.RTCPeerConnection.prototype.getReceivers) {
+ receiver = _this.getReceivers().find(function (r) {
+ return r.track && r.track.id === track.id;
+ });
+ } else {
+ receiver = {
+ track: track
+ };
+ }
+ var event = new Event('track');
+ event.track = track;
+ event.receiver = receiver;
+ event.transceiver = {
+ receiver: receiver
+ };
+ event.streams = [e.stream];
+ _this.dispatchEvent(event);
+ });
+ };
+ this.addEventListener('addstream', this._ontrackpoly);
+ }
+ return origSetRemoteDescription.apply(this, arguments);
+ };
+ } else {
+ // even if RTCRtpTransceiver is in window, it is only used and
+ // emitted in unified-plan. Unfortunately this means we need
+ // to unconditionally wrap the event.
+ utils.wrapPeerConnectionEvent(window, 'track', function (e) {
+ if (!e.transceiver) {
+ Object.defineProperty(e, 'transceiver', {
+ value: {
+ receiver: e.receiver
+ }
+ });
+ }
+ return e;
+ });
+ }
+}
+function shimGetSendersWithDtmf(window) {
+ // Overrides addTrack/removeTrack, depends on shimAddTrackRemoveTrack.
+ if (_typeof(window) === 'object' && window.RTCPeerConnection && !('getSenders' in window.RTCPeerConnection.prototype) && 'createDTMFSender' in window.RTCPeerConnection.prototype) {
+ var shimSenderWithDtmf = function shimSenderWithDtmf(pc, track) {
+ return {
+ track: track,
+ get dtmf() {
+ if (this._dtmf === undefined) {
+ if (track.kind === 'audio') {
+ this._dtmf = pc.createDTMFSender(track);
+ } else {
+ this._dtmf = null;
+ }
+ }
+ return this._dtmf;
+ },
+ _pc: pc
+ };
+ };
+
+ // augment addTrack when getSenders is not available.
+ if (!window.RTCPeerConnection.prototype.getSenders) {
+ window.RTCPeerConnection.prototype.getSenders = function getSenders() {
+ this._senders = this._senders || [];
+ return this._senders.slice(); // return a copy of the internal state.
+ };
+ var origAddTrack = window.RTCPeerConnection.prototype.addTrack;
+ window.RTCPeerConnection.prototype.addTrack = function addTrack(track, stream) {
+ var sender = origAddTrack.apply(this, arguments);
+ if (!sender) {
+ sender = shimSenderWithDtmf(this, track);
+ this._senders.push(sender);
+ }
+ return sender;
+ };
+ var origRemoveTrack = window.RTCPeerConnection.prototype.removeTrack;
+ window.RTCPeerConnection.prototype.removeTrack = function removeTrack(sender) {
+ origRemoveTrack.apply(this, arguments);
+ var idx = this._senders.indexOf(sender);
+ if (idx !== -1) {
+ this._senders.splice(idx, 1);
+ }
+ };
+ }
+ var origAddStream = window.RTCPeerConnection.prototype.addStream;
+ window.RTCPeerConnection.prototype.addStream = function addStream(stream) {
+ var _this2 = this;
+ this._senders = this._senders || [];
+ origAddStream.apply(this, [stream]);
+ stream.getTracks().forEach(function (track) {
+ _this2._senders.push(shimSenderWithDtmf(_this2, track));
+ });
+ };
+ var origRemoveStream = window.RTCPeerConnection.prototype.removeStream;
+ window.RTCPeerConnection.prototype.removeStream = function removeStream(stream) {
+ var _this3 = this;
+ this._senders = this._senders || [];
+ origRemoveStream.apply(this, [stream]);
+ stream.getTracks().forEach(function (track) {
+ var sender = _this3._senders.find(function (s) {
+ return s.track === track;
+ });
+ if (sender) {
+ // remove sender
+ _this3._senders.splice(_this3._senders.indexOf(sender), 1);
+ }
+ });
+ };
+ } else if (_typeof(window) === 'object' && window.RTCPeerConnection && 'getSenders' in window.RTCPeerConnection.prototype && 'createDTMFSender' in window.RTCPeerConnection.prototype && window.RTCRtpSender && !('dtmf' in window.RTCRtpSender.prototype)) {
+ var origGetSenders = window.RTCPeerConnection.prototype.getSenders;
+ window.RTCPeerConnection.prototype.getSenders = function getSenders() {
+ var _this4 = this;
+ var senders = origGetSenders.apply(this, []);
+ senders.forEach(function (sender) {
+ return sender._pc = _this4;
+ });
+ return senders;
+ };
+ Object.defineProperty(window.RTCRtpSender.prototype, 'dtmf', {
+ get: function get() {
+ if (this._dtmf === undefined) {
+ if (this.track.kind === 'audio') {
+ this._dtmf = this._pc.createDTMFSender(this.track);
+ } else {
+ this._dtmf = null;
+ }
+ }
+ return this._dtmf;
+ }
+ });
+ }
+}
+function shimSenderReceiverGetStats(window) {
+ if (!(_typeof(window) === 'object' && window.RTCPeerConnection && window.RTCRtpSender && window.RTCRtpReceiver)) {
+ return;
+ }
+
+ // shim sender stats.
+ if (!('getStats' in window.RTCRtpSender.prototype)) {
+ var origGetSenders = window.RTCPeerConnection.prototype.getSenders;
+ if (origGetSenders) {
+ window.RTCPeerConnection.prototype.getSenders = function getSenders() {
+ var _this5 = this;
+ var senders = origGetSenders.apply(this, []);
+ senders.forEach(function (sender) {
+ return sender._pc = _this5;
+ });
+ return senders;
+ };
+ }
+ var origAddTrack = window.RTCPeerConnection.prototype.addTrack;
+ if (origAddTrack) {
+ window.RTCPeerConnection.prototype.addTrack = function addTrack() {
+ var sender = origAddTrack.apply(this, arguments);
+ sender._pc = this;
+ return sender;
+ };
+ }
+ window.RTCRtpSender.prototype.getStats = function getStats() {
+ var sender = this;
+ return this._pc.getStats().then(function (result) {
+ return (
+ /* Note: this will include stats of all senders that
+ * send a track with the same id as sender.track as
+ * it is not possible to identify the RTCRtpSender.
+ */
+ utils.filterStats(result, sender.track, true)
+ );
+ });
+ };
+ }
+
+ // shim receiver stats.
+ if (!('getStats' in window.RTCRtpReceiver.prototype)) {
+ var origGetReceivers = window.RTCPeerConnection.prototype.getReceivers;
+ if (origGetReceivers) {
+ window.RTCPeerConnection.prototype.getReceivers = function getReceivers() {
+ var _this6 = this;
+ var receivers = origGetReceivers.apply(this, []);
+ receivers.forEach(function (receiver) {
+ return receiver._pc = _this6;
+ });
+ return receivers;
+ };
+ }
+ utils.wrapPeerConnectionEvent(window, 'track', function (e) {
+ e.receiver._pc = e.srcElement;
+ return e;
+ });
+ window.RTCRtpReceiver.prototype.getStats = function getStats() {
+ var receiver = this;
+ return this._pc.getStats().then(function (result) {
+ return utils.filterStats(result, receiver.track, false);
+ });
+ };
+ }
+ if (!('getStats' in window.RTCRtpSender.prototype && 'getStats' in window.RTCRtpReceiver.prototype)) {
+ return;
+ }
+
+ // shim RTCPeerConnection.getStats(track).
+ var origGetStats = window.RTCPeerConnection.prototype.getStats;
+ window.RTCPeerConnection.prototype.getStats = function getStats() {
+ if (arguments.length > 0 && arguments[0] instanceof window.MediaStreamTrack) {
+ var track = arguments[0];
+ var sender;
+ var receiver;
+ var err;
+ this.getSenders().forEach(function (s) {
+ if (s.track === track) {
+ if (sender) {
+ err = true;
+ } else {
+ sender = s;
+ }
+ }
+ });
+ this.getReceivers().forEach(function (r) {
+ if (r.track === track) {
+ if (receiver) {
+ err = true;
+ } else {
+ receiver = r;
+ }
+ }
+ return r.track === track;
+ });
+ if (err || sender && receiver) {
+ return Promise.reject(new DOMException('There are more than one sender or receiver for the track.', 'InvalidAccessError'));
+ } else if (sender) {
+ return sender.getStats();
+ } else if (receiver) {
+ return receiver.getStats();
+ }
+ return Promise.reject(new DOMException('There is no sender or receiver for the track.', 'InvalidAccessError'));
+ }
+ return origGetStats.apply(this, arguments);
+ };
+}
+function shimAddTrackRemoveTrackWithNative(window) {
+ // shim addTrack/removeTrack with native variants in order to make
+ // the interactions with legacy getLocalStreams behave as in other browsers.
+ // Keeps a mapping stream.id => [stream, rtpsenders...]
+ window.RTCPeerConnection.prototype.getLocalStreams = function getLocalStreams() {
+ var _this7 = this;
+ this._shimmedLocalStreams = this._shimmedLocalStreams || {};
+ return Object.keys(this._shimmedLocalStreams).map(function (streamId) {
+ return _this7._shimmedLocalStreams[streamId][0];
+ });
+ };
+ var origAddTrack = window.RTCPeerConnection.prototype.addTrack;
+ window.RTCPeerConnection.prototype.addTrack = function addTrack(track, stream) {
+ if (!stream) {
+ return origAddTrack.apply(this, arguments);
+ }
+ this._shimmedLocalStreams = this._shimmedLocalStreams || {};
+ var sender = origAddTrack.apply(this, arguments);
+ if (!this._shimmedLocalStreams[stream.id]) {
+ this._shimmedLocalStreams[stream.id] = [stream, sender];
+ } else if (this._shimmedLocalStreams[stream.id].indexOf(sender) === -1) {
+ this._shimmedLocalStreams[stream.id].push(sender);
+ }
+ return sender;
+ };
+ var origAddStream = window.RTCPeerConnection.prototype.addStream;
+ window.RTCPeerConnection.prototype.addStream = function addStream(stream) {
+ var _this8 = this;
+ this._shimmedLocalStreams = this._shimmedLocalStreams || {};
+ stream.getTracks().forEach(function (track) {
+ var alreadyExists = _this8.getSenders().find(function (s) {
+ return s.track === track;
+ });
+ if (alreadyExists) {
+ throw new DOMException('Track already exists.', 'InvalidAccessError');
+ }
+ });
+ var existingSenders = this.getSenders();
+ origAddStream.apply(this, arguments);
+ var newSenders = this.getSenders().filter(function (newSender) {
+ return existingSenders.indexOf(newSender) === -1;
+ });
+ this._shimmedLocalStreams[stream.id] = [stream].concat(newSenders);
+ };
+ var origRemoveStream = window.RTCPeerConnection.prototype.removeStream;
+ window.RTCPeerConnection.prototype.removeStream = function removeStream(stream) {
+ this._shimmedLocalStreams = this._shimmedLocalStreams || {};
+ delete this._shimmedLocalStreams[stream.id];
+ return origRemoveStream.apply(this, arguments);
+ };
+ var origRemoveTrack = window.RTCPeerConnection.prototype.removeTrack;
+ window.RTCPeerConnection.prototype.removeTrack = function removeTrack(sender) {
+ var _this9 = this;
+ this._shimmedLocalStreams = this._shimmedLocalStreams || {};
+ if (sender) {
+ Object.keys(this._shimmedLocalStreams).forEach(function (streamId) {
+ var idx = _this9._shimmedLocalStreams[streamId].indexOf(sender);
+ if (idx !== -1) {
+ _this9._shimmedLocalStreams[streamId].splice(idx, 1);
+ }
+ if (_this9._shimmedLocalStreams[streamId].length === 1) {
+ delete _this9._shimmedLocalStreams[streamId];
+ }
+ });
+ }
+ return origRemoveTrack.apply(this, arguments);
+ };
+}
+function shimAddTrackRemoveTrack(window, browserDetails) {
+ if (!window.RTCPeerConnection) {
+ return;
+ }
+ // shim addTrack and removeTrack.
+ if (window.RTCPeerConnection.prototype.addTrack && browserDetails.version >= 65) {
+ return shimAddTrackRemoveTrackWithNative(window);
+ }
+
+ // also shim pc.getLocalStreams when addTrack is shimmed
+ // to return the original streams.
+ var origGetLocalStreams = window.RTCPeerConnection.prototype.getLocalStreams;
+ window.RTCPeerConnection.prototype.getLocalStreams = function getLocalStreams() {
+ var _this10 = this;
+ var nativeStreams = origGetLocalStreams.apply(this);
+ this._reverseStreams = this._reverseStreams || {};
+ return nativeStreams.map(function (stream) {
+ return _this10._reverseStreams[stream.id];
+ });
+ };
+ var origAddStream = window.RTCPeerConnection.prototype.addStream;
+ window.RTCPeerConnection.prototype.addStream = function addStream(stream) {
+ var _this11 = this;
+ this._streams = this._streams || {};
+ this._reverseStreams = this._reverseStreams || {};
+ stream.getTracks().forEach(function (track) {
+ var alreadyExists = _this11.getSenders().find(function (s) {
+ return s.track === track;
+ });
+ if (alreadyExists) {
+ throw new DOMException('Track already exists.', 'InvalidAccessError');
+ }
+ });
+ // Add identity mapping for consistency with addTrack.
+ // Unless this is being used with a stream from addTrack.
+ if (!this._reverseStreams[stream.id]) {
+ var newStream = new window.MediaStream(stream.getTracks());
+ this._streams[stream.id] = newStream;
+ this._reverseStreams[newStream.id] = stream;
+ stream = newStream;
+ }
+ origAddStream.apply(this, [stream]);
+ };
+ var origRemoveStream = window.RTCPeerConnection.prototype.removeStream;
+ window.RTCPeerConnection.prototype.removeStream = function removeStream(stream) {
+ this._streams = this._streams || {};
+ this._reverseStreams = this._reverseStreams || {};
+ origRemoveStream.apply(this, [this._streams[stream.id] || stream]);
+ delete this._reverseStreams[this._streams[stream.id] ? this._streams[stream.id].id : stream.id];
+ delete this._streams[stream.id];
+ };
+ window.RTCPeerConnection.prototype.addTrack = function addTrack(track, stream) {
+ var _this12 = this;
+ if (this.signalingState === 'closed') {
+ throw new DOMException('The RTCPeerConnection\'s signalingState is \'closed\'.', 'InvalidStateError');
+ }
+ var streams = [].slice.call(arguments, 1);
+ if (streams.length !== 1 || !streams[0].getTracks().find(function (t) {
+ return t === track;
+ })) {
+ // this is not fully correct but all we can manage without
+ // [[associated MediaStreams]] internal slot.
+ throw new DOMException('The adapter.js addTrack polyfill only supports a single ' + ' stream which is associated with the specified track.', 'NotSupportedError');
+ }
+ var alreadyExists = this.getSenders().find(function (s) {
+ return s.track === track;
+ });
+ if (alreadyExists) {
+ throw new DOMException('Track already exists.', 'InvalidAccessError');
+ }
+ this._streams = this._streams || {};
+ this._reverseStreams = this._reverseStreams || {};
+ var oldStream = this._streams[stream.id];
+ if (oldStream) {
+ // this is using odd Chrome behaviour, use with caution:
+ // https://bugs.chromium.org/p/webrtc/issues/detail?id=7815
+ // Note: we rely on the high-level addTrack/dtmf shim to
+ // create the sender with a dtmf sender.
+ oldStream.addTrack(track);
+
+ // Trigger ONN async.
+ Promise.resolve().then(function () {
+ _this12.dispatchEvent(new Event('negotiationneeded'));
+ });
+ } else {
+ var newStream = new window.MediaStream([track]);
+ this._streams[stream.id] = newStream;
+ this._reverseStreams[newStream.id] = stream;
+ this.addStream(newStream);
+ }
+ return this.getSenders().find(function (s) {
+ return s.track === track;
+ });
+ };
+
+ // replace the internal stream id with the external one and
+ // vice versa.
+ function replaceInternalStreamId(pc, description) {
+ var sdp = description.sdp;
+ Object.keys(pc._reverseStreams || []).forEach(function (internalId) {
+ var externalStream = pc._reverseStreams[internalId];
+ var internalStream = pc._streams[externalStream.id];
+ sdp = sdp.replace(new RegExp(internalStream.id, 'g'), externalStream.id);
+ });
+ return new RTCSessionDescription({
+ type: description.type,
+ sdp: sdp
+ });
+ }
+ function replaceExternalStreamId(pc, description) {
+ var sdp = description.sdp;
+ Object.keys(pc._reverseStreams || []).forEach(function (internalId) {
+ var externalStream = pc._reverseStreams[internalId];
+ var internalStream = pc._streams[externalStream.id];
+ sdp = sdp.replace(new RegExp(externalStream.id, 'g'), internalStream.id);
+ });
+ return new RTCSessionDescription({
+ type: description.type,
+ sdp: sdp
+ });
+ }
+ ['createOffer', 'createAnswer'].forEach(function (method) {
+ var nativeMethod = window.RTCPeerConnection.prototype[method];
+ var methodObj = _defineProperty({}, method, function () {
+ var _this13 = this;
+ var args = arguments;
+ var isLegacyCall = arguments.length && typeof arguments[0] === 'function';
+ if (isLegacyCall) {
+ return nativeMethod.apply(this, [function (description) {
+ var desc = replaceInternalStreamId(_this13, description);
+ args[0].apply(null, [desc]);
+ }, function (err) {
+ if (args[1]) {
+ args[1].apply(null, err);
+ }
+ }, arguments[2]]);
+ }
+ return nativeMethod.apply(this, arguments).then(function (description) {
+ return replaceInternalStreamId(_this13, description);
+ });
+ });
+ window.RTCPeerConnection.prototype[method] = methodObj[method];
+ });
+ var origSetLocalDescription = window.RTCPeerConnection.prototype.setLocalDescription;
+ window.RTCPeerConnection.prototype.setLocalDescription = function setLocalDescription() {
+ if (!arguments.length || !arguments[0].type) {
+ return origSetLocalDescription.apply(this, arguments);
+ }
+ arguments[0] = replaceExternalStreamId(this, arguments[0]);
+ return origSetLocalDescription.apply(this, arguments);
+ };
+
+ // TODO: mangle getStats: https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamstats-streamidentifier
+
+ var origLocalDescription = Object.getOwnPropertyDescriptor(window.RTCPeerConnection.prototype, 'localDescription');
+ Object.defineProperty(window.RTCPeerConnection.prototype, 'localDescription', {
+ get: function get() {
+ var description = origLocalDescription.get.apply(this);
+ if (description.type === '') {
+ return description;
+ }
+ return replaceInternalStreamId(this, description);
+ }
+ });
+ window.RTCPeerConnection.prototype.removeTrack = function removeTrack(sender) {
+ var _this14 = this;
+ if (this.signalingState === 'closed') {
+ throw new DOMException('The RTCPeerConnection\'s signalingState is \'closed\'.', 'InvalidStateError');
+ }
+ // We can not yet check for sender instanceof RTCRtpSender
+ // since we shim RTPSender. So we check if sender._pc is set.
+ if (!sender._pc) {
+ throw new DOMException('Argument 1 of RTCPeerConnection.removeTrack ' + 'does not implement interface RTCRtpSender.', 'TypeError');
+ }
+ var isLocal = sender._pc === this;
+ if (!isLocal) {
+ throw new DOMException('Sender was not created by this connection.', 'InvalidAccessError');
+ }
+
+ // Search for the native stream the senders track belongs to.
+ this._streams = this._streams || {};
+ var stream;
+ Object.keys(this._streams).forEach(function (streamid) {
+ var hasTrack = _this14._streams[streamid].getTracks().find(function (track) {
+ return sender.track === track;
+ });
+ if (hasTrack) {
+ stream = _this14._streams[streamid];
+ }
+ });
+ if (stream) {
+ if (stream.getTracks().length === 1) {
+ // if this is the last track of the stream, remove the stream. This
+ // takes care of any shimmed _senders.
+ this.removeStream(this._reverseStreams[stream.id]);
+ } else {
+ // relying on the same odd chrome behaviour as above.
+ stream.removeTrack(sender.track);
+ }
+ this.dispatchEvent(new Event('negotiationneeded'));
+ }
+ };
+}
+function shimPeerConnection(window, browserDetails) {
+ if (!window.RTCPeerConnection && window.webkitRTCPeerConnection) {
+ // very basic support for old versions.
+ window.RTCPeerConnection = window.webkitRTCPeerConnection;
+ }
+ if (!window.RTCPeerConnection) {
+ return;
+ }
+
+ // shim implicit creation of RTCSessionDescription/RTCIceCandidate
+ if (browserDetails.version < 53) {
+ ['setLocalDescription', 'setRemoteDescription', 'addIceCandidate'].forEach(function (method) {
+ var nativeMethod = window.RTCPeerConnection.prototype[method];
+ var methodObj = _defineProperty({}, method, function () {
+ arguments[0] = new (method === 'addIceCandidate' ? window.RTCIceCandidate : window.RTCSessionDescription)(arguments[0]);
+ return nativeMethod.apply(this, arguments);
+ });
+ window.RTCPeerConnection.prototype[method] = methodObj[method];
+ });
+ }
+}
+
+// Attempt to fix ONN in plan-b mode.
+function fixNegotiationNeeded(window, browserDetails) {
+ utils.wrapPeerConnectionEvent(window, 'negotiationneeded', function (e) {
+ var pc = e.target;
+ if (browserDetails.version < 72 || pc.getConfiguration && pc.getConfiguration().sdpSemantics === 'plan-b') {
+ if (pc.signalingState !== 'stable') {
+ return;
+ }
+ }
+ return e;
+ });
+}
+
+},{"../utils.js":10,"./getusermedia":4}],4:[function(require,module,exports){
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+/* eslint-env node */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.shimGetUserMedia = shimGetUserMedia;
+var utils = _interopRequireWildcard(require("../utils.js"));
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
+var logging = utils.log;
+function shimGetUserMedia(window, browserDetails) {
+ var navigator = window && window.navigator;
+ if (!navigator.mediaDevices) {
+ return;
+ }
+ var constraintsToChrome_ = function constraintsToChrome_(c) {
+ if (_typeof(c) !== 'object' || c.mandatory || c.optional) {
+ return c;
+ }
+ var cc = {};
+ Object.keys(c).forEach(function (key) {
+ if (key === 'require' || key === 'advanced' || key === 'mediaSource') {
+ return;
+ }
+ var r = _typeof(c[key]) === 'object' ? c[key] : {
+ ideal: c[key]
+ };
+ if (r.exact !== undefined && typeof r.exact === 'number') {
+ r.min = r.max = r.exact;
+ }
+ var oldname_ = function oldname_(prefix, name) {
+ if (prefix) {
+ return prefix + name.charAt(0).toUpperCase() + name.slice(1);
+ }
+ return name === 'deviceId' ? 'sourceId' : name;
+ };
+ if (r.ideal !== undefined) {
+ cc.optional = cc.optional || [];
+ var oc = {};
+ if (typeof r.ideal === 'number') {
+ oc[oldname_('min', key)] = r.ideal;
+ cc.optional.push(oc);
+ oc = {};
+ oc[oldname_('max', key)] = r.ideal;
+ cc.optional.push(oc);
+ } else {
+ oc[oldname_('', key)] = r.ideal;
+ cc.optional.push(oc);
+ }
+ }
+ if (r.exact !== undefined && typeof r.exact !== 'number') {
+ cc.mandatory = cc.mandatory || {};
+ cc.mandatory[oldname_('', key)] = r.exact;
+ } else {
+ ['min', 'max'].forEach(function (mix) {
+ if (r[mix] !== undefined) {
+ cc.mandatory = cc.mandatory || {};
+ cc.mandatory[oldname_(mix, key)] = r[mix];
+ }
+ });
+ }
+ });
+ if (c.advanced) {
+ cc.optional = (cc.optional || []).concat(c.advanced);
+ }
+ return cc;
+ };
+ var shimConstraints_ = function shimConstraints_(constraints, func) {
+ if (browserDetails.version >= 61) {
+ return func(constraints);
+ }
+ constraints = JSON.parse(JSON.stringify(constraints));
+ if (constraints && _typeof(constraints.audio) === 'object') {
+ var remap = function remap(obj, a, b) {
+ if (a in obj && !(b in obj)) {
+ obj[b] = obj[a];
+ delete obj[a];
+ }
+ };
+ constraints = JSON.parse(JSON.stringify(constraints));
+ remap(constraints.audio, 'autoGainControl', 'googAutoGainControl');
+ remap(constraints.audio, 'noiseSuppression', 'googNoiseSuppression');
+ constraints.audio = constraintsToChrome_(constraints.audio);
+ }
+ if (constraints && _typeof(constraints.video) === 'object') {
+ // Shim facingMode for mobile & surface pro.
+ var face = constraints.video.facingMode;
+ face = face && (_typeof(face) === 'object' ? face : {
+ ideal: face
+ });
+ var getSupportedFacingModeLies = browserDetails.version < 66;
+ if (face && (face.exact === 'user' || face.exact === 'environment' || face.ideal === 'user' || face.ideal === 'environment') && !(navigator.mediaDevices.getSupportedConstraints && navigator.mediaDevices.getSupportedConstraints().facingMode && !getSupportedFacingModeLies)) {
+ delete constraints.video.facingMode;
+ var matches;
+ if (face.exact === 'environment' || face.ideal === 'environment') {
+ matches = ['back', 'rear'];
+ } else if (face.exact === 'user' || face.ideal === 'user') {
+ matches = ['front'];
+ }
+ if (matches) {
+ // Look for matches in label, or use last cam for back (typical).
+ return navigator.mediaDevices.enumerateDevices().then(function (devices) {
+ devices = devices.filter(function (d) {
+ return d.kind === 'videoinput';
+ });
+ var dev = devices.find(function (d) {
+ return matches.some(function (match) {
+ return d.label.toLowerCase().includes(match);
+ });
+ });
+ if (!dev && devices.length && matches.includes('back')) {
+ dev = devices[devices.length - 1]; // more likely the back cam
+ }
+ if (dev) {
+ constraints.video.deviceId = face.exact ? {
+ exact: dev.deviceId
+ } : {
+ ideal: dev.deviceId
+ };
+ }
+ constraints.video = constraintsToChrome_(constraints.video);
+ logging('chrome: ' + JSON.stringify(constraints));
+ return func(constraints);
+ });
+ }
+ }
+ constraints.video = constraintsToChrome_(constraints.video);
+ }
+ logging('chrome: ' + JSON.stringify(constraints));
+ return func(constraints);
+ };
+ var shimError_ = function shimError_(e) {
+ if (browserDetails.version >= 64) {
+ return e;
+ }
+ return {
+ name: {
+ PermissionDeniedError: 'NotAllowedError',
+ PermissionDismissedError: 'NotAllowedError',
+ InvalidStateError: 'NotAllowedError',
+ DevicesNotFoundError: 'NotFoundError',
+ ConstraintNotSatisfiedError: 'OverconstrainedError',
+ TrackStartError: 'NotReadableError',
+ MediaDeviceFailedDueToShutdown: 'NotAllowedError',
+ MediaDeviceKillSwitchOn: 'NotAllowedError',
+ TabCaptureError: 'AbortError',
+ ScreenCaptureError: 'AbortError',
+ DeviceCaptureError: 'AbortError'
+ }[e.name] || e.name,
+ message: e.message,
+ constraint: e.constraint || e.constraintName,
+ toString: function toString() {
+ return this.name + (this.message && ': ') + this.message;
+ }
+ };
+ };
+ var getUserMedia_ = function getUserMedia_(constraints, onSuccess, onError) {
+ shimConstraints_(constraints, function (c) {
+ navigator.webkitGetUserMedia(c, onSuccess, function (e) {
+ if (onError) {
+ onError(shimError_(e));
+ }
+ });
+ });
+ };
+ navigator.getUserMedia = getUserMedia_.bind(navigator);
+
+ // Even though Chrome 45 has navigator.mediaDevices and a getUserMedia
+ // function which returns a Promise, it does not accept spec-style
+ // constraints.
+ if (navigator.mediaDevices.getUserMedia) {
+ var origGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
+ navigator.mediaDevices.getUserMedia = function (cs) {
+ return shimConstraints_(cs, function (c) {
+ return origGetUserMedia(c).then(function (stream) {
+ if (c.audio && !stream.getAudioTracks().length || c.video && !stream.getVideoTracks().length) {
+ stream.getTracks().forEach(function (track) {
+ track.stop();
+ });
+ throw new DOMException('', 'NotFoundError');
+ }
+ return stream;
+ }, function (e) {
+ return Promise.reject(shimError_(e));
+ });
+ });
+ };
+ }
+}
+
+},{"../utils.js":10}],5:[function(require,module,exports){
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+/* eslint-env node */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.removeExtmapAllowMixed = removeExtmapAllowMixed;
+exports.shimAddIceCandidateNullOrEmpty = shimAddIceCandidateNullOrEmpty;
+exports.shimConnectionState = shimConnectionState;
+exports.shimMaxMessageSize = shimMaxMessageSize;
+exports.shimParameterlessSetLocalDescription = shimParameterlessSetLocalDescription;
+exports.shimRTCIceCandidate = shimRTCIceCandidate;
+exports.shimRTCIceCandidateRelayProtocol = shimRTCIceCandidateRelayProtocol;
+exports.shimSendThrowTypeError = shimSendThrowTypeError;
+var _sdp = _interopRequireDefault(require("sdp"));
+var utils = _interopRequireWildcard(require("./utils"));
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
+function shimRTCIceCandidate(window) {
+ // foundation is arbitrarily chosen as an indicator for full support for
+ // https://w3c.github.io/webrtc-pc/#rtcicecandidate-interface
+ if (!window.RTCIceCandidate || window.RTCIceCandidate && 'foundation' in window.RTCIceCandidate.prototype) {
+ return;
+ }
+ var NativeRTCIceCandidate = window.RTCIceCandidate;
+ window.RTCIceCandidate = function RTCIceCandidate(args) {
+ // Remove the a= which shouldn't be part of the candidate string.
+ if (_typeof(args) === 'object' && args.candidate && args.candidate.indexOf('a=') === 0) {
+ args = JSON.parse(JSON.stringify(args));
+ args.candidate = args.candidate.substring(2);
+ }
+ if (args.candidate && args.candidate.length) {
+ // Augment the native candidate with the parsed fields.
+ var nativeCandidate = new NativeRTCIceCandidate(args);
+ var parsedCandidate = _sdp["default"].parseCandidate(args.candidate);
+ for (var key in parsedCandidate) {
+ if (!(key in nativeCandidate)) {
+ Object.defineProperty(nativeCandidate, key, {
+ value: parsedCandidate[key]
+ });
+ }
+ }
+
+ // Override serializer to not serialize the extra attributes.
+ nativeCandidate.toJSON = function toJSON() {
+ return {
+ candidate: nativeCandidate.candidate,
+ sdpMid: nativeCandidate.sdpMid,
+ sdpMLineIndex: nativeCandidate.sdpMLineIndex,
+ usernameFragment: nativeCandidate.usernameFragment
+ };
+ };
+ return nativeCandidate;
+ }
+ return new NativeRTCIceCandidate(args);
+ };
+ window.RTCIceCandidate.prototype = NativeRTCIceCandidate.prototype;
+
+ // Hook up the augmented candidate in onicecandidate and
+ // addEventListener('icecandidate', ...)
+ utils.wrapPeerConnectionEvent(window, 'icecandidate', function (e) {
+ if (e.candidate) {
+ Object.defineProperty(e, 'candidate', {
+ value: new window.RTCIceCandidate(e.candidate),
+ writable: 'false'
+ });
+ }
+ return e;
+ });
+}
+function shimRTCIceCandidateRelayProtocol(window) {
+ if (!window.RTCIceCandidate || window.RTCIceCandidate && 'relayProtocol' in window.RTCIceCandidate.prototype) {
+ return;
+ }
+
+ // Hook up the augmented candidate in onicecandidate and
+ // addEventListener('icecandidate', ...)
+ utils.wrapPeerConnectionEvent(window, 'icecandidate', function (e) {
+ if (e.candidate) {
+ var parsedCandidate = _sdp["default"].parseCandidate(e.candidate.candidate);
+ if (parsedCandidate.type === 'relay') {
+ // This is a libwebrtc-specific mapping of local type preference
+ // to relayProtocol.
+ e.candidate.relayProtocol = {
+ 0: 'tls',
+ 1: 'tcp',
+ 2: 'udp'
+ }[parsedCandidate.priority >> 24];
+ }
+ }
+ return e;
+ });
+}
+function shimMaxMessageSize(window, browserDetails) {
+ if (!window.RTCPeerConnection) {
+ return;
+ }
+ if (!('sctp' in window.RTCPeerConnection.prototype)) {
+ Object.defineProperty(window.RTCPeerConnection.prototype, 'sctp', {
+ get: function get() {
+ return typeof this._sctp === 'undefined' ? null : this._sctp;
+ }
+ });
+ }
+ var sctpInDescription = function sctpInDescription(description) {
+ if (!description || !description.sdp) {
+ return false;
+ }
+ var sections = _sdp["default"].splitSections(description.sdp);
+ sections.shift();
+ return sections.some(function (mediaSection) {
+ var mLine = _sdp["default"].parseMLine(mediaSection);
+ return mLine && mLine.kind === 'application' && mLine.protocol.indexOf('SCTP') !== -1;
+ });
+ };
+ var getRemoteFirefoxVersion = function getRemoteFirefoxVersion(description) {
+ // TODO: Is there a better solution for detecting Firefox?
+ var match = description.sdp.match(/mozilla...THIS_IS_SDPARTA-(\d+)/);
+ if (match === null || match.length < 2) {
+ return -1;
+ }
+ var version = parseInt(match[1], 10);
+ // Test for NaN (yes, this is ugly)
+ return version !== version ? -1 : version;
+ };
+ var getCanSendMaxMessageSize = function getCanSendMaxMessageSize(remoteIsFirefox) {
+ // Every implementation we know can send at least 64 KiB.
+ // Note: Although Chrome is technically able to send up to 256 KiB, the
+ // data does not reach the other peer reliably.
+ // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=8419
+ var canSendMaxMessageSize = 65536;
+ if (browserDetails.browser === 'firefox') {
+ if (browserDetails.version < 57) {
+ if (remoteIsFirefox === -1) {
+ // FF < 57 will send in 16 KiB chunks using the deprecated PPID
+ // fragmentation.
+ canSendMaxMessageSize = 16384;
+ } else {
+ // However, other FF (and RAWRTC) can reassemble PPID-fragmented
+ // messages. Thus, supporting ~2 GiB when sending.
+ canSendMaxMessageSize = 2147483637;
+ }
+ } else if (browserDetails.version < 60) {
+ // Currently, all FF >= 57 will reset the remote maximum message size
+ // to the default value when a data channel is created at a later
+ // stage. :(
+ // See: https://bugzilla.mozilla.org/show_bug.cgi?id=1426831
+ canSendMaxMessageSize = browserDetails.version === 57 ? 65535 : 65536;
+ } else {
+ // FF >= 60 supports sending ~2 GiB
+ canSendMaxMessageSize = 2147483637;
+ }
+ }
+ return canSendMaxMessageSize;
+ };
+ var getMaxMessageSize = function getMaxMessageSize(description, remoteIsFirefox) {
+ // Note: 65536 bytes is the default value from the SDP spec. Also,
+ // every implementation we know supports receiving 65536 bytes.
+ var maxMessageSize = 65536;
+
+ // FF 57 has a slightly incorrect default remote max message size, so
+ // we need to adjust it here to avoid a failure when sending.
+ // See: https://bugzilla.mozilla.org/show_bug.cgi?id=1425697
+ if (browserDetails.browser === 'firefox' && browserDetails.version === 57) {
+ maxMessageSize = 65535;
+ }
+ var match = _sdp["default"].matchPrefix(description.sdp, 'a=max-message-size:');
+ if (match.length > 0) {
+ maxMessageSize = parseInt(match[0].substring(19), 10);
+ } else if (browserDetails.browser === 'firefox' && remoteIsFirefox !== -1) {
+ // If the maximum message size is not present in the remote SDP and
+ // both local and remote are Firefox, the remote peer can receive
+ // ~2 GiB.
+ maxMessageSize = 2147483637;
+ }
+ return maxMessageSize;
+ };
+ var origSetRemoteDescription = window.RTCPeerConnection.prototype.setRemoteDescription;
+ window.RTCPeerConnection.prototype.setRemoteDescription = function setRemoteDescription() {
+ this._sctp = null;
+ // Chrome decided to not expose .sctp in plan-b mode.
+ // As usual, adapter.js has to do an 'ugly worakaround'
+ // to cover up the mess.
+ if (browserDetails.browser === 'chrome' && browserDetails.version >= 76) {
+ var _this$getConfiguratio = this.getConfiguration(),
+ sdpSemantics = _this$getConfiguratio.sdpSemantics;
+ if (sdpSemantics === 'plan-b') {
+ Object.defineProperty(this, 'sctp', {
+ get: function get() {
+ return typeof this._sctp === 'undefined' ? null : this._sctp;
+ },
+ enumerable: true,
+ configurable: true
+ });
+ }
+ }
+ if (sctpInDescription(arguments[0])) {
+ // Check if the remote is FF.
+ var isFirefox = getRemoteFirefoxVersion(arguments[0]);
+
+ // Get the maximum message size the local peer is capable of sending
+ var canSendMMS = getCanSendMaxMessageSize(isFirefox);
+
+ // Get the maximum message size of the remote peer.
+ var remoteMMS = getMaxMessageSize(arguments[0], isFirefox);
+
+ // Determine final maximum message size
+ var maxMessageSize;
+ if (canSendMMS === 0 && remoteMMS === 0) {
+ maxMessageSize = Number.POSITIVE_INFINITY;
+ } else if (canSendMMS === 0 || remoteMMS === 0) {
+ maxMessageSize = Math.max(canSendMMS, remoteMMS);
+ } else {
+ maxMessageSize = Math.min(canSendMMS, remoteMMS);
+ }
+
+ // Create a dummy RTCSctpTransport object and the 'maxMessageSize'
+ // attribute.
+ var sctp = {};
+ Object.defineProperty(sctp, 'maxMessageSize', {
+ get: function get() {
+ return maxMessageSize;
+ }
+ });
+ this._sctp = sctp;
+ }
+ return origSetRemoteDescription.apply(this, arguments);
+ };
+}
+function shimSendThrowTypeError(window) {
+ if (!(window.RTCPeerConnection && 'createDataChannel' in window.RTCPeerConnection.prototype)) {
+ return;
+ }
+
+ // Note: Although Firefox >= 57 has a native implementation, the maximum
+ // message size can be reset for all data channels at a later stage.
+ // See: https://bugzilla.mozilla.org/show_bug.cgi?id=1426831
+
+ function wrapDcSend(dc, pc) {
+ var origDataChannelSend = dc.send;
+ dc.send = function send() {
+ var data = arguments[0];
+ var length = data.length || data.size || data.byteLength;
+ if (dc.readyState === 'open' && pc.sctp && length > pc.sctp.maxMessageSize) {
+ throw new TypeError('Message too large (can send a maximum of ' + pc.sctp.maxMessageSize + ' bytes)');
+ }
+ return origDataChannelSend.apply(dc, arguments);
+ };
+ }
+ var origCreateDataChannel = window.RTCPeerConnection.prototype.createDataChannel;
+ window.RTCPeerConnection.prototype.createDataChannel = function createDataChannel() {
+ var dataChannel = origCreateDataChannel.apply(this, arguments);
+ wrapDcSend(dataChannel, this);
+ return dataChannel;
+ };
+ utils.wrapPeerConnectionEvent(window, 'datachannel', function (e) {
+ wrapDcSend(e.channel, e.target);
+ return e;
+ });
+}
+
+/* shims RTCConnectionState by pretending it is the same as iceConnectionState.
+ * See https://bugs.chromium.org/p/webrtc/issues/detail?id=6145#c12
+ * for why this is a valid hack in Chrome. In Firefox it is slightly incorrect
+ * since DTLS failures would be hidden. See
+ * https://bugzilla.mozilla.org/show_bug.cgi?id=1265827
+ * for the Firefox tracking bug.
+ */
+function shimConnectionState(window) {
+ if (!window.RTCPeerConnection || 'connectionState' in window.RTCPeerConnection.prototype) {
+ return;
+ }
+ var proto = window.RTCPeerConnection.prototype;
+ Object.defineProperty(proto, 'connectionState', {
+ get: function get() {
+ return {
+ completed: 'connected',
+ checking: 'connecting'
+ }[this.iceConnectionState] || this.iceConnectionState;
+ },
+ enumerable: true,
+ configurable: true
+ });
+ Object.defineProperty(proto, 'onconnectionstatechange', {
+ get: function get() {
+ return this._onconnectionstatechange || null;
+ },
+ set: function set(cb) {
+ if (this._onconnectionstatechange) {
+ this.removeEventListener('connectionstatechange', this._onconnectionstatechange);
+ delete this._onconnectionstatechange;
+ }
+ if (cb) {
+ this.addEventListener('connectionstatechange', this._onconnectionstatechange = cb);
+ }
+ },
+ enumerable: true,
+ configurable: true
+ });
+ ['setLocalDescription', 'setRemoteDescription'].forEach(function (method) {
+ var origMethod = proto[method];
+ proto[method] = function () {
+ if (!this._connectionstatechangepoly) {
+ this._connectionstatechangepoly = function (e) {
+ var pc = e.target;
+ if (pc._lastConnectionState !== pc.connectionState) {
+ pc._lastConnectionState = pc.connectionState;
+ var newEvent = new Event('connectionstatechange', e);
+ pc.dispatchEvent(newEvent);
+ }
+ return e;
+ };
+ this.addEventListener('iceconnectionstatechange', this._connectionstatechangepoly);
+ }
+ return origMethod.apply(this, arguments);
+ };
+ });
+}
+function removeExtmapAllowMixed(window, browserDetails) {
+ /* remove a=extmap-allow-mixed for webrtc.org < M71 */
+ if (!window.RTCPeerConnection) {
+ return;
+ }
+ if (browserDetails.browser === 'chrome' && browserDetails.version >= 71) {
+ return;
+ }
+ if (browserDetails.browser === 'safari' && browserDetails.version >= 605) {
+ return;
+ }
+ var nativeSRD = window.RTCPeerConnection.prototype.setRemoteDescription;
+ window.RTCPeerConnection.prototype.setRemoteDescription = function setRemoteDescription(desc) {
+ if (desc && desc.sdp && desc.sdp.indexOf('\na=extmap-allow-mixed') !== -1) {
+ var sdp = desc.sdp.split('\n').filter(function (line) {
+ return line.trim() !== 'a=extmap-allow-mixed';
+ }).join('\n');
+ // Safari enforces read-only-ness of RTCSessionDescription fields.
+ if (window.RTCSessionDescription && desc instanceof window.RTCSessionDescription) {
+ arguments[0] = new window.RTCSessionDescription({
+ type: desc.type,
+ sdp: sdp
+ });
+ } else {
+ desc.sdp = sdp;
+ }
+ }
+ return nativeSRD.apply(this, arguments);
+ };
+}
+function shimAddIceCandidateNullOrEmpty(window, browserDetails) {
+ // Support for addIceCandidate(null or undefined)
+ // as well as addIceCandidate({candidate: "", ...})
+ // https://bugs.chromium.org/p/chromium/issues/detail?id=978582
+ // Note: must be called before other polyfills which change the signature.
+ if (!(window.RTCPeerConnection && window.RTCPeerConnection.prototype)) {
+ return;
+ }
+ var nativeAddIceCandidate = window.RTCPeerConnection.prototype.addIceCandidate;
+ if (!nativeAddIceCandidate || nativeAddIceCandidate.length === 0) {
+ return;
+ }
+ window.RTCPeerConnection.prototype.addIceCandidate = function addIceCandidate() {
+ if (!arguments[0]) {
+ if (arguments[1]) {
+ arguments[1].apply(null);
+ }
+ return Promise.resolve();
+ }
+ // Firefox 68+ emits and processes {candidate: "", ...}, ignore
+ // in older versions.
+ // Native support for ignoring exists for Chrome M77+.
+ // Safari ignores as well, exact version unknown but works in the same
+ // version that also ignores addIceCandidate(null).
+ if ((browserDetails.browser === 'chrome' && browserDetails.version < 78 || browserDetails.browser === 'firefox' && browserDetails.version < 68 || browserDetails.browser === 'safari') && arguments[0] && arguments[0].candidate === '') {
+ return Promise.resolve();
+ }
+ return nativeAddIceCandidate.apply(this, arguments);
+ };
+}
+
+// Note: Make sure to call this ahead of APIs that modify
+// setLocalDescription.length
+function shimParameterlessSetLocalDescription(window, browserDetails) {
+ if (!(window.RTCPeerConnection && window.RTCPeerConnection.prototype)) {
+ return;
+ }
+ var nativeSetLocalDescription = window.RTCPeerConnection.prototype.setLocalDescription;
+ if (!nativeSetLocalDescription || nativeSetLocalDescription.length === 0) {
+ return;
+ }
+ window.RTCPeerConnection.prototype.setLocalDescription = function setLocalDescription() {
+ var _this = this;
+ var desc = arguments[0] || {};
+ if (_typeof(desc) !== 'object' || desc.type && desc.sdp) {
+ return nativeSetLocalDescription.apply(this, arguments);
+ }
+ // The remaining steps should technically happen when SLD comes off the
+ // RTCPeerConnection's operations chain (not ahead of going on it), but
+ // this is too difficult to shim. Instead, this shim only covers the
+ // common case where the operations chain is empty. This is imperfect, but
+ // should cover many cases. Rationale: Even if we can't reduce the glare
+ // window to zero on imperfect implementations, there's value in tapping
+ // into the perfect negotiation pattern that several browsers support.
+ desc = {
+ type: desc.type,
+ sdp: desc.sdp
+ };
+ if (!desc.type) {
+ switch (this.signalingState) {
+ case 'stable':
+ case 'have-local-offer':
+ case 'have-remote-pranswer':
+ desc.type = 'offer';
+ break;
+ default:
+ desc.type = 'answer';
+ break;
+ }
+ }
+ if (desc.sdp || desc.type !== 'offer' && desc.type !== 'answer') {
+ return nativeSetLocalDescription.apply(this, [desc]);
+ }
+ var func = desc.type === 'offer' ? this.createOffer : this.createAnswer;
+ return func.apply(this).then(function (d) {
+ return nativeSetLocalDescription.apply(_this, [d]);
+ });
+ };
+}
+
+},{"./utils":10,"sdp":11}],6:[function(require,module,exports){
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+/* eslint-env node */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.shimAddTransceiver = shimAddTransceiver;
+exports.shimCreateAnswer = shimCreateAnswer;
+exports.shimCreateOffer = shimCreateOffer;
+Object.defineProperty(exports, "shimGetDisplayMedia", {
+ enumerable: true,
+ get: function get() {
+ return _getdisplaymedia.shimGetDisplayMedia;
+ }
+});
+exports.shimGetParameters = shimGetParameters;
+Object.defineProperty(exports, "shimGetUserMedia", {
+ enumerable: true,
+ get: function get() {
+ return _getusermedia.shimGetUserMedia;
+ }
+});
+exports.shimOnTrack = shimOnTrack;
+exports.shimPeerConnection = shimPeerConnection;
+exports.shimRTCDataChannel = shimRTCDataChannel;
+exports.shimReceiverGetStats = shimReceiverGetStats;
+exports.shimRemoveStream = shimRemoveStream;
+exports.shimSenderGetStats = shimSenderGetStats;
+var utils = _interopRequireWildcard(require("../utils"));
+var _getusermedia = require("./getusermedia");
+var _getdisplaymedia = require("./getdisplaymedia");
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
+function _toConsumableArray(arr) { return _arrayWithoutHoles(arr) || _iterableToArray(arr) || _unsupportedIterableToArray(arr) || _nonIterableSpread(); }
+function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+function _iterableToArray(iter) { if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) return Array.from(iter); }
+function _arrayWithoutHoles(arr) { if (Array.isArray(arr)) return _arrayLikeToArray(arr); }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
+function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
+function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
+function shimOnTrack(window) {
+ if (_typeof(window) === 'object' && window.RTCTrackEvent && 'receiver' in window.RTCTrackEvent.prototype && !('transceiver' in window.RTCTrackEvent.prototype)) {
+ Object.defineProperty(window.RTCTrackEvent.prototype, 'transceiver', {
+ get: function get() {
+ return {
+ receiver: this.receiver
+ };
+ }
+ });
+ }
+}
+function shimPeerConnection(window, browserDetails) {
+ if (_typeof(window) !== 'object' || !(window.RTCPeerConnection || window.mozRTCPeerConnection)) {
+ return; // probably media.peerconnection.enabled=false in about:config
+ }
+ if (!window.RTCPeerConnection && window.mozRTCPeerConnection) {
+ // very basic support for old versions.
+ window.RTCPeerConnection = window.mozRTCPeerConnection;
+ }
+ if (browserDetails.version < 53) {
+ // shim away need for obsolete RTCIceCandidate/RTCSessionDescription.
+ ['setLocalDescription', 'setRemoteDescription', 'addIceCandidate'].forEach(function (method) {
+ var nativeMethod = window.RTCPeerConnection.prototype[method];
+ var methodObj = _defineProperty({}, method, function () {
+ arguments[0] = new (method === 'addIceCandidate' ? window.RTCIceCandidate : window.RTCSessionDescription)(arguments[0]);
+ return nativeMethod.apply(this, arguments);
+ });
+ window.RTCPeerConnection.prototype[method] = methodObj[method];
+ });
+ }
+ var modernStatsTypes = {
+ inboundrtp: 'inbound-rtp',
+ outboundrtp: 'outbound-rtp',
+ candidatepair: 'candidate-pair',
+ localcandidate: 'local-candidate',
+ remotecandidate: 'remote-candidate'
+ };
+ var nativeGetStats = window.RTCPeerConnection.prototype.getStats;
+ window.RTCPeerConnection.prototype.getStats = function getStats() {
+ var _arguments = Array.prototype.slice.call(arguments),
+ selector = _arguments[0],
+ onSucc = _arguments[1],
+ onErr = _arguments[2];
+ return nativeGetStats.apply(this, [selector || null]).then(function (stats) {
+ if (browserDetails.version < 53 && !onSucc) {
+ // Shim only promise getStats with spec-hyphens in type names
+ // Leave callback version alone; misc old uses of forEach before Map
+ try {
+ stats.forEach(function (stat) {
+ stat.type = modernStatsTypes[stat.type] || stat.type;
+ });
+ } catch (e) {
+ if (e.name !== 'TypeError') {
+ throw e;
+ }
+ // Avoid TypeError: "type" is read-only, in old versions. 34-43ish
+ stats.forEach(function (stat, i) {
+ stats.set(i, Object.assign({}, stat, {
+ type: modernStatsTypes[stat.type] || stat.type
+ }));
+ });
+ }
+ }
+ return stats;
+ }).then(onSucc, onErr);
+ };
+}
+function shimSenderGetStats(window) {
+ if (!(_typeof(window) === 'object' && window.RTCPeerConnection && window.RTCRtpSender)) {
+ return;
+ }
+ if (window.RTCRtpSender && 'getStats' in window.RTCRtpSender.prototype) {
+ return;
+ }
+ var origGetSenders = window.RTCPeerConnection.prototype.getSenders;
+ if (origGetSenders) {
+ window.RTCPeerConnection.prototype.getSenders = function getSenders() {
+ var _this = this;
+ var senders = origGetSenders.apply(this, []);
+ senders.forEach(function (sender) {
+ return sender._pc = _this;
+ });
+ return senders;
+ };
+ }
+ var origAddTrack = window.RTCPeerConnection.prototype.addTrack;
+ if (origAddTrack) {
+ window.RTCPeerConnection.prototype.addTrack = function addTrack() {
+ var sender = origAddTrack.apply(this, arguments);
+ sender._pc = this;
+ return sender;
+ };
+ }
+ window.RTCRtpSender.prototype.getStats = function getStats() {
+ return this.track ? this._pc.getStats(this.track) : Promise.resolve(new Map());
+ };
+}
+function shimReceiverGetStats(window) {
+ if (!(_typeof(window) === 'object' && window.RTCPeerConnection && window.RTCRtpSender)) {
+ return;
+ }
+ if (window.RTCRtpSender && 'getStats' in window.RTCRtpReceiver.prototype) {
+ return;
+ }
+ var origGetReceivers = window.RTCPeerConnection.prototype.getReceivers;
+ if (origGetReceivers) {
+ window.RTCPeerConnection.prototype.getReceivers = function getReceivers() {
+ var _this2 = this;
+ var receivers = origGetReceivers.apply(this, []);
+ receivers.forEach(function (receiver) {
+ return receiver._pc = _this2;
+ });
+ return receivers;
+ };
+ }
+ utils.wrapPeerConnectionEvent(window, 'track', function (e) {
+ e.receiver._pc = e.srcElement;
+ return e;
+ });
+ window.RTCRtpReceiver.prototype.getStats = function getStats() {
+ return this._pc.getStats(this.track);
+ };
+}
+function shimRemoveStream(window) {
+ if (!window.RTCPeerConnection || 'removeStream' in window.RTCPeerConnection.prototype) {
+ return;
+ }
+ window.RTCPeerConnection.prototype.removeStream = function removeStream(stream) {
+ var _this3 = this;
+ utils.deprecated('removeStream', 'removeTrack');
+ this.getSenders().forEach(function (sender) {
+ if (sender.track && stream.getTracks().includes(sender.track)) {
+ _this3.removeTrack(sender);
+ }
+ });
+ };
+}
+function shimRTCDataChannel(window) {
+ // rename DataChannel to RTCDataChannel (native fix in FF60):
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1173851
+ if (window.DataChannel && !window.RTCDataChannel) {
+ window.RTCDataChannel = window.DataChannel;
+ }
+}
+function shimAddTransceiver(window) {
+ // https://github.com/webrtcHacks/adapter/issues/998#issuecomment-516921647
+ // Firefox ignores the init sendEncodings options passed to addTransceiver
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1396918
+ if (!(_typeof(window) === 'object' && window.RTCPeerConnection)) {
+ return;
+ }
+ var origAddTransceiver = window.RTCPeerConnection.prototype.addTransceiver;
+ if (origAddTransceiver) {
+ window.RTCPeerConnection.prototype.addTransceiver = function addTransceiver() {
+ this.setParametersPromises = [];
+ // WebIDL input coercion and validation
+ var sendEncodings = arguments[1] && arguments[1].sendEncodings;
+ if (sendEncodings === undefined) {
+ sendEncodings = [];
+ }
+ sendEncodings = _toConsumableArray(sendEncodings);
+ var shouldPerformCheck = sendEncodings.length > 0;
+ if (shouldPerformCheck) {
+ // If sendEncodings params are provided, validate grammar
+ sendEncodings.forEach(function (encodingParam) {
+ if ('rid' in encodingParam) {
+ var ridRegex = /^[a-z0-9]{0,16}$/i;
+ if (!ridRegex.test(encodingParam.rid)) {
+ throw new TypeError('Invalid RID value provided.');
+ }
+ }
+ if ('scaleResolutionDownBy' in encodingParam) {
+ if (!(parseFloat(encodingParam.scaleResolutionDownBy) >= 1.0)) {
+ throw new RangeError('scale_resolution_down_by must be >= 1.0');
+ }
+ }
+ if ('maxFramerate' in encodingParam) {
+ if (!(parseFloat(encodingParam.maxFramerate) >= 0)) {
+ throw new RangeError('max_framerate must be >= 0.0');
+ }
+ }
+ });
+ }
+ var transceiver = origAddTransceiver.apply(this, arguments);
+ if (shouldPerformCheck) {
+ // Check if the init options were applied. If not we do this in an
+ // asynchronous way and save the promise reference in a global object.
+ // This is an ugly hack, but at the same time is way more robust than
+ // checking the sender parameters before and after the createOffer
+ // Also note that after the createoffer we are not 100% sure that
+ // the params were asynchronously applied so we might miss the
+ // opportunity to recreate offer.
+ var sender = transceiver.sender;
+ var params = sender.getParameters();
+ if (!('encodings' in params) ||
+ // Avoid being fooled by patched getParameters() below.
+ params.encodings.length === 1 && Object.keys(params.encodings[0]).length === 0) {
+ params.encodings = sendEncodings;
+ sender.sendEncodings = sendEncodings;
+ this.setParametersPromises.push(sender.setParameters(params).then(function () {
+ delete sender.sendEncodings;
+ })["catch"](function () {
+ delete sender.sendEncodings;
+ }));
+ }
+ }
+ return transceiver;
+ };
+ }
+}
+function shimGetParameters(window) {
+ if (!(_typeof(window) === 'object' && window.RTCRtpSender)) {
+ return;
+ }
+ var origGetParameters = window.RTCRtpSender.prototype.getParameters;
+ if (origGetParameters) {
+ window.RTCRtpSender.prototype.getParameters = function getParameters() {
+ var params = origGetParameters.apply(this, arguments);
+ if (!('encodings' in params)) {
+ params.encodings = [].concat(this.sendEncodings || [{}]);
+ }
+ return params;
+ };
+ }
+}
+function shimCreateOffer(window) {
+ // https://github.com/webrtcHacks/adapter/issues/998#issuecomment-516921647
+ // Firefox ignores the init sendEncodings options passed to addTransceiver
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1396918
+ if (!(_typeof(window) === 'object' && window.RTCPeerConnection)) {
+ return;
+ }
+ var origCreateOffer = window.RTCPeerConnection.prototype.createOffer;
+ window.RTCPeerConnection.prototype.createOffer = function createOffer() {
+ var _arguments2 = arguments,
+ _this4 = this;
+ if (this.setParametersPromises && this.setParametersPromises.length) {
+ return Promise.all(this.setParametersPromises).then(function () {
+ return origCreateOffer.apply(_this4, _arguments2);
+ })["finally"](function () {
+ _this4.setParametersPromises = [];
+ });
+ }
+ return origCreateOffer.apply(this, arguments);
+ };
+}
+function shimCreateAnswer(window) {
+ // https://github.com/webrtcHacks/adapter/issues/998#issuecomment-516921647
+ // Firefox ignores the init sendEncodings options passed to addTransceiver
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1396918
+ if (!(_typeof(window) === 'object' && window.RTCPeerConnection)) {
+ return;
+ }
+ var origCreateAnswer = window.RTCPeerConnection.prototype.createAnswer;
+ window.RTCPeerConnection.prototype.createAnswer = function createAnswer() {
+ var _arguments3 = arguments,
+ _this5 = this;
+ if (this.setParametersPromises && this.setParametersPromises.length) {
+ return Promise.all(this.setParametersPromises).then(function () {
+ return origCreateAnswer.apply(_this5, _arguments3);
+ })["finally"](function () {
+ _this5.setParametersPromises = [];
+ });
+ }
+ return origCreateAnswer.apply(this, arguments);
+ };
+}
+
+},{"../utils":10,"./getdisplaymedia":7,"./getusermedia":8}],7:[function(require,module,exports){
+/*
+ * Copyright (c) 2018 The adapter.js project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+/* eslint-env node */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.shimGetDisplayMedia = shimGetDisplayMedia;
+function shimGetDisplayMedia(window, preferredMediaSource) {
+ if (window.navigator.mediaDevices && 'getDisplayMedia' in window.navigator.mediaDevices) {
+ return;
+ }
+ if (!window.navigator.mediaDevices) {
+ return;
+ }
+ window.navigator.mediaDevices.getDisplayMedia = function getDisplayMedia(constraints) {
+ if (!(constraints && constraints.video)) {
+ var err = new DOMException('getDisplayMedia without video ' + 'constraints is undefined');
+ err.name = 'NotFoundError';
+ // from https://heycam.github.io/webidl/#idl-DOMException-error-names
+ err.code = 8;
+ return Promise.reject(err);
+ }
+ if (constraints.video === true) {
+ constraints.video = {
+ mediaSource: preferredMediaSource
+ };
+ } else {
+ constraints.video.mediaSource = preferredMediaSource;
+ }
+ return window.navigator.mediaDevices.getUserMedia(constraints);
+ };
+}
+
+},{}],8:[function(require,module,exports){
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+/* eslint-env node */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.shimGetUserMedia = shimGetUserMedia;
+var utils = _interopRequireWildcard(require("../utils"));
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
+function shimGetUserMedia(window, browserDetails) {
+ var navigator = window && window.navigator;
+ var MediaStreamTrack = window && window.MediaStreamTrack;
+ navigator.getUserMedia = function (constraints, onSuccess, onError) {
+ // Replace Firefox 44+'s deprecation warning with unprefixed version.
+ utils.deprecated('navigator.getUserMedia', 'navigator.mediaDevices.getUserMedia');
+ navigator.mediaDevices.getUserMedia(constraints).then(onSuccess, onError);
+ };
+ if (!(browserDetails.version > 55 && 'autoGainControl' in navigator.mediaDevices.getSupportedConstraints())) {
+ var remap = function remap(obj, a, b) {
+ if (a in obj && !(b in obj)) {
+ obj[b] = obj[a];
+ delete obj[a];
+ }
+ };
+ var nativeGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
+ navigator.mediaDevices.getUserMedia = function (c) {
+ if (_typeof(c) === 'object' && _typeof(c.audio) === 'object') {
+ c = JSON.parse(JSON.stringify(c));
+ remap(c.audio, 'autoGainControl', 'mozAutoGainControl');
+ remap(c.audio, 'noiseSuppression', 'mozNoiseSuppression');
+ }
+ return nativeGetUserMedia(c);
+ };
+ if (MediaStreamTrack && MediaStreamTrack.prototype.getSettings) {
+ var nativeGetSettings = MediaStreamTrack.prototype.getSettings;
+ MediaStreamTrack.prototype.getSettings = function () {
+ var obj = nativeGetSettings.apply(this, arguments);
+ remap(obj, 'mozAutoGainControl', 'autoGainControl');
+ remap(obj, 'mozNoiseSuppression', 'noiseSuppression');
+ return obj;
+ };
+ }
+ if (MediaStreamTrack && MediaStreamTrack.prototype.applyConstraints) {
+ var nativeApplyConstraints = MediaStreamTrack.prototype.applyConstraints;
+ MediaStreamTrack.prototype.applyConstraints = function (c) {
+ if (this.kind === 'audio' && _typeof(c) === 'object') {
+ c = JSON.parse(JSON.stringify(c));
+ remap(c, 'autoGainControl', 'mozAutoGainControl');
+ remap(c, 'noiseSuppression', 'mozNoiseSuppression');
+ }
+ return nativeApplyConstraints.apply(this, [c]);
+ };
+ }
+ }
+}
+
+},{"../utils":10}],9:[function(require,module,exports){
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.shimAudioContext = shimAudioContext;
+exports.shimCallbacksAPI = shimCallbacksAPI;
+exports.shimConstraints = shimConstraints;
+exports.shimCreateOfferLegacy = shimCreateOfferLegacy;
+exports.shimGetUserMedia = shimGetUserMedia;
+exports.shimLocalStreamsAPI = shimLocalStreamsAPI;
+exports.shimRTCIceServerUrls = shimRTCIceServerUrls;
+exports.shimRemoteStreamsAPI = shimRemoteStreamsAPI;
+exports.shimTrackEventTransceiver = shimTrackEventTransceiver;
+var utils = _interopRequireWildcard(require("../utils"));
+function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
+function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
+function shimLocalStreamsAPI(window) {
+ if (_typeof(window) !== 'object' || !window.RTCPeerConnection) {
+ return;
+ }
+ if (!('getLocalStreams' in window.RTCPeerConnection.prototype)) {
+ window.RTCPeerConnection.prototype.getLocalStreams = function getLocalStreams() {
+ if (!this._localStreams) {
+ this._localStreams = [];
+ }
+ return this._localStreams;
+ };
+ }
+ if (!('addStream' in window.RTCPeerConnection.prototype)) {
+ var _addTrack = window.RTCPeerConnection.prototype.addTrack;
+ window.RTCPeerConnection.prototype.addStream = function addStream(stream) {
+ var _this = this;
+ if (!this._localStreams) {
+ this._localStreams = [];
+ }
+ if (!this._localStreams.includes(stream)) {
+ this._localStreams.push(stream);
+ }
+ // Try to emulate Chrome's behaviour of adding in audio-video order.
+ // Safari orders by track id.
+ stream.getAudioTracks().forEach(function (track) {
+ return _addTrack.call(_this, track, stream);
+ });
+ stream.getVideoTracks().forEach(function (track) {
+ return _addTrack.call(_this, track, stream);
+ });
+ };
+ window.RTCPeerConnection.prototype.addTrack = function addTrack(track) {
+ var _this2 = this;
+ for (var _len = arguments.length, streams = new Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) {
+ streams[_key - 1] = arguments[_key];
+ }
+ if (streams) {
+ streams.forEach(function (stream) {
+ if (!_this2._localStreams) {
+ _this2._localStreams = [stream];
+ } else if (!_this2._localStreams.includes(stream)) {
+ _this2._localStreams.push(stream);
+ }
+ });
+ }
+ return _addTrack.apply(this, arguments);
+ };
+ }
+ if (!('removeStream' in window.RTCPeerConnection.prototype)) {
+ window.RTCPeerConnection.prototype.removeStream = function removeStream(stream) {
+ var _this3 = this;
+ if (!this._localStreams) {
+ this._localStreams = [];
+ }
+ var index = this._localStreams.indexOf(stream);
+ if (index === -1) {
+ return;
+ }
+ this._localStreams.splice(index, 1);
+ var tracks = stream.getTracks();
+ this.getSenders().forEach(function (sender) {
+ if (tracks.includes(sender.track)) {
+ _this3.removeTrack(sender);
+ }
+ });
+ };
+ }
+}
+function shimRemoteStreamsAPI(window) {
+ if (_typeof(window) !== 'object' || !window.RTCPeerConnection) {
+ return;
+ }
+ if (!('getRemoteStreams' in window.RTCPeerConnection.prototype)) {
+ window.RTCPeerConnection.prototype.getRemoteStreams = function getRemoteStreams() {
+ return this._remoteStreams ? this._remoteStreams : [];
+ };
+ }
+ if (!('onaddstream' in window.RTCPeerConnection.prototype)) {
+ Object.defineProperty(window.RTCPeerConnection.prototype, 'onaddstream', {
+ get: function get() {
+ return this._onaddstream;
+ },
+ set: function set(f) {
+ var _this4 = this;
+ if (this._onaddstream) {
+ this.removeEventListener('addstream', this._onaddstream);
+ this.removeEventListener('track', this._onaddstreampoly);
+ }
+ this.addEventListener('addstream', this._onaddstream = f);
+ this.addEventListener('track', this._onaddstreampoly = function (e) {
+ e.streams.forEach(function (stream) {
+ if (!_this4._remoteStreams) {
+ _this4._remoteStreams = [];
+ }
+ if (_this4._remoteStreams.includes(stream)) {
+ return;
+ }
+ _this4._remoteStreams.push(stream);
+ var event = new Event('addstream');
+ event.stream = stream;
+ _this4.dispatchEvent(event);
+ });
+ });
+ }
+ });
+ var origSetRemoteDescription = window.RTCPeerConnection.prototype.setRemoteDescription;
+ window.RTCPeerConnection.prototype.setRemoteDescription = function setRemoteDescription() {
+ var pc = this;
+ if (!this._onaddstreampoly) {
+ this.addEventListener('track', this._onaddstreampoly = function (e) {
+ e.streams.forEach(function (stream) {
+ if (!pc._remoteStreams) {
+ pc._remoteStreams = [];
+ }
+ if (pc._remoteStreams.indexOf(stream) >= 0) {
+ return;
+ }
+ pc._remoteStreams.push(stream);
+ var event = new Event('addstream');
+ event.stream = stream;
+ pc.dispatchEvent(event);
+ });
+ });
+ }
+ return origSetRemoteDescription.apply(pc, arguments);
+ };
+ }
+}
+function shimCallbacksAPI(window) {
+ if (_typeof(window) !== 'object' || !window.RTCPeerConnection) {
+ return;
+ }
+ var prototype = window.RTCPeerConnection.prototype;
+ var origCreateOffer = prototype.createOffer;
+ var origCreateAnswer = prototype.createAnswer;
+ var setLocalDescription = prototype.setLocalDescription;
+ var setRemoteDescription = prototype.setRemoteDescription;
+ var addIceCandidate = prototype.addIceCandidate;
+ prototype.createOffer = function createOffer(successCallback, failureCallback) {
+ var options = arguments.length >= 2 ? arguments[2] : arguments[0];
+ var promise = origCreateOffer.apply(this, [options]);
+ if (!failureCallback) {
+ return promise;
+ }
+ promise.then(successCallback, failureCallback);
+ return Promise.resolve();
+ };
+ prototype.createAnswer = function createAnswer(successCallback, failureCallback) {
+ var options = arguments.length >= 2 ? arguments[2] : arguments[0];
+ var promise = origCreateAnswer.apply(this, [options]);
+ if (!failureCallback) {
+ return promise;
+ }
+ promise.then(successCallback, failureCallback);
+ return Promise.resolve();
+ };
+ var withCallback = function withCallback(description, successCallback, failureCallback) {
+ var promise = setLocalDescription.apply(this, [description]);
+ if (!failureCallback) {
+ return promise;
+ }
+ promise.then(successCallback, failureCallback);
+ return Promise.resolve();
+ };
+ prototype.setLocalDescription = withCallback;
+ withCallback = function withCallback(description, successCallback, failureCallback) {
+ var promise = setRemoteDescription.apply(this, [description]);
+ if (!failureCallback) {
+ return promise;
+ }
+ promise.then(successCallback, failureCallback);
+ return Promise.resolve();
+ };
+ prototype.setRemoteDescription = withCallback;
+ withCallback = function withCallback(candidate, successCallback, failureCallback) {
+ var promise = addIceCandidate.apply(this, [candidate]);
+ if (!failureCallback) {
+ return promise;
+ }
+ promise.then(successCallback, failureCallback);
+ return Promise.resolve();
+ };
+ prototype.addIceCandidate = withCallback;
+}
+function shimGetUserMedia(window) {
+ var navigator = window && window.navigator;
+ if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+ // shim not needed in Safari 12.1
+ var mediaDevices = navigator.mediaDevices;
+ var _getUserMedia = mediaDevices.getUserMedia.bind(mediaDevices);
+ navigator.mediaDevices.getUserMedia = function (constraints) {
+ return _getUserMedia(shimConstraints(constraints));
+ };
+ }
+ if (!navigator.getUserMedia && navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+ navigator.getUserMedia = function getUserMedia(constraints, cb, errcb) {
+ navigator.mediaDevices.getUserMedia(constraints).then(cb, errcb);
+ }.bind(navigator);
+ }
+}
+function shimConstraints(constraints) {
+ if (constraints && constraints.video !== undefined) {
+ return Object.assign({}, constraints, {
+ video: utils.compactObject(constraints.video)
+ });
+ }
+ return constraints;
+}
+function shimRTCIceServerUrls(window) {
+ if (!window.RTCPeerConnection) {
+ return;
+ }
+ // migrate from non-spec RTCIceServer.url to RTCIceServer.urls
+ var OrigPeerConnection = window.RTCPeerConnection;
+ window.RTCPeerConnection = function RTCPeerConnection(pcConfig, pcConstraints) {
+ if (pcConfig && pcConfig.iceServers) {
+ var newIceServers = [];
+ for (var i = 0; i < pcConfig.iceServers.length; i++) {
+ var server = pcConfig.iceServers[i];
+ if (server.urls === undefined && server.url) {
+ utils.deprecated('RTCIceServer.url', 'RTCIceServer.urls');
+ server = JSON.parse(JSON.stringify(server));
+ server.urls = server.url;
+ delete server.url;
+ newIceServers.push(server);
+ } else {
+ newIceServers.push(pcConfig.iceServers[i]);
+ }
+ }
+ pcConfig.iceServers = newIceServers;
+ }
+ return new OrigPeerConnection(pcConfig, pcConstraints);
+ };
+ window.RTCPeerConnection.prototype = OrigPeerConnection.prototype;
+ // wrap static methods. Currently just generateCertificate.
+ if ('generateCertificate' in OrigPeerConnection) {
+ Object.defineProperty(window.RTCPeerConnection, 'generateCertificate', {
+ get: function get() {
+ return OrigPeerConnection.generateCertificate;
+ }
+ });
+ }
+}
+function shimTrackEventTransceiver(window) {
+ // Add event.transceiver member over deprecated event.receiver
+ if (_typeof(window) === 'object' && window.RTCTrackEvent && 'receiver' in window.RTCTrackEvent.prototype && !('transceiver' in window.RTCTrackEvent.prototype)) {
+ Object.defineProperty(window.RTCTrackEvent.prototype, 'transceiver', {
+ get: function get() {
+ return {
+ receiver: this.receiver
+ };
+ }
+ });
+ }
+}
+function shimCreateOfferLegacy(window) {
+ var origCreateOffer = window.RTCPeerConnection.prototype.createOffer;
+ window.RTCPeerConnection.prototype.createOffer = function createOffer(offerOptions) {
+ if (offerOptions) {
+ if (typeof offerOptions.offerToReceiveAudio !== 'undefined') {
+ // support bit values
+ offerOptions.offerToReceiveAudio = !!offerOptions.offerToReceiveAudio;
+ }
+ var audioTransceiver = this.getTransceivers().find(function (transceiver) {
+ return transceiver.receiver.track.kind === 'audio';
+ });
+ if (offerOptions.offerToReceiveAudio === false && audioTransceiver) {
+ if (audioTransceiver.direction === 'sendrecv') {
+ if (audioTransceiver.setDirection) {
+ audioTransceiver.setDirection('sendonly');
+ } else {
+ audioTransceiver.direction = 'sendonly';
+ }
+ } else if (audioTransceiver.direction === 'recvonly') {
+ if (audioTransceiver.setDirection) {
+ audioTransceiver.setDirection('inactive');
+ } else {
+ audioTransceiver.direction = 'inactive';
+ }
+ }
+ } else if (offerOptions.offerToReceiveAudio === true && !audioTransceiver) {
+ this.addTransceiver('audio', {
+ direction: 'recvonly'
+ });
+ }
+ if (typeof offerOptions.offerToReceiveVideo !== 'undefined') {
+ // support bit values
+ offerOptions.offerToReceiveVideo = !!offerOptions.offerToReceiveVideo;
+ }
+ var videoTransceiver = this.getTransceivers().find(function (transceiver) {
+ return transceiver.receiver.track.kind === 'video';
+ });
+ if (offerOptions.offerToReceiveVideo === false && videoTransceiver) {
+ if (videoTransceiver.direction === 'sendrecv') {
+ if (videoTransceiver.setDirection) {
+ videoTransceiver.setDirection('sendonly');
+ } else {
+ videoTransceiver.direction = 'sendonly';
+ }
+ } else if (videoTransceiver.direction === 'recvonly') {
+ if (videoTransceiver.setDirection) {
+ videoTransceiver.setDirection('inactive');
+ } else {
+ videoTransceiver.direction = 'inactive';
+ }
+ }
+ } else if (offerOptions.offerToReceiveVideo === true && !videoTransceiver) {
+ this.addTransceiver('video', {
+ direction: 'recvonly'
+ });
+ }
+ }
+ return origCreateOffer.apply(this, arguments);
+ };
+}
+function shimAudioContext(window) {
+ if (_typeof(window) !== 'object' || window.AudioContext) {
+ return;
+ }
+ window.AudioContext = window.webkitAudioContext;
+}
+
+},{"../utils":10}],10:[function(require,module,exports){
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree.
+ */
+/* eslint-env node */
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+ value: true
+});
+exports.compactObject = compactObject;
+exports.deprecated = deprecated;
+exports.detectBrowser = detectBrowser;
+exports.disableLog = disableLog;
+exports.disableWarnings = disableWarnings;
+exports.extractVersion = extractVersion;
+exports.filterStats = filterStats;
+exports.log = log;
+exports.walkStats = walkStats;
+exports.wrapPeerConnectionEvent = wrapPeerConnectionEvent;
+function _defineProperty(obj, key, value) { key = _toPropertyKey(key); if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
+function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
+function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
+function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
+var logDisabled_ = true;
+var deprecationWarnings_ = true;
+
+/**
+ * Extract browser version out of the provided user agent string.
+ *
+ * @param {!string} uastring userAgent string.
+ * @param {!string} expr Regular expression used as match criteria.
+ * @param {!number} pos position in the version string to be returned.
+ * @return {!number} browser version.
+ */
+function extractVersion(uastring, expr, pos) {
+ var match = uastring.match(expr);
+ return match && match.length >= pos && parseInt(match[pos], 10);
+}
+
+// Wraps the peerconnection event eventNameToWrap in a function
+// which returns the modified event object (or false to prevent
+// the event).
+function wrapPeerConnectionEvent(window, eventNameToWrap, wrapper) {
+ if (!window.RTCPeerConnection) {
+ return;
+ }
+ var proto = window.RTCPeerConnection.prototype;
+ var nativeAddEventListener = proto.addEventListener;
+ proto.addEventListener = function (nativeEventName, cb) {
+ if (nativeEventName !== eventNameToWrap) {
+ return nativeAddEventListener.apply(this, arguments);
+ }
+ var wrappedCallback = function wrappedCallback(e) {
+ var modifiedEvent = wrapper(e);
+ if (modifiedEvent) {
+ if (cb.handleEvent) {
+ cb.handleEvent(modifiedEvent);
+ } else {
+ cb(modifiedEvent);
+ }
+ }
+ };
+ this._eventMap = this._eventMap || {};
+ if (!this._eventMap[eventNameToWrap]) {
+ this._eventMap[eventNameToWrap] = new Map();
+ }
+ this._eventMap[eventNameToWrap].set(cb, wrappedCallback);
+ return nativeAddEventListener.apply(this, [nativeEventName, wrappedCallback]);
+ };
+ var nativeRemoveEventListener = proto.removeEventListener;
+ proto.removeEventListener = function (nativeEventName, cb) {
+ if (nativeEventName !== eventNameToWrap || !this._eventMap || !this._eventMap[eventNameToWrap]) {
+ return nativeRemoveEventListener.apply(this, arguments);
+ }
+ if (!this._eventMap[eventNameToWrap].has(cb)) {
+ return nativeRemoveEventListener.apply(this, arguments);
+ }
+ var unwrappedCb = this._eventMap[eventNameToWrap].get(cb);
+ this._eventMap[eventNameToWrap]["delete"](cb);
+ if (this._eventMap[eventNameToWrap].size === 0) {
+ delete this._eventMap[eventNameToWrap];
+ }
+ if (Object.keys(this._eventMap).length === 0) {
+ delete this._eventMap;
+ }
+ return nativeRemoveEventListener.apply(this, [nativeEventName, unwrappedCb]);
+ };
+ Object.defineProperty(proto, 'on' + eventNameToWrap, {
+ get: function get() {
+ return this['_on' + eventNameToWrap];
+ },
+ set: function set(cb) {
+ if (this['_on' + eventNameToWrap]) {
+ this.removeEventListener(eventNameToWrap, this['_on' + eventNameToWrap]);
+ delete this['_on' + eventNameToWrap];
+ }
+ if (cb) {
+ this.addEventListener(eventNameToWrap, this['_on' + eventNameToWrap] = cb);
+ }
+ },
+ enumerable: true,
+ configurable: true
+ });
+}
+function disableLog(bool) {
+ if (typeof bool !== 'boolean') {
+ return new Error('Argument type: ' + _typeof(bool) + '. Please use a boolean.');
+ }
+ logDisabled_ = bool;
+ return bool ? 'adapter.js logging disabled' : 'adapter.js logging enabled';
+}
+
+/**
+ * Disable or enable deprecation warnings
+ * @param {!boolean} bool set to true to disable warnings.
+ */
+function disableWarnings(bool) {
+ if (typeof bool !== 'boolean') {
+ return new Error('Argument type: ' + _typeof(bool) + '. Please use a boolean.');
+ }
+ deprecationWarnings_ = !bool;
+ return 'adapter.js deprecation warnings ' + (bool ? 'disabled' : 'enabled');
+}
+function log() {
+ if ((typeof window === "undefined" ? "undefined" : _typeof(window)) === 'object') {
+ if (logDisabled_) {
+ return;
+ }
+ if (typeof console !== 'undefined' && typeof console.log === 'function') {
+ console.log.apply(console, arguments);
+ }
+ }
+}
+
+/**
+ * Shows a deprecation warning suggesting the modern and spec-compatible API.
+ */
+function deprecated(oldMethod, newMethod) {
+ if (!deprecationWarnings_) {
+ return;
+ }
+ console.warn(oldMethod + ' is deprecated, please use ' + newMethod + ' instead.');
+}
+
+/**
+ * Browser detector.
+ *
+ * @return {object} result containing browser and version
+ * properties.
+ */
+function detectBrowser(window) {
+ // Returned result object.
+ var result = {
+ browser: null,
+ version: null
+ };
+
+ // Fail early if it's not a browser
+ if (typeof window === 'undefined' || !window.navigator || !window.navigator.userAgent) {
+ result.browser = 'Not a browser.';
+ return result;
+ }
+ var navigator = window.navigator;
+
+ // Prefer navigator.userAgentData.
+ if (navigator.userAgentData && navigator.userAgentData.brands) {
+ var chromium = navigator.userAgentData.brands.find(function (brand) {
+ return brand.brand === 'Chromium';
+ });
+ if (chromium) {
+ return {
+ browser: 'chrome',
+ version: parseInt(chromium.version, 10)
+ };
+ }
+ }
+ if (navigator.mozGetUserMedia) {
+ // Firefox.
+ result.browser = 'firefox';
+ result.version = extractVersion(navigator.userAgent, /Firefox\/(\d+)\./, 1);
+ } else if (navigator.webkitGetUserMedia || window.isSecureContext === false && window.webkitRTCPeerConnection) {
+ // Chrome, Chromium, Webview, Opera.
+ // Version matches Chrome/WebRTC version.
+ // Chrome 74 removed webkitGetUserMedia on http as well so we need the
+ // more complicated fallback to webkitRTCPeerConnection.
+ result.browser = 'chrome';
+ result.version = extractVersion(navigator.userAgent, /Chrom(e|ium)\/(\d+)\./, 2);
+ } else if (window.RTCPeerConnection && navigator.userAgent.match(/AppleWebKit\/(\d+)\./)) {
+ // Safari.
+ result.browser = 'safari';
+ result.version = extractVersion(navigator.userAgent, /AppleWebKit\/(\d+)\./, 1);
+ result.supportsUnifiedPlan = window.RTCRtpTransceiver && 'currentDirection' in window.RTCRtpTransceiver.prototype;
+ } else {
+ // Default fallthrough: not supported.
+ result.browser = 'Not a supported browser.';
+ return result;
+ }
+ return result;
+}
+
+/**
+ * Checks if something is an object.
+ *
+ * @param {*} val The something you want to check.
+ * @return true if val is an object, false otherwise.
+ */
+function isObject(val) {
+ return Object.prototype.toString.call(val) === '[object Object]';
+}
+
+/**
+ * Remove all empty objects and undefined values
+ * from a nested object -- an enhanced and vanilla version
+ * of Lodash's `compact`.
+ */
+function compactObject(data) {
+ if (!isObject(data)) {
+ return data;
+ }
+ return Object.keys(data).reduce(function (accumulator, key) {
+ var isObj = isObject(data[key]);
+ var value = isObj ? compactObject(data[key]) : data[key];
+ var isEmptyObject = isObj && !Object.keys(value).length;
+ if (value === undefined || isEmptyObject) {
+ return accumulator;
+ }
+ return Object.assign(accumulator, _defineProperty({}, key, value));
+ }, {});
+}
+
+/* iterates the stats graph recursively. */
+function walkStats(stats, base, resultSet) {
+ if (!base || resultSet.has(base.id)) {
+ return;
+ }
+ resultSet.set(base.id, base);
+ Object.keys(base).forEach(function (name) {
+ if (name.endsWith('Id')) {
+ walkStats(stats, stats.get(base[name]), resultSet);
+ } else if (name.endsWith('Ids')) {
+ base[name].forEach(function (id) {
+ walkStats(stats, stats.get(id), resultSet);
+ });
+ }
+ });
+}
+
+/* filter getStats for a sender/receiver track. */
+function filterStats(result, track, outbound) {
+ var streamStatsType = outbound ? 'outbound-rtp' : 'inbound-rtp';
+ var filteredResult = new Map();
+ if (track === null) {
+ return filteredResult;
+ }
+ var trackStats = [];
+ result.forEach(function (value) {
+ if (value.type === 'track' && value.trackIdentifier === track.id) {
+ trackStats.push(value);
+ }
+ });
+ trackStats.forEach(function (trackStat) {
+ result.forEach(function (stats) {
+ if (stats.type === streamStatsType && stats.trackId === trackStat.id) {
+ walkStats(result, stats, filteredResult);
+ }
+ });
+ });
+ return filteredResult;
+}
+
+},{}],11:[function(require,module,exports){
+/* eslint-env node */
+'use strict';
+
+// SDP helpers.
+
+var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; };
+
+var SDPUtils = {};
+
+// Generate an alphanumeric identifier for cname or mids.
+// TODO: use UUIDs instead? https://gist.github.com/jed/982883
+SDPUtils.generateIdentifier = function () {
+ return Math.random().toString(36).substring(2, 12);
+};
+
+// The RTCP CNAME used by all peerconnections from the same JS.
+SDPUtils.localCName = SDPUtils.generateIdentifier();
+
+// Splits SDP into lines, dealing with both CRLF and LF.
+SDPUtils.splitLines = function (blob) {
+ return blob.trim().split('\n').map(function (line) {
+ return line.trim();
+ });
+};
+// Splits SDP into sessionpart and mediasections. Ensures CRLF.
+SDPUtils.splitSections = function (blob) {
+ var parts = blob.split('\nm=');
+ return parts.map(function (part, index) {
+ return (index > 0 ? 'm=' + part : part).trim() + '\r\n';
+ });
+};
+
+// Returns the session description.
+SDPUtils.getDescription = function (blob) {
+ var sections = SDPUtils.splitSections(blob);
+ return sections && sections[0];
+};
+
+// Returns the individual media sections.
+SDPUtils.getMediaSections = function (blob) {
+ var sections = SDPUtils.splitSections(blob);
+ sections.shift();
+ return sections;
+};
+
+// Returns lines that start with a certain prefix.
+SDPUtils.matchPrefix = function (blob, prefix) {
+ return SDPUtils.splitLines(blob).filter(function (line) {
+ return line.indexOf(prefix) === 0;
+ });
+};
+
+// Parses an ICE candidate line. Sample input:
+// candidate:702786350 2 udp 41819902 8.8.8.8 60769 typ relay raddr 8.8.8.8
+// rport 55996"
+// Input can be prefixed with a=.
+SDPUtils.parseCandidate = function (line) {
+ var parts = void 0;
+ // Parse both variants.
+ if (line.indexOf('a=candidate:') === 0) {
+ parts = line.substring(12).split(' ');
+ } else {
+ parts = line.substring(10).split(' ');
+ }
+
+ var candidate = {
+ foundation: parts[0],
+ component: { 1: 'rtp', 2: 'rtcp' }[parts[1]] || parts[1],
+ protocol: parts[2].toLowerCase(),
+ priority: parseInt(parts[3], 10),
+ ip: parts[4],
+ address: parts[4], // address is an alias for ip.
+ port: parseInt(parts[5], 10),
+ // skip parts[6] == 'typ'
+ type: parts[7]
+ };
+
+ for (var i = 8; i < parts.length; i += 2) {
+ switch (parts[i]) {
+ case 'raddr':
+ candidate.relatedAddress = parts[i + 1];
+ break;
+ case 'rport':
+ candidate.relatedPort = parseInt(parts[i + 1], 10);
+ break;
+ case 'tcptype':
+ candidate.tcpType = parts[i + 1];
+ break;
+ case 'ufrag':
+ candidate.ufrag = parts[i + 1]; // for backward compatibility.
+ candidate.usernameFragment = parts[i + 1];
+ break;
+ default:
+ // extension handling, in particular ufrag. Don't overwrite.
+ if (candidate[parts[i]] === undefined) {
+ candidate[parts[i]] = parts[i + 1];
+ }
+ break;
+ }
+ }
+ return candidate;
+};
+
+// Translates a candidate object into SDP candidate attribute.
+// This does not include the a= prefix!
+SDPUtils.writeCandidate = function (candidate) {
+ var sdp = [];
+ sdp.push(candidate.foundation);
+
+ var component = candidate.component;
+ if (component === 'rtp') {
+ sdp.push(1);
+ } else if (component === 'rtcp') {
+ sdp.push(2);
+ } else {
+ sdp.push(component);
+ }
+ sdp.push(candidate.protocol.toUpperCase());
+ sdp.push(candidate.priority);
+ sdp.push(candidate.address || candidate.ip);
+ sdp.push(candidate.port);
+
+ var type = candidate.type;
+ sdp.push('typ');
+ sdp.push(type);
+ if (type !== 'host' && candidate.relatedAddress && candidate.relatedPort) {
+ sdp.push('raddr');
+ sdp.push(candidate.relatedAddress);
+ sdp.push('rport');
+ sdp.push(candidate.relatedPort);
+ }
+ if (candidate.tcpType && candidate.protocol.toLowerCase() === 'tcp') {
+ sdp.push('tcptype');
+ sdp.push(candidate.tcpType);
+ }
+ if (candidate.usernameFragment || candidate.ufrag) {
+ sdp.push('ufrag');
+ sdp.push(candidate.usernameFragment || candidate.ufrag);
+ }
+ return 'candidate:' + sdp.join(' ');
+};
+
+// Parses an ice-options line, returns an array of option tags.
+// Sample input:
+// a=ice-options:foo bar
+SDPUtils.parseIceOptions = function (line) {
+ return line.substring(14).split(' ');
+};
+
+// Parses a rtpmap line, returns RTCRtpCoddecParameters. Sample input:
+// a=rtpmap:111 opus/48000/2
+SDPUtils.parseRtpMap = function (line) {
+ var parts = line.substring(9).split(' ');
+ var parsed = {
+ payloadType: parseInt(parts.shift(), 10) // was: id
+ };
+
+ parts = parts[0].split('/');
+
+ parsed.name = parts[0];
+ parsed.clockRate = parseInt(parts[1], 10); // was: clockrate
+ parsed.channels = parts.length === 3 ? parseInt(parts[2], 10) : 1;
+ // legacy alias, got renamed back to channels in ORTC.
+ parsed.numChannels = parsed.channels;
+ return parsed;
+};
+
+// Generates a rtpmap line from RTCRtpCodecCapability or
+// RTCRtpCodecParameters.
+SDPUtils.writeRtpMap = function (codec) {
+ var pt = codec.payloadType;
+ if (codec.preferredPayloadType !== undefined) {
+ pt = codec.preferredPayloadType;
+ }
+ var channels = codec.channels || codec.numChannels || 1;
+ return 'a=rtpmap:' + pt + ' ' + codec.name + '/' + codec.clockRate + (channels !== 1 ? '/' + channels : '') + '\r\n';
+};
+
+// Parses a extmap line (headerextension from RFC 5285). Sample input:
+// a=extmap:2 urn:ietf:params:rtp-hdrext:toffset
+// a=extmap:2/sendonly urn:ietf:params:rtp-hdrext:toffset
+SDPUtils.parseExtmap = function (line) {
+ var parts = line.substring(9).split(' ');
+ return {
+ id: parseInt(parts[0], 10),
+ direction: parts[0].indexOf('/') > 0 ? parts[0].split('/')[1] : 'sendrecv',
+ uri: parts[1],
+ attributes: parts.slice(2).join(' ')
+ };
+};
+
+// Generates an extmap line from RTCRtpHeaderExtensionParameters or
+// RTCRtpHeaderExtension.
+SDPUtils.writeExtmap = function (headerExtension) {
+ return 'a=extmap:' + (headerExtension.id || headerExtension.preferredId) + (headerExtension.direction && headerExtension.direction !== 'sendrecv' ? '/' + headerExtension.direction : '') + ' ' + headerExtension.uri + (headerExtension.attributes ? ' ' + headerExtension.attributes : '') + '\r\n';
+};
+
+// Parses a fmtp line, returns dictionary. Sample input:
+// a=fmtp:96 vbr=on;cng=on
+// Also deals with vbr=on; cng=on
+SDPUtils.parseFmtp = function (line) {
+ var parsed = {};
+ var kv = void 0;
+ var parts = line.substring(line.indexOf(' ') + 1).split(';');
+ for (var j = 0; j < parts.length; j++) {
+ kv = parts[j].trim().split('=');
+ parsed[kv[0].trim()] = kv[1];
+ }
+ return parsed;
+};
+
+// Generates a fmtp line from RTCRtpCodecCapability or RTCRtpCodecParameters.
+SDPUtils.writeFmtp = function (codec) {
+ var line = '';
+ var pt = codec.payloadType;
+ if (codec.preferredPayloadType !== undefined) {
+ pt = codec.preferredPayloadType;
+ }
+ if (codec.parameters && Object.keys(codec.parameters).length) {
+ var params = [];
+ Object.keys(codec.parameters).forEach(function (param) {
+ if (codec.parameters[param] !== undefined) {
+ params.push(param + '=' + codec.parameters[param]);
+ } else {
+ params.push(param);
+ }
+ });
+ line += 'a=fmtp:' + pt + ' ' + params.join(';') + '\r\n';
+ }
+ return line;
+};
+
+// Parses a rtcp-fb line, returns RTCPRtcpFeedback object. Sample input:
+// a=rtcp-fb:98 nack rpsi
+SDPUtils.parseRtcpFb = function (line) {
+ var parts = line.substring(line.indexOf(' ') + 1).split(' ');
+ return {
+ type: parts.shift(),
+ parameter: parts.join(' ')
+ };
+};
+
+// Generate a=rtcp-fb lines from RTCRtpCodecCapability or RTCRtpCodecParameters.
+SDPUtils.writeRtcpFb = function (codec) {
+ var lines = '';
+ var pt = codec.payloadType;
+ if (codec.preferredPayloadType !== undefined) {
+ pt = codec.preferredPayloadType;
+ }
+ if (codec.rtcpFeedback && codec.rtcpFeedback.length) {
+ // FIXME: special handling for trr-int?
+ codec.rtcpFeedback.forEach(function (fb) {
+ lines += 'a=rtcp-fb:' + pt + ' ' + fb.type + (fb.parameter && fb.parameter.length ? ' ' + fb.parameter : '') + '\r\n';
+ });
+ }
+ return lines;
+};
+
+// Parses a RFC 5576 ssrc media attribute. Sample input:
+// a=ssrc:3735928559 cname:something
+SDPUtils.parseSsrcMedia = function (line) {
+ var sp = line.indexOf(' ');
+ var parts = {
+ ssrc: parseInt(line.substring(7, sp), 10)
+ };
+ var colon = line.indexOf(':', sp);
+ if (colon > -1) {
+ parts.attribute = line.substring(sp + 1, colon);
+ parts.value = line.substring(colon + 1);
+ } else {
+ parts.attribute = line.substring(sp + 1);
+ }
+ return parts;
+};
+
+// Parse a ssrc-group line (see RFC 5576). Sample input:
+// a=ssrc-group:semantics 12 34
+SDPUtils.parseSsrcGroup = function (line) {
+ var parts = line.substring(13).split(' ');
+ return {
+ semantics: parts.shift(),
+ ssrcs: parts.map(function (ssrc) {
+ return parseInt(ssrc, 10);
+ })
+ };
+};
+
+// Extracts the MID (RFC 5888) from a media section.
+// Returns the MID or undefined if no mid line was found.
+SDPUtils.getMid = function (mediaSection) {
+ var mid = SDPUtils.matchPrefix(mediaSection, 'a=mid:')[0];
+ if (mid) {
+ return mid.substring(6);
+ }
+};
+
+// Parses a fingerprint line for DTLS-SRTP.
+SDPUtils.parseFingerprint = function (line) {
+ var parts = line.substring(14).split(' ');
+ return {
+ algorithm: parts[0].toLowerCase(), // algorithm is case-sensitive in Edge.
+ value: parts[1].toUpperCase() // the definition is upper-case in RFC 4572.
+ };
+};
+
+// Extracts DTLS parameters from SDP media section or sessionpart.
+// FIXME: for consistency with other functions this should only
+// get the fingerprint line as input. See also getIceParameters.
+SDPUtils.getDtlsParameters = function (mediaSection, sessionpart) {
+ var lines = SDPUtils.matchPrefix(mediaSection + sessionpart, 'a=fingerprint:');
+ // Note: a=setup line is ignored since we use the 'auto' role in Edge.
+ return {
+ role: 'auto',
+ fingerprints: lines.map(SDPUtils.parseFingerprint)
+ };
+};
+
+// Serializes DTLS parameters to SDP.
+SDPUtils.writeDtlsParameters = function (params, setupType) {
+ var sdp = 'a=setup:' + setupType + '\r\n';
+ params.fingerprints.forEach(function (fp) {
+ sdp += 'a=fingerprint:' + fp.algorithm + ' ' + fp.value + '\r\n';
+ });
+ return sdp;
+};
+
+// Parses a=crypto lines into
+// https://rawgit.com/aboba/edgertc/master/msortc-rs4.html#dictionary-rtcsrtpsdesparameters-members
+SDPUtils.parseCryptoLine = function (line) {
+ var parts = line.substring(9).split(' ');
+ return {
+ tag: parseInt(parts[0], 10),
+ cryptoSuite: parts[1],
+ keyParams: parts[2],
+ sessionParams: parts.slice(3)
+ };
+};
+
+SDPUtils.writeCryptoLine = function (parameters) {
+ return 'a=crypto:' + parameters.tag + ' ' + parameters.cryptoSuite + ' ' + (_typeof(parameters.keyParams) === 'object' ? SDPUtils.writeCryptoKeyParams(parameters.keyParams) : parameters.keyParams) + (parameters.sessionParams ? ' ' + parameters.sessionParams.join(' ') : '') + '\r\n';
+};
+
+// Parses the crypto key parameters into
+// https://rawgit.com/aboba/edgertc/master/msortc-rs4.html#rtcsrtpkeyparam*
+SDPUtils.parseCryptoKeyParams = function (keyParams) {
+ if (keyParams.indexOf('inline:') !== 0) {
+ return null;
+ }
+ var parts = keyParams.substring(7).split('|');
+ return {
+ keyMethod: 'inline',
+ keySalt: parts[0],
+ lifeTime: parts[1],
+ mkiValue: parts[2] ? parts[2].split(':')[0] : undefined,
+ mkiLength: parts[2] ? parts[2].split(':')[1] : undefined
+ };
+};
+
+SDPUtils.writeCryptoKeyParams = function (keyParams) {
+ return keyParams.keyMethod + ':' + keyParams.keySalt + (keyParams.lifeTime ? '|' + keyParams.lifeTime : '') + (keyParams.mkiValue && keyParams.mkiLength ? '|' + keyParams.mkiValue + ':' + keyParams.mkiLength : '');
+};
+
+// Extracts all SDES parameters.
+SDPUtils.getCryptoParameters = function (mediaSection, sessionpart) {
+ var lines = SDPUtils.matchPrefix(mediaSection + sessionpart, 'a=crypto:');
+ return lines.map(SDPUtils.parseCryptoLine);
+};
+
+// Parses ICE information from SDP media section or sessionpart.
+// FIXME: for consistency with other functions this should only
+// get the ice-ufrag and ice-pwd lines as input.
+SDPUtils.getIceParameters = function (mediaSection, sessionpart) {
+ var ufrag = SDPUtils.matchPrefix(mediaSection + sessionpart, 'a=ice-ufrag:')[0];
+ var pwd = SDPUtils.matchPrefix(mediaSection + sessionpart, 'a=ice-pwd:')[0];
+ if (!(ufrag && pwd)) {
+ return null;
+ }
+ return {
+ usernameFragment: ufrag.substring(12),
+ password: pwd.substring(10)
+ };
+};
+
+// Serializes ICE parameters to SDP.
+SDPUtils.writeIceParameters = function (params) {
+ var sdp = 'a=ice-ufrag:' + params.usernameFragment + '\r\n' + 'a=ice-pwd:' + params.password + '\r\n';
+ if (params.iceLite) {
+ sdp += 'a=ice-lite\r\n';
+ }
+ return sdp;
+};
+
+// Parses the SDP media section and returns RTCRtpParameters.
+SDPUtils.parseRtpParameters = function (mediaSection) {
+ var description = {
+ codecs: [],
+ headerExtensions: [],
+ fecMechanisms: [],
+ rtcp: []
+ };
+ var lines = SDPUtils.splitLines(mediaSection);
+ var mline = lines[0].split(' ');
+ description.profile = mline[2];
+ for (var i = 3; i < mline.length; i++) {
+ // find all codecs from mline[3..]
+ var pt = mline[i];
+ var rtpmapline = SDPUtils.matchPrefix(mediaSection, 'a=rtpmap:' + pt + ' ')[0];
+ if (rtpmapline) {
+ var codec = SDPUtils.parseRtpMap(rtpmapline);
+ var fmtps = SDPUtils.matchPrefix(mediaSection, 'a=fmtp:' + pt + ' ');
+ // Only the first a=fmtp: is considered.
+ codec.parameters = fmtps.length ? SDPUtils.parseFmtp(fmtps[0]) : {};
+ codec.rtcpFeedback = SDPUtils.matchPrefix(mediaSection, 'a=rtcp-fb:' + pt + ' ').map(SDPUtils.parseRtcpFb);
+ description.codecs.push(codec);
+ // parse FEC mechanisms from rtpmap lines.
+ switch (codec.name.toUpperCase()) {
+ case 'RED':
+ case 'ULPFEC':
+ description.fecMechanisms.push(codec.name.toUpperCase());
+ break;
+ default:
+ // only RED and ULPFEC are recognized as FEC mechanisms.
+ break;
+ }
+ }
+ }
+ SDPUtils.matchPrefix(mediaSection, 'a=extmap:').forEach(function (line) {
+ description.headerExtensions.push(SDPUtils.parseExtmap(line));
+ });
+ var wildcardRtcpFb = SDPUtils.matchPrefix(mediaSection, 'a=rtcp-fb:* ').map(SDPUtils.parseRtcpFb);
+ description.codecs.forEach(function (codec) {
+ wildcardRtcpFb.forEach(function (fb) {
+ var duplicate = codec.rtcpFeedback.find(function (existingFeedback) {
+ return existingFeedback.type === fb.type && existingFeedback.parameter === fb.parameter;
+ });
+ if (!duplicate) {
+ codec.rtcpFeedback.push(fb);
+ }
+ });
+ });
+ // FIXME: parse rtcp.
+ return description;
+};
+
+// Generates parts of the SDP media section describing the capabilities /
+// parameters.
+SDPUtils.writeRtpDescription = function (kind, caps) {
+ var sdp = '';
+
+ // Build the mline.
+ sdp += 'm=' + kind + ' ';
+ sdp += caps.codecs.length > 0 ? '9' : '0'; // reject if no codecs.
+ sdp += ' ' + (caps.profile || 'UDP/TLS/RTP/SAVPF') + ' ';
+ sdp += caps.codecs.map(function (codec) {
+ if (codec.preferredPayloadType !== undefined) {
+ return codec.preferredPayloadType;
+ }
+ return codec.payloadType;
+ }).join(' ') + '\r\n';
+
+ sdp += 'c=IN IP4 0.0.0.0\r\n';
+ sdp += 'a=rtcp:9 IN IP4 0.0.0.0\r\n';
+
+ // Add a=rtpmap lines for each codec. Also fmtp and rtcp-fb.
+ caps.codecs.forEach(function (codec) {
+ sdp += SDPUtils.writeRtpMap(codec);
+ sdp += SDPUtils.writeFmtp(codec);
+ sdp += SDPUtils.writeRtcpFb(codec);
+ });
+ var maxptime = 0;
+ caps.codecs.forEach(function (codec) {
+ if (codec.maxptime > maxptime) {
+ maxptime = codec.maxptime;
+ }
+ });
+ if (maxptime > 0) {
+ sdp += 'a=maxptime:' + maxptime + '\r\n';
+ }
+
+ if (caps.headerExtensions) {
+ caps.headerExtensions.forEach(function (extension) {
+ sdp += SDPUtils.writeExtmap(extension);
+ });
+ }
+ // FIXME: write fecMechanisms.
+ return sdp;
+};
+
+// Parses the SDP media section and returns an array of
+// RTCRtpEncodingParameters.
+SDPUtils.parseRtpEncodingParameters = function (mediaSection) {
+ var encodingParameters = [];
+ var description = SDPUtils.parseRtpParameters(mediaSection);
+ var hasRed = description.fecMechanisms.indexOf('RED') !== -1;
+ var hasUlpfec = description.fecMechanisms.indexOf('ULPFEC') !== -1;
+
+ // filter a=ssrc:... cname:, ignore PlanB-msid
+ var ssrcs = SDPUtils.matchPrefix(mediaSection, 'a=ssrc:').map(function (line) {
+ return SDPUtils.parseSsrcMedia(line);
+ }).filter(function (parts) {
+ return parts.attribute === 'cname';
+ });
+ var primarySsrc = ssrcs.length > 0 && ssrcs[0].ssrc;
+ var secondarySsrc = void 0;
+
+ var flows = SDPUtils.matchPrefix(mediaSection, 'a=ssrc-group:FID').map(function (line) {
+ var parts = line.substring(17).split(' ');
+ return parts.map(function (part) {
+ return parseInt(part, 10);
+ });
+ });
+ if (flows.length > 0 && flows[0].length > 1 && flows[0][0] === primarySsrc) {
+ secondarySsrc = flows[0][1];
+ }
+
+ description.codecs.forEach(function (codec) {
+ if (codec.name.toUpperCase() === 'RTX' && codec.parameters.apt) {
+ var encParam = {
+ ssrc: primarySsrc,
+ codecPayloadType: parseInt(codec.parameters.apt, 10)
+ };
+ if (primarySsrc && secondarySsrc) {
+ encParam.rtx = { ssrc: secondarySsrc };
+ }
+ encodingParameters.push(encParam);
+ if (hasRed) {
+ encParam = JSON.parse(JSON.stringify(encParam));
+ encParam.fec = {
+ ssrc: primarySsrc,
+ mechanism: hasUlpfec ? 'red+ulpfec' : 'red'
+ };
+ encodingParameters.push(encParam);
+ }
+ }
+ });
+ if (encodingParameters.length === 0 && primarySsrc) {
+ encodingParameters.push({
+ ssrc: primarySsrc
+ });
+ }
+
+ // we support both b=AS and b=TIAS but interpret AS as TIAS.
+ var bandwidth = SDPUtils.matchPrefix(mediaSection, 'b=');
+ if (bandwidth.length) {
+ if (bandwidth[0].indexOf('b=TIAS:') === 0) {
+ bandwidth = parseInt(bandwidth[0].substring(7), 10);
+ } else if (bandwidth[0].indexOf('b=AS:') === 0) {
+ // use formula from JSEP to convert b=AS to TIAS value.
+ bandwidth = parseInt(bandwidth[0].substring(5), 10) * 1000 * 0.95 - 50 * 40 * 8;
+ } else {
+ bandwidth = undefined;
+ }
+ encodingParameters.forEach(function (params) {
+ params.maxBitrate = bandwidth;
+ });
+ }
+ return encodingParameters;
+};
+
+// parses http://draft.ortc.org/#rtcrtcpparameters*
+SDPUtils.parseRtcpParameters = function (mediaSection) {
+ var rtcpParameters = {};
+
+ // Gets the first SSRC. Note that with RTX there might be multiple
+ // SSRCs.
+ var remoteSsrc = SDPUtils.matchPrefix(mediaSection, 'a=ssrc:').map(function (line) {
+ return SDPUtils.parseSsrcMedia(line);
+ }).filter(function (obj) {
+ return obj.attribute === 'cname';
+ })[0];
+ if (remoteSsrc) {
+ rtcpParameters.cname = remoteSsrc.value;
+ rtcpParameters.ssrc = remoteSsrc.ssrc;
+ }
+
+ // Edge uses the compound attribute instead of reducedSize
+ // compound is !reducedSize
+ var rsize = SDPUtils.matchPrefix(mediaSection, 'a=rtcp-rsize');
+ rtcpParameters.reducedSize = rsize.length > 0;
+ rtcpParameters.compound = rsize.length === 0;
+
+ // parses the rtcp-mux attrіbute.
+ // Note that Edge does not support unmuxed RTCP.
+ var mux = SDPUtils.matchPrefix(mediaSection, 'a=rtcp-mux');
+ rtcpParameters.mux = mux.length > 0;
+
+ return rtcpParameters;
+};
+
+SDPUtils.writeRtcpParameters = function (rtcpParameters) {
+ var sdp = '';
+ if (rtcpParameters.reducedSize) {
+ sdp += 'a=rtcp-rsize\r\n';
+ }
+ if (rtcpParameters.mux) {
+ sdp += 'a=rtcp-mux\r\n';
+ }
+ if (rtcpParameters.ssrc !== undefined && rtcpParameters.cname) {
+ sdp += 'a=ssrc:' + rtcpParameters.ssrc + ' cname:' + rtcpParameters.cname + '\r\n';
+ }
+ return sdp;
+};
+
+// parses either a=msid: or a=ssrc:... msid lines and returns
+// the id of the MediaStream and MediaStreamTrack.
+SDPUtils.parseMsid = function (mediaSection) {
+ var parts = void 0;
+ var spec = SDPUtils.matchPrefix(mediaSection, 'a=msid:');
+ if (spec.length === 1) {
+ parts = spec[0].substring(7).split(' ');
+ return { stream: parts[0], track: parts[1] };
+ }
+ var planB = SDPUtils.matchPrefix(mediaSection, 'a=ssrc:').map(function (line) {
+ return SDPUtils.parseSsrcMedia(line);
+ }).filter(function (msidParts) {
+ return msidParts.attribute === 'msid';
+ });
+ if (planB.length > 0) {
+ parts = planB[0].value.split(' ');
+ return { stream: parts[0], track: parts[1] };
+ }
+};
+
+// SCTP
+// parses draft-ietf-mmusic-sctp-sdp-26 first and falls back
+// to draft-ietf-mmusic-sctp-sdp-05
+SDPUtils.parseSctpDescription = function (mediaSection) {
+ var mline = SDPUtils.parseMLine(mediaSection);
+ var maxSizeLine = SDPUtils.matchPrefix(mediaSection, 'a=max-message-size:');
+ var maxMessageSize = void 0;
+ if (maxSizeLine.length > 0) {
+ maxMessageSize = parseInt(maxSizeLine[0].substring(19), 10);
+ }
+ if (isNaN(maxMessageSize)) {
+ maxMessageSize = 65536;
+ }
+ var sctpPort = SDPUtils.matchPrefix(mediaSection, 'a=sctp-port:');
+ if (sctpPort.length > 0) {
+ return {
+ port: parseInt(sctpPort[0].substring(12), 10),
+ protocol: mline.fmt,
+ maxMessageSize: maxMessageSize
+ };
+ }
+ var sctpMapLines = SDPUtils.matchPrefix(mediaSection, 'a=sctpmap:');
+ if (sctpMapLines.length > 0) {
+ var parts = sctpMapLines[0].substring(10).split(' ');
+ return {
+ port: parseInt(parts[0], 10),
+ protocol: parts[1],
+ maxMessageSize: maxMessageSize
+ };
+ }
+};
+
+// SCTP
+// outputs the draft-ietf-mmusic-sctp-sdp-26 version that all browsers
+// support by now receiving in this format, unless we originally parsed
+// as the draft-ietf-mmusic-sctp-sdp-05 format (indicated by the m-line
+// protocol of DTLS/SCTP -- without UDP/ or TCP/)
+SDPUtils.writeSctpDescription = function (media, sctp) {
+ var output = [];
+ if (media.protocol !== 'DTLS/SCTP') {
+ output = ['m=' + media.kind + ' 9 ' + media.protocol + ' ' + sctp.protocol + '\r\n', 'c=IN IP4 0.0.0.0\r\n', 'a=sctp-port:' + sctp.port + '\r\n'];
+ } else {
+ output = ['m=' + media.kind + ' 9 ' + media.protocol + ' ' + sctp.port + '\r\n', 'c=IN IP4 0.0.0.0\r\n', 'a=sctpmap:' + sctp.port + ' ' + sctp.protocol + ' 65535\r\n'];
+ }
+ if (sctp.maxMessageSize !== undefined) {
+ output.push('a=max-message-size:' + sctp.maxMessageSize + '\r\n');
+ }
+ return output.join('');
+};
+
+// Generate a session ID for SDP.
+// https://tools.ietf.org/html/draft-ietf-rtcweb-jsep-20#section-5.2.1
+// recommends using a cryptographically random +ve 64-bit value
+// but right now this should be acceptable and within the right range
+SDPUtils.generateSessionId = function () {
+ return Math.random().toString().substr(2, 22);
+};
+
+// Write boiler plate for start of SDP
+// sessId argument is optional - if not supplied it will
+// be generated randomly
+// sessVersion is optional and defaults to 2
+// sessUser is optional and defaults to 'thisisadapterortc'
+SDPUtils.writeSessionBoilerplate = function (sessId, sessVer, sessUser) {
+ var sessionId = void 0;
+ var version = sessVer !== undefined ? sessVer : 2;
+ if (sessId) {
+ sessionId = sessId;
+ } else {
+ sessionId = SDPUtils.generateSessionId();
+ }
+ var user = sessUser || 'thisisadapterortc';
+ // FIXME: sess-id should be an NTP timestamp.
+ return 'v=0\r\n' + 'o=' + user + ' ' + sessionId + ' ' + version + ' IN IP4 127.0.0.1\r\n' + 's=-\r\n' + 't=0 0\r\n';
+};
+
+// Gets the direction from the mediaSection or the sessionpart.
+SDPUtils.getDirection = function (mediaSection, sessionpart) {
+ // Look for sendrecv, sendonly, recvonly, inactive, default to sendrecv.
+ var lines = SDPUtils.splitLines(mediaSection);
+ for (var i = 0; i < lines.length; i++) {
+ switch (lines[i]) {
+ case 'a=sendrecv':
+ case 'a=sendonly':
+ case 'a=recvonly':
+ case 'a=inactive':
+ return lines[i].substring(2);
+ default:
+ // FIXME: What should happen here?
+ }
+ }
+ if (sessionpart) {
+ return SDPUtils.getDirection(sessionpart);
+ }
+ return 'sendrecv';
+};
+
+SDPUtils.getKind = function (mediaSection) {
+ var lines = SDPUtils.splitLines(mediaSection);
+ var mline = lines[0].split(' ');
+ return mline[0].substring(2);
+};
+
+SDPUtils.isRejected = function (mediaSection) {
+ return mediaSection.split(' ', 2)[1] === '0';
+};
+
+SDPUtils.parseMLine = function (mediaSection) {
+ var lines = SDPUtils.splitLines(mediaSection);
+ var parts = lines[0].substring(2).split(' ');
+ return {
+ kind: parts[0],
+ port: parseInt(parts[1], 10),
+ protocol: parts[2],
+ fmt: parts.slice(3).join(' ')
+ };
+};
+
+SDPUtils.parseOLine = function (mediaSection) {
+ var line = SDPUtils.matchPrefix(mediaSection, 'o=')[0];
+ var parts = line.substring(2).split(' ');
+ return {
+ username: parts[0],
+ sessionId: parts[1],
+ sessionVersion: parseInt(parts[2], 10),
+ netType: parts[3],
+ addressType: parts[4],
+ address: parts[5]
+ };
+};
+
+// a very naive interpretation of a valid SDP.
+SDPUtils.isValidSDP = function (blob) {
+ if (typeof blob !== 'string' || blob.length === 0) {
+ return false;
+ }
+ var lines = SDPUtils.splitLines(blob);
+ for (var i = 0; i < lines.length; i++) {
+ if (lines[i].length < 2 || lines[i].charAt(1) !== '=') {
+ return false;
+ }
+ // TODO: check the modifier a bit more.
+ }
+ return true;
+};
+
+// Expose public methods.
+if ((typeof module === 'undefined' ? 'undefined' : _typeof(module)) === 'object') {
+ module.exports = SDPUtils;
+}
+},{}]},{},[1])(1)
+});
diff --git a/content_styles/calendar.css b/content_styles/calendar.css
new file mode 100644
index 00000000..89e63a68
--- /dev/null
+++ b/content_styles/calendar.css
@@ -0,0 +1,101 @@
+.calendar {
+ position: absolute;
+ left: 0;
+ top: 0;
+ width: 300px;
+ height: 300px;
+ background-color: rgba(0,0,0,0.7);
+ border-radius: 10px;
+ color: white;
+ font-family: "Avenir", "Futura", Helvetica Neue, Helvetica, Arial, sans-serif;
+ transform-origin: left bottom;
+ transform: scale(0.8);
+}
+.calendar > div {
+ position: absolute;
+ user-select: none;
+ -webkit-user-select: none;
+}
+#calHeader {
+ left: 0;
+ top: 0;
+ width: 100%;
+ height: 30px;
+}
+#calHeader > div {
+ position: absolute;
+ text-align: center;
+ line-height: 30px;
+ vertical-align: middle;
+ border-radius: 15px;
+ user-select: none;
+ -webkit-user-select: none;
+}
+#calPrevMonth {
+ left: 0;
+ top: 0;
+ width: 30px;
+ height: 30px;
+ cursor: pointer;
+}
+#calNextMonth {
+ right: 0;
+ top: 0;
+ width: 30px;
+ height: 30px;
+ cursor: pointer;
+}
+#calNextMonth:hover, #calPrevMonth:hover {
+ background-color: rgba(255,255,255,0.3);
+}
+#calMonthName {
+ left: 90px;
+ top: 0;
+ width: calc(100% - 180px);
+ height: 100%;
+ cursor: pointer;
+}
+#calLabels {
+ left: 0;
+ top: 30px;
+ width: 100%;
+ height: 30px;
+}
+#calLabels > div {
+ position: absolute;
+ top: 0;
+ height: 30px;
+ text-align: center;
+ line-height: 30px;
+ vertical-align: middle;
+ color: gray;
+ border-radius: 10px;
+}
+#calDates {
+ left: 0;
+ top: 60px;
+ width: 100%;
+ height: 240px;
+}
+.calDate {
+ position: absolute;
+ text-align: center;
+ vertical-align: middle;
+ cursor: pointer;
+ user-select: none;
+ -webkit-user-select: none;
+}
+.highlightedDate {
+ background-color: rgba(255,255,255,0.1);
+}
+.calDate:hover {
+ background-color: rgba(255,255,255,0.3);
+}
+.otherMonthDate {
+ color: gray;
+}
+.selectedDate {
+ color: yellow;
+ background-color: rgba(255,255,255, 0.2) !important;
+ cursor: default;
+}
diff --git a/content_styles/remoteOperator.css b/content_styles/remoteOperator.css
new file mode 100644
index 00000000..8e933907
--- /dev/null
+++ b/content_styles/remoteOperator.css
@@ -0,0 +1,292 @@
+body {
+ pointer-events: none;
+ /* If unfamiliar with svw/svh, see: https://developer.mozilla.org/en-US/docs/Web/CSS/length#small */
+ width: 100svw; /* "small" view width and height safely work with the iPhone floating url bar (sadly vw/vh don't) */
+ height: 100svh;
+ overflow: hidden;
+ -webkit-user-select: none;
+ touch-action: none;
+ -webkit-touch-callout: none;
+}
+
+body > * {
+ pointer-events: auto;
+ /* overflow:hidden is necessary to prevent unintentional native scrolling/zooming within iOS safari app */
+ /* but we should limit its scope as much as possible instead of adding it here */
+}
+
+/* necessary to prevent native scrolling/zooming within iOS safari app */
+.canvas-node-connections {
+ overflow: hidden;
+}
+
+/* necessary to prevent native scrolling/zooming within iOS safari app */
+.canvas-main-threejs {
+ overflow: hidden;
+}
+
+#UIButtons > * {
+ cursor: pointer;
+}
+
+.hiddenDesktopButton {
+ display: none; /* totally hide this button on desktop because it doesn't do anything */
+}
+
+#canvas {
+ background-color: transparent;
+ transform: matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0.1, 1);
+ pointer-events: auto;
+}
+
+#interactionCursor {
+ pointer-events: none;
+ position: absolute;
+ width: 30px;
+ height: 30px;
+ /*border: 3px solid cyan;*/
+ display: none;
+}
+
+#staticInteractionCursor {
+ pointer-events: none;
+ position: absolute;
+ width: 30px;
+ height: 30px;
+ display: none;
+ opacity: 0.3;
+}
+
+#mainThreejsCanvas {
+ pointer-events: auto !important;
+}
+
+.blockIconTinted {
+ background-color: rgba(100,255,255,0.25) !important; /*on remote operator, override background to have more contrast*/
+}
+
+.desktopMenuBar {
+ position: absolute;
+ left: 0;
+ top: 0;
+ width: 100%;
+ height: 30px;
+ background-color: rgba(0,0,0, 0.9);
+ color: lightgray;
+ transform: matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 3000, 1);
+ z-index: 3000;
+}
+
+.desktopMenuBarMenu {
+ position: absolute;
+ top: 0;
+ left: 0;
+ width: 100px;
+}
+
+.desktopMenuBarMenuTitle {
+ height: 30px;
+ text-align: left;
+ line-height: 30px;
+ padding-left: 10px;
+ cursor: pointer;
+ border-radius: 5px;
+}
+
+.desktopMenuBarMenuTitle:hover {
+ background-color: rgba(255,255,255, 0.1);
+}
+
+.desktopMenuBarMenuTitleOpen {
+ background-color: rgba(255,255,255, 0.2) !important;
+}
+
+.desktopMenuBarMenuTitleDisabled {
+ color: rgb(75, 75, 75);
+ pointer-events: none;
+}
+
+.desktopMenuBarMenuDropdown {
+ /* TODO: how to best adapt width of menubar to comfortably fit contents? (not an issue now, but may want to) */
+ width: 250px;
+ background-color: rgba(0,0,0, 0.9);
+ border-radius: 5px;
+ /*overflow: hidden;*/
+}
+
+.desktopMenuBarLight {
+ background-color: rgba(64,170,29,0.9);
+}
+
+.hiddenDropdown {
+ display: none;
+}
+
+.desktopMenuBarItem {
+ width: calc(100% - 10px);
+ padding-left: 10px;
+ height: 30px;
+ cursor: pointer;
+ line-height: 30px;
+}
+
+.desktopMenuBarItem:hover {
+ background-color: rgba(55, 55, 55, 0.9);
+}
+
+.desktopMenuBarItemTextToggle {
+ padding-left: 20px;
+}
+
+.desktopMenuBarItemSeparator {
+ pointer-events: none;
+ cursor: unset;
+ height: 8px;
+ padding: 0;
+}
+
+.desktopMenuBarItemDisabled {
+ cursor: not-allowed;
+ color: gray;
+}
+
+.desktopMenuBarItemCheckmark {
+ position: absolute;
+ left: 10px;
+ top: 0;
+}
+
+.desktopMenuBarItemCheckmarkHidden {
+ visibility: hidden;
+}
+
+.desktopMenuBarItemCheckmarkDisabled {
+ opacity: 0.5;
+}
+
+.desktopMenuBarItemArrow {
+ position: absolute;
+ right: 15px;
+ top: 0;
+ width: 30px;
+ text-align: center;
+}
+
+.desktopMenuBarSubmenu {
+ left: 100%;
+ width: 250px;
+ background-color: rgba(0, 0, 0, 0.9);
+}
+
+.desktopMenuBarItemShortcut {
+ position: absolute;
+ right: 15px;
+ top: 0;
+ width: 30px;
+ text-align: center;
+ color: rgb(155, 155, 155);
+}
+
+.desktopMenuBarItemShortcutModifier {
+ position: absolute;
+ right: 45px;
+ top: 0;
+ text-align: right;
+ color: rgb(155, 155, 155);
+}
+
+.mode-prompt-container {
+ position: absolute;
+ right: 0;
+ bottom: 20px;
+ width: 320px;
+ height: max-content;
+ margin-right: 20px;
+ pointer-events: none;
+}
+
+.mode-prompt {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ justify-content: center;
+ align-items: flex-start;
+ width: 320px;
+ height: max-content;
+ background: rgba(0, 0, 0, 0.5);
+ border-radius: 5px;
+ margin-top: 10px;
+ font-size: 1em;
+ pointer-events: none;
+ padding: 16px;
+ opacity: 0;
+ box-sizing: border-box;
+ animation: 5s ease-in forwards promptFadeIn;
+}
+
+.mode-prompt.remove-prompt {
+ animation: promptFadeOut ease-in .3s forwards;
+}
+
+.mode-prompt ul {
+ padding-inline-start: 20px;
+}
+
+.mode-prompt-big-font {
+ font-size: 1.1em;
+ font-weight: 600;
+}
+
+@keyframes promptFadeIn {
+ 0% { opacity: 0 }
+ 20% { opacity: 1 }
+ 80% { opacity: 1 }
+ 100% { opacity: 0 }
+}
+
+#avatar-follow-border {
+ position: fixed;
+ top: 0;
+ left: 0;
+ right: 0;
+ bottom: 0;
+ border: 8px solid white; /* change the color with JS */
+ pointer-events: none;
+ z-index: 9999; /* go on top of EVERYTHING */
+}
+
+#avatar-follow-border > .fullscreenSubtitle {
+ position: absolute;
+ left: 50%;
+ transform: translateX(-50%);
+ text-align: center;
+ bottom: 50px;
+ background-color: rgba(0,0,0,0.5);
+ padding: 8px 16px;
+ border-radius: 15px;
+}
+
+#touchControlsContainer {
+ position: absolute;
+ bottom: 20px;
+ right: 20px;
+ z-index: 1000;
+ transform: translateZ(1000px);
+}
+
+.touchControlButtonContainer {
+ width: 44px;
+ height: 44px;
+ border-radius: 5px;
+ background-color: rgba(0, 0, 0, 0.7);
+ margin-bottom: 10px;
+ &.selected {
+ background-color: rgba(150, 150, 150, 0.9);
+ }
+}
+
+.touchControlButtonIcon {
+ width: 100%;
+ height: 100%;
+ cursor: pointer;
+}
diff --git a/content_styles/videoPlayback.css b/content_styles/videoPlayback.css
new file mode 100644
index 00000000..0bddc820
--- /dev/null
+++ b/content_styles/videoPlayback.css
@@ -0,0 +1,405 @@
+.videoPreview {
+ position: absolute;
+ left: 0;
+ top: 0;
+ transform: matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1000, 1);
+ z-index: 1000;
+ background-color: rgba(255,255,255,0.5);
+}
+
+.videoPreviewContainer {
+ position: absolute;
+ left: 0;
+ top: 0;
+ transform: matrix3d(-1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0, 0, 1000, 1);
+ z-index: 1000;
+ width: 256px;
+ height: 144px;
+ background-color: black;
+}
+
+.timelineBox {
+ position: absolute;
+ left: 0;
+ top: 0;
+ transform: matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1000, 1);
+ z-index: 1000;
+ /*background-color: rgba(0,0,0,0.8);*/
+ /*border: 1px solid rgba(0, 255, 255, 0.5);*/
+ width: 100px;
+ height: 100px;
+}
+
+#timelineContainer {
+ position: absolute;
+ left: 0;
+ bottom: 0;
+ width: 100vw;
+ height: 130px;
+ /*pointer-events: none;*/
+ background-color: rgba(0, 0, 0, 0.8);
+ transform: matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 3000, 1);
+}
+
+#timelineVisibilityBox {
+ position: relative;
+ left: 0;
+ top: 0;
+ width: 120px;
+ height: 100%;
+ border-radius: 5px 0 0 5px;
+}
+
+#timelineTrackBox {
+ position: absolute;
+ left: calc(120px + 4px);
+ top: 0;
+ width: calc(100% - 120px - 60px - 2 * 4px);
+ height: 100%;
+}
+
+#timelineTrackScrollBox {
+ position: absolute;
+ left: 0;
+ top: -300px;
+ width: 100%;
+ height: calc(100% + 300px);
+ overflow: hidden;
+ pointer-events: none;
+}
+
+#timelineTrackScrollBoxInner {
+ position: absolute;
+ left: 0;
+ top: 300px;
+ width: 100%;
+ height: calc(100% - 300px);
+ pointer-events: auto;
+}
+
+#timelineTracksContainer {
+ position: absolute;
+ left: 0;
+ width: 100%;
+ height: calc(100% - 10px);
+}
+
+#timelinePlayhead {
+ position: absolute;
+ left: 10px;
+ top: calc(-20px + 4px);
+ /*height: calc(100% + 20px);*/
+ height: 100%;
+ width: 20px;
+ cursor: pointer;
+ filter: saturate(0) brightness(1.5);
+}
+
+#timelinePlayheadDot {
+ position: absolute;
+ left: 20px;
+ bottom: 3px;
+ height: 10px;
+ width: 10px;
+ border-radius: 5px;
+ background-color: rgba(255, 255, 255, 1.0); /* rgba(255, 210, 0, 1.0); */
+ z-index: 1; /* go in front of the scroll bar */
+ pointer-events: none;
+}
+
+#timelinePlayhead:hover {
+ filter: brightness(120%) saturate(60%);
+ -webkit-filter: brightness(120%) saturate(60%);
+}
+
+.timelinePlayheadPlaying {
+ filter: hue-rotate(-60deg) saturate(400%) !important;
+ -webkit-filter: hue-rotate(-120deg) saturate(400%) !important;
+}
+
+.timelinePlayheadSelected {
+ filter: hue-rotate(-120deg) saturate(400%) !important;
+ -webkit-filter: hue-rotate(-120deg) saturate(400%) !important;
+}
+
+#timelineVideoPreviewContainer {
+ position: absolute;
+ left: -128px;
+ top: -168px;
+ width: 256px;
+ height: 144px;
+ /*display: none;*/
+ transform-origin: left bottom;
+ transform: scale(0.6);
+ border: 4px solid rgba(0, 0, 0, 0.8);
+}
+
+#timelineTimestampDisplay {
+ position: absolute;
+ left: 0;
+ top: 10px;
+ width: 100%;
+ text-align: center;
+ height: 48px;
+ color: rgb(255,255,255);
+}
+
+#timelineDateDisplay {
+ position: absolute;
+ left: 0;
+ top: 30px;
+ font-size: 11px;
+ width: 100%;
+ text-align: center;
+ height: 48px;
+ color: rgb(255,255,255);
+}
+
+#timelineCalendarButton {
+ position: absolute;
+ left: calc((120px - 48px)/2);
+ top: 50px;
+ width: 48px;
+ height: 48px;
+ text-align: center;
+ cursor: pointer;
+ filter: saturate(0) brightness(1.5);
+}
+
+#timelineCalendarButton:hover {
+ /*opacity: 0.7;*/
+ filter: saturate(0.5) brightness(1) !important;
+}
+
+/*.timelineVideoPreviewHidden {*/
+/*}*/
+
+.timelineVideoPreviewPlaying {
+ display: inline !important;
+}
+
+.timelineVideoPreviewSelected {
+ display: inline !important;
+}
+
+.timelineVideoPreviewNoTrack {
+ /*display: none !important;*/
+ visibility: hidden;
+}
+
+.timelineVideoPreviewNoSource {
+ /*display: none !important;*/
+ opacity: 0.2;
+}
+
+.timelineInnerVideoNoSource {
+ display: none !important;
+}
+
+.timelineTrack {
+ position: absolute;
+ background-color: rgba(75, 75, 75, 0.5);
+ /*border-radius: 7px;*/
+ left: 20px;
+ width: calc(100% - 2 * (20px + 4px));
+ /*cursor: pointer;*/
+ overflow: hidden;
+}
+
+/*.timelineTrack:hover {*/
+/* background-color: rgba(0,255,255, 0.2);*/
+/*}*/
+
+.selectedTrack {
+ background-color: rgba(255,255,255, 0.2);
+ border: 2px solid rgba(255,255,255, 0.5);
+ margin: -2px;
+}
+
+.timelineSegment {
+ position: absolute;
+ background-color: rgba(255,255,255, 0.5);
+ border-radius: 5px;
+ top: 0;
+ height: 100%;
+ z-index: 100;
+}
+
+/*.timelineSegment:hover {*/
+/* !*background-color: rgba(0,255,255, 0.7);*!*/
+/* !*border: 2px solid white;*!*/
+/* box-shadow: 0 0 5px black;*/
+/*}*/
+
+.timelineSegmentPlaying {
+ background-color: rgba(255, 255, 255, 0.9);
+}
+
+.selectedSegment {
+ background-color: rgba(255,255,255, 0.9);
+ /*border: 2px solid rgba(255,255,255, 0.5);*/
+}
+
+#timelineControlsBox {
+ position: absolute;
+ left: calc(100% - 60px);
+ top: 0;
+ width: 60px;
+ height: 100%;
+ border-radius: 0 5px 5px 0;
+}
+
+.timelineControlButton {
+ position: absolute;
+ width: calc(60px - 2 * 5px);
+ height: calc(60px - 2 * 5px);
+ cursor: pointer;
+}
+
+.timelineControlButton:hover {
+ /*opacity: 0.7;*/
+ filter: saturate(0.5) brightness(1) !important;
+}
+
+#timelinePlayButton {
+ left: 5px;
+ top: 10px;
+ filter: saturate(0) brightness(1.5);
+}
+
+/*#timelineSeekButton {*/
+/* left: 5px;*/
+/* top: calc(2 * 5px + (60px - 2 * 5px));*/
+/*}*/
+
+#timelineSpeedButton {
+ left: 5px;
+ top: calc(2 * 10px + (60px - 2 * 5px));
+ /*top: calc(3 * 5px + 2 * (60px - 2 * 5px));*/
+ filter: saturate(0) brightness(1.5);
+}
+
+.hiddenTimeline {
+ display: none;
+}
+
+#colorVideoCanvas {
+ position: absolute;
+ left: 20px;
+ top: 100px;
+ width: 960px;
+ height: 540px;
+ transform-origin: left top;
+ transform: matrix3d(0.5, 0, 0, 0, 0, 0.5, 0, 0, 0, 0, 1, 0, 0, 0, 1000, 1);
+ z-index: 1000;
+ background-color: rgba(255,255,255, 0.1);
+}
+
+#depthVideoCanvas {
+ position: absolute;
+ left: 520px;
+ top: 100px;
+ width: 256px;
+ height: 144px;
+ transform-origin: left top;
+ transform: matrix3d(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1000, 1);
+ z-index: 1000;
+ background-color: rgba(255,255,255, 0.1);
+}
+
+#poseCanvas {
+ position: absolute;
+ left: 20px;
+ top: 100px;
+ width: 8px;
+ height: 8px;
+ transform-origin: left top;
+ transform: matrix3d(10, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 1000, 1);
+ z-index: 1000;
+ background-color: rgba(255,255,255, 0.2);
+}
+
+.timelineCalendarVisible {
+ display: inline;
+ top: -325px !important;
+}
+
+.timelineCalendarHidden {
+ display: none;
+}
+
+#timelineZoomBar {
+ position: absolute;
+ left: 10px;
+ bottom: 5px;
+ filter: saturate(0) brightness(1.5);
+}
+
+#timelineZoomBar:hover {
+ filter: saturate(0.5) brightness(1) !important;
+}
+
+#zoomSliderBackground {
+ position: relative;
+ width: 100px;
+ height: 20px;
+}
+
+#zoomSliderHandle {
+ position: absolute;
+ width: 16px;
+ height: 16px;
+ left: 15px;
+ top: 2px;
+ cursor: pointer;
+}
+
+.timelineScrollBarContainer {
+ position: absolute;
+ background-color: rgba(255,255,255,0.3);
+ border-radius: 5px;
+ height: 10px;
+ /*transition: opacity 0.5s ease-in-out;*/
+}
+
+.timelineScrollBarHandle {
+ position: relative;
+ left: 0;
+ top: 0;
+ background-color: rgba(155,155,155,1.0);
+ border-radius: 5px;
+ height: 10px;
+ cursor: pointer;
+}
+
+.timelineScrollBarHandle:hover {
+ background-color: rgba(125,185,185,1.0);
+}
+
+.timelineScrollBarFadeout {
+ opacity: 0;
+ animation: scrollbarFadeout 0.5s;
+}
+
+.timelineScrollBarNoData {
+ visibility: hidden;
+}
+
+@keyframes scrollbarFadeout {
+ 0% {
+ opacity: 1;
+ }
+ 100% {
+ opacity: 0;
+ }
+}
+
+#timelineScrollBar {
+ left: 20px;
+ bottom: 3px;
+ width: calc(100% - 2 * (20px + 4px));
+}
+
+.hiddenScrollBar {
+ display: none;
+}
diff --git a/interfaces/remoteOperatorUI/VideoFileManager.js b/interfaces/remoteOperatorUI/VideoFileManager.js
new file mode 100644
index 00000000..66387c32
--- /dev/null
+++ b/interfaces/remoteOperatorUI/VideoFileManager.js
@@ -0,0 +1,169 @@
+const fs = require('fs');
+const path = require('path');
+const constants = require('./videoConstants');
+const utils = require('./utilities');
+
+/**
+ * @fileOverview
+ * The VideoFileManager is initialized with an outputPath where all video recordings and their derivatives are stored,
+ * and contains a variety of utilities for reading and writing the files to a specific nested file structure,
+ * and to convert this into a json blob containing the file tree (the persistentInfo).
+ * All state is reconstructed from the file structure when the server restarts.
+ */
+
+let outputPath = null;
+let persistentInfo = null;
+
+const createMissingDirs = (devicePath) => {
+ utils.mkdirIfNeeded(devicePath, true);
+ let dir = constants.DIR_NAMES;
+
+ let sessionVideosPath = path.join(devicePath, dir.session_videos);
+ let unprocessedChunksPath = path.join(devicePath, dir.unprocessed_chunks);
+ let processedChunksPath = path.join(devicePath, dir.processed_chunks);
+
+ [dir.color, dir.depth, dir.pose].forEach(name => {
+ utils.mkdirIfNeeded(path.join(sessionVideosPath, name), true);
+ utils.mkdirIfNeeded(path.join(unprocessedChunksPath, name), true);
+ utils.mkdirIfNeeded(path.join(processedChunksPath, name), true);
+ });
+};
+
+const parseDeviceDirectory = (devicePath) => {
+ let info = {};
+ createMissingDirs(devicePath);
+
+ // add fully-concatenated color and depth videos
+ let sessionVideosPath = path.join(devicePath, constants.DIR_NAMES.session_videos);
+ fs.readdirSync(path.join(sessionVideosPath, constants.DIR_NAMES.color)).forEach(filepath => {
+ let sessionId = getSessionIdFromFilename(filepath, 'session_');
+ if (sessionId && sessionId.length === 8) {
+ if (typeof info[sessionId] === 'undefined') {
+ info[sessionId] = {};
+ }
+ info[sessionId].color = filepath;
+
+ if (fs.existsSync(path.join(sessionVideosPath, constants.DIR_NAMES.depth, filepath))) {
+ info[sessionId].depth = filepath;
+ }
+ }
+ });
+ // append pose data separately from logic for color, since pose may be available before color & depth
+ fs.readdirSync(path.join(sessionVideosPath, 'pose')).forEach(filepath => {
+ let sessionId = getSessionIdFromFilename(filepath, 'session_');
+ if (sessionId && sessionId.length === 8) {
+ if (typeof info[sessionId] === 'undefined') {
+ info[sessionId] = {};
+ }
+ info[sessionId].pose = filepath;
+ }
+ });
+
+ // add the list of chunks (processed and unprocessed) to the json output
+ [constants.DIR_NAMES.processed_chunks, constants.DIR_NAMES.unprocessed_chunks].forEach(dirName => {
+ let thisPath = path.join(devicePath, dirName);
+ fs.readdirSync(path.join(thisPath, constants.DIR_NAMES.color)).forEach(filepath => { // do for color, but check for depth within the block to ensure both exist
+ let sessionId = getSessionIdFromFilename(filepath, 'chunk_');
+ if (sessionId && sessionId.length === 8) {
+ if (typeof info[sessionId] === 'undefined') {
+ info[sessionId] = {};
+ }
+ if (typeof info[sessionId][dirName] === 'undefined') {
+ info[sessionId][dirName] = [];
+ }
+ if (fs.existsSync(path.join(thisPath, constants.DIR_NAMES.depth, filepath))) {
+ info[sessionId][dirName].push(filepath);
+ }
+ }
+ });
+ });
+
+ return info;
+};
+
+const getSessionIdFromFilename = (filename, prefix) => {
+ let re = new RegExp(prefix + '[a-zA-Z0-9]{8}');
+ let matches = filename.match(re);
+ if (!matches || matches.length === 0) { return null; }
+ return (prefix ? matches[0].replace(prefix, '') : matches[0]);
+};
+
+const getNestedFilePaths = (deviceId, dirName, colorOrDepth) => {
+ if (!outputPath) { console.warn('You never called initWithOutputPath on VideoFileManager'); }
+
+ let dirPath = path.join(outputPath, deviceId, dirName, colorOrDepth);
+ utils.mkdirIfNeeded(dirPath, true);
+ let filetype = '.' + ((colorOrDepth === constants.DIR_NAMES.depth) ? constants.DEPTH_FILETYPE : constants.COLOR_FILETYPE);
+ return fs.readdirSync(dirPath).filter(filename => filename.includes(filetype));
+};
+
+const deleteChunksForSession = (deviceId, sessionId) => {
+ if (!outputPath) { console.warn('You never called initWithOutputPath on VideoFileManager'); }
+
+ let counter = 0;
+ [constants.DIR_NAMES.unprocessed_chunks, constants.DIR_NAMES.processed_chunks].forEach(dirName => {
+ [constants.DIR_NAMES.color, constants.DIR_NAMES.depth].forEach(colorOrDepth => {
+ getNestedFilePaths(deviceId, dirName, colorOrDepth).filter(path => {
+ return path.includes(sessionId);
+ }).map(filename => {
+ return path.join(outputPath, deviceId, dirName, colorOrDepth, filename);
+ }).forEach(chunkPath => {
+ fs.rmSync(chunkPath);
+ counter++;
+ });
+ });
+ });
+ if (counter > 0) {
+ console.log('deleted ' + counter + ' leftover video chunk files for recorded video ' + sessionId);
+ }
+};
+
+module.exports = {
+ initWithOutputPath: (path) => {
+ outputPath = path;
+ utils.mkdirIfNeeded(path, true);
+ },
+ createMissingDirs: createMissingDirs,
+ // we rebuild the json blob each time by parsing the filesystem, so this is stored mainly as a means for other systems to retrieve the data
+ buildPersistentInfo: () => {
+ if (!outputPath) { console.warn('You never called initWithOutputPath on VideoFileManager'); }
+
+ let info = {};
+ // each folder in outputPath is a device
+ // check that folder's session_videos, processed_chunks, and unprocessed_chunks to determine how many sessions there are and what state they're in
+ fs.readdirSync(outputPath).filter((filename) => {
+ let isHidden = filename[0] === '.';
+ return fs.statSync(path.join(outputPath, filename)).isDirectory() && !isHidden;
+ }).forEach(deviceDirName => {
+ info[deviceDirName] = parseDeviceDirectory(path.join(outputPath, deviceDirName));
+ });
+ persistentInfo = info;
+ },
+ savePersistentInfo: () => {
+ if (!outputPath) { console.warn('You never called initWithOutputPath on VideoFileManager'); }
+
+ let jsonPath = path.join(outputPath, 'videoInfo.json');
+ fs.writeFileSync(jsonPath, JSON.stringify(persistentInfo, null, 4));
+ },
+ getUnprocessedChunkFilePaths: (deviceId, colorOrDepth = constants.DIR_NAMES.color) => {
+ return getNestedFilePaths(deviceId, constants.DIR_NAMES.unprocessed_chunks, colorOrDepth);
+ },
+ getProcessedChunkFilePaths: (deviceId, colorOrDepth = constants.DIR_NAMES.color) => {
+ return getNestedFilePaths(deviceId, constants.DIR_NAMES.processed_chunks, colorOrDepth);
+ },
+ getSessionFilePaths: (deviceId, colorOrDepth = constants.DIR_NAMES.color) => {
+ return getNestedFilePaths(deviceId, constants.DIR_NAMES.session_videos, colorOrDepth);
+ },
+ deleteLeftoverChunks(deviceId) {
+ let colorSessionPaths = module.exports.getSessionFilePaths(deviceId, constants.DIR_NAMES.color);
+ let depthSessionPaths = module.exports.getSessionFilePaths(deviceId, constants.DIR_NAMES.depth);
+ colorSessionPaths.forEach(path => {
+ if (depthSessionPaths.includes(path)) {
+ let sessionId = getSessionIdFromFilename(path, 'session_');
+ deleteChunksForSession(deviceId, sessionId);
+ }
+ });
+ },
+ get outputPath() { return outputPath; },
+ get persistentInfo() { return persistentInfo; }
+};
diff --git a/interfaces/remoteOperatorUI/VideoProcessManager.js b/interfaces/remoteOperatorUI/VideoProcessManager.js
new file mode 100644
index 00000000..7d3f8e54
--- /dev/null
+++ b/interfaces/remoteOperatorUI/VideoProcessManager.js
@@ -0,0 +1,210 @@
+const fs = require('fs');
+const path = require('path');
+const ffmpegInterface = require('./ffmpegInterface');
+const VideoFileManager = require('./VideoFileManager');
+const constants = require('./videoConstants');
+
+/**
+ * @fileOverview
+ * The VideoProcessManager listens for messages originating from virtualizer devices,
+ * and creates a new Connection to manage the raw video recording process for each camera device
+ * The Connection spawns a ffmpeg process when a startRecording message is received, and continuously
+ * appends frame data to this process until the device sends stopRecording or disconnects.
+ * Frame data is written to a new video "chunk" file each 15 seconds, to reduce memory footprint and point of failure.
+ * When the Connection's process is done, it triggers a callback where the VideoServer can make use of the chunk files.
+ */
+
+let connections = {};
+let callbacks = {
+ recordingDone: null
+};
+
+module.exports = {
+ onConnection: (deviceId) => {
+ console.log('-- on connection: ' + deviceId);
+ connections[deviceId] = new Connection(deviceId, callbacks);
+ },
+ onDisconnection: (deviceId) => {
+ console.log('-- on disconnection: ' + deviceId);
+ if (connections[deviceId]) {
+ connections[deviceId].stopRecording(true);
+ // TODO: should we also delete this.connections[deviceId]?
+ }
+ },
+ startRecording: (deviceId) => {
+ console.log('-- start recording: ' + deviceId);
+ if (connections[deviceId]) {
+ connections[deviceId].startRecording();
+ }
+ },
+ stopRecording: (deviceId) => {
+ console.log('-- stop recording: ' + deviceId);
+ if (connections[deviceId]) {
+ connections[deviceId].stopRecording(false);
+ }
+ },
+ onFrame: (rgb, depth, pose, deviceId) => {
+ if (connections[deviceId]) {
+ connections[deviceId].onFrame(rgb, depth, pose);
+ }
+ },
+ setRecordingDoneCallback: (callback) => {
+ callbacks.recordingDone = callback;
+ }
+};
+
+class Connection {
+ constructor(deviceId, callbacks) {
+ this.deviceId = deviceId;
+ this.sessionId = null;
+ this.STATUS = Object.freeze({
+ NOT_STARTED: 'NOT_STARTED',
+ STARTED: 'STARTED',
+ ENDING: 'ENDING',
+ DISCONNECTED: 'DISCONNECTED',
+ STOPPED: 'STOPPED'
+ });
+ this.isRecording = false;
+ this.processes = {
+ color: null,
+ depth: null,
+ pose: null
+ };
+ this.processStatuses = {
+ color: this.STATUS.NOT_STARTED,
+ depth: this.STATUS.NOT_STARTED,
+ pose: this.STATUS.NOT_STARTED
+ };
+ this.poses = [];
+ this.chunkCount = 0;
+ this.callbacks = callbacks;
+ }
+ startRecording() {
+ this.sessionId = this.uuidTimeShort();
+ this.chunkCount = 0;
+ this.isRecording = true;
+ // fileManager setup persistent data and directories
+
+ this.spawnProcesses();
+ this.waitUntilNextChunk();
+ }
+ // restart every 15 seconds (unless socket disconnected, just process data and stop)
+ waitUntilNextChunk() {
+ setTimeout(_ => {
+ this.stopProcesses();
+ if (this.processStatuses.color !== this.STATUS.DISCONNECTED &&
+ this.processStatuses.color !== this.STATUS.STOPPED) {
+ setTimeout(_ => {
+ this.recordNextChunk();
+ }, 10); // not sure if this delay is necessary between chunks but doesnt seem unreasonable
+ }
+ }, constants.SEGMENT_LENGTH);
+ }
+ recordNextChunk() {
+ this.chunkCount += 1;
+ this.isRecording = true;
+
+ this.spawnProcesses();
+ this.waitUntilNextChunk();
+ }
+ spawnProcesses() {
+ let index = this.chunkCount;
+
+ // start color stream process
+ // depth images are 1920x1080 lossy JPG images
+ let chunkTimestamp = Date.now();
+ let colorFilename = 'chunk_' + this.sessionId + '_' + index + '_' + chunkTimestamp + '.' + constants.COLOR_FILETYPE;
+ let colorOutputPath = path.join(VideoFileManager.outputPath, this.deviceId, constants.DIR_NAMES.unprocessed_chunks, constants.DIR_NAMES.color, colorFilename);
+ this.processes.color = ffmpegInterface.ffmpeg_image2mp4(colorOutputPath, constants.RECORDING_FPS, 'mjpeg', constants.COLOR_WIDTH, constants.COLOR_HEIGHT, constants.COLOR_CRF, constants.COLOR_SCALE);
+ if (this.processes.color) {
+ this.processStatuses.color = this.STATUS.STARTED;
+ }
+
+ // start depth stream process
+ // depth images are 256x144 lossless PNG buffers
+ let depthFilename = 'chunk_' + this.sessionId + '_' + index + '_' + chunkTimestamp + '.' + constants.DEPTH_FILETYPE;
+ let depthOutputPath = path.join(VideoFileManager.outputPath, this.deviceId, constants.DIR_NAMES.unprocessed_chunks, constants.DIR_NAMES.depth, depthFilename);
+ this.processes.depth = ffmpegInterface.ffmpeg_image2mp4(depthOutputPath, constants.RECORDING_FPS, 'png', constants.DEPTH_WIDTH, constants.DEPTH_HEIGHT, constants.DEPTH_CRF, constants.DEPTH_SCALE);
+ // this.processes[deviceId][this.PROCESS.DEPTH] = ffmpeg_image2losslessVideo(depthOutputPath, 10, 'png', 256, 144); // this version isn't working as reliably
+ if (this.processes.depth) {
+ this.processStatuses.depth = this.STATUS.STARTED;
+ }
+
+ this.processStatuses.pose = this.STATUS.STARTED;
+ this.poses = [];
+ }
+ onFrame(rgb, depth, pose) {
+ if (!this.isRecording) { return; }
+
+ if (this.processes.color && this.processStatuses.color === this.STATUS.STARTED) {
+ this.processes.color.stdin.write(rgb);
+ }
+ if (this.processes.depth && this.processStatuses.depth === this.STATUS.STARTED) {
+ this.processes.depth.stdin.write(depth);
+ }
+ if (this.processStatuses.pose === this.STATUS.STARTED) {
+ this.poses.push({
+ pose: pose.toString('base64'),
+ time: Date.now()
+ });
+ }
+ }
+ stopProcesses() {
+ this.isRecording = false;
+
+ if (this.processes.color !== 'undefined' && this.processStatuses.color === this.STATUS.STARTED) {
+ console.log('end color process');
+ this.processes.color.stdin.setEncoding('utf8');
+ this.processes.color.stdin.write('q');
+ this.processes.color.stdin.end();
+ this.processStatuses.color = this.STATUS.ENDING;
+ }
+
+ if (this.processes.depth !== 'undefined' && this.processStatuses.depth === this.STATUS.STARTED) {
+ console.log('end depth process');
+ this.processes.depth.stdin.setEncoding('utf8');
+ this.processes.depth.stdin.write('q');
+ this.processes.depth.stdin.end();
+ this.processStatuses.depth = this.STATUS.ENDING;
+ }
+
+ if (this.processStatuses.pose === this.STATUS.STARTED) {
+ console.log('end pose process');
+ this.processStatuses.pose = this.STATUS.ENDING;
+
+ // immediately write the poses from memory to storage, and reset the poses in memory
+ let index = this.chunkCount;
+ let poseFilename = 'chunk_' + this.sessionId + '_' + index + '_' + Date.now() + '.json';
+ let poseOutputPath = path.join(VideoFileManager.outputPath, this.deviceId, constants.DIR_NAMES.unprocessed_chunks, 'pose', poseFilename);
+ fs.writeFileSync(poseOutputPath, JSON.stringify(this.poses));
+ this.poses = [];
+ }
+ }
+ stopRecording(didDisconnect) {
+ if (!this.isRecording) { return; }
+
+ this.stopProcesses();
+
+ if (didDisconnect) {
+ this.processStatuses.color = this.STATUS.DISCONNECTED;
+ this.processStatuses.depth = this.STATUS.DISCONNECTED;
+ this.processStatuses.pose = this.STATUS.DISCONNECTED;
+ } else {
+ this.processStatuses.color = this.STATUS.STOPPED;
+ this.processStatuses.depth = this.STATUS.STOPPED;
+ this.processStatuses.pose = this.STATUS.STOPPED;
+ }
+
+ // when the video processes are done, other modules can do what they'd like with it
+ if (this.callbacks.recordingDone) {
+ this.callbacks.recordingDone(this.deviceId, this.sessionId, this.chunkCount);
+ }
+ }
+ uuidTimeShort() {
+ var dateUuidTime = new Date();
+ var abcUuidTime = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+ var stampUuidTime = parseInt('' + dateUuidTime.getMilliseconds() + dateUuidTime.getMinutes() + dateUuidTime.getHours() + dateUuidTime.getDay()).toString(36);
+ while (stampUuidTime.length < 8) stampUuidTime = abcUuidTime.charAt(Math.floor(Math.random() * abcUuidTime.length)) + stampUuidTime;
+ return stampUuidTime;
+ }
+}
diff --git a/interfaces/remoteOperatorUI/VideoServer.js b/interfaces/remoteOperatorUI/VideoServer.js
new file mode 100644
index 00000000..6de6b044
--- /dev/null
+++ b/interfaces/remoteOperatorUI/VideoServer.js
@@ -0,0 +1,244 @@
+const fs = require('fs');
+const path = require('path');
+const ffmpegInterface = require('./ffmpegInterface');
+const VideoFileManager = require('./VideoFileManager');
+const VideoProcessManager = require('./VideoProcessManager');
+const constants = require('./videoConstants');
+const utils = require('./utilities');
+
+/**
+ * @fileOverview
+ * The VideoServer handles logic to record point cloud videos captured by the streamRouter
+ * It passes messages to the VideoProcessManager (onConnection, onDisconnection, startRecording, stopRecording, onFrame)
+ * and it utilizes the VideoFileManager to save the recorded video chunks (and json poses) to a nested directory at the outputPath
+ *
+ * Recorded videos pass through three stages:
+ * 1. unprocessed_chunks: these are 15 second video segments created on a rolling basis while the video is recording
+ * 2. processed_chunks: when the video has stopped recording, chunks have an optional post-processing step and are copied here
+ * 3. session_videos: when all chunks have been processed, they are concatenated into a final video here
+ *
+ * RGB and depth videos are created with ffmpeg child processes. Parameters can be configured in videoConstants and ffmpegInterface.
+ * Poses are written to json files as a list of timestamped base64-encoded matrices.
+ *
+ * Metadata for all recordings can be accessed at localhost:8081/virtualizer_recordings
+ */
+class VideoServer {
+ constructor(outputPath) {
+ this.outputPath = outputPath;
+ console.info('Virtualizer recordings path: ' + this.outputPath);
+
+ VideoFileManager.initWithOutputPath(outputPath);
+ // we rebuild/save the persistentInfo on server restart so that it doesn't get out-of-sync with filesystem state
+ VideoFileManager.buildPersistentInfo();
+ VideoFileManager.savePersistentInfo();
+
+ // the process manager is mostly self-sufficient at creating the recordings, but we need to know when it's done.
+ // it records in 15 second chunks, and when it's done we concatenate the chunks together
+ VideoProcessManager.setRecordingDoneCallback(this.onRecordingDone.bind(this));
+
+ // each time the server restarts, we check for chunks that were never concatenated and try to concat them into finished videos
+ // this way, if the server unexpectedly stops while the videos are processing, they can be recovered
+ Object.keys(VideoFileManager.persistentInfo).forEach(deviceId => {
+ this.concatChunksIntoSessionVideo(deviceId);
+ });
+
+ // this copies over chunk files from unprocessed_chunks to processed_chunks
+ // processed_chunks is a way to future-proof the system such that if we want to do any post-processing of the
+ // chunk files, we have a place to reliably do that before making the videos available to the system in a concatenated form
+ // for example, we can rescale each chunk to a certain length or re-encode them based on info we have only after all the chunks have been recorded
+ Object.keys(VideoFileManager.persistentInfo).forEach(deviceId => {
+ this.processUnprocessedChunks(deviceId);
+ });
+
+ // after chunks have been concatenated into a session video, we can delete them
+ Object.keys(VideoFileManager.persistentInfo).forEach(deviceId => {
+ VideoFileManager.deleteLeftoverChunks(deviceId);
+ });
+ }
+ startRecording(deviceId) {
+ VideoFileManager.createMissingDirs(path.join(this.outputPath, deviceId));
+ VideoProcessManager.startRecording(deviceId);
+ }
+ stopRecording(deviceId) {
+ VideoProcessManager.stopRecording(deviceId);
+ }
+ onConnection(deviceId) {
+ VideoProcessManager.onConnection(deviceId);
+ }
+ onDisconnection(deviceId) {
+ VideoProcessManager.onDisconnection(deviceId);
+ }
+ onFrame(rgb, depth, pose, deviceId) {
+ VideoProcessManager.onFrame(rgb, depth, pose, deviceId);
+ }
+ // TODO: this could be optimized by listening for files to finish writing, rather than waiting a fixed time
+ onRecordingDone(deviceId, sessionId, lastChunkIndex) {
+ setTimeout(() => { // wait for final video to finish processing
+ utils.mkdirIfNeeded(path.join(this.outputPath, deviceId, 'tmp'), true);
+ let tmpOutputPath = path.join(this.outputPath, deviceId, 'tmp', sessionId + '_done_' + lastChunkIndex + '.json');
+ fs.writeFileSync(tmpOutputPath, JSON.stringify({ success: true}));
+
+ this.processUnprocessedChunks(deviceId);
+
+ // try concatenating after a longer delay. if not all chunks have finished processing by then...
+ // ...the chunk count won't match up and it will skip concatenating for now...
+ // ...it will reattempt each time you restart the server.
+ setTimeout(() => {
+ // try to concat rescaled videos after waiting awhile after recording stopped
+ VideoFileManager.buildPersistentInfo(); // recompile persistent info so session metadata contains new chunks
+ this.concatChunksIntoSessionVideo(deviceId);
+ }, 1000 * (lastChunkIndex + 1)); // delay 1 second per chunk we need to process, should give plenty of time
+ }, 5000);
+ }
+ concatChunksIntoSessionVideo(deviceId) {
+ if (!fs.existsSync(path.join(this.outputPath, deviceId))) {
+ console.error('Failed to concat video chunks, directory does not exist', path.join(this.outputPath, deviceId));
+ return;
+ }
+
+ // we use a tmp text file as a lock to ensure that all of the unprocessed chunks are represented in the processed chunks
+ let tmpFiles = [];
+ if (fs.existsSync(path.join(this.outputPath, deviceId, 'tmp'))) {
+ tmpFiles = fs.readdirSync(path.join(this.outputPath, deviceId, 'tmp'));
+ }
+
+ let sessions = VideoFileManager.persistentInfo[deviceId];
+ Object.keys(sessions).forEach(sessionId => {
+ let s = sessions[sessionId];
+ if (s.color && s.depth && s.pose) { return; }
+ if (s.processed_chunks && s.processed_chunks.length > 0) {
+ let matchingFiles = tmpFiles.filter(filename => { return filename.includes(sessionId + '_done'); });
+ let tmpFilename = matchingFiles.length > 0 ? matchingFiles[0] : null;
+ if (tmpFilename) {
+ let numberOfChunks = parseInt(tmpFilename.match(/_\d+.json/)[0].match(/\d+/)[0]) + 1;
+ if (s.processed_chunks.length !== numberOfChunks && s.unprocessed_chunks.length === numberOfChunks) {
+ // there are some unprocessed chunks not present in the processed chunks
+ return; // skip concatenating this session
+ }
+ }
+
+ if (!s.color) { s.color = this.concatFiles(deviceId, sessionId, constants.DIR_NAMES.color, s.processed_chunks); }
+ if (!s.depth) { s.depth = this.concatFiles(deviceId, sessionId, constants.DIR_NAMES.depth, s.processed_chunks); }
+ if (!s.pose) { s.pose = this.concatPosesIfNeeded(deviceId, sessionId); }
+ }
+ });
+
+ VideoFileManager.savePersistentInfo();
+ }
+ extractTimeInformation(fileList) { // we could also probably just use the SEGMENT_LENGTH * fileList.length, but this works too
+ let fileRecordingTimes = fileList.map(filename => parseInt(filename.match(/[0-9]{13,}/))); // extract timestamp
+ let firstTimestamp = Math.min(...fileRecordingTimes) - constants.SEGMENT_LENGTH; // estimate, since this is at the end of the first video
+ let lastTimestamp = Math.max(...fileRecordingTimes);
+ return {
+ start: firstTimestamp,
+ end: lastTimestamp,
+ duration: lastTimestamp - firstTimestamp
+ };
+ }
+ concatFiles(deviceId, sessionId, colorOrDepth = constants.DIR_NAMES.color, files) {
+ // passing a list of videos into ffmpeg is most easily done with a txt file listing the files in a specific format
+ let fileText = '';
+ for (let i = 0; i < files.length; i++) {
+ fileText += 'file \'' + path.join(this.outputPath, deviceId, constants.DIR_NAMES.processed_chunks, colorOrDepth, files[i]) + '\'\n';
+ }
+
+ // write file list to txt file so it can be used by ffmpeg as input
+ let txt_filename = colorOrDepth + '_filenames_' + sessionId + '.txt';
+ utils.mkdirIfNeeded(path.join(this.outputPath, deviceId, 'tmp'), true);
+ let txtFilePath = path.join(this.outputPath, deviceId, 'tmp', txt_filename);
+ if (fs.existsSync(txtFilePath)) {
+ fs.unlinkSync(txtFilePath);
+ }
+ fs.writeFileSync(txtFilePath, fileText);
+
+ let filetype = (colorOrDepth === constants.DIR_NAMES.depth) ? constants.DEPTH_FILETYPE : constants.COLOR_FILETYPE;
+ // we store the video timestamp directly in its filename, so this info never gets lost
+ let timeInfo = this.extractTimeInformation(files);
+ let filename = 'device_' + deviceId + '_session_' + sessionId + '_start_' + timeInfo.start + '_end_' + timeInfo.end + '.' + filetype;
+ let outputPath = path.join(this.outputPath, deviceId, constants.DIR_NAMES.session_videos, colorOrDepth, filename);
+ ffmpegInterface.ffmpeg_concat_mp4s(outputPath, txtFilePath);
+
+ return filename;
+ }
+ concatPosesIfNeeded(deviceId, sessionId) {
+ // check if output file exists for this device/session pair
+ let filename = 'device_' + deviceId + '_session_' + sessionId + '.json';
+ let outputPath = path.join(this.outputPath, deviceId, constants.DIR_NAMES.session_videos, 'pose', filename);
+ if (fs.existsSync(outputPath)) {
+ return filename; // already exists, return early
+ }
+
+ // load all pose chunks. each is a json file with a timestamped list of poses for 15 seconds of the video
+ let files = fs.readdirSync(path.join(this.outputPath, deviceId, constants.DIR_NAMES.unprocessed_chunks, 'pose'));
+ files = files.filter(filename => {
+ return filename.includes(sessionId);
+ });
+
+ // the concatenated file contains a correctly-ordered array with all of the poses from each chunk's array
+ let poseData = [];
+ files.forEach(filename => {
+ let filePath = path.join(this.outputPath, deviceId, constants.DIR_NAMES.unprocessed_chunks, 'pose', filename);
+ poseData.push(JSON.parse(fs.readFileSync(filePath, 'utf-8')));
+ });
+ let flattened = poseData.flat();
+ fs.writeFileSync(outputPath, JSON.stringify(flattened));
+
+ return filename;
+ }
+ processUnprocessedChunks(deviceId) {
+ let unprocessedPath = path.join(this.outputPath, deviceId, constants.DIR_NAMES.unprocessed_chunks);
+ let processedPath = path.join(this.outputPath, deviceId, constants.DIR_NAMES.processed_chunks);
+ let fileMap = {
+ color: {
+ processed: VideoFileManager.getProcessedChunkFilePaths(deviceId, constants.DIR_NAMES.color),
+ unprocessed: VideoFileManager.getUnprocessedChunkFilePaths(deviceId, constants.DIR_NAMES.color)
+ },
+ depth: {
+ processed: VideoFileManager.getProcessedChunkFilePaths(deviceId, constants.DIR_NAMES.depth),
+ unprocessed: VideoFileManager.getUnprocessedChunkFilePaths(deviceId, constants.DIR_NAMES.depth)
+ }
+ };
+
+ Object.keys(fileMap).forEach(colorOrDepth => {
+ let filesToScale = [];
+
+ // ignore any video chunks that are corrupted (~0 bytes), perhaps due to stopping recording before it wrote any frames
+ fileMap[colorOrDepth].unprocessed.forEach(filename => {
+ let timestamp = filename.match(/[0-9]{13,}/);
+ if (!fileMap[colorOrDepth].processed.some(resizedFilename => resizedFilename.includes(timestamp))) {
+ let colorFilename = filename.replace(/\.[^/.]+$/, '') + '.' + constants.COLOR_FILETYPE;
+ let depthFilename = filename.replace(/\.[^/.]+$/, '') + '.' + constants.DEPTH_FILETYPE;
+ let colorFilePath = path.join(unprocessedPath, constants.DIR_NAMES.color, colorFilename);
+ let depthFilePath = path.join(unprocessedPath, constants.DIR_NAMES.depth, depthFilename);
+ if (fs.existsSync(colorFilePath) && fs.existsSync(depthFilePath)) {
+ let byteSizeColor = fs.statSync(colorFilePath).size;
+ let byteSizeDepth = fs.statSync(depthFilePath).size;
+ if (byteSizeColor > 48 && byteSizeDepth > 48) {
+ filesToScale.push(filename);
+ }
+ }
+ }
+ });
+
+ // note: why is ffmpeg_adjust_length (sometimes) part of the process:
+ // the incoming image stream, while it approximates 10 fps, is not exactly that.
+ // so the resulting "15 second" chunks are sometimes 8 seconds, sometimes 10, etc.
+ // one way to fix this time warping is to rescale each chunk to be 15 seconds exactly.
+ // the other method (currently used) is to leave the chunks as-is, and correct it in the video playback system:
+ // use the timestamps stored in the filename, rather than the length of the video, during playback
+
+ // either copy the files directly to processed_chunks, or do some postprocessing (like ffmpeg_adjust_length)
+ filesToScale.forEach(filename => {
+ let inputPath = path.join(unprocessedPath, colorOrDepth, filename);
+ let outputPath = path.join(processedPath, colorOrDepth, filename);
+ if (constants.RESCALE_VIDEOS) {
+ ffmpegInterface.ffmpeg_adjust_length(outputPath, inputPath, constants.SEGMENT_LENGTH / 1000);
+ } else {
+ fs.copyFileSync(inputPath, outputPath, fs.constants.COPYFILE_EXCL);
+ }
+ });
+ });
+ }
+}
+
+module.exports = VideoServer;
diff --git a/interfaces/remoteOperatorUI/config.html b/interfaces/remoteOperatorUI/config.html
new file mode 100644
index 00000000..9732683a
--- /dev/null
+++ b/interfaces/remoteOperatorUI/config.html
@@ -0,0 +1,387 @@
+
+
+
+
+ Remote Operator Configurator
+
+
+
+
+
+
+ Help text
+
+
+
Setting Name
+
+
Enter Name
+
+
+
+
+
+
+
+
+
+
+ The remoteOperatorUI interface allows you to host a Remote Operator userinterface on localhost:8081. Just configure
+ the absolute path to your vuforia-spatial-toolbox-userinterface directory and restart your server.
+
+
+
+
+
+
+
+
+
+
diff --git a/interfaces/remoteOperatorUI/ffmpegInterface.js b/interfaces/remoteOperatorUI/ffmpegInterface.js
new file mode 100644
index 00000000..514c63dc
--- /dev/null
+++ b/interfaces/remoteOperatorUI/ffmpegInterface.js
@@ -0,0 +1,205 @@
+const fs = require('fs');
+const cp = require('child_process');
+const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path;
+const constants = require('./videoConstants');
+
+/**
+ * @fileOverview
+ * This contains a set of utility functions that will spawn an ffmpeg child process to perform the specified task
+ * Enable constants.DEBUG_LOG_FFMPEG to print information from these processes
+ *
+ * Note: I've intentionally left the rubble of some experiments in here, that it may help future thought processes
+ *
+ * TODO: how can we listen for when the process finishes writing the output file, to reliably continue only when safe?
+ */
+
+module.exports = {
+ // Create a process that converts a stream of jpeg or png images into a video.
+ // To add a frame, call process.stdin.write(rgb_buffer). To finish, call process.stdin.end()
+ ffmpeg_image2mp4: (output_path, framerate = 10, input_vcodec = 'mjpeg', input_width = 1920, input_height = 1080, crf = 25, output_scale = 0.25) => {
+ let outputWidth = input_width * output_scale;
+ let outputHeight = input_height * output_scale;
+
+ let args = [
+ '-r', framerate,
+ // '-framerate', framerate,
+ // '-probesize', '5000',
+ // '-analyzeduration', '5000',
+ '-f', 'image2pipe',
+ '-vcodec', input_vcodec,
+ '-s', input_width + 'x' + input_height,
+ '-i', '-',
+ '-vcodec', 'libx264',
+ '-crf', crf,
+ '-pix_fmt', 'yuv420p',
+ '-vf', 'scale=' + outputWidth + ':' + outputHeight + ', setsar=1:1', //, realtime, fps=' + framerate,
+ // '-preset', 'ultrafast',
+ // '-copyts',
+ // '-tune', 'zerolatency',
+ // '-r', framerate, // will duplicate frames to meet this but still look like the framerate set before -i,
+ output_path
+ ];
+
+ let process = cp.spawn(ffmpegPath, args);
+
+ if (constants.DEBUG_LOG_FFMPEG) {
+ // process.stdout.on('data', function(data) {
+ // console.log('stdout data', data);
+ // });
+ process.stderr.setEncoding('utf8');
+ process.stderr.on('data', function(data, err) {
+ console.log('stderr data', data);
+ console.warn(err);
+ });
+ // process.on('close', function() {
+ // console.log('finished');
+ // });
+ }
+
+ return process;
+ },
+ // experimental alternative to ffmpeg_image2mp4 that tries to use lossless codecs and parameters
+ // not quite successful, but could be iterated upon
+ ffmpeg_image2losslessVideo: (output_path, framerate = 10, input_vcodec = 'png', input_width = 256, input_height = 144) => {
+ // let outputWidth = input_width;
+ // let outputHeight = input_height;
+
+ // ffmpeg -i video.avi -c:v libx265 \
+ // -x265-params "profile=monochrome12:crf=0:lossless=1:preset=veryslow:qp=0" \
+ // video.mkv
+
+ let args = [
+ '-r', framerate,
+ '-f', 'image2pipe',
+ '-vcodec', input_vcodec,
+ '-pix_fmt', 'argb',
+ '-s', input_width + 'x' + input_height,
+ '-i', '-',
+ // '-vcodec', 'libx265',
+ '-vcodec', 'libvpx-vp9',
+ '-lossless', '1',
+ // '-x265-params', 'lossless=1',
+ // '-pix_fmt', 'yuv420p',
+ '-pix_fmt', 'argb',
+ // '-vf', 'scale=' + outputWidth + ':' + outputHeight + ', setsar=1:1', //, realtime, fps=' + framerate,
+ // '-preset', 'ultrafast',
+ // '-copyts',
+ // '-tune', 'zerolatency',
+ // '-r', framerate, // will duplicate frames to meet this but still look like the framerate set before -i,
+ output_path
+ ];
+
+ // not working with MP4 ... works losslessly with .MKV but not playable in HTML video element
+ // let args = [
+ // '-r', framerate,
+ // '-f', 'image2pipe',
+ // '-vcodec', input_vcodec,
+ // '-s', input_width + 'x' + input_height,
+ // '-i', '-',
+ // '-vcodec', 'libx265',
+ // '-x265-params', 'crf=0:lossless=1:preset=veryslow:qp=0',
+ // // '-pix_fmt', 'yuv420p',
+ // // '-vf', 'scale=' + outputWidth + ':' + outputHeight + ', setsar=1:1', //, realtime, fps=' + framerate,
+ // // '-preset', 'ultrafast',
+ // // '-copyts',
+ // // '-tune', 'zerolatency',
+ // // '-r', framerate, // will duplicate frames to meet this but still look like the framerate set before -i,
+ // output_path
+ // ];
+
+ let process = cp.spawn(ffmpegPath, args);
+
+ if (constants.DEBUG_LOG_FFMPEG) {
+ // process.stdout.on('data', function(data) {
+ // console.log('stdout data', data);
+ // });
+ process.stderr.setEncoding('utf8');
+ process.stderr.on('data', function (data) {
+ console.log('stderr data', data);
+ });
+ // process.on('close', function() {
+ // console.log('finished');
+ // });
+ }
+
+ return process;
+ },
+ // takes in a textfile with a list of filepaths, one per line, e.g:
+ // file '/my/file/path1.mp4'
+ // file '/my/file/path2.mp4'
+ // and concatenates the specified video segments into a single video
+ ffmpeg_concat_mp4s: (output_path, file_list_path) => {
+ // ffmpeg -f concat -safe 0 -i fileList.txt -c copy mergedVideo.mp4
+ // we pass in a timestamp so we can use an identical one in the color and depth videos that match up
+ let args = [
+ '-f', 'concat',
+ '-safe', '0',
+ '-i', file_list_path,
+ '-c', 'copy',
+ output_path
+ ];
+
+ let process = cp.spawn(ffmpegPath, args);
+ return process;
+ },
+ // takes a video and resamples the timestamps of each frame to make it last the specified duration (in seconds)
+ ffmpeg_adjust_length: (output_path, input_path, newDuration) => {
+ let filesize = fs.statSync(input_path); // size in bytes
+ if (filesize.size <= 48) {
+ console.warn('corrupted video has ~0 bytes, cant resize: ' + input_path);
+ return;
+ }
+ fs.open(input_path, 'r', function(_err, _fd) {
+ module.exports.ffmpeg_get_duration(input_path, (currentDurationInSeconds) => {
+ console.log('change duration from ' + currentDurationInSeconds + 's to ' + newDuration + 's', input_path);
+ let args = [
+ '-i', input_path,
+ '-filter:v', 'setpts=' + newDuration / currentDurationInSeconds + '*PTS',
+ output_path
+ ];
+ let process = cp.spawn(ffmpegPath, args);
+
+ if (constants.DEBUG_LOG_FFMPEG) {
+ process.stderr.setEncoding('utf8');
+ process.stderr.on('data', function (data) {
+ console.log('stderr data', data);
+ });
+ }
+ console.log('file with adjusted length: ' + output_path);
+ });
+ });
+ },
+ ffmpeg_get_duration(filepath, completionHandler) {
+ let args = [ '-i', filepath ]; // this doesn't have an output path, but stderr will print info about the input video
+ // can also use ffprobe, just install @ffprobe-installer/ffprobe
+ let process = cp.spawn(ffmpegPath, args);
+
+ process.stderr.setEncoding('utf8');
+ process.stderr.on('data', function(data) {
+ if (constants.DEBUG_LOG_FFMPEG) {
+ console.log('ffmpeg_get_duration stderr data', data);
+ }
+ let matches = data.match(/Duration: \d\d:\d\d:\d\d.\d\d/);
+ if (!matches || matches.length === 0) { console.log('couldnt get duration of video'); return; }
+ let durationString = matches[0].replace(/^Duration: /, '');
+ // convert format, e.g. 00:01:03.60 => 63.6 seconds
+ let parts = durationString.split(':');
+ let seconds = parseFloat(parts[2]);
+ seconds += 60 * parseInt(parts[1]);
+ seconds += 60 * 60 * parseInt(parts[0]);
+ completionHandler(seconds);
+ });
+ if (constants.DEBUG_LOG_FFMPEG) {
+ process.stdout.on('data', function(data) {
+ console.log('ffmpeg_get_duration stdout data', data);
+ });
+ process.on('close', function () {
+ console.log('ffmpeg_get_duration finished');
+ });
+ process.on('exit', function () {
+ console.log('ffmpeg_get_duration exit');
+ });
+ }
+ return process;
+ }
+};
diff --git a/interfaces/remoteOperatorUI/index.js b/interfaces/remoteOperatorUI/index.js
new file mode 100644
index 00000000..d6dbbf73
--- /dev/null
+++ b/interfaces/remoteOperatorUI/index.js
@@ -0,0 +1,317 @@
+/*
+* Copyright © 2018 PTC
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+/**
+ * Set to true to enable the hardware interface
+ **/
+
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+
+const server = require('@libraries/hardwareInterfaces');
+const utilities = require('@libraries/utilities');
+const Addons = require('@libraries/addons/Addons');
+const LocalUIApp = require('@libraries/LocalUIApp');
+const server8080 = require('../../../../server.js');
+
+const settings = server.loadHardwareInterface(__dirname);
+
+exports.enabled = os.platform() === 'ios' || settings('enabled');
+exports.configurable = true; // can be turned on/off/adjusted from the web frontend
+
+/**
+ * These settings will be exposed to the webFrontend to potentially be modified
+ */
+exports.settings = {
+ userinterfacePath: {
+ value: settings('userinterfacePath'),
+ type: 'text',
+ helpText: 'The absolute path to the vuforia-spatial-toolbox-userinterface.'
+ }
+};
+
+if (exports.enabled) {
+
+ if (os.platform() !== 'ios') {
+ const addonPaths = [
+ path.join(__dirname, '../../../'),
+ path.join(os.homedir(), 'Documents/toolbox/addons'),
+ ];
+
+ const addons = new Addons(addonPaths);
+ const addonFolders = addons.listAddonFolders();
+
+ // Set this in the web frontend, e.g.:
+ // /Users/Benjamin/Documents/github/vuforia-spatial-toolbox-ios/bin/data/userinterface
+ const userinterfacePath = settings('userinterfacePath');
+
+ // Load the userinterface codebase (including all add-ons) using the server's LocalUIApp class
+ // and serve the userinterface on port 8081
+ try {
+ console.info(`UI path for Remote Operator: ${userinterfacePath}`);
+ const localUIApp = new LocalUIApp(userinterfacePath, addonFolders);
+ localUIApp.setup();
+ startHTTPServer(localUIApp, 8081);
+ } catch (e) {
+ console.error('Failed to start Remote Operator: ', e);
+ }
+ }
+
+ try {
+ const rzvServer = require('./server.js');
+ rzvServer.start();
+ } catch (e) {
+ console.error('Unable to start Reality Zone Viewer video/skeleton server', e);
+ }
+}
+
+function startHTTPServer(localUIApp, port) {
+ let mouseTranslation = null;
+ let mouseRotation = null;
+ let mouseConnected = false;
+ var callibrationFrames = 100;
+
+ const io = server8080.io;
+
+ let http = null;
+ if (server8080.useHTTPS) {
+ const fs = require('fs');
+ let options = {
+ key: fs.readFileSync('key.pem'),
+ cert: fs.readFileSync('cert.pem')
+ };
+ http = require('https').Server(options, localUIApp.app);
+ } else {
+ http = require('http').Server(localUIApp.app);
+ }
+ http.on('upgrade', function(req, socket, head) {
+ io.server.handleUpgrade(req, socket, head, (ws) => {
+ io.server.emit('connection', ws, req);
+ });
+ });
+
+ // const wrapServer = new WebSocket.Server({server: http});
+ // Slightly janky shim to put the 8081 connections over onto the 8080 handler
+ // wrapServer.on('connection', (...args) => {
+ // io.onConnection(...args);
+ // });
+
+ function ioBroadcast(route, msg) {
+ for (const socket of io.sockets) {
+ socket.emit(route, msg);
+ }
+ }
+
+ const objectsPath = server.getObjectsPath();
+ const identityFolderName = '.identity';
+
+ http.listen(port, function() {
+ console.info('Remote Operator listening on port (http' + (server8080.useHTTPS ? 's' : '') + ') ' + port);
+
+ // serves the camera poses that correspond to a recorded rgb+depth 3d video
+ localUIApp.app.use('/virtualizer_recording/:deviceId/pose/:filename', function (req, res) {
+ let deviceId = req.params.deviceId;
+ let filename = req.params.filename;
+
+ const jsonFilePath = path.join(objectsPath, identityFolderName, 'virtualizer_recordings', deviceId, 'session_videos', 'pose', filename);
+
+ if (!fs.existsSync(jsonFilePath)) {
+ res.status(404).send('No file at path: ' + jsonFilePath);
+ return;
+ }
+
+ res.sendFile(jsonFilePath);
+ });
+
+ // serves the color and depth video files in streaming format, if range headers are provided
+ localUIApp.app.use('/virtualizer_recording/:deviceId/:colorOrDepth/:filename', function (req, res) {
+ let deviceId = req.params.deviceId;
+ let videoType = req.params.colorOrDepth;
+ let filename = req.params.filename;
+ const videoPath = path.join(objectsPath, identityFolderName, 'virtualizer_recordings', deviceId, 'session_videos', videoType, filename);
+
+ if (!fs.existsSync(videoPath)) {
+ res.status(404).send('No video at path: ' + videoPath);
+ return;
+ }
+
+ const range = req.headers.range;
+ if (!range) {
+ res.sendFile(videoPath); // send video normally if no range headers
+ return;
+ }
+
+ const videoSize = fs.statSync(videoPath).size;
+
+ // Parse Range (example: "bytes=32324-")
+ const CHUNK_SIZE = 10 ** 6; // 1 MB
+ const start = Number(range.replace(/\D/g, ''));
+ const end = Math.min(start + CHUNK_SIZE, videoSize - 1);
+
+ // Create Headers
+ const contentLength = end - start + 1;
+ const headers = {
+ 'Content-Range': `bytes ${start}-${end}/${videoSize}`,
+ 'Accept-Ranges': 'bytes',
+ 'Content-Length': contentLength,
+ 'Content-Type': 'video/mp4',
+ };
+
+ // HTTP Status 206 for partial content
+ res.writeHead(206, headers);
+ const videoStream = fs.createReadStream(videoPath, {start, end});
+ videoStream.pipe(res);
+ });
+
+ // serves the json file containing all of the file paths to the different 3d video recordings
+ localUIApp.app.use('/virtualizer_recordings', function (req, res) {
+ const jsonPath = path.join(objectsPath, identityFolderName, 'virtualizer_recordings', 'videoInfo.json');
+ if (!fs.existsSync(jsonPath)) {
+ res.json({});
+ } else {
+ res.json(JSON.parse(fs.readFileSync(jsonPath, { encoding: 'utf8', flag: 'r' })));
+ }
+ });
+
+ server8080.webServer.use('/userinterface', localUIApp.app);
+
+ // pass visibleObjects messages to the userinterface
+ server.subscribeToMatrixStream(function(visibleObjects) {
+ ioBroadcast('visibleObjects', visibleObjects);
+ });
+
+ // pass UDP messages to the userinterface
+ server.subscribeToUDPMessages(function(msgContent) {
+ ioBroadcast('udpMessage', msgContent);
+ });
+
+ function socketServer() {
+ io.on('connection', function (socket) {
+ socket.on('/subscribe/editorUpdates', function (msg) {
+ var _msgContent = JSON.parse(msg);
+ // console.log('/subscribe/editorUpdates', msgContent);
+
+ // realityEditorSocketArray[socket.id] = {object: msgContent.object, protocol: thisProtocol};
+
+ // socket.emit('object', JSON.stringify({
+ // object: msgContent.object,
+ // frame: msgContent.frame,
+ // node: key,
+ // data: objects[msgContent.object].frames[msgContent.frame].nodes[key].data
+ // }));
+
+ // socket.emit('object/publicData', JSON.stringify({
+ // object: msgContent.object,
+ // frame: msgContent.frame,
+ // publicData: publicData
+ // }));
+
+ });
+
+ socket.on('/matrix/visibleObjects', function (msg) {
+ var msgContent = JSON.parse(msg);
+ ioBroadcast('/matrix/visibleObjects', msgContent);
+ });
+
+ socket.on('/update', function(msg) {
+ var objectKey;
+ var frameKey;
+ var nodeKey;
+
+ var msgContent = JSON.parse(msg);
+ if (typeof msgContent.objectKey !== 'undefined') {
+ objectKey = msgContent.objectKey;
+ }
+ if (typeof msgContent.frameKey !== 'undefined') {
+ frameKey = msgContent.frameKey;
+ }
+ if (typeof msgContent.nodeKey !== 'undefined') {
+ nodeKey = msgContent.nodeKey;
+ }
+
+ if (objectKey && frameKey && nodeKey) {
+ ioBroadcast('/update/node', msgContent);
+ } else if (objectKey && frameKey) {
+ ioBroadcast('/update/frame', msgContent);
+ } else if (objectKey) {
+ ioBroadcast('/update/object', msgContent);
+ }
+
+ });
+
+ /**
+ * Implements the native API functionality of UDP sending for the hosted reality editor desktop app
+ */
+ socket.on('/nativeAPI/sendUDPMessage', function(msg) {
+ var msgContent = JSON.parse(msg);
+ utilities.actionSender(msgContent);
+ });
+
+ // try to connect to custom input device
+ try {
+ connectTo6DMouse();
+ } catch (e) {
+ // Did not connect to input hardware. Control remote operator with mouse + scroll wheel, right-click drag and shift-right-click-drag
+ }
+
+ function connectTo6DMouse() {
+ if (!mouseConnected) {
+ mouseConnected = true;
+ var sm = require('../6DMouse/3DConnexion.js');
+ var callibration = null;
+ sm.spaceMice.onData = function(mouse) {
+ mouseTranslation = mouse.mice[0]['translate'];
+ mouseRotation = mouse.mice[0]['rotate'];
+
+ // try calibrating for the first 100 time-steps, then send translation and rotation
+ // updates to the userinterface using socket messages
+ if (!callibration) {
+ callibrationFrames--;
+ if (callibrationFrames === 0) {
+
+ if (mouseTranslation.x < 1.0 && mouseTranslation.x > -1.0) mouseTranslation.x = 0;
+ if (mouseTranslation.y < 1.0 && mouseTranslation.y > -1.0) mouseTranslation.y = 0;
+ if (mouseTranslation.z < 1.0 && mouseTranslation.z > -1.0) mouseTranslation.z = 0;
+
+ mouseTranslation.x *= 20;
+ mouseTranslation.y *= 20;
+ mouseTranslation.z *= 20;
+
+ callibration = {
+
+ x: mouseTranslation.x * 20,
+ y: mouseTranslation.y * 20,
+ z: mouseTranslation.z * 20
+ };
+ }
+ } else {
+ ioBroadcast('/mouse/transformation', {
+ translation: {
+ x: mouseTranslation.x - callibration.x,
+ y: mouseTranslation.y - callibration.y,
+ z: mouseTranslation.z - callibration.z
+ },
+ rotation: mouseRotation
+ });
+ }
+ };
+ }
+ }
+
+ });
+ }
+
+ socketServer();
+ });
+
+ server.addEventListener('shutdown', () => {
+ http.close();
+ });
+}
+
diff --git a/interfaces/remoteOperatorUI/makeStreamRouter.js b/interfaces/remoteOperatorUI/makeStreamRouter.js
new file mode 100644
index 00000000..2930420d
--- /dev/null
+++ b/interfaces/remoteOperatorUI/makeStreamRouter.js
@@ -0,0 +1,248 @@
+const sceneGraph = require('./sceneGraph/index.js');
+const SceneNode = require('./sceneGraph/SceneNode.js');
+sceneGraph.initService();
+
+function requestId(req) {
+ return parseInt(req.ip.split(/\./g)[3]);
+}
+
+function messageWithId(msg, id) {
+ let idBuf = Buffer.alloc(1);
+ idBuf.writeUint8(id, 0);
+ return Buffer.concat([idBuf, msg]);
+}
+
+module.exports = function makeStreamRouter(app) {
+ let colorPool = [];
+ let depthPool = [];
+ let matrixPool = [];
+ let callbacks = {
+ onFrame: [],
+ onConnection: [],
+ onDisconnection: [],
+ onError: []
+ };
+ app.ws('/colorProvider', function(ws, req) {
+ const id = requestId(req);
+ ws.on('message', function(msg, _isBinary) {
+ const msgWithId = messageWithId(msg, id);
+ for (let wsC of colorPool) {
+ if (wsC.bufferedAmount > 10 * 1024) {
+ continue;
+ }
+ wsC.send(msgWithId);
+ }
+ processFrame(id, msg, null, null);
+ });
+ ws.on('close', function(_event) {
+ callbacks.onDisconnection.forEach(function(cb) {
+ cb(id);
+ });
+ });
+ ws.on('error', function(_event) {
+ callbacks.onError.forEach(function(cb) {
+ cb(id);
+ });
+ });
+
+ callbacks.onConnection.forEach(function(cb) {
+ cb(id);
+ });
+ });
+
+ app.ws('/depthProvider', function(ws, req) {
+ const id = requestId(req);
+ ws.on('message', function(msg, _isBinary) {
+ const msgWithId = messageWithId(msg, id);
+ for (let wsD of depthPool) {
+ if (wsD.bufferedAmount > 10 * 1024) {
+ continue;
+ }
+ wsD.send(msgWithId);
+ }
+ processFrame(id, null, msg, null);
+ });
+ });
+
+ app.ws('/matrixProvider', function(ws, req) {
+ const id = requestId(req);
+ ws.on('message', function(matricesMsg, _isBinary) {
+ const matrices = JSON.parse(matricesMsg);
+ let cameraNode = sceneGraph.getSceneNodeById(sceneGraph.NAMES.CAMERA);
+ cameraNode.setLocalMatrix(matrices.camera);
+ cameraNode.updateWorldMatrix();
+ let gpNode = sceneGraph.getSceneNodeById(sceneGraph.NAMES.GROUNDPLANE + sceneGraph.TAGS.ROTATE_X);
+ if (!gpNode) {
+ gpNode = sceneGraph.getSceneNodeById(sceneGraph.NAMES.GROUNDPLANE);
+ }
+ sceneGraph.getSceneNodeById(sceneGraph.NAMES.GROUNDPLANE).setLocalMatrix(matrices.groundplane);
+ sceneGraph.getSceneNodeById(sceneGraph.NAMES.GROUNDPLANE).updateWorldMatrix();
+
+ let sceneNode = new SceneNode('posePixel');
+ sceneNode.setParent(sceneGraph.getSceneNodeById('ROOT'));
+
+ let initialVehicleMatrix = [
+ -1, 0, 0, 0,
+ 0, 1, 0, 0,
+ 0, 0, -1, 0,
+ 0, 0, 0, 1,
+ ];
+
+ sceneNode.setPositionRelativeTo(cameraNode, initialVehicleMatrix);
+ sceneNode.updateWorldMatrix();
+
+ let cameraMat = sceneNode.getMatrixRelativeTo(gpNode);
+
+ const msg = Buffer.from(new Float32Array(cameraMat).buffer);
+ const msgWithId = messageWithId(msg, id);
+ for (const wsM of matrixPool) {
+ wsM.send(msgWithId);
+ }
+ processFrame(id, null, null, msg);
+ });
+ });
+
+ app.ws('/color', function(ws) {
+ colorPool.push(ws);
+ });
+
+ app.ws('/depth', function(ws) {
+ depthPool.push(ws);
+ });
+
+ app.ws('/matrix', function(ws) {
+ matrixPool.push(ws);
+ });
+
+ let providers = [];
+ let consumers = [];
+ let idToSocket = {};
+
+ app.ws('/signalling', function(ws, _req) {
+ let wsId;
+
+ ws.on('message', function(msgRaw) {
+ let msg;
+ try {
+ msg = JSON.parse(msgRaw);
+ } catch (e) {
+ console.warn('unable to parse ws msg', e, msgRaw);
+ return;
+ }
+
+ if (msg.command === 'joinNetwork') {
+ wsId = msg.src;
+ idToSocket[wsId] = ws;
+
+ if (msg.role === 'consumer') {
+ // new remote operator, send list of iphones
+ ws.send(JSON.stringify({
+ command: 'discoverPeers',
+ dest: msg.src,
+ providers: providers,
+ consumers: consumers,
+ }));
+
+ consumers.push(wsId);
+ }
+
+ if (msg.role === 'provider') {
+ if (!providers.includes(wsId)) {
+ providers.push(wsId);
+ }
+ // idToSocket may have duplicates
+ let socketsSentTo = [];
+ for (let peerId in idToSocket) {
+ if (providers.includes(peerId)) {
+ continue;
+ }
+ let sock = idToSocket[peerId];
+ if (socketsSentTo.includes(sock)) {
+ continue;
+ }
+ socketsSentTo.push(sock);
+ sock.send(JSON.stringify(msg));
+ }
+ }
+ }
+
+ if (msg.command === 'leaveNetwork') {
+ onClose(msg.src);
+ return;
+ }
+
+ if (msg.dest) {
+ if (!idToSocket.hasOwnProperty(msg.dest)) {
+ console.warn('missing dest', msg.dest);
+ return;
+ }
+ idToSocket[msg.dest].send(JSON.stringify(msg));
+ }
+ });
+
+ const onClose = function(wsIdClosed) {
+ delete idToSocket[wsIdClosed];
+ providers = providers.filter(provId => provId !== wsIdClosed);
+ consumers = consumers.filter(conId => conId !== wsIdClosed);
+ };
+
+ ws.on('close', function() {
+ onClose(wsId);
+ });
+
+ ws.on('error', function(e) {
+ console.error('signalling ws error', e);
+ });
+ });
+
+ let frameData = {};
+ function processFrame(id, color, depth, matrix) {
+ if (typeof frameData[id] === 'undefined') {
+ frameData[id] = {
+ color: null,
+ depth: null,
+ matrix: null
+ };
+ return;
+ }
+ if (color) {
+ frameData[id].color = color;
+ }
+ if (depth) {
+ frameData[id].depth = depth;
+ }
+ if (matrix) {
+ frameData[id].matrix = matrix;
+ }
+ if (frameData[id].color && frameData[id].depth && frameData[id].matrix) {
+ callbacks.onFrame.forEach(function(cb) {
+ cb(frameData[id].color, frameData[id].depth, frameData[id].matrix, id);
+ });
+ delete frameData[id];
+ }
+ }
+
+ const onFrame = function(callback) {
+ callbacks.onFrame.push(callback);
+ };
+
+ const onConnection = function(callback) {
+ callbacks.onConnection.push(callback);
+ };
+
+ const onDisconnection = function(callback) {
+ callbacks.onDisconnection.push(callback);
+ };
+
+ const onError = function(callback) {
+ callbacks.onError.push(callback);
+ };
+
+ return {
+ onFrame: onFrame,
+ onConnection: onConnection,
+ onDisconnection: onDisconnection,
+ onError: onError
+ };
+};
+
diff --git a/interfaces/remoteOperatorUI/sceneGraph/SceneNode.js b/interfaces/remoteOperatorUI/sceneGraph/SceneNode.js
new file mode 100644
index 00000000..188595cd
--- /dev/null
+++ b/interfaces/remoteOperatorUI/sceneGraph/SceneNode.js
@@ -0,0 +1,200 @@
+/*
+* Created by Ben Reynolds on 07/13/20.
+*
+* Copyright (c) 2020 PTC Inc
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+const utils = require('./utilities.js');
+
+module.exports = class SceneNode {
+ /**
+ * Defines a node in our scene graph
+ * @constructor
+ */
+ constructor(id) {
+ this.localMatrix = utils.newIdentityMatrix();
+ this.worldMatrix = utils.newIdentityMatrix();
+ this.children = [];
+ this.id = id; // mostly attached for debugging
+ this.parent = null;
+ this.tags = {}; // can be used to label nodes and query the graph
+
+ // if true, any nodes added to this will instead be added to a child of this rotating 90deg
+ this.needsRotateX = false;
+
+ this.needsRecompute = true; // if true, triggers recompute on sub-tree
+ this.needsRerender = true;
+ this.anythingInSubtreeNeedsRerender = true;
+ this.anythingInSubtreeNeedsRecompute = true;
+ this.needsUploadToServer = false;
+
+ // this can be set true when sceneGraph is updated as a result of remote activity
+ this.dontBroadcastNext = false;
+ }
+
+ /**
+ * Sets the parent node of this node, so that it is positioned relative to that
+ * @param {SceneNode} parent
+ */
+ setParent(parent) {
+ if (parent && this.parent && parent === this.parent) {
+ return; // ignore duplicate function calls
+ }
+
+ // remove us from our parent
+ if (this.parent) {
+ let index = this.parent.children.indexOf(this);
+ if (index > -1) {
+ this.parent.children.splice(index, 1);
+ }
+ }
+
+ // add us to our new parent
+ if (parent) {
+ parent.children.push(this);
+ }
+ this.parent = parent;
+
+ // recompute now that we're part of a new parent subtree
+ this.flagForRecompute();
+ }
+
+ getAccumulatedParentScale() {
+ let totalParentScale = 1;
+ let parentPointer = this.parent;
+ while (parentPointer) {
+ let thisParentScale = parentPointer.getVehicleScale(true); // important: avoid infinite loop with "true"
+ totalParentScale *= thisParentScale;
+ parentPointer = parentPointer.parent;
+ }
+ return totalParentScale;
+ }
+
+ getTransformMatrix() {
+ // extracts correctly for frames or nodes
+ let x = this.getVehicleX();
+ let y = this.getVehicleY();
+ let scale = this.getVehicleScale();
+ return [scale, 0, 0, 0,
+ 0, scale, 0, 0,
+ 0, 0, scale, 0,
+ x, y, 0, 1];
+ }
+
+ /**
+ * Compute where this node is relative to the scene origin
+ * @param {Array.} parentWorldMatrix
+ */
+ updateWorldMatrix(parentWorldMatrix) {
+ if (this.needsRecompute) {
+ if (parentWorldMatrix) {
+ // this.worldMatrix stores fully-multiplied position relative to origin
+ utils.multiplyMatrix(this.localMatrix, parentWorldMatrix, this.worldMatrix);
+ } else {
+ // if no parent, localMatrix is worldMatrix
+ utils.copyMatrixInPlace(this.localMatrix, this.worldMatrix);
+ }
+
+ this.needsRecompute = false; // reset dirty flag so we don't repeat this redundantly
+ this.flagForRerender();
+ }
+
+ // process all of its children to update entire subtree
+ if (this.anythingInSubtreeNeedsRecompute) {
+ this.children.forEach(function(childNode) {
+ childNode.updateWorldMatrix(this.worldMatrix);
+ }.bind(this));
+ }
+
+ this.anythingInSubtreeNeedsRecompute = false;
+ }
+
+ setLocalMatrix(matrix) {
+ if (!matrix || matrix.length !== 16) { return; } // ignore malformed/empty input
+ utils.copyMatrixInPlace(matrix, this.localMatrix);
+
+ // flagging this will eventually set the other necessary flags for this and parent/children nodes
+ this.flagForRecompute();
+ }
+
+ flagForRerender() {
+ this.needsRerender = true;
+ this.flagContainingSubtreeForRerender();
+ }
+
+ flagContainingSubtreeForRerender() {
+ this.anythingInSubtreeNeedsRerender = true;
+ if (this.parent) {
+ this.parent.flagContainingSubtreeForRerender();
+ }
+ }
+
+ flagForRecompute() {
+ this.needsRecompute = true;
+ this.flagContainingSubtreeForRecompute();
+
+ // make sure all children get recomputed too, because they are relative to this
+ this.children.forEach(function(childNode) {
+ childNode.flagForRecompute();
+ }.bind(this));
+ }
+
+ flagContainingSubtreeForRecompute() {
+ this.anythingInSubtreeNeedsRecompute = true;
+ if (this.parent && !this.parent.anythingInSubtreeNeedsRecompute) {
+ this.parent.flagContainingSubtreeForRecompute();
+ }
+ }
+
+ getMatrixRelativeTo(otherNode) {
+ // note that this could be one frame out-of-date if this is flaggedForRecompute
+ let thisWorldMatrix = this.worldMatrix;
+ let thatWorldMatrix = otherNode.worldMatrix;
+
+ // if they're the same, we should get identity matrix
+ let relativeMatrix = [];
+ utils.multiplyMatrix(thisWorldMatrix, utils.invertMatrix(thatWorldMatrix), relativeMatrix);
+
+ return relativeMatrix;
+ }
+
+ // figures out what local matrix this node would need to position it globally at the provided world matrix
+ calculateLocalMatrix(worldMatrix) {
+ // get the world matrix of the node's parent = parentWorldMatrix
+ let parentWorldMatrix = this.parent.worldMatrix;
+ // compute the difference between desired worldMatrix and parentWorldMatrix
+ let relativeMatrix = [];
+ utils.multiplyMatrix(worldMatrix, utils.invertMatrix(parentWorldMatrix), relativeMatrix);
+ // return that difference
+
+ return relativeMatrix;
+ }
+
+ setPositionRelativeTo(otherSceneNode, relativeMatrix) {
+ if (typeof relativeMatrix === 'undefined') { relativeMatrix = utils.newIdentityMatrix(); }
+
+ // compute new localMatrix so that
+ // this.localMatrix * parentNode.worldMatrix = relativeMatrix * otherSceneNode.worldMatrix
+ // solving for localMatrix yields:
+ // this.localMatrix = relativeMatrix * otherSceneNode.worldMatrix * inv(parentNode.worldMatrix)
+
+ let temp = [];
+ let result = [];
+ utils.multiplyMatrix(otherSceneNode.worldMatrix, utils.invertMatrix(this.parent.worldMatrix), temp);
+ utils.multiplyMatrix(relativeMatrix, temp, result);
+
+ this.setLocalMatrix(result);
+ }
+
+ addTag(tagName) {
+ this.tags[tagName] = true;
+ }
+
+ removeTag(tagName) {
+ delete this.tags[tagName];
+ }
+};
diff --git a/interfaces/remoteOperatorUI/sceneGraph/index.js b/interfaces/remoteOperatorUI/sceneGraph/index.js
new file mode 100644
index 00000000..53807b11
--- /dev/null
+++ b/interfaces/remoteOperatorUI/sceneGraph/index.js
@@ -0,0 +1,118 @@
+/*
+* Created by Ben Reynolds on 07/13/20.
+*
+* Copyright (c) 2020 PTC Inc
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+/**
+ * This is the new positioning API for objects, tools, and nodes
+ * Scene Graph implementation was inspired by:
+ * https://webglfundamentals.org/webgl/lessons/webgl-scene-graph.html
+ */
+
+const SceneNode = require('./SceneNode.js');
+
+let sceneGraph = {};
+let rootNode;
+let cameraNode;
+let groundPlaneNode;
+
+// TODO ben: use this enum in other modules instead of having any string names
+const NAMES = Object.freeze({
+ ROOT: 'ROOT',
+ CAMERA: 'CAMERA',
+ GROUNDPLANE: 'GROUNDPLANE'
+});
+
+const TAGS = Object.freeze({
+ OBJECT: 'object',
+ TOOL: 'tool',
+ NODE: 'node',
+ ROTATE_X: 'rotateX'
+});
+
+function initService() {
+ // create root node for scene located at phone's (0,0,0) coordinate system
+ rootNode = new SceneNode(NAMES.ROOT);
+ sceneGraph[NAMES.ROOT] = rootNode;
+
+ // create node for camera outside the tree of the main scene
+ cameraNode = new SceneNode(NAMES.CAMERA);
+ sceneGraph[NAMES.CAMERA] = cameraNode;
+
+ // create a node representing the ground plane coordinate system
+ groundPlaneNode = new SceneNode(NAMES.GROUNDPLANE);
+ groundPlaneNode.needsRotateX = true;
+ addRotateX(groundPlaneNode, NAMES.GROUNDPLANE, true);
+ sceneGraph[NAMES.GROUNDPLANE] = groundPlaneNode;
+}
+
+function setCameraPosition(cameraMatrix) {
+ if (!cameraNode) { return; }
+ cameraNode.setLocalMatrix(cameraMatrix);
+}
+
+function setGroundPlanePosition(groundPlaneMatrix) {
+ groundPlaneNode.setLocalMatrix(groundPlaneMatrix);
+}
+
+/**
+ * @param {string} id
+ * @return {SceneNode}
+ */
+function getSceneNodeById(id) {
+ return sceneGraph[id];
+}
+
+/************ Private Functions ************/
+function addRotateX(sceneNodeObject, objectId, groundPlaneVariation) {
+ let sceneNodeRotateX;
+ let thisNodeId = objectId + 'rotateX';
+ if (typeof sceneGraph[thisNodeId] !== 'undefined') {
+ sceneNodeRotateX = sceneGraph[thisNodeId];
+ } else {
+ sceneNodeRotateX = new SceneNode(thisNodeId);
+ sceneNodeRotateX.addTag(TAGS.ROTATE_X);
+ sceneGraph[thisNodeId] = sceneNodeRotateX;
+ }
+
+ sceneNodeRotateX.setParent(sceneNodeObject);
+
+ // image target objects require one coordinate system rotation. ground plane requires another.
+ if (groundPlaneVariation) {
+ sceneNodeRotateX.setLocalMatrix(makeGroundPlaneRotationX(-(Math.PI / 2)));
+ } else {
+ sceneNodeRotateX.setLocalMatrix([ // transform coordinate system by rotateX
+ 1, 0, 0, 0,
+ 0, -1, 0, 0,
+ 0, 0, 1, 0,
+ 0, 0, 0, 1
+ ]);
+ }
+}
+
+function makeGroundPlaneRotationX(theta) {
+ var c = Math.cos(theta), s = Math.sin(theta);
+ return [ 1, 0, 0, 0,
+ 0, c, -s, 0,
+ 0, s, c, 0,
+ 0, 0, 0, 1];
+}
+
+module.exports = {
+ // public init method
+ initService,
+
+ // public methods to update the positions of things in the sceneGraph
+ setCameraPosition,
+ setGroundPlanePosition,
+ // TODO: can we get rid of full/direct access to sceneGraph?
+ getSceneNodeById,
+
+ NAMES,
+ TAGS,
+};
diff --git a/interfaces/remoteOperatorUI/sceneGraph/utilities.js b/interfaces/remoteOperatorUI/sceneGraph/utilities.js
new file mode 100644
index 00000000..5cacde70
--- /dev/null
+++ b/interfaces/remoteOperatorUI/sceneGraph/utilities.js
@@ -0,0 +1,158 @@
+/**
+ * @preserve
+ *
+ * .,,,;;,'''..
+ * .'','... ..',,,.
+ * .,,,,,,',,',;;:;,. .,l,
+ * .,',. ... ,;, :l.
+ * ':;. .'.:do;;. .c ol;'.
+ * ';;' ;.; ', .dkl';, .c :; .'.',::,,'''.
+ * ',,;;;,. ; .,' .'''. .'. .d;''.''''.
+ * .oxddl;::,,. ', .'''. .... .'. ,:;..
+ * .'cOX0OOkdoc. .,'. .. ..... 'lc.
+ * .:;,,::co0XOko' ....''..'.'''''''.
+ * .dxk0KKdc:cdOXKl............. .. ..,c....
+ * .',lxOOxl:'':xkl,',......'.... ,'.
+ * .';:oo:... .
+ * .cd, ╔═╗┌┬┐┬┌┬┐┌─┐┬─┐ .
+ * .l; ║╣ │││ │ │ │├┬┘ '
+ * 'l. ╚═╝─┴┘┴ ┴ └─┘┴└─ '.
+ * .o. ...
+ * .''''','.;:''.........
+ * .' .l
+ * .:. l'
+ * .:. .l.
+ * .x: :k;,.
+ * cxlc; cdc,,;;.
+ * 'l :.. .c ,
+ * o.
+ * .,
+ *
+ * ╦═╗┌─┐┌─┐┬ ┬┌┬┐┬ ┬ ╔═╗┌┬┐┬┌┬┐┌─┐┬─┐ ╔═╗┬─┐┌─┐ ┬┌─┐┌─┐┌┬┐
+ * ╠╦╝├┤ ├─┤│ │ │ └┬┘ ║╣ │││ │ │ │├┬┘ ╠═╝├┬┘│ │ │├┤ │ │
+ * ╩╚═└─┘┴ ┴┴─┘┴ ┴ ┴ ╚═╝─┴┘┴ ┴ └─┘┴└─ ╩ ┴└─└─┘└┘└─┘└─┘ ┴
+ *
+ *
+ * Created by Valentin on 10/22/14.
+ *
+ * Copyright (c) 2015 Valentin Heun
+ * Modified by Valentin Heun 2014, 2015, 2016, 2017
+ * Modified by Benjamin Reynholds 2016, 2017
+ * Modified by James Hobin 2016, 2017
+ *
+ * All ascii characters above must be included in any redistribution.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+/**
+ * @fileOverview js
+ * Various utility functions, mostly mathematical, for calculating AR geometry.
+ * Includes simply utilities like multiplying and inverting a matrix,
+ * as well as sophisticated algorithms for marker-plane intersections and raycasting points onto a plane.
+ */
+
+/**
+ * @desc This function multiplies one m16 matrix with a second m16 matrix
+ * @param {Array.} m2 - origin matrix to be multiplied with
+ * @param {Array.} m1 - second matrix that multiplies.
+ * @return {Array.} m16 matrix result of the multiplication
+ */
+function multiplyMatrix(m2, m1, r) {
+ // var r = [];
+ // Cm1che only the current line of the second mm1trix
+ r[0] = m2[0] * m1[0] + m2[1] * m1[4] + m2[2] * m1[8] + m2[3] * m1[12];
+ r[1] = m2[0] * m1[1] + m2[1] * m1[5] + m2[2] * m1[9] + m2[3] * m1[13];
+ r[2] = m2[0] * m1[2] + m2[1] * m1[6] + m2[2] * m1[10] + m2[3] * m1[14];
+ r[3] = m2[0] * m1[3] + m2[1] * m1[7] + m2[2] * m1[11] + m2[3] * m1[15];
+
+ r[4] = m2[4] * m1[0] + m2[5] * m1[4] + m2[6] * m1[8] + m2[7] * m1[12];
+ r[5] = m2[4] * m1[1] + m2[5] * m1[5] + m2[6] * m1[9] + m2[7] * m1[13];
+ r[6] = m2[4] * m1[2] + m2[5] * m1[6] + m2[6] * m1[10] + m2[7] * m1[14];
+ r[7] = m2[4] * m1[3] + m2[5] * m1[7] + m2[6] * m1[11] + m2[7] * m1[15];
+
+ r[8] = m2[8] * m1[0] + m2[9] * m1[4] + m2[10] * m1[8] + m2[11] * m1[12];
+ r[9] = m2[8] * m1[1] + m2[9] * m1[5] + m2[10] * m1[9] + m2[11] * m1[13];
+ r[10] = m2[8] * m1[2] + m2[9] * m1[6] + m2[10] * m1[10] + m2[11] * m1[14];
+ r[11] = m2[8] * m1[3] + m2[9] * m1[7] + m2[10] * m1[11] + m2[11] * m1[15];
+
+ r[12] = m2[12] * m1[0] + m2[13] * m1[4] + m2[14] * m1[8] + m2[15] * m1[12];
+ r[13] = m2[12] * m1[1] + m2[13] * m1[5] + m2[14] * m1[9] + m2[15] * m1[13];
+ r[14] = m2[12] * m1[2] + m2[13] * m1[6] + m2[14] * m1[10] + m2[15] * m1[14];
+ r[15] = m2[12] * m1[3] + m2[13] * m1[7] + m2[14] * m1[11] + m2[15] * m1[15];
+ // return r;
+}
+
+/**
+ * @desc copies one m16 matrix in to another m16 matrix
+ * Use instead of copyMatrix function when speed is very important - this is faster
+ * @param {Array.} m1 - source matrix
+ * @param {Array.} m2 - resulting copy of the matrix
+ */
+function copyMatrixInPlace(m1, m2) {
+ m2[0] = m1[0];
+ m2[1] = m1[1];
+ m2[2] = m1[2];
+ m2[3] = m1[3];
+ m2[4] = m1[4];
+ m2[5] = m1[5];
+ m2[6] = m1[6];
+ m2[7] = m1[7];
+ m2[8] = m1[8];
+ m2[9] = m1[9];
+ m2[10] = m1[10];
+ m2[11] = m1[11];
+ m2[12] = m1[12];
+ m2[13] = m1[13];
+ m2[14] = m1[14];
+ m2[15] = m1[15];
+}
+
+/**
+ * @desc inverting a matrix
+ * @param {Array.} a origin matrix
+ * @return {Array.} a inverted copy of the origin matrix
+ */
+function invertMatrix (a) {
+ var b = [];
+ var c = a[0], d = a[1], e = a[2], g = a[3], f = a[4], h = a[5], i = a[6], j = a[7], k = a[8], l = a[9], o = a[10], m = a[11], n = a[12], p = a[13], r = a[14], s = a[15], A = c * h - d * f, B = c * i - e * f, t = c * j - g * f, u = d * i - e * h, v = d * j - g * h, w = e * j - g * i, x = k * p - l * n, y = k * r - o * n, z = k * s - m * n, C = l * r - o * p, D = l * s - m * p, E = o * s - m * r, q = 1 / (A * E - B * D + t * C + u * z - v * y + w * x);
+ b[0] = (h * E - i * D + j * C) * q;
+ b[1] = ( -d * E + e * D - g * C) * q;
+ b[2] = (p * w - r * v + s * u) * q;
+ b[3] = ( -l * w + o * v - m * u) * q;
+ b[4] = ( -f * E + i * z - j * y) * q;
+ b[5] = (c * E - e * z + g * y) * q;
+ b[6] = ( -n * w + r * t - s * B) * q;
+ b[7] = (k * w - o * t + m * B) * q;
+ b[8] = (f * D - h * z + j * x) * q;
+ b[9] = ( -c * D + d * z - g * x) * q;
+ b[10] = (n * v - p * t + s * A) * q;
+ b[11] = ( -k * v + l * t - m * A) * q;
+ b[12] = ( -f * C + h * y - i * x) * q;
+ b[13] = (c * C - d * y + e * x) * q;
+ b[14] = ( -n * u + p * B - r * A) * q;
+ b[15] = (k * u - l * B + o * A) * q;
+ return b;
+}
+
+/**
+ * Helper method for creating a new 4x4 identity matrix
+ * @return {Array.}
+ */
+function newIdentityMatrix() {
+ return [
+ 1, 0, 0, 0,
+ 0, 1, 0, 0,
+ 0, 0, 1, 0,
+ 0, 0, 0, 1
+ ];
+}
+
+module.exports = {
+ newIdentityMatrix,
+ multiplyMatrix,
+ copyMatrixInPlace,
+ invertMatrix,
+};
diff --git a/interfaces/remoteOperatorUI/server.js b/interfaces/remoteOperatorUI/server.js
new file mode 100644
index 00000000..dcc3faa5
--- /dev/null
+++ b/interfaces/remoteOperatorUI/server.js
@@ -0,0 +1,224 @@
+const cors = require('cors');
+const express = require('express');
+const expressWs = require('express-ws');
+const makeStreamRouter = require('./makeStreamRouter.js');
+const path = require('path');
+const os = require('os');
+const server = require('@libraries/hardwareInterfaces');
+
+let DEBUG_DISABLE_VIDEO_RECORDING = os.platform() === 'ios' || process.env.NODE_ENV === 'test';
+
+let VideoServer;
+if (!DEBUG_DISABLE_VIDEO_RECORDING) {
+ try {
+ VideoServer = require('./VideoServer.js');
+ } catch (e) {
+ console.warn('VideoServer unavailable', e);
+ DEBUG_DISABLE_VIDEO_RECORDING = true;
+ }
+}
+
+module.exports.start = function start() {
+ const app = express();
+
+ const identityFolderName = '.identity';
+ const DEVICE_ID_PREFIX = 'device';
+
+ app.use(cors());
+ expressWs(app);
+ const streamRouter = makeStreamRouter(app);
+
+ // rgb+depth videos are stored in the Documents/spatialToolbox/.identity/virtualizer_recordings
+ let videoServer = null;
+ if (!DEBUG_DISABLE_VIDEO_RECORDING) {
+ videoServer = new VideoServer(path.join(server.getObjectsPath(), identityFolderName, '/virtualizer_recordings'));
+ // trigger events in VideoServer whenever sockets connect, disconnect, or send data
+ streamRouter.onFrame((rgb, depth, pose, deviceId) => {
+ videoServer.onFrame(rgb, depth, pose, DEVICE_ID_PREFIX + deviceId);
+ });
+ streamRouter.onConnection((deviceId) => {
+ videoServer.onConnection(DEVICE_ID_PREFIX + deviceId);
+ });
+ streamRouter.onDisconnection((deviceId) => {
+ videoServer.onDisconnection(DEVICE_ID_PREFIX + deviceId);
+ });
+ streamRouter.onError((deviceId) => {
+ console.log('on error: ' + deviceId); // haven't seen this trigger yet but probably good to also disconnect
+ videoServer.onDisconnection(DEVICE_ID_PREFIX + deviceId);
+ });
+ }
+
+ let allWebsockets = [];
+ let sensorDescriptions = {};
+
+ function broadcast(broadcaster, msgStr) {
+ for (let ws of allWebsockets) {
+ if (ws === broadcaster) {
+ continue;
+ }
+ ws.send(msgStr);
+ }
+ }
+
+ let activeSkels = {};
+
+ function requestId(req) {
+ return parseInt(req.ip.split(/\./g)[3]);
+ }
+
+ app.ws('/', (ws, req) => {
+ console.log('an attempt /');
+ allWebsockets.push(ws);
+ let wsId = '' + (Math.random() * 9999);
+ let deviceId = requestId(req);
+
+ ws.addEventListener('close', () => {
+ allWebsockets = allWebsockets.filter(a => a !== ws);
+ });
+
+ let playback = null;
+ ws.on('message', (msgStr, _isBinary) => {
+
+ try {
+ const msg = JSON.parse(msgStr);
+ switch (msg.command) {
+ case '/update/humanPoses':
+ doUpdateHumanPoses(msg);
+ break;
+ case '/update/sensorDescription':
+ doUpdateSensorDescription(msg);
+ break;
+ case '/videoRecording/start':
+ if (videoServer) {
+ videoServer.startRecording(DEVICE_ID_PREFIX + deviceId);
+ }
+ break;
+ case '/videoRecording/stop':
+ if (videoServer) {
+ videoServer.stopRecording(DEVICE_ID_PREFIX + deviceId);
+ }
+ break;
+ }
+ } catch (error) {
+ console.warn('Could not parse message: ', error);
+ }
+
+ });
+
+ let cleared = false;
+ function doUpdateHumanPoses(msg) {
+ if (playback && !playback.running) {
+ playback = null;
+ }
+ if (msg.hasOwnProperty('length')) {
+ msg = {
+ time: Date.now(),
+ pose: msg,
+ };
+ }
+ let poses = msg.pose;
+ for (let skel of poses) {
+ activeSkels[skel.id] = {
+ msgId: wsId,
+ skel,
+ lastUpdate: Date.now(),
+ };
+ }
+ for (let activeSkel of Object.values(activeSkels)) {
+ if (activeSkel.skel.joints.length === 0 ||
+ Date.now() - activeSkel.lastUpdate > 1500) {
+ delete activeSkels[activeSkel.skel.id];
+ continue;
+ }
+ if (activeSkel.msgId !== wsId) {
+ poses.push(activeSkel.skel);
+ }
+ }
+ if (poses.length === 0) {
+ if (cleared) {
+ return;
+ } else {
+ cleared = true;
+ }
+ } else {
+ cleared = false;
+ }
+
+ if (!playback) {
+ broadcast(ws, JSON.stringify(msg));
+ }
+
+ processSensorActivations(poses);
+ }
+
+ function doUpdateSensorDescription(desc) {
+ sensorDescriptions[desc.id] = JSON.parse(JSON.stringify(desc)); // desc;
+ // const t = desc.x;
+ // desc.x = -desc.x;
+ // desc.z = -desc.z;
+ console.log('sensorDesc', desc);
+ // sock.broadcast.emit('/update/sensorDescription', JSON.stringify(desc));
+ broadcast(ws, JSON.stringify(desc));
+ }
+
+ function processSensorActivations(poses) {
+ // for (let pose of poses) {
+ // for (let joint of pose.joints) {
+ // joint.z = -joint.z;
+ // }
+ // }
+
+ for (let id in sensorDescriptions) {
+ let sensorDesc = sensorDescriptions[id];
+ let oldCount = sensorDesc.count;
+ sensorDesc.count = 0;
+
+ for (let pose of poses) {
+ for (let joint of pose.joints) {
+ if (joint.x < sensorDesc.x - sensorDesc.width / 2) {
+ continue;
+ }
+ if (joint.x > sensorDesc.x + sensorDesc.width / 2) {
+ continue;
+ }
+ if (joint.y < sensorDesc.y - sensorDesc.height / 2) {
+ continue;
+ }
+ if (joint.y > sensorDesc.y + sensorDesc.height / 2) {
+ continue;
+ }
+ if (joint.z < sensorDesc.z - sensorDesc.depth / 2) {
+ continue;
+ }
+ if (joint.z > sensorDesc.z + sensorDesc.depth / 2) {
+ continue;
+ }
+
+ sensorDesc.count += 1;
+ break;
+ }
+ }
+
+ let sendActivation = oldCount !== sensorDesc.count || Math.random() < 0.03;
+
+ if (sendActivation) {
+ // console.log('yey', sensorDesc);
+ broadcast(ws, JSON.stringify({
+ command: '/update/sensorActivation',
+ id: sensorDesc.id,
+ count: Math.floor(sensorDesc.count),
+ active: sensorDesc.count > 0,
+ }));
+ }
+ }
+ }
+
+ });
+
+ const port = 31337;
+ const httpServer = app.listen(port);
+ server.addEventListener('shutdown', () => {
+ httpServer.close();
+ });
+ console.info("Reality Zone Viewer video/skeleton server listening on port " + port);
+};
diff --git a/interfaces/remoteOperatorUI/utilities.js b/interfaces/remoteOperatorUI/utilities.js
new file mode 100644
index 00000000..525b2e75
--- /dev/null
+++ b/interfaces/remoteOperatorUI/utilities.js
@@ -0,0 +1,11 @@
+const fs = require('fs');
+
+module.exports = {
+ mkdirIfNeeded: (path, recursive) => { // helper function since we do this so often
+ let options = recursive ? {recursive: true} : undefined;
+ if (!fs.existsSync(path)) {
+ fs.mkdirSync(path, options);
+ console.log('created directory: ' + path);
+ }
+ }
+};
diff --git a/interfaces/remoteOperatorUI/videoConstants.js b/interfaces/remoteOperatorUI/videoConstants.js
new file mode 100644
index 00000000..61e8f1e8
--- /dev/null
+++ b/interfaces/remoteOperatorUI/videoConstants.js
@@ -0,0 +1,35 @@
+module.exports = Object.freeze({
+ // how long should each video chunk be (ms). they are concatenated when recording stops.
+ SEGMENT_LENGTH: 15000,
+ // note: if we change this here, also need to change timeline playback,
+ // because pose/image synchronization implementation depends on the FPS to locate the correct frame
+ RECORDING_FPS: 10,
+
+ // files are stored in the outputPath (currently Documents/spatialToolbox/.identity/virtualizer_recordings)
+ // an example filepath looks something like virtualizer_recordings/deviceId/session_videos/color/filename.mp4
+ DIR_NAMES: {
+ unprocessed_chunks: 'unprocessed_chunks', // where to store raw 15 second recording chunks
+ processed_chunks: 'processed_chunks', // where to store chunks after optional post-processing
+ session_videos: 'session_videos', // where to store concatenated chunks as a final video
+ color: 'color', // subdirectory for rgb video files
+ depth: 'depth', // subdirectory for depth video files
+ pose: 'pose' // subdirectory for pose json files
+ },
+
+ // adjustable ffmpeg parameters
+ COLOR_FILETYPE: 'mp4',
+ DEPTH_FILETYPE: 'mp4', // previously tried webm and mkv for lossless encoding but it never quite worked
+ COLOR_CRF: 25, // 0 is pseudo-lossless, 51 is worst quality possible, 23 is considered default
+ DEPTH_CRF: 0, // a subjectively sane range for crf is 17-28 (https://trac.ffmpeg.org/)
+ COLOR_SCALE: 0.5, // camera image is initially (COLOR_WIDTH x COLOR_HEIGHT), this will scale down the video dimensions
+ DEPTH_SCALE: 1, // depth sensor image should probably stay at scale=1 to preserve information.
+ COLOR_WIDTH: 1920,
+ COLOR_HEIGHT: 1080,
+ DEPTH_WIDTH: 256,
+ DEPTH_HEIGHT: 144,
+
+ // disable to prevent lossy transformation, enable to stretch videos back to correct time length
+ // (it's ok to be false, the video playback system can adjust for this)
+ RESCALE_VIDEOS: false,
+ DEBUG_LOG_FFMPEG: false
+});
diff --git a/interfaces/virtualizer/index.js b/interfaces/virtualizer/index.js
new file mode 100644
index 00000000..fdc6626e
--- /dev/null
+++ b/interfaces/virtualizer/index.js
@@ -0,0 +1,723 @@
+/*
+* Copyright © 2018 PTC
+*
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+var server = require('@libraries/hardwareInterfaces');
+var utilities = require('@libraries/utilities');
+
+var settings = server.loadHardwareInterface(__dirname);
+
+exports.enabled = settings('enabled');
+exports.configurable = true; // can be turned on/off/adjusted from the web frontend
+
+if (exports.enabled) {
+ var express = require('express');
+ var app = require('express')();
+ var cors = require('cors'); // Library for HTTP Cross-Origin-Resource-Sharing
+ app.use(cors());
+ // allow requests from all origins with '*'. TODO make it dependent on the local network. this is important for security
+ app.options('*', cors());
+
+ app.use(express.static(__dirname + '/public'));
+ var http = require('http').Server(app);
+ var ip = require('ip');
+ var glob = require('glob');
+ var io = require('socket.io')(http, { wsEngine: 'ws' });
+
+ var socket_list = [];
+
+ var stations = [];
+ var viewers = [];
+ var editorToSocketId = {};
+ var socketToEditorId = {};
+ var viewer_width = null;
+ var viewer_height = null;
+
+ //custom vuforia server:
+ var vuforiaResultClient = null;
+ var vuforiaCamClient = null;
+ var reality_zone_gifurl_list = ['bogus_url1', 'bogus_url2'];
+ //var system = null;
+
+ var tic = 0;
+ var toc = 0;
+
+ var desktop_connected_hack = 0;
+
+ let obj = null;
+ let realityZoneControlInterface = null;
+
+ server.subscribeToUDPMessages(function(msgContent) {
+ //console.log('received udp message with content: ' + msgContent);
+ //console.dir(msgContent);
+
+ try {
+ obj = msgContent;
+ //var obj = JSON.parse(msgContent);
+ /*
+ id: thisId,
+ ip: thisIp,
+ vn: thisVersionNumber,
+ pr: protocol,
+ tcs: objects[thisId].tcs,
+ zone: zone
+ */
+
+ //check if its an object on the network:
+ if (obj.id && obj.ip && obj.vn ) {
+
+
+ var objectName = obj.id.slice(0, -12);
+ var httpPort = 8080;
+ var xmlAddress = 'http://' + obj.ip + ':' + httpPort + '/obj/' + objectName + '/target/target.xml';
+ var datAddress = 'http://' + obj.ip + ':' + httpPort + '/obj/' + objectName + '/target/target.dat';
+
+ var messageObject = {
+ id: obj.id,
+ ip: obj.ip,
+ versionNumber: obj.vn,
+ protocol: obj.pr,
+ temporaryChecksum: obj.tcs,
+ zone: obj.zone,
+ xmlAddress: xmlAddress,
+ datAddress: datAddress
+ };
+
+ for (let station of stations) {
+ station.emit('realityEditorObject_server_system', JSON.stringify(messageObject));
+ }
+
+ //console.log('received udp message: ' + obj.id + " " + obj.ip + " " + obj.vn + " " + obj.pr + " " + obj.tcs + " " + obj.zone);
+ //console.log('xml address: ' + xmlAddress);
+ //console.log('data address: ' + datAddress);
+ }
+
+
+
+ if (obj.action != null) {
+ //console.dir(obj);
+ //console.log('received action: ' + obj.action);
+
+ var action = JSON.parse(obj.action);
+ if (action.action == 'advertiseEditor') {
+ console.log('action advertise editor received');
+ console.log('width: ' + action.resolution.width + ' height: ' + action.resolution.height);
+ console.log('editor id: ' + action.editorId);
+ console.log('completed reading json');
+
+ viewer_width = action.resolution.width;
+ viewer_height = action.resolution.height;
+
+ //send a message:
+ var zoneResponseMessage = {
+ action: 'zoneDiscovered',
+ ip: ip.address(),
+ port: 3020
+ };
+ utilities.actionSender(JSON.stringify(zoneResponseMessage));
+ }
+ }
+
+
+ } catch (e) {
+ //console.log('could not parse json message: ' + e);
+ }
+
+
+
+
+ });
+
+ io.on('connection', function(socket) {
+ console.log('my ip address: ' + ip.address());
+ console.log('Client has connected');
+ socket.emit('message', 'hello client');
+ socket.id = Math.floor(Math.random() * 1000000);
+ var temp_ip = socket.request.connection.remoteAddress;
+ var parts = temp_ip.split(':');
+ temp_ip = parts[parts.length - 1];
+ socket_list.push(socket);
+
+ socket.viewer_width = viewer_width;
+ socket.viewer_height = viewer_height;
+
+ socket.resolution_set = 0;
+
+ socket.on('disconnect', function () {
+ //if it's in the viewer list, remove it:
+ var index = viewers.indexOf(socket);
+ if (index > -1) {
+ viewers.splice(index, 1);
+ }
+
+ index = socket_list.indexOf(socket);
+ if (index > -1) {
+ socket_list.splice(index, 1);
+ }
+
+ if (vuforiaCamClient == socket) {
+ console.log('vuforiaCamClient disconnecting');
+ vuforiaCamClient = null;
+ }
+
+ if (vuforiaResultClient == socket) {
+ console.log('vuforiaResultClient disconnecting');
+ vuforiaCamClient = null;
+ }
+
+ });
+
+
+ socket.on('image', function (data) {
+ //console.log('station sent image. viewer list length: ' + viewers.length + ' datalength: ' + data.length);
+
+ // extract editorId if included
+ let editorId = null;
+ try {
+ let partsData = data.split(';_;');
+ editorId = partsData[2];
+ // console.log('received image for ' + editorId);
+ } catch (e) {
+ // console.log('error extracting editorId from image data', e);
+ }
+
+ if (stations.length > 0) {
+ for (let viewer of viewers) {
+ let viewerEditorId = socketToEditorId[viewer.id];
+ // console.log('send to viewer ' + viewerEditorId + '?');
+ if (editorId && viewerEditorId) {
+ if (viewerEditorId !== editorId) {
+ // console.log('skipping mismatching editorIds');
+ continue;
+ }
+ }
+
+ viewer.emit('image', data);
+ /*
+ //write it to file for debug purposes
+ fs.writeFile('debug.jpg', data,'base64', function(err, data){
+ if (err) console.log(err);
+ console.log("Successfully Written to File.");
+ });
+ */
+ }
+ }
+ });
+
+ socket.on('name', function(data) {
+ console.log('name function called with: ' + data);
+ //console.log('data length: ' + data.length);
+ //console.log('data = viewer? ' + (data === '"viewer"'));
+ let parsedData = {};
+ try {
+ parsedData = JSON.parse(data);
+ console.log(parsedData.type);
+ } catch (e) {
+ console.log('cant parse ' + data);
+ }
+ if (parsedData.type === 'viewer') {
+ console.log('client identified as viewer. assigning it latest viewer and id: ' + socket.id + ' and editorId: ' + parsedData.editorId);
+
+ socket.type = 'viewer';
+ viewers.push(socket);
+
+ editorToSocketId[parsedData.editorId] = socket.id;
+ socketToEditorId[socket.id] = parsedData.editorId;
+
+ for (let station of stations) {
+ console.log('setting resolution for the first time: ' + viewer_width + ',' + viewer_height );
+ station.emit('resolution', '' + viewer_width + ',' + viewer_height);
+ }
+
+ //send some matrices to the RDV:
+ console.log('now that a desktop has connected');
+
+ var visibleObjectsString = '{"amazonBox0zbc6yetuoyj":[-0.9964230765028328,0.009800613580158324,-0.08393736781840644,0,0.022929297584320937,0.987345281122669,-0.15691860457929643,0,-0.08133670600902992,0.15828222984430484,0.9840378965122027,0,329.2388939106902,77.77425852082308,-1489.0291313193022,1],"kepwareBox4Qimhnuea3n6":[-0.9913746548884379,0.034050970326370084,-0.12655601222953172,0,0.051082427979348755,0.9896809533465255,-0.13387410555924745,0,-0.12069159903807483,0.1391844414830788,0.9828837698713899,0,212.63741144996703,206.15960431449824,-1826.5693898311488,1],"_WORLD_OBJECT_local":[-0.9742120258704184,-0.00437900631612313,-0.22559212287700664,0,-0.035464931453757634,-0.9844127588621392,0.17226278091636868,0,-0.22282991544549868,0.17582138398908906,0.9588711290537871,0,161.99950094104497,-166.5114134489016,-519.0149842693205,1],"_WORLD_OBJECT_PF5x8fv3zcgm":[-0.9742120258704184,-0.00437900631612313,-0.22559212287700664,0,-0.035464931453757634,-0.9844127588621392,0.17226278091636868,0,-0.22282991544549868,0.17582138398908906,0.9588711290537871,0,161.99950094104497,-166.5114134489016,-519.0149842693205,1]}';
+ var _visibleObjects = JSON.parse(visibleObjectsString);
+
+ var zoneMatrixMessage = {
+ action: 'zoneMatrixMessage',
+ matrices: visibleObjectsString,
+ ip: ip.address(),
+ port: 3020
+ };
+
+ utilities.actionSender(JSON.stringify(zoneMatrixMessage));
+
+ console.log('looking for objects: ');
+ var pingMessage = {
+ action: 'ping'
+ };
+ for (var i = 0; i < 3; i++) {
+ setTimeout(function() {
+ utilities.actionSender(JSON.stringify(pingMessage));
+ //realityEditor.app.sendUDPMessage({action: 'ping'});
+ }, 500 * i); // space out each message by 500ms
+ }
+ }
+
+ if (data === 'station') {
+ console.log('client identified as station');
+ stations.push(socket);
+ socket.type = 'station';
+ if (viewer_width != null) {
+ console.log('sending resolution to station: ' + viewer_width + ',' + viewer_height );
+ socket.emit('resolution', '' + viewer_width + ',' + viewer_height);
+ }
+ }
+
+ if (data === 'vuforiaCamClient') {
+ console.log('client identified as vuforiaCamClient');
+ vuforiaCamClient = socket;
+ socket.type = 'vuforiaCamClient';
+ }
+
+ if (data === 'vuforiaResultClient') {
+ console.log('client identified as vuforiaResultClient');
+ vuforiaResultClient = socket;
+ socket.type = 'vuforiaResultClient';
+ }
+
+ if (data === 'realityZoneControlInterface') {
+ console.log('client identified as realityZoneControlInterface');
+ realityZoneControlInterface = socket;
+ socket.type = 'realityZoneControlInterface';
+
+
+ //find all the gifs:
+ var gif_url_token = __dirname + '/public/gifs/*/*.gif';
+ reality_zone_gifurl_list = [];
+ glob(gif_url_token, function (er, files) {
+ //console.log('dir name: ' + __dirname);
+ //console.log('gif urls: ' + files);
+ for (var j = 0; j < files.length; j++) {
+ var clean_file = files[j];
+ var pos = clean_file.indexOf('/gifs/');
+ clean_file = clean_file.substring(pos);
+ console.log('adding: ' + clean_file);
+ reality_zone_gifurl_list.push(clean_file);
+ }
+
+
+ //console.log('reality_zone_gifurl_list: ');
+ //console.dir(reality_zone_gifurl_list);
+ //send it a list of image urls!
+ for (var ii = 0; ii < reality_zone_gifurl_list.length; ii++) {
+
+ var gifurl_object = new Object();
+ gifurl_object.gifurl = reality_zone_gifurl_list[ii];
+ //console.log('sending: ' + JSON.stringify(gifurl_object));
+ socket.emit('realityZoneGif_server_realityZoneControlInterface', JSON.stringify(gifurl_object));
+ }
+
+ // files is an array of filenames.
+ // If the `nonull` option is set, and nothing
+ // was found, then files is ["**/*.js"]
+ // er is an error object or null.
+ });
+ }
+ });
+
+ socket.on('message', function(data) {
+ console.log('message send', data);
+ if (socket.type !== 'viewer') {
+ console.log('not a viewer but we will let it slide for now');
+ }
+ for (let station of stations) {
+ station.emit('message', data);
+ }
+ });
+
+ socket.on('realityZoneGif_system_server', function(data) {
+ //add to reality_zone_gifurl_list
+ if (realityZoneControlInterface != null) {
+ var data_object = JSON.parse(data);
+ var clean_file = data_object.gifurl;
+ var pos = clean_file.indexOf('gifs');
+ clean_file = clean_file.substring(pos - 1);
+
+ var gifurl_object = new Object();
+ gifurl_object.gifurl = clean_file;
+ realityZoneControlInterface.emit('realityZoneGif_server_realityZoneControlInterface', JSON.stringify(gifurl_object));
+ }
+ });
+
+ socket.on('stopRecording_realityZoneControlInterface_server', function(_data) {
+ console.log('got stop recording command from website');
+ for (let station of stations) {
+ station.emit('stopRecording_server_system');
+ }
+ });
+
+ socket.on('startRecording_realityZoneControlInterface_server', function(_data) {
+ console.log('got start recording command from website');
+ for (let station of stations) {
+ station.emit('startRecording_server_system');
+ }
+ });
+
+ socket.on('twin_realityZoneControlInterface_server', function(_data) {
+ console.log('creating twin');
+ for (let station of stations) {
+ station.emit('twin_server_system');
+ }
+ });
+
+ socket.on('clearTwins_realityZoneControlInterface_server', function(_data) {
+ console.log('clearing twins');
+ for (let station of stations) {
+ station.emit('clearTwins_server_system');
+ }
+ });
+
+ socket.on('zoneInteractionMessage_realityZoneControlInterface_server', function(data) {
+ console.log('zone interaction message: ' + data);
+ for (let station of stations) {
+ station.emit('zoneInteractionMessage_server_system', data);
+ }
+ });
+
+
+
+ //viewer resolution information:
+ socket.on('resolution', function(data) {
+ console.log('received resolution from phone viewer: ' + data);
+ var result = data.split(',');
+ viewer_width = parseFloat(result[0]);
+ viewer_height = parseFloat(result[1]);
+ socket.viewer_width = viewer_width;
+ socket.viewer_height = viewer_height;
+
+ console.log('width: ' + viewer_width);
+ console.log('height: ' + viewer_height);
+
+
+
+ for (let station of stations) {
+ console.log('found resolution from a phone!');
+ station.emit('resolutionPhone', '' + viewer_width + ',' + viewer_height);
+ /*
+ if(socket.id!=ip.address()){ //this is necessary to fix the issue wiht desktop. todo: explain this better.
+ console.log("found resolution from a phone!");
+ station.emit('resolutionPhone',""+viewer_width+','+viewer_height);
+ }
+ */
+
+ console.log('sending resolution to station from phone: ' + viewer_width + ',' + viewer_height );
+ station.emit('resolution', '' + viewer_width + ',' + viewer_height);
+ }
+ });
+
+
+ //viewer resolution information:
+ socket.on('resolutionDesktop', function(data) {
+ console.log('received resolution from desktop viewer: ' + data);
+ desktop_connected_hack = 1;
+ console.log('TURNIGN DESKTOP CONNECTED HACK TO 1');
+ var result = data.split(',');
+ viewer_width = parseFloat(result[0]);
+ viewer_height = parseFloat(result[1]);
+ socket.viewer_width = viewer_width;
+ socket.viewer_height = viewer_height;
+
+ console.log('width: ' + viewer_width);
+ console.log('height: ' + viewer_height);
+
+
+
+ for (let station of stations) {
+ /*
+ if(socket.id!=ip.address()){ //this is necessary to fix the issue wiht desktop. todo: explain this better.
+ console.log("found resolution from a phone!");
+ station.emit('resolutionPhone',""+viewer_width+','+viewer_height);
+ }
+ */
+ console.log('sending resolution to station from desktop: ' + viewer_width + ',' + viewer_height );
+ station.emit('resolution', '' + viewer_width + ',' + viewer_height);
+ }
+ });
+
+
+ //viewer pose information:
+ socket.on('pose', function(data) {
+ //console.log('received position from viewer: ' + socket.id);
+ //console.log('current viewer is: ' + viewers[0].id);
+ for (let station of stations) {
+ if (socket.id == viewers[0].id) { //only capture the most recent one
+ //console.log('sending viewer data to station');
+ station.emit('pose', data);
+ } else {
+ //console.log('not sending viewer data to station');
+ }
+ }
+ });
+
+ // this is for the 2020 scene graph version
+ socket.on('cameraPosition', function(data) {
+ if (stations.length > 0) {
+ try {
+ var poseInfo = JSON.parse(data);
+
+ if (poseInfo.cameraPoseMatrix && poseInfo.projectionMatrix) {
+
+ if ((socket.viewer_width !== poseInfo.resolution.width) || (socket.viewer_height !== poseInfo.resolution.height)) {
+ socket.viewer_width = poseInfo.resolution.width;
+ socket.viewer_height = poseInfo.resolution.height;
+ console.log('setting unity resolution from valid matrix to: ' + socket.viewer_width + ',' + socket.viewer_height);
+ for (let station of stations) {
+ station.emit('resolution', '' + socket.viewer_width + ',' + socket.viewer_height);
+ }
+ }
+
+ for (let station of stations) {
+ station.emit('cameraPosition', JSON.stringify(poseInfo));
+ }
+ }
+
+ } catch (e) {
+ console.log('error! could not parse camera pose info: ' + e);
+ }
+ }
+ });
+
+ // ben is testing this with the new version of the RDV camera system - 11/1/19
+ socket.on('poseVuforiaCamera', function(data) {
+ if (stations.length > 0) {
+ try {
+ var poseInfo = JSON.parse(data);
+
+ var cameraPoseMatrix = poseInfo.cameraPoseMatrix;
+ let validMatrix = cameraPoseMatrix;
+
+ //with RDV origin mode: overwrite with RDV camera to RDV origin matrix
+ if (poseInfo.cameraMode === 'REALITY_ZONE_ORIGIN') {
+ validMatrix = poseInfo.RDVCameraPoseMatrix;
+ }
+
+ if (validMatrix != -1) {
+ var data_for_old_unity_version = poseInfo;
+ data_for_old_unity_version.modelViewMatrix = validMatrix;
+ data_for_old_unity_version.projectionMatrix = poseInfo.projectionMatrix;
+ data_for_old_unity_version.realProjectionMatrix = poseInfo.realProjectionMatrix;
+
+ if ((socket.viewer_width != poseInfo.resolution.width) || (socket.viewer_height != poseInfo.resolution.height)) {
+ socket.viewer_width = poseInfo.resolution.width;
+ socket.viewer_height = poseInfo.resolution.height;
+ console.log('setting unity resolution from valid matrix to: ' + socket.viewer_width + ',' + socket.viewer_height);
+ for (let station of stations) {
+ station.emit('resolution', '' + socket.viewer_width + ',' + socket.viewer_height);
+ }
+ }
+
+ data_for_old_unity_version = JSON.stringify(data_for_old_unity_version);
+ for (let station of stations) {
+ station.emit('poseVuforiaDesktop', data_for_old_unity_version);
+ }
+ }
+
+ } catch (e) {
+ console.log('error! could not parse camera pose info: ' + e);
+ }
+ }
+ });
+
+ //use this for now on, hisham - 5/22/2019
+ socket.on('poseVuforia', function(data) {
+ //console.log('pose vuforia called: ' + data);
+ if (stations.length > 0) {
+ try {
+ var poseInfo = JSON.parse(data);
+
+
+
+ //socket.viewer_width = 640;
+ //socket.viewer_height = 480;
+
+ /*
+ if(socket.resolution_set == 0){
+ console.log('setting unity resolution to: ' +socket.viewer_width+','+socket.viewer_height);
+ station.emit('resolution',""+socket.viewer_width+','+socket.viewer_height);
+ socket.resolution_set = 1;
+ }
+ */
+
+
+ var _vuforiaObjectList = poseInfo.visibleObjectMatrices;
+ var objectNameList = Object.keys(poseInfo.visibleObjectMatrices);
+ var validMatrix = -1;
+ for (var i = 0; i < objectNameList.length; i++) {
+ //if(!objectNameList[i].includes('_WORLD_OBJECT')){
+ //if(!objectNameList[i].includes('feederZoneTwo')){
+ if (objectNameList[i].includes('kepwareBox4Qimhnuea3n6')) {
+ validMatrix = poseInfo.visibleObjectMatrices[objectNameList[i]];
+ }
+ }
+
+ //with RDV origin mode: overwrite with RDV camera to RDV origin matrix
+ if (poseInfo.cameraMode === 'REALITY_ZONE_ORIGIN') {
+ //console.log("reality zone origin!");
+ validMatrix = poseInfo.RDVCameraPoseMatrix;
+ }
+
+ //console.dir(validMatrix);
+ //vuforiaObjectList = JSON.parse(vuforiaObjectList);
+ //console.dir(vuforiaObjectList);
+
+
+
+ if (validMatrix != -1) {
+ var data_for_old_unity_version = poseInfo;
+ data_for_old_unity_version.modelViewMatrix = validMatrix;
+ data_for_old_unity_version.projectionMatrix = poseInfo.projectionMatrix;
+ data_for_old_unity_version.realProjectionMatrix = poseInfo.realProjectionMatrix;
+
+ //console.log(poseInfo.cameraPoseMatrix);
+ //console.log(poseInfo.projectionMatrix);
+ //console.log("received resolution: " , poseInfo.resolution.width , "," , poseInfo.resolution.height);
+ //console.log('socket viewer resolution: ',socket.viewer_width , ",",socket.viewer_height);
+
+ if ((socket.viewer_width != poseInfo.resolution.width) || (socket.viewer_height != poseInfo.resolution.height)) {
+
+ socket.viewer_width = poseInfo.resolution.width;
+ socket.viewer_height = poseInfo.resolution.height;
+ console.log('change in resolution: ');
+ console.log('setting unity resolution from valid matrix to: ' + socket.viewer_width + ',' + socket.viewer_height);
+ for (let station of stations) {
+ station.emit('resolution', '' + socket.viewer_width + ',' + socket.viewer_height);
+ }
+ }
+
+
+
+ data_for_old_unity_version = JSON.stringify(data_for_old_unity_version);
+ //console.log('sending pose vuforia desktop');
+ for (let station of stations) {
+ station.emit('poseVuforiaDesktop', data_for_old_unity_version);
+ }
+ }
+
+ } catch (e) {
+ console.log('error! could not parse pose info: ' + e);
+ }
+ }
+
+
+
+ });
+
+
+
+
+ //viewer pose information (from vuforia matrix):
+ socket.on('poseVuforiaOld', function(data) {
+ //console.log('sending pose information from phone to station');
+ //console.log('received position from viewer: ' + socket.id + " " + data);
+ //console.log('current viewer is: ' + viewers[0].id);
+ //console.log('pose vuforia phone: ' + data);
+
+ for (let station of stations) {
+ //console.log('sending viewer data to station from: ' + socket.id + " " + data);
+ //console.log('station is not null, desktop connected hack: ' + desktop_connected_hack);
+ if (desktop_connected_hack == 0) {
+ station.emit('poseVuforia', data);
+ }
+ }
+ });
+
+ socket.on('poseVuforiaDesktop', function(data) {
+ //console.log('pose vuforia desktop: ' + data);
+ if (desktop_connected_hack == 1) {
+ for (let station of stations) {
+ station.emit('poseVuforiaDesktop', data);
+ }
+ }
+ });
+
+ socket.on('debug', function(data) {
+ console.log('viewer debug: ' + data);
+ });
+
+ socket.on('startPing', function(data) {
+ console.log('received start ping from object. emitting data to station');
+ //console.log(data);
+ for (let station of stations) {
+ station.emit('ping', data);
+ }
+ });
+
+ socket.on('stationPong', function(data) {
+ //console.log('got data from station');
+ for (let viewer of viewers) {
+ viewer.emit('endPong', data);
+ }
+ });
+
+ socket.on('pong2', function(_data) {
+ //console.log('got pong!')
+ var d = new Date();
+ toc = d.getTime();
+ console.log('round trip ping pong: ' + (toc - tic) + ' ms');
+ });
+
+ //reality zone vuforia module:
+ socket.on('vuforiaModuleUpdate_system_server', function(data) {
+ //console.log('received vuforia module update: ' + data);
+ if (viewers.length > 0) {
+ //console.log('data after parsing json: ');
+ //console.log(JSON.parse(data));
+ //data = JSON.stringify(JSON.parse(data));
+ data = JSON.parse(data);
+
+ var zoneMatrixMessage = {
+ action: 'zoneMatrixMessage',
+ matrices: data,
+ ip: ip.address(),
+ port: 3020
+ };
+
+ utilities.actionSender(JSON.stringify(zoneMatrixMessage));
+ }
+ });
+
+
+
+ //custom vuforia server:
+ socket.on('vuforiaImage_system_server', function(data) {
+ console.log('image received from system');
+ console.log('data length: ' + data.length);
+ console.log('first 100: ' + data.substring(0, 99));
+ if ((vuforiaCamClient != null)) {
+ //if((vuforiaCamClient != null) && (vuforiaResultClient!=null)){
+ console.log('passing image to vuforia camera');
+ vuforiaCamClient.emit('cameraData_server_vuforiaCameraClient', data);
+ } else {
+ console.log('could not find vuforia camera');
+ var response = new Object();
+ response.message = 'there was no vuforia cam processor';
+ socket.emit('vuforiaResult_server_system', JSON.stringify(response));
+ }
+ });
+
+ socket.on('vuforiaResult_vuforiaResultClient_server', function(data) {
+ if (stations.length === 0) {
+ console.log('ERROR: received vuforia result but there is not reality-zone unity system connected');
+ return;
+ }
+
+ for (let station of stations) {
+ station.emit('vuforiaResult_server_system', data);
+ }
+ });
+ });
+
+ const httpServer = http.listen(3020, function() {
+ console.log('listening on *:3020');
+ });
+ server.addEventListener('shutdown', () => {
+ httpServer.close();
+ });
+}
diff --git a/package-lock.json b/package-lock.json
new file mode 100644
index 00000000..a34bff74
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,4395 @@
+{
+ "name": "vuforia-spatial-remote-operator-addon",
+ "version": "1.0.0",
+ "lockfileVersion": 2,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "vuforia-spatial-remote-operator-addon",
+ "version": "1.0.0",
+ "license": "MPL-2.0",
+ "dependencies": {
+ "@ffmpeg-installer/ffmpeg": "^1.1.0",
+ "cors": "^2.8.5",
+ "express": "^4.17.1",
+ "express-ws": "^5.0.2",
+ "socket.io": "~2.2.0",
+ "socket.io-client": "~2.3.1",
+ "toolsocket": "github:ptcrealitylab/toolsocket"
+ },
+ "devDependencies": {
+ "eslint": "^8.57.0",
+ "webrtc-adapter": "^9.0.1"
+ }
+ },
+ "node_modules/@aashutoshrathi/word-wrap": {
+ "version": "1.2.6",
+ "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
+ "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/@eslint-community/eslint-utils": {
+ "version": "4.4.0",
+ "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
+ "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==",
+ "dev": true,
+ "dependencies": {
+ "eslint-visitor-keys": "^3.3.0"
+ },
+ "engines": {
+ "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ },
+ "peerDependencies": {
+ "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0"
+ }
+ },
+ "node_modules/@eslint-community/regexpp": {
+ "version": "4.10.0",
+ "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz",
+ "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==",
+ "dev": true,
+ "engines": {
+ "node": "^12.0.0 || ^14.0.0 || >=16.0.0"
+ }
+ },
+ "node_modules/@eslint/eslintrc": {
+ "version": "2.1.4",
+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz",
+ "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==",
+ "dev": true,
+ "dependencies": {
+ "ajv": "^6.12.4",
+ "debug": "^4.3.2",
+ "espree": "^9.6.0",
+ "globals": "^13.19.0",
+ "ignore": "^5.2.0",
+ "import-fresh": "^3.2.1",
+ "js-yaml": "^4.1.0",
+ "minimatch": "^3.1.2",
+ "strip-json-comments": "^3.1.1"
+ },
+ "engines": {
+ "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://opencollective.com/eslint"
+ }
+ },
+ "node_modules/@eslint/eslintrc/node_modules/debug": {
+ "version": "4.3.4",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+ "dev": true,
+ "dependencies": {
+ "ms": "2.1.2"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/@eslint/eslintrc/node_modules/ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ },
+ "node_modules/@eslint/js": {
+ "version": "8.57.0",
+ "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz",
+ "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==",
+ "dev": true,
+ "engines": {
+ "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ }
+ },
+ "node_modules/@ffmpeg-installer/darwin-arm64": {
+ "version": "4.1.5",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz",
+ "integrity": "sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==",
+ "cpu": [
+ "arm64"
+ ],
+ "hasInstallScript": true,
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@ffmpeg-installer/darwin-x64": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz",
+ "integrity": "sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==",
+ "cpu": [
+ "x64"
+ ],
+ "hasInstallScript": true,
+ "optional": true,
+ "os": [
+ "darwin"
+ ]
+ },
+ "node_modules/@ffmpeg-installer/ffmpeg": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz",
+ "integrity": "sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==",
+ "optionalDependencies": {
+ "@ffmpeg-installer/darwin-arm64": "4.1.5",
+ "@ffmpeg-installer/darwin-x64": "4.1.0",
+ "@ffmpeg-installer/linux-arm": "4.1.3",
+ "@ffmpeg-installer/linux-arm64": "4.1.4",
+ "@ffmpeg-installer/linux-ia32": "4.1.0",
+ "@ffmpeg-installer/linux-x64": "4.1.0",
+ "@ffmpeg-installer/win32-ia32": "4.1.0",
+ "@ffmpeg-installer/win32-x64": "4.1.0"
+ }
+ },
+ "node_modules/@ffmpeg-installer/linux-arm": {
+ "version": "4.1.3",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz",
+ "integrity": "sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==",
+ "cpu": [
+ "arm"
+ ],
+ "hasInstallScript": true,
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@ffmpeg-installer/linux-arm64": {
+ "version": "4.1.4",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz",
+ "integrity": "sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==",
+ "cpu": [
+ "arm64"
+ ],
+ "hasInstallScript": true,
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@ffmpeg-installer/linux-ia32": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz",
+ "integrity": "sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==",
+ "cpu": [
+ "ia32"
+ ],
+ "hasInstallScript": true,
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@ffmpeg-installer/linux-x64": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz",
+ "integrity": "sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==",
+ "cpu": [
+ "x64"
+ ],
+ "hasInstallScript": true,
+ "optional": true,
+ "os": [
+ "linux"
+ ]
+ },
+ "node_modules/@ffmpeg-installer/win32-ia32": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz",
+ "integrity": "sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==",
+ "cpu": [
+ "ia32"
+ ],
+ "optional": true,
+ "os": [
+ "win32"
+ ]
+ },
+ "node_modules/@ffmpeg-installer/win32-x64": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz",
+ "integrity": "sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==",
+ "cpu": [
+ "x64"
+ ],
+ "optional": true,
+ "os": [
+ "win32"
+ ]
+ },
+ "node_modules/@humanwhocodes/config-array": {
+ "version": "0.11.14",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz",
+ "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==",
+ "dev": true,
+ "dependencies": {
+ "@humanwhocodes/object-schema": "^2.0.2",
+ "debug": "^4.3.1",
+ "minimatch": "^3.0.5"
+ },
+ "engines": {
+ "node": ">=10.10.0"
+ }
+ },
+ "node_modules/@humanwhocodes/config-array/node_modules/debug": {
+ "version": "4.3.4",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+ "dev": true,
+ "dependencies": {
+ "ms": "2.1.2"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/@humanwhocodes/config-array/node_modules/ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ },
+ "node_modules/@humanwhocodes/module-importer": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
+ "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
+ "dev": true,
+ "engines": {
+ "node": ">=12.22"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/nzakas"
+ }
+ },
+ "node_modules/@humanwhocodes/object-schema": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz",
+ "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==",
+ "dev": true
+ },
+ "node_modules/@nodelib/fs.scandir": {
+ "version": "2.1.5",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+ "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+ "dev": true,
+ "dependencies": {
+ "@nodelib/fs.stat": "2.0.5",
+ "run-parallel": "^1.1.9"
+ },
+ "engines": {
+ "node": ">= 8"
+ }
+ },
+ "node_modules/@nodelib/fs.stat": {
+ "version": "2.0.5",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+ "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+ "dev": true,
+ "engines": {
+ "node": ">= 8"
+ }
+ },
+ "node_modules/@nodelib/fs.walk": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+ "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+ "dev": true,
+ "dependencies": {
+ "@nodelib/fs.scandir": "2.1.5",
+ "fastq": "^1.6.0"
+ },
+ "engines": {
+ "node": ">= 8"
+ }
+ },
+ "node_modules/@ungap/structured-clone": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
+ "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==",
+ "dev": true
+ },
+ "node_modules/accepts": {
+ "version": "1.3.8",
+ "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
+ "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
+ "dependencies": {
+ "mime-types": "~2.1.34",
+ "negotiator": "0.6.3"
+ },
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/acorn": {
+ "version": "8.11.3",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
+ "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
+ "dev": true,
+ "bin": {
+ "acorn": "bin/acorn"
+ },
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
+ "node_modules/acorn-jsx": {
+ "version": "5.3.2",
+ "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
+ "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
+ "dev": true,
+ "peerDependencies": {
+ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
+ }
+ },
+ "node_modules/after": {
+ "version": "0.8.2",
+ "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz",
+ "integrity": "sha512-QbJ0NTQ/I9DI3uSJA4cbexiwQeRAfjPScqIbSjUDd9TOrcg6pTkdgziesOqxBMBzit8vFCTwrP27t13vFOORRA=="
+ },
+ "node_modules/ajv": {
+ "version": "6.12.6",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+ "dev": true,
+ "dependencies": {
+ "fast-deep-equal": "^3.1.1",
+ "fast-json-stable-stringify": "^2.0.0",
+ "json-schema-traverse": "^0.4.1",
+ "uri-js": "^4.2.2"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/epoberezkin"
+ }
+ },
+ "node_modules/ansi-regex": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/ansi-styles": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+ "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+ "dev": true,
+ "dependencies": {
+ "color-convert": "^2.0.1"
+ },
+ "engines": {
+ "node": ">=8"
+ },
+ "funding": {
+ "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+ }
+ },
+ "node_modules/argparse": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+ "dev": true
+ },
+ "node_modules/array-flatten": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
+ },
+ "node_modules/arraybuffer.slice": {
+ "version": "0.0.7",
+ "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
+ "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
+ },
+ "node_modules/async-limiter": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz",
+ "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ=="
+ },
+ "node_modules/backo2": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz",
+ "integrity": "sha512-zj6Z6M7Eq+PBZ7PQxl5NT665MvJdAkzp0f60nAJ+sLaSCBPMwVak5ZegFbgVCzFcCJTKFoMizvM5Ld7+JrRJHA=="
+ },
+ "node_modules/balanced-match": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+ "dev": true
+ },
+ "node_modules/base64-arraybuffer": {
+ "version": "0.1.5",
+ "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz",
+ "integrity": "sha512-437oANT9tP582zZMwSvZGy2nmSeAb8DW2me3y+Uv1Wp2Rulr8Mqlyrv3E7MLxmsiaPSMMDmiDVzgE+e8zlMx9g==",
+ "engines": {
+ "node": ">= 0.6.0"
+ }
+ },
+ "node_modules/base64id": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz",
+ "integrity": "sha512-rz8L+d/xByiB/vLVftPkyY215fqNrmasrcJsYkVcm4TgJNz+YXKrFaFAWibSaHkiKoSgMDCb+lipOIRQNGYesw==",
+ "engines": {
+ "node": ">= 0.4.0"
+ }
+ },
+ "node_modules/better-assert": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/better-assert/-/better-assert-1.0.2.tgz",
+ "integrity": "sha512-bYeph2DFlpK1XmGs6fvlLRUN29QISM3GBuUwSFsMY2XRx4AvC0WNCS57j4c/xGrK2RS24C1w3YoBOsw9fT46tQ==",
+ "dependencies": {
+ "callsite": "1.0.0"
+ },
+ "engines": {
+ "node": "*"
+ }
+ },
+ "node_modules/blob": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz",
+ "integrity": "sha512-gaqbzQPqOoamawKg0LGVd7SzLgXS+JH61oWprSLH+P+abTczqJbhTR8CmJ2u9/bUYNmHTGJx/UEmn6doAvvuig=="
+ },
+ "node_modules/body-parser": {
+ "version": "1.20.1",
+ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
+ "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
+ "dependencies": {
+ "bytes": "3.1.2",
+ "content-type": "~1.0.4",
+ "debug": "2.6.9",
+ "depd": "2.0.0",
+ "destroy": "1.2.0",
+ "http-errors": "2.0.0",
+ "iconv-lite": "0.4.24",
+ "on-finished": "2.4.1",
+ "qs": "6.11.0",
+ "raw-body": "2.5.1",
+ "type-is": "~1.6.18",
+ "unpipe": "1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.8",
+ "npm": "1.2.8000 || >= 1.4.16"
+ }
+ },
+ "node_modules/body-parser/node_modules/debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/body-parser/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/brace-expansion": {
+ "version": "1.1.11",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+ "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+ "dev": true,
+ "dependencies": {
+ "balanced-match": "^1.0.0",
+ "concat-map": "0.0.1"
+ }
+ },
+ "node_modules/bytes": {
+ "version": "3.1.2",
+ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+ "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/call-bind": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz",
+ "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==",
+ "dependencies": {
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.1",
+ "set-function-length": "^1.1.1"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/callsite": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz",
+ "integrity": "sha512-0vdNRFXn5q+dtOqjfFtmtlI9N2eVZ7LMyEV2iKC5mEEFvSg/69Ml6b/WU2qF8W1nLRa0wiSrDT3Y5jOHZCwKPQ==",
+ "engines": {
+ "node": "*"
+ }
+ },
+ "node_modules/callsites": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=6"
+ }
+ },
+ "node_modules/chalk": {
+ "version": "4.1.2",
+ "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+ "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+ "dev": true,
+ "dependencies": {
+ "ansi-styles": "^4.1.0",
+ "supports-color": "^7.1.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/chalk/chalk?sponsor=1"
+ }
+ },
+ "node_modules/color-convert": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+ "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+ "dev": true,
+ "dependencies": {
+ "color-name": "~1.1.4"
+ },
+ "engines": {
+ "node": ">=7.0.0"
+ }
+ },
+ "node_modules/color-name": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+ "dev": true
+ },
+ "node_modules/component-bind": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/component-bind/-/component-bind-1.0.0.tgz",
+ "integrity": "sha512-WZveuKPeKAG9qY+FkYDeADzdHyTYdIboXS59ixDeRJL5ZhxpqUnxSOwop4FQjMsiYm3/Or8cegVbpAHNA7pHxw=="
+ },
+ "node_modules/component-emitter": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz",
+ "integrity": "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==",
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/component-inherit": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/component-inherit/-/component-inherit-0.0.3.tgz",
+ "integrity": "sha512-w+LhYREhatpVqTESyGFg3NlP6Iu0kEKUHETY9GoZP/pQyW4mHFZuFWRUCIqVPZ36ueVLtoOEZaAqbCF2RDndaA=="
+ },
+ "node_modules/concat-map": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
+ "dev": true
+ },
+ "node_modules/content-disposition": {
+ "version": "0.5.4",
+ "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
+ "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
+ "dependencies": {
+ "safe-buffer": "5.2.1"
+ },
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/content-type": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
+ "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/cookie": {
+ "version": "0.5.0",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
+ "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/cookie-signature": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
+ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
+ },
+ "node_modules/cors": {
+ "version": "2.8.5",
+ "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
+ "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
+ "dependencies": {
+ "object-assign": "^4",
+ "vary": "^1"
+ },
+ "engines": {
+ "node": ">= 0.10"
+ }
+ },
+ "node_modules/cross-spawn": {
+ "version": "7.0.3",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+ "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+ "dev": true,
+ "dependencies": {
+ "path-key": "^3.1.0",
+ "shebang-command": "^2.0.0",
+ "which": "^2.0.1"
+ },
+ "engines": {
+ "node": ">= 8"
+ }
+ },
+ "node_modules/debug": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+ "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+ "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)",
+ "dependencies": {
+ "ms": "^2.1.1"
+ }
+ },
+ "node_modules/deep-is": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
+ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
+ "dev": true
+ },
+ "node_modules/define-data-property": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz",
+ "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==",
+ "dependencies": {
+ "get-intrinsic": "^1.2.1",
+ "gopd": "^1.0.1",
+ "has-property-descriptors": "^1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
+ "node_modules/depd": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+ "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/destroy": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
+ "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
+ "engines": {
+ "node": ">= 0.8",
+ "npm": "1.2.8000 || >= 1.4.16"
+ }
+ },
+ "node_modules/doctrine": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
+ "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+ "dev": true,
+ "dependencies": {
+ "esutils": "^2.0.2"
+ },
+ "engines": {
+ "node": ">=6.0.0"
+ }
+ },
+ "node_modules/ee-first": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="
+ },
+ "node_modules/encodeurl": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
+ "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/engine.io": {
+ "version": "3.3.2",
+ "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.3.2.tgz",
+ "integrity": "sha512-AsaA9KG7cWPXWHp5FvHdDWY3AMWeZ8x+2pUVLcn71qE5AtAzgGbxuclOytygskw8XGmiQafTmnI9Bix3uihu2w==",
+ "dependencies": {
+ "accepts": "~1.3.4",
+ "base64id": "1.0.0",
+ "cookie": "0.3.1",
+ "debug": "~3.1.0",
+ "engine.io-parser": "~2.1.0",
+ "ws": "~6.1.0"
+ }
+ },
+ "node_modules/engine.io-client": {
+ "version": "3.4.4",
+ "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.4.4.tgz",
+ "integrity": "sha512-iU4CRr38Fecj8HoZEnFtm2EiKGbYZcPn3cHxqNGl/tmdWRf60KhK+9vE0JeSjgnlS/0oynEfLgKbT9ALpim0sQ==",
+ "dependencies": {
+ "component-emitter": "~1.3.0",
+ "component-inherit": "0.0.3",
+ "debug": "~3.1.0",
+ "engine.io-parser": "~2.2.0",
+ "has-cors": "1.1.0",
+ "indexof": "0.0.1",
+ "parseqs": "0.0.6",
+ "parseuri": "0.0.6",
+ "ws": "~6.1.0",
+ "xmlhttprequest-ssl": "~1.5.4",
+ "yeast": "0.1.2"
+ }
+ },
+ "node_modules/engine.io-client/node_modules/base64-arraybuffer": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz",
+ "integrity": "sha512-a1eIFi4R9ySrbiMuyTGx5e92uRH5tQY6kArNcFaKBUleIoLjdjBg7Zxm3Mqm3Kmkf27HLR/1fnxX9q8GQ7Iavg==",
+ "engines": {
+ "node": ">= 0.6.0"
+ }
+ },
+ "node_modules/engine.io-client/node_modules/debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/engine.io-client/node_modules/engine.io-parser": {
+ "version": "2.2.1",
+ "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.2.1.tgz",
+ "integrity": "sha512-x+dN/fBH8Ro8TFwJ+rkB2AmuVw9Yu2mockR/p3W8f8YtExwFgDvBDi0GWyb4ZLkpahtDGZgtr3zLovanJghPqg==",
+ "dependencies": {
+ "after": "0.8.2",
+ "arraybuffer.slice": "~0.0.7",
+ "base64-arraybuffer": "0.1.4",
+ "blob": "0.0.5",
+ "has-binary2": "~1.0.2"
+ }
+ },
+ "node_modules/engine.io-client/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/engine.io-client/node_modules/ws": {
+ "version": "6.1.4",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
+ "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+ "dependencies": {
+ "async-limiter": "~1.0.0"
+ }
+ },
+ "node_modules/engine.io-parser": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.3.tgz",
+ "integrity": "sha512-6HXPre2O4Houl7c4g7Ic/XzPnHBvaEmN90vtRO9uLmwtRqQmTOw0QMevL1TOfL2Cpu1VzsaTmMotQgMdkzGkVA==",
+ "dependencies": {
+ "after": "0.8.2",
+ "arraybuffer.slice": "~0.0.7",
+ "base64-arraybuffer": "0.1.5",
+ "blob": "0.0.5",
+ "has-binary2": "~1.0.2"
+ }
+ },
+ "node_modules/engine.io/node_modules/cookie": {
+ "version": "0.3.1",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
+ "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/engine.io/node_modules/debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/engine.io/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/engine.io/node_modules/ws": {
+ "version": "6.1.4",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
+ "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+ "dependencies": {
+ "async-limiter": "~1.0.0"
+ }
+ },
+ "node_modules/escape-html": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+ "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
+ },
+ "node_modules/escape-string-regexp": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+ "dev": true,
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/eslint": {
+ "version": "8.57.0",
+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz",
+ "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==",
+ "dev": true,
+ "dependencies": {
+ "@eslint-community/eslint-utils": "^4.2.0",
+ "@eslint-community/regexpp": "^4.6.1",
+ "@eslint/eslintrc": "^2.1.4",
+ "@eslint/js": "8.57.0",
+ "@humanwhocodes/config-array": "^0.11.14",
+ "@humanwhocodes/module-importer": "^1.0.1",
+ "@nodelib/fs.walk": "^1.2.8",
+ "@ungap/structured-clone": "^1.2.0",
+ "ajv": "^6.12.4",
+ "chalk": "^4.0.0",
+ "cross-spawn": "^7.0.2",
+ "debug": "^4.3.2",
+ "doctrine": "^3.0.0",
+ "escape-string-regexp": "^4.0.0",
+ "eslint-scope": "^7.2.2",
+ "eslint-visitor-keys": "^3.4.3",
+ "espree": "^9.6.1",
+ "esquery": "^1.4.2",
+ "esutils": "^2.0.2",
+ "fast-deep-equal": "^3.1.3",
+ "file-entry-cache": "^6.0.1",
+ "find-up": "^5.0.0",
+ "glob-parent": "^6.0.2",
+ "globals": "^13.19.0",
+ "graphemer": "^1.4.0",
+ "ignore": "^5.2.0",
+ "imurmurhash": "^0.1.4",
+ "is-glob": "^4.0.0",
+ "is-path-inside": "^3.0.3",
+ "js-yaml": "^4.1.0",
+ "json-stable-stringify-without-jsonify": "^1.0.1",
+ "levn": "^0.4.1",
+ "lodash.merge": "^4.6.2",
+ "minimatch": "^3.1.2",
+ "natural-compare": "^1.4.0",
+ "optionator": "^0.9.3",
+ "strip-ansi": "^6.0.1",
+ "text-table": "^0.2.0"
+ },
+ "bin": {
+ "eslint": "bin/eslint.js"
+ },
+ "engines": {
+ "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://opencollective.com/eslint"
+ }
+ },
+ "node_modules/eslint-scope": {
+ "version": "7.2.2",
+ "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz",
+ "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==",
+ "dev": true,
+ "dependencies": {
+ "esrecurse": "^4.3.0",
+ "estraverse": "^5.2.0"
+ },
+ "engines": {
+ "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://opencollective.com/eslint"
+ }
+ },
+ "node_modules/eslint-visitor-keys": {
+ "version": "3.4.3",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz",
+ "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==",
+ "dev": true,
+ "engines": {
+ "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://opencollective.com/eslint"
+ }
+ },
+ "node_modules/eslint/node_modules/debug": {
+ "version": "4.3.4",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+ "dev": true,
+ "dependencies": {
+ "ms": "2.1.2"
+ },
+ "engines": {
+ "node": ">=6.0"
+ },
+ "peerDependenciesMeta": {
+ "supports-color": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/eslint/node_modules/ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ },
+ "node_modules/espree": {
+ "version": "9.6.1",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
+ "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==",
+ "dev": true,
+ "dependencies": {
+ "acorn": "^8.9.0",
+ "acorn-jsx": "^5.3.2",
+ "eslint-visitor-keys": "^3.4.1"
+ },
+ "engines": {
+ "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://opencollective.com/eslint"
+ }
+ },
+ "node_modules/esquery": {
+ "version": "1.5.0",
+ "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz",
+ "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==",
+ "dev": true,
+ "dependencies": {
+ "estraverse": "^5.1.0"
+ },
+ "engines": {
+ "node": ">=0.10"
+ }
+ },
+ "node_modules/esrecurse": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+ "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+ "dev": true,
+ "dependencies": {
+ "estraverse": "^5.2.0"
+ },
+ "engines": {
+ "node": ">=4.0"
+ }
+ },
+ "node_modules/estraverse": {
+ "version": "5.3.0",
+ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+ "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+ "dev": true,
+ "engines": {
+ "node": ">=4.0"
+ }
+ },
+ "node_modules/esutils": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
+ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/etag": {
+ "version": "1.8.1",
+ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
+ "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/express": {
+ "version": "4.18.2",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
+ "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
+ "dependencies": {
+ "accepts": "~1.3.8",
+ "array-flatten": "1.1.1",
+ "body-parser": "1.20.1",
+ "content-disposition": "0.5.4",
+ "content-type": "~1.0.4",
+ "cookie": "0.5.0",
+ "cookie-signature": "1.0.6",
+ "debug": "2.6.9",
+ "depd": "2.0.0",
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "etag": "~1.8.1",
+ "finalhandler": "1.2.0",
+ "fresh": "0.5.2",
+ "http-errors": "2.0.0",
+ "merge-descriptors": "1.0.1",
+ "methods": "~1.1.2",
+ "on-finished": "2.4.1",
+ "parseurl": "~1.3.3",
+ "path-to-regexp": "0.1.7",
+ "proxy-addr": "~2.0.7",
+ "qs": "6.11.0",
+ "range-parser": "~1.2.1",
+ "safe-buffer": "5.2.1",
+ "send": "0.18.0",
+ "serve-static": "1.15.0",
+ "setprototypeof": "1.2.0",
+ "statuses": "2.0.1",
+ "type-is": "~1.6.18",
+ "utils-merge": "1.0.1",
+ "vary": "~1.1.2"
+ },
+ "engines": {
+ "node": ">= 0.10.0"
+ }
+ },
+ "node_modules/express-ws": {
+ "version": "5.0.2",
+ "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz",
+ "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==",
+ "dependencies": {
+ "ws": "^7.4.6"
+ },
+ "engines": {
+ "node": ">=4.5.0"
+ },
+ "peerDependencies": {
+ "express": "^4.0.0 || ^5.0.0-alpha.1"
+ }
+ },
+ "node_modules/express/node_modules/debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/express/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/fast-deep-equal": {
+ "version": "3.1.3",
+ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+ "dev": true
+ },
+ "node_modules/fast-json-stable-stringify": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+ "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+ "dev": true
+ },
+ "node_modules/fast-levenshtein": {
+ "version": "2.0.6",
+ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
+ "dev": true
+ },
+ "node_modules/fastq": {
+ "version": "1.17.1",
+ "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz",
+ "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==",
+ "dev": true,
+ "dependencies": {
+ "reusify": "^1.0.4"
+ }
+ },
+ "node_modules/file-entry-cache": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
+ "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+ "dev": true,
+ "dependencies": {
+ "flat-cache": "^3.0.4"
+ },
+ "engines": {
+ "node": "^10.12.0 || >=12.0.0"
+ }
+ },
+ "node_modules/finalhandler": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
+ "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
+ "dependencies": {
+ "debug": "2.6.9",
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "on-finished": "2.4.1",
+ "parseurl": "~1.3.3",
+ "statuses": "2.0.1",
+ "unpipe": "~1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/finalhandler/node_modules/debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/finalhandler/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/find-up": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
+ "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
+ "dev": true,
+ "dependencies": {
+ "locate-path": "^6.0.0",
+ "path-exists": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/flat-cache": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz",
+ "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==",
+ "dev": true,
+ "dependencies": {
+ "flatted": "^3.2.9",
+ "keyv": "^4.5.3",
+ "rimraf": "^3.0.2"
+ },
+ "engines": {
+ "node": "^10.12.0 || >=12.0.0"
+ }
+ },
+ "node_modules/flatted": {
+ "version": "3.3.1",
+ "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz",
+ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==",
+ "dev": true
+ },
+ "node_modules/forwarded": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+ "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/fresh": {
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
+ "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/fs.realpath": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+ "dev": true
+ },
+ "node_modules/function-bind": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+ "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/get-intrinsic": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz",
+ "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==",
+ "dependencies": {
+ "function-bind": "^1.1.2",
+ "has-proto": "^1.0.1",
+ "has-symbols": "^1.0.3",
+ "hasown": "^2.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/glob": {
+ "version": "7.2.3",
+ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+ "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+ "dev": true,
+ "dependencies": {
+ "fs.realpath": "^1.0.0",
+ "inflight": "^1.0.4",
+ "inherits": "2",
+ "minimatch": "^3.1.1",
+ "once": "^1.3.0",
+ "path-is-absolute": "^1.0.0"
+ },
+ "engines": {
+ "node": "*"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/isaacs"
+ }
+ },
+ "node_modules/glob-parent": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+ "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+ "dev": true,
+ "dependencies": {
+ "is-glob": "^4.0.3"
+ },
+ "engines": {
+ "node": ">=10.13.0"
+ }
+ },
+ "node_modules/globals": {
+ "version": "13.24.0",
+ "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz",
+ "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==",
+ "dev": true,
+ "dependencies": {
+ "type-fest": "^0.20.2"
+ },
+ "engines": {
+ "node": ">=8"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/gopd": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+ "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+ "dependencies": {
+ "get-intrinsic": "^1.1.3"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/graphemer": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz",
+ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
+ "dev": true
+ },
+ "node_modules/has-binary2": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-binary2/-/has-binary2-1.0.3.tgz",
+ "integrity": "sha512-G1LWKhDSvhGeAQ8mPVQlqNcOB2sJdwATtZKl2pDKKHfpf/rYj24lkinxf69blJbnsvtqqNU+L3SL50vzZhXOnw==",
+ "dependencies": {
+ "isarray": "2.0.1"
+ }
+ },
+ "node_modules/has-cors": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz",
+ "integrity": "sha512-g5VNKdkFuUuVCP9gYfDJHjK2nqdQJ7aDLTnycnc2+RvsOQbuLdF5pm7vuE5J76SEBIQjs4kQY/BWq74JUmjbXA=="
+ },
+ "node_modules/has-flag": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/has-property-descriptors": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz",
+ "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==",
+ "dependencies": {
+ "get-intrinsic": "^1.2.2"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/has-proto": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
+ "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/has-symbols": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+ "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/hasown": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
+ "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
+ "dependencies": {
+ "function-bind": "^1.1.2"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
+ "node_modules/http-errors": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
+ "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
+ "dependencies": {
+ "depd": "2.0.0",
+ "inherits": "2.0.4",
+ "setprototypeof": "1.2.0",
+ "statuses": "2.0.1",
+ "toidentifier": "1.0.1"
+ },
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/iconv-lite": {
+ "version": "0.4.24",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+ "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+ "dependencies": {
+ "safer-buffer": ">= 2.1.2 < 3"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/ignore": {
+ "version": "5.3.1",
+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
+ "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==",
+ "dev": true,
+ "engines": {
+ "node": ">= 4"
+ }
+ },
+ "node_modules/import-fresh": {
+ "version": "3.3.0",
+ "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+ "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+ "dev": true,
+ "dependencies": {
+ "parent-module": "^1.0.0",
+ "resolve-from": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=6"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/imurmurhash": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
+ "dev": true,
+ "engines": {
+ "node": ">=0.8.19"
+ }
+ },
+ "node_modules/indexof": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz",
+ "integrity": "sha512-i0G7hLJ1z0DE8dsqJa2rycj9dBmNKgXBvotXtZYXakU9oivfB9Uj2ZBC27qqef2U58/ZLwalxa1X/RDCdkHtVg=="
+ },
+ "node_modules/inflight": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+ "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+ "dev": true,
+ "dependencies": {
+ "once": "^1.3.0",
+ "wrappy": "1"
+ }
+ },
+ "node_modules/inherits": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+ },
+ "node_modules/ipaddr.js": {
+ "version": "1.9.1",
+ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
+ "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+ "engines": {
+ "node": ">= 0.10"
+ }
+ },
+ "node_modules/is-extglob": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+ "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/is-glob": {
+ "version": "4.0.3",
+ "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+ "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
+ "dev": true,
+ "dependencies": {
+ "is-extglob": "^2.1.1"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/is-path-inside": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
+ "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/isarray": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz",
+ "integrity": "sha512-c2cu3UxbI+b6kR3fy0nRnAhodsvR9dx7U5+znCOzdj6IfP3upFURTr0Xl5BlQZNKZjEtxrmVyfSdeE3O57smoQ=="
+ },
+ "node_modules/isexe": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+ "dev": true
+ },
+ "node_modules/js-yaml": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+ "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+ "dev": true,
+ "dependencies": {
+ "argparse": "^2.0.1"
+ },
+ "bin": {
+ "js-yaml": "bin/js-yaml.js"
+ }
+ },
+ "node_modules/json-buffer": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
+ "dev": true
+ },
+ "node_modules/json-schema-traverse": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+ "dev": true
+ },
+ "node_modules/json-stable-stringify-without-jsonify": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
+ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
+ "dev": true
+ },
+ "node_modules/keyv": {
+ "version": "4.5.4",
+ "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
+ "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
+ "dev": true,
+ "dependencies": {
+ "json-buffer": "3.0.1"
+ }
+ },
+ "node_modules/levn": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
+ "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+ "dev": true,
+ "dependencies": {
+ "prelude-ls": "^1.2.1",
+ "type-check": "~0.4.0"
+ },
+ "engines": {
+ "node": ">= 0.8.0"
+ }
+ },
+ "node_modules/locate-path": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
+ "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
+ "dev": true,
+ "dependencies": {
+ "p-locate": "^5.0.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/lodash.merge": {
+ "version": "4.6.2",
+ "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
+ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
+ "dev": true
+ },
+ "node_modules/media-typer": {
+ "version": "0.3.0",
+ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+ "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/merge-descriptors": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
+ "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w=="
+ },
+ "node_modules/methods": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+ "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/mime": {
+ "version": "1.6.0",
+ "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+ "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
+ "bin": {
+ "mime": "cli.js"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/mime-db": {
+ "version": "1.52.0",
+ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+ "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/mime-types": {
+ "version": "2.1.35",
+ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+ "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+ "dependencies": {
+ "mime-db": "1.52.0"
+ },
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/minimatch": {
+ "version": "3.1.2",
+ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+ "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+ "dev": true,
+ "dependencies": {
+ "brace-expansion": "^1.1.7"
+ },
+ "engines": {
+ "node": "*"
+ }
+ },
+ "node_modules/ms": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+ },
+ "node_modules/natural-compare": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
+ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
+ "dev": true
+ },
+ "node_modules/negotiator": {
+ "version": "0.6.3",
+ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
+ "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/object-assign": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+ "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/object-component": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/object-component/-/object-component-0.0.3.tgz",
+ "integrity": "sha512-S0sN3agnVh2SZNEIGc0N1X4Z5K0JeFbGBrnuZpsxuUh5XLF0BnvWkMjRXo/zGKLd/eghvNIKcx1pQkmUjXIyrA=="
+ },
+ "node_modules/object-inspect": {
+ "version": "1.13.1",
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
+ "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/on-finished": {
+ "version": "2.4.1",
+ "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
+ "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
+ "dependencies": {
+ "ee-first": "1.1.1"
+ },
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/once": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+ "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+ "dev": true,
+ "dependencies": {
+ "wrappy": "1"
+ }
+ },
+ "node_modules/optionator": {
+ "version": "0.9.3",
+ "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
+ "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
+ "dev": true,
+ "dependencies": {
+ "@aashutoshrathi/word-wrap": "^1.2.3",
+ "deep-is": "^0.1.3",
+ "fast-levenshtein": "^2.0.6",
+ "levn": "^0.4.1",
+ "prelude-ls": "^1.2.1",
+ "type-check": "^0.4.0"
+ },
+ "engines": {
+ "node": ">= 0.8.0"
+ }
+ },
+ "node_modules/p-limit": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+ "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+ "dev": true,
+ "dependencies": {
+ "yocto-queue": "^0.1.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/p-locate": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
+ "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+ "dev": true,
+ "dependencies": {
+ "p-limit": "^3.0.2"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/parent-module": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
+ "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
+ "dev": true,
+ "dependencies": {
+ "callsites": "^3.0.0"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
+ "node_modules/parseqs": {
+ "version": "0.0.6",
+ "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.6.tgz",
+ "integrity": "sha512-jeAGzMDbfSHHA091hr0r31eYfTig+29g3GKKE/PPbEQ65X0lmMwlEoqmhzu0iztID5uJpZsFlUPDP8ThPL7M8w=="
+ },
+ "node_modules/parseuri": {
+ "version": "0.0.6",
+ "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.6.tgz",
+ "integrity": "sha512-AUjen8sAkGgao7UyCX6Ahv0gIK2fABKmYjvP4xmy5JaKvcbTRueIqIPHLAfq30xJddqSE033IOMUSOMCcK3Sow=="
+ },
+ "node_modules/parseurl": {
+ "version": "1.3.3",
+ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
+ "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/path-exists": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+ "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/path-is-absolute": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+ "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+ "dev": true,
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/path-key": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+ "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/path-to-regexp": {
+ "version": "0.1.7",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
+ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
+ },
+ "node_modules/prelude-ls": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
+ "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+ "dev": true,
+ "engines": {
+ "node": ">= 0.8.0"
+ }
+ },
+ "node_modules/proxy-addr": {
+ "version": "2.0.7",
+ "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
+ "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
+ "dependencies": {
+ "forwarded": "0.2.0",
+ "ipaddr.js": "1.9.1"
+ },
+ "engines": {
+ "node": ">= 0.10"
+ }
+ },
+ "node_modules/punycode": {
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+ "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
+ "dev": true,
+ "engines": {
+ "node": ">=6"
+ }
+ },
+ "node_modules/qs": {
+ "version": "6.11.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+ "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
+ "dependencies": {
+ "side-channel": "^1.0.4"
+ },
+ "engines": {
+ "node": ">=0.6"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/queue-microtask": {
+ "version": "1.2.3",
+ "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
+ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
+ "node_modules/range-parser": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
+ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==",
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/raw-body": {
+ "version": "2.5.1",
+ "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
+ "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
+ "dependencies": {
+ "bytes": "3.1.2",
+ "http-errors": "2.0.0",
+ "iconv-lite": "0.4.24",
+ "unpipe": "1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/resolve-from": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+ "dev": true,
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/reusify": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+ "dev": true,
+ "engines": {
+ "iojs": ">=1.0.0",
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/rimraf": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+ "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+ "dev": true,
+ "dependencies": {
+ "glob": "^7.1.3"
+ },
+ "bin": {
+ "rimraf": "bin.js"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/isaacs"
+ }
+ },
+ "node_modules/run-parallel": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
+ "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+ "dev": true,
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ],
+ "dependencies": {
+ "queue-microtask": "^1.2.2"
+ }
+ },
+ "node_modules/safe-buffer": {
+ "version": "5.2.1",
+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/feross"
+ },
+ {
+ "type": "patreon",
+ "url": "https://www.patreon.com/feross"
+ },
+ {
+ "type": "consulting",
+ "url": "https://feross.org/support"
+ }
+ ]
+ },
+ "node_modules/safer-buffer": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+ },
+ "node_modules/sdp": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.0.tgz",
+ "integrity": "sha512-d7wDPgDV3DDiqulJjKiV2865wKsJ34YI+NDREbm+FySq6WuKOikwyNQcm+doLAZ1O6ltdO0SeKle2xMpN3Brgw==",
+ "dev": true
+ },
+ "node_modules/send": {
+ "version": "0.18.0",
+ "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
+ "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
+ "dependencies": {
+ "debug": "2.6.9",
+ "depd": "2.0.0",
+ "destroy": "1.2.0",
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "etag": "~1.8.1",
+ "fresh": "0.5.2",
+ "http-errors": "2.0.0",
+ "mime": "1.6.0",
+ "ms": "2.1.3",
+ "on-finished": "2.4.1",
+ "range-parser": "~1.2.1",
+ "statuses": "2.0.1"
+ },
+ "engines": {
+ "node": ">= 0.8.0"
+ }
+ },
+ "node_modules/send/node_modules/debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/send/node_modules/debug/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/serve-static": {
+ "version": "1.15.0",
+ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
+ "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
+ "dependencies": {
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "parseurl": "~1.3.3",
+ "send": "0.18.0"
+ },
+ "engines": {
+ "node": ">= 0.8.0"
+ }
+ },
+ "node_modules/set-function-length": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz",
+ "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==",
+ "dependencies": {
+ "define-data-property": "^1.1.1",
+ "get-intrinsic": "^1.2.1",
+ "gopd": "^1.0.1",
+ "has-property-descriptors": "^1.0.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
+ "node_modules/setprototypeof": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
+ },
+ "node_modules/shebang-command": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+ "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+ "dev": true,
+ "dependencies": {
+ "shebang-regex": "^3.0.0"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/shebang-regex": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/side-channel": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
+ "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+ "dependencies": {
+ "call-bind": "^1.0.0",
+ "get-intrinsic": "^1.0.2",
+ "object-inspect": "^1.9.0"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
+ "node_modules/socket.io": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.2.0.tgz",
+ "integrity": "sha512-wxXrIuZ8AILcn+f1B4ez4hJTPG24iNgxBBDaJfT6MsyOhVYiTXWexGoPkd87ktJG8kQEcL/NBvRi64+9k4Kc0w==",
+ "dependencies": {
+ "debug": "~4.1.0",
+ "engine.io": "~3.3.1",
+ "has-binary2": "~1.0.2",
+ "socket.io-adapter": "~1.1.0",
+ "socket.io-client": "2.2.0",
+ "socket.io-parser": "~3.3.0"
+ }
+ },
+ "node_modules/socket.io-adapter": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.2.tgz",
+ "integrity": "sha512-WzZRUj1kUjrTIrUKpZLEzFZ1OLj5FwLlAFQs9kuZJzJi5DKdU7FsWc36SNmA8iDOtwBQyT8FkrriRM8vXLYz8g=="
+ },
+ "node_modules/socket.io-client": {
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.3.1.tgz",
+ "integrity": "sha512-YXmXn3pA8abPOY//JtYxou95Ihvzmg8U6kQyolArkIyLd0pgVhrfor/iMsox8cn07WCOOvvuJ6XKegzIucPutQ==",
+ "dependencies": {
+ "backo2": "1.0.2",
+ "component-bind": "1.0.0",
+ "component-emitter": "~1.3.0",
+ "debug": "~3.1.0",
+ "engine.io-client": "~3.4.0",
+ "has-binary2": "~1.0.2",
+ "indexof": "0.0.1",
+ "parseqs": "0.0.6",
+ "parseuri": "0.0.6",
+ "socket.io-parser": "~3.3.0",
+ "to-array": "0.1.4"
+ }
+ },
+ "node_modules/socket.io-client/node_modules/debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/socket.io-client/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/socket.io-parser": {
+ "version": "3.3.3",
+ "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.3.3.tgz",
+ "integrity": "sha512-qOg87q1PMWWTeO01768Yh9ogn7chB9zkKtQnya41Y355S0UmpXgpcrFwAgjYJxu9BdKug5r5e9YtVSeWhKBUZg==",
+ "dependencies": {
+ "component-emitter": "~1.3.0",
+ "debug": "~3.1.0",
+ "isarray": "2.0.1"
+ }
+ },
+ "node_modules/socket.io-parser/node_modules/debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/socket.io-parser/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/socket.io/node_modules/component-emitter": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
+ "integrity": "sha512-jPatnhd33viNplKjqXKRkGU345p263OIWzDL2wH3LGIGp5Kojo+uXizHmOADRvhGFFTnJqX3jBAKP6vvmSDKcA=="
+ },
+ "node_modules/socket.io/node_modules/engine.io-client": {
+ "version": "3.3.3",
+ "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.3.3.tgz",
+ "integrity": "sha512-PXIgpzb1brtBzh8Q6vCjzCMeu4nfEPmaDm+L3Qb2sVHwLkxC1qRiBMSjOB0NJNjZ0hbPNUKQa+s8J2XxLOIEeQ==",
+ "dependencies": {
+ "component-emitter": "1.2.1",
+ "component-inherit": "0.0.3",
+ "debug": "~3.1.0",
+ "engine.io-parser": "~2.1.1",
+ "has-cors": "1.1.0",
+ "indexof": "0.0.1",
+ "parseqs": "0.0.5",
+ "parseuri": "0.0.5",
+ "ws": "~6.1.0",
+ "xmlhttprequest-ssl": "~1.6.3",
+ "yeast": "0.1.2"
+ }
+ },
+ "node_modules/socket.io/node_modules/engine.io-client/node_modules/debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/socket.io/node_modules/ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "node_modules/socket.io/node_modules/parseqs": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz",
+ "integrity": "sha512-B3Nrjw2aL7aI4TDujOzfA4NsEc4u1lVcIRE0xesutH8kjeWF70uk+W5cBlIQx04zUH9NTBvuN36Y9xLRPK6Jjw==",
+ "dependencies": {
+ "better-assert": "~1.0.0"
+ }
+ },
+ "node_modules/socket.io/node_modules/parseuri": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz",
+ "integrity": "sha512-ijhdxJu6l5Ru12jF0JvzXVPvsC+VibqeaExlNoMhWN6VQ79PGjkmc7oA4W1lp00sFkNyj0fx6ivPLdV51/UMog==",
+ "dependencies": {
+ "better-assert": "~1.0.0"
+ }
+ },
+ "node_modules/socket.io/node_modules/socket.io-client": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.2.0.tgz",
+ "integrity": "sha512-56ZrkTDbdTLmBIyfFYesgOxsjcLnwAKoN4CiPyTVkMQj3zTUh0QAx3GbvIvLpFEOvQWu92yyWICxB0u7wkVbYA==",
+ "dependencies": {
+ "backo2": "1.0.2",
+ "base64-arraybuffer": "0.1.5",
+ "component-bind": "1.0.0",
+ "component-emitter": "1.2.1",
+ "debug": "~3.1.0",
+ "engine.io-client": "~3.3.1",
+ "has-binary2": "~1.0.2",
+ "has-cors": "1.1.0",
+ "indexof": "0.0.1",
+ "object-component": "0.0.3",
+ "parseqs": "0.0.5",
+ "parseuri": "0.0.5",
+ "socket.io-parser": "~3.3.0",
+ "to-array": "0.1.4"
+ }
+ },
+ "node_modules/socket.io/node_modules/socket.io-client/node_modules/debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "dependencies": {
+ "ms": "2.0.0"
+ }
+ },
+ "node_modules/socket.io/node_modules/ws": {
+ "version": "6.1.4",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
+ "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+ "dependencies": {
+ "async-limiter": "~1.0.0"
+ }
+ },
+ "node_modules/socket.io/node_modules/xmlhttprequest-ssl": {
+ "version": "1.6.3",
+ "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.6.3.tgz",
+ "integrity": "sha512-3XfeQE/wNkvrIktn2Kf0869fC0BN6UpydVasGIeSm2B1Llihf7/0UfZM+eCkOw3P7bP4+qPgqhm7ZoxuJtFU0Q==",
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
+ "node_modules/statuses": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+ "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/strip-ansi": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+ "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+ "dev": true,
+ "dependencies": {
+ "ansi-regex": "^5.0.1"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/strip-json-comments": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+ "dev": true,
+ "engines": {
+ "node": ">=8"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/supports-color": {
+ "version": "7.2.0",
+ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+ "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+ "dev": true,
+ "dependencies": {
+ "has-flag": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/text-table": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
+ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
+ "dev": true
+ },
+ "node_modules/to-array": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/to-array/-/to-array-0.1.4.tgz",
+ "integrity": "sha512-LhVdShQD/4Mk4zXNroIQZJC+Ap3zgLcDuwEdcmLv9CCO73NWockQDwyUnW/m8VX/EElfL6FcYx7EeutN4HJA6A=="
+ },
+ "node_modules/toidentifier": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+ "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
+ "engines": {
+ "node": ">=0.6"
+ }
+ },
+ "node_modules/toolsocket": {
+ "version": "2.0.1",
+ "resolved": "git+ssh://git@github.com/ptcrealitylab/toolsocket.git#b4271baf7214d1c5810ed8ecd1bddebd0d786384",
+ "license": "MPL-2.0",
+ "dependencies": {
+ "ws": "^8.2.3"
+ }
+ },
+ "node_modules/toolsocket/node_modules/ws": {
+ "version": "8.14.2",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
+ "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
+ "engines": {
+ "node": ">=10.0.0"
+ },
+ "peerDependencies": {
+ "bufferutil": "^4.0.1",
+ "utf-8-validate": ">=5.0.2"
+ },
+ "peerDependenciesMeta": {
+ "bufferutil": {
+ "optional": true
+ },
+ "utf-8-validate": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/type-check": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
+ "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+ "dev": true,
+ "dependencies": {
+ "prelude-ls": "^1.2.1"
+ },
+ "engines": {
+ "node": ">= 0.8.0"
+ }
+ },
+ "node_modules/type-fest": {
+ "version": "0.20.2",
+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+ "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+ "dev": true,
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/type-is": {
+ "version": "1.6.18",
+ "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
+ "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
+ "dependencies": {
+ "media-typer": "0.3.0",
+ "mime-types": "~2.1.24"
+ },
+ "engines": {
+ "node": ">= 0.6"
+ }
+ },
+ "node_modules/unpipe": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+ "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/uri-js": {
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+ "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+ "dev": true,
+ "dependencies": {
+ "punycode": "^2.1.0"
+ }
+ },
+ "node_modules/utils-merge": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
+ "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==",
+ "engines": {
+ "node": ">= 0.4.0"
+ }
+ },
+ "node_modules/vary": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
+ "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/webrtc-adapter": {
+ "version": "9.0.1",
+ "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.1.tgz",
+ "integrity": "sha512-1AQO+d4ElfVSXyzNVTOewgGT/tAomwwztX/6e3totvyyzXPvXIIuUUjAmyZGbKBKbZOXauuJooZm3g6IuFuiNQ==",
+ "dev": true,
+ "dependencies": {
+ "sdp": "^3.2.0"
+ },
+ "engines": {
+ "node": ">=6.0.0",
+ "npm": ">=3.10.0"
+ }
+ },
+ "node_modules/which": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+ "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+ "dev": true,
+ "dependencies": {
+ "isexe": "^2.0.0"
+ },
+ "bin": {
+ "node-which": "bin/node-which"
+ },
+ "engines": {
+ "node": ">= 8"
+ }
+ },
+ "node_modules/wrappy": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+ "dev": true
+ },
+ "node_modules/ws": {
+ "version": "7.5.9",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
+ "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
+ "engines": {
+ "node": ">=8.3.0"
+ },
+ "peerDependencies": {
+ "bufferutil": "^4.0.1",
+ "utf-8-validate": "^5.0.2"
+ },
+ "peerDependenciesMeta": {
+ "bufferutil": {
+ "optional": true
+ },
+ "utf-8-validate": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/xmlhttprequest-ssl": {
+ "version": "1.5.5",
+ "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
+ "integrity": "sha512-/bFPLUgJrfGUL10AIv4Y7/CUt6so9CLtB/oFxQSHseSDNNCdC6vwwKEqwLN6wNPBg9YWXAiMu8jkf6RPRS/75Q==",
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
+ "node_modules/yeast": {
+ "version": "0.1.2",
+ "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz",
+ "integrity": "sha512-8HFIh676uyGYP6wP13R/j6OJ/1HwJ46snpvzE7aHAN3Ryqh2yX6Xox2B4CUmTwwOIzlG3Bs7ocsP5dZH/R1Qbg=="
+ },
+ "node_modules/yocto-queue": {
+ "version": "0.1.0",
+ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
+ "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+ "dev": true,
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ }
+ },
+ "dependencies": {
+ "@aashutoshrathi/word-wrap": {
+ "version": "1.2.6",
+ "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
+ "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==",
+ "dev": true
+ },
+ "@eslint-community/eslint-utils": {
+ "version": "4.4.0",
+ "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
+ "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==",
+ "dev": true,
+ "requires": {
+ "eslint-visitor-keys": "^3.3.0"
+ }
+ },
+ "@eslint-community/regexpp": {
+ "version": "4.10.0",
+ "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz",
+ "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==",
+ "dev": true
+ },
+ "@eslint/eslintrc": {
+ "version": "2.1.4",
+ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz",
+ "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==",
+ "dev": true,
+ "requires": {
+ "ajv": "^6.12.4",
+ "debug": "^4.3.2",
+ "espree": "^9.6.0",
+ "globals": "^13.19.0",
+ "ignore": "^5.2.0",
+ "import-fresh": "^3.2.1",
+ "js-yaml": "^4.1.0",
+ "minimatch": "^3.1.2",
+ "strip-json-comments": "^3.1.1"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "4.3.4",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+ "dev": true,
+ "requires": {
+ "ms": "2.1.2"
+ }
+ },
+ "ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ }
+ }
+ },
+ "@eslint/js": {
+ "version": "8.57.0",
+ "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz",
+ "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==",
+ "dev": true
+ },
+ "@ffmpeg-installer/darwin-arm64": {
+ "version": "4.1.5",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz",
+ "integrity": "sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==",
+ "optional": true
+ },
+ "@ffmpeg-installer/darwin-x64": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz",
+ "integrity": "sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==",
+ "optional": true
+ },
+ "@ffmpeg-installer/ffmpeg": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz",
+ "integrity": "sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==",
+ "requires": {
+ "@ffmpeg-installer/darwin-arm64": "4.1.5",
+ "@ffmpeg-installer/darwin-x64": "4.1.0",
+ "@ffmpeg-installer/linux-arm": "4.1.3",
+ "@ffmpeg-installer/linux-arm64": "4.1.4",
+ "@ffmpeg-installer/linux-ia32": "4.1.0",
+ "@ffmpeg-installer/linux-x64": "4.1.0",
+ "@ffmpeg-installer/win32-ia32": "4.1.0",
+ "@ffmpeg-installer/win32-x64": "4.1.0"
+ }
+ },
+ "@ffmpeg-installer/linux-arm": {
+ "version": "4.1.3",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz",
+ "integrity": "sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==",
+ "optional": true
+ },
+ "@ffmpeg-installer/linux-arm64": {
+ "version": "4.1.4",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz",
+ "integrity": "sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==",
+ "optional": true
+ },
+ "@ffmpeg-installer/linux-ia32": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz",
+ "integrity": "sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==",
+ "optional": true
+ },
+ "@ffmpeg-installer/linux-x64": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz",
+ "integrity": "sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==",
+ "optional": true
+ },
+ "@ffmpeg-installer/win32-ia32": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz",
+ "integrity": "sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==",
+ "optional": true
+ },
+ "@ffmpeg-installer/win32-x64": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz",
+ "integrity": "sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==",
+ "optional": true
+ },
+ "@humanwhocodes/config-array": {
+ "version": "0.11.14",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz",
+ "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==",
+ "dev": true,
+ "requires": {
+ "@humanwhocodes/object-schema": "^2.0.2",
+ "debug": "^4.3.1",
+ "minimatch": "^3.0.5"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "4.3.4",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+ "dev": true,
+ "requires": {
+ "ms": "2.1.2"
+ }
+ },
+ "ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ }
+ }
+ },
+ "@humanwhocodes/module-importer": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz",
+ "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==",
+ "dev": true
+ },
+ "@humanwhocodes/object-schema": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz",
+ "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==",
+ "dev": true
+ },
+ "@nodelib/fs.scandir": {
+ "version": "2.1.5",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+ "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+ "dev": true,
+ "requires": {
+ "@nodelib/fs.stat": "2.0.5",
+ "run-parallel": "^1.1.9"
+ }
+ },
+ "@nodelib/fs.stat": {
+ "version": "2.0.5",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+ "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+ "dev": true
+ },
+ "@nodelib/fs.walk": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+ "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+ "dev": true,
+ "requires": {
+ "@nodelib/fs.scandir": "2.1.5",
+ "fastq": "^1.6.0"
+ }
+ },
+ "@ungap/structured-clone": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
+ "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==",
+ "dev": true
+ },
+ "accepts": {
+ "version": "1.3.8",
+ "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
+ "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
+ "requires": {
+ "mime-types": "~2.1.34",
+ "negotiator": "0.6.3"
+ }
+ },
+ "acorn": {
+ "version": "8.11.3",
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
+ "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
+ "dev": true
+ },
+ "acorn-jsx": {
+ "version": "5.3.2",
+ "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
+ "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
+ "dev": true,
+ "requires": {}
+ },
+ "after": {
+ "version": "0.8.2",
+ "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz",
+ "integrity": "sha512-QbJ0NTQ/I9DI3uSJA4cbexiwQeRAfjPScqIbSjUDd9TOrcg6pTkdgziesOqxBMBzit8vFCTwrP27t13vFOORRA=="
+ },
+ "ajv": {
+ "version": "6.12.6",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+ "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+ "dev": true,
+ "requires": {
+ "fast-deep-equal": "^3.1.1",
+ "fast-json-stable-stringify": "^2.0.0",
+ "json-schema-traverse": "^0.4.1",
+ "uri-js": "^4.2.2"
+ }
+ },
+ "ansi-regex": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+ "dev": true
+ },
+ "ansi-styles": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+ "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+ "dev": true,
+ "requires": {
+ "color-convert": "^2.0.1"
+ }
+ },
+ "argparse": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+ "dev": true
+ },
+ "array-flatten": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="
+ },
+ "arraybuffer.slice": {
+ "version": "0.0.7",
+ "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
+ "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
+ },
+ "async-limiter": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz",
+ "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ=="
+ },
+ "backo2": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz",
+ "integrity": "sha512-zj6Z6M7Eq+PBZ7PQxl5NT665MvJdAkzp0f60nAJ+sLaSCBPMwVak5ZegFbgVCzFcCJTKFoMizvM5Ld7+JrRJHA=="
+ },
+ "balanced-match": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
+ "dev": true
+ },
+ "base64-arraybuffer": {
+ "version": "0.1.5",
+ "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz",
+ "integrity": "sha512-437oANT9tP582zZMwSvZGy2nmSeAb8DW2me3y+Uv1Wp2Rulr8Mqlyrv3E7MLxmsiaPSMMDmiDVzgE+e8zlMx9g=="
+ },
+ "base64id": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz",
+ "integrity": "sha512-rz8L+d/xByiB/vLVftPkyY215fqNrmasrcJsYkVcm4TgJNz+YXKrFaFAWibSaHkiKoSgMDCb+lipOIRQNGYesw=="
+ },
+ "better-assert": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/better-assert/-/better-assert-1.0.2.tgz",
+ "integrity": "sha512-bYeph2DFlpK1XmGs6fvlLRUN29QISM3GBuUwSFsMY2XRx4AvC0WNCS57j4c/xGrK2RS24C1w3YoBOsw9fT46tQ==",
+ "requires": {
+ "callsite": "1.0.0"
+ }
+ },
+ "blob": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz",
+ "integrity": "sha512-gaqbzQPqOoamawKg0LGVd7SzLgXS+JH61oWprSLH+P+abTczqJbhTR8CmJ2u9/bUYNmHTGJx/UEmn6doAvvuig=="
+ },
+ "body-parser": {
+ "version": "1.20.1",
+ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
+ "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
+ "requires": {
+ "bytes": "3.1.2",
+ "content-type": "~1.0.4",
+ "debug": "2.6.9",
+ "depd": "2.0.0",
+ "destroy": "1.2.0",
+ "http-errors": "2.0.0",
+ "iconv-lite": "0.4.24",
+ "on-finished": "2.4.1",
+ "qs": "6.11.0",
+ "raw-body": "2.5.1",
+ "type-is": "~1.6.18",
+ "unpipe": "1.0.0"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ }
+ }
+ },
+ "brace-expansion": {
+ "version": "1.1.11",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+ "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+ "dev": true,
+ "requires": {
+ "balanced-match": "^1.0.0",
+ "concat-map": "0.0.1"
+ }
+ },
+ "bytes": {
+ "version": "3.1.2",
+ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+ "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="
+ },
+ "call-bind": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz",
+ "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==",
+ "requires": {
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.1",
+ "set-function-length": "^1.1.1"
+ }
+ },
+ "callsite": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz",
+ "integrity": "sha512-0vdNRFXn5q+dtOqjfFtmtlI9N2eVZ7LMyEV2iKC5mEEFvSg/69Ml6b/WU2qF8W1nLRa0wiSrDT3Y5jOHZCwKPQ=="
+ },
+ "callsites": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
+ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
+ "dev": true
+ },
+ "chalk": {
+ "version": "4.1.2",
+ "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+ "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+ "dev": true,
+ "requires": {
+ "ansi-styles": "^4.1.0",
+ "supports-color": "^7.1.0"
+ }
+ },
+ "color-convert": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+ "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+ "dev": true,
+ "requires": {
+ "color-name": "~1.1.4"
+ }
+ },
+ "color-name": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+ "dev": true
+ },
+ "component-bind": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/component-bind/-/component-bind-1.0.0.tgz",
+ "integrity": "sha512-WZveuKPeKAG9qY+FkYDeADzdHyTYdIboXS59ixDeRJL5ZhxpqUnxSOwop4FQjMsiYm3/Or8cegVbpAHNA7pHxw=="
+ },
+ "component-emitter": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz",
+ "integrity": "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ=="
+ },
+ "component-inherit": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/component-inherit/-/component-inherit-0.0.3.tgz",
+ "integrity": "sha512-w+LhYREhatpVqTESyGFg3NlP6Iu0kEKUHETY9GoZP/pQyW4mHFZuFWRUCIqVPZ36ueVLtoOEZaAqbCF2RDndaA=="
+ },
+ "concat-map": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
+ "dev": true
+ },
+ "content-disposition": {
+ "version": "0.5.4",
+ "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
+ "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
+ "requires": {
+ "safe-buffer": "5.2.1"
+ }
+ },
+ "content-type": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
+ "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="
+ },
+ "cookie": {
+ "version": "0.5.0",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
+ "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw=="
+ },
+ "cookie-signature": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
+ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ=="
+ },
+ "cors": {
+ "version": "2.8.5",
+ "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
+ "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
+ "requires": {
+ "object-assign": "^4",
+ "vary": "^1"
+ }
+ },
+ "cross-spawn": {
+ "version": "7.0.3",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+ "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+ "dev": true,
+ "requires": {
+ "path-key": "^3.1.0",
+ "shebang-command": "^2.0.0",
+ "which": "^2.0.1"
+ }
+ },
+ "debug": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+ "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+ "requires": {
+ "ms": "^2.1.1"
+ }
+ },
+ "deep-is": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
+ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
+ "dev": true
+ },
+ "define-data-property": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz",
+ "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==",
+ "requires": {
+ "get-intrinsic": "^1.2.1",
+ "gopd": "^1.0.1",
+ "has-property-descriptors": "^1.0.0"
+ }
+ },
+ "depd": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+ "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="
+ },
+ "destroy": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
+ "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="
+ },
+ "doctrine": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
+ "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==",
+ "dev": true,
+ "requires": {
+ "esutils": "^2.0.2"
+ }
+ },
+ "ee-first": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="
+ },
+ "encodeurl": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
+ "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="
+ },
+ "engine.io": {
+ "version": "3.3.2",
+ "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.3.2.tgz",
+ "integrity": "sha512-AsaA9KG7cWPXWHp5FvHdDWY3AMWeZ8x+2pUVLcn71qE5AtAzgGbxuclOytygskw8XGmiQafTmnI9Bix3uihu2w==",
+ "requires": {
+ "accepts": "~1.3.4",
+ "base64id": "1.0.0",
+ "cookie": "0.3.1",
+ "debug": "~3.1.0",
+ "engine.io-parser": "~2.1.0",
+ "ws": "~6.1.0"
+ },
+ "dependencies": {
+ "cookie": {
+ "version": "0.3.1",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
+ "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw=="
+ },
+ "debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "ws": {
+ "version": "6.1.4",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
+ "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+ "requires": {
+ "async-limiter": "~1.0.0"
+ }
+ }
+ }
+ },
+ "engine.io-client": {
+ "version": "3.4.4",
+ "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.4.4.tgz",
+ "integrity": "sha512-iU4CRr38Fecj8HoZEnFtm2EiKGbYZcPn3cHxqNGl/tmdWRf60KhK+9vE0JeSjgnlS/0oynEfLgKbT9ALpim0sQ==",
+ "requires": {
+ "component-emitter": "~1.3.0",
+ "component-inherit": "0.0.3",
+ "debug": "~3.1.0",
+ "engine.io-parser": "~2.2.0",
+ "has-cors": "1.1.0",
+ "indexof": "0.0.1",
+ "parseqs": "0.0.6",
+ "parseuri": "0.0.6",
+ "ws": "~6.1.0",
+ "xmlhttprequest-ssl": "~1.5.4",
+ "yeast": "0.1.2"
+ },
+ "dependencies": {
+ "base64-arraybuffer": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.4.tgz",
+ "integrity": "sha512-a1eIFi4R9ySrbiMuyTGx5e92uRH5tQY6kArNcFaKBUleIoLjdjBg7Zxm3Mqm3Kmkf27HLR/1fnxX9q8GQ7Iavg=="
+ },
+ "debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "engine.io-parser": {
+ "version": "2.2.1",
+ "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.2.1.tgz",
+ "integrity": "sha512-x+dN/fBH8Ro8TFwJ+rkB2AmuVw9Yu2mockR/p3W8f8YtExwFgDvBDi0GWyb4ZLkpahtDGZgtr3zLovanJghPqg==",
+ "requires": {
+ "after": "0.8.2",
+ "arraybuffer.slice": "~0.0.7",
+ "base64-arraybuffer": "0.1.4",
+ "blob": "0.0.5",
+ "has-binary2": "~1.0.2"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "ws": {
+ "version": "6.1.4",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
+ "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+ "requires": {
+ "async-limiter": "~1.0.0"
+ }
+ }
+ }
+ },
+ "engine.io-parser": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.3.tgz",
+ "integrity": "sha512-6HXPre2O4Houl7c4g7Ic/XzPnHBvaEmN90vtRO9uLmwtRqQmTOw0QMevL1TOfL2Cpu1VzsaTmMotQgMdkzGkVA==",
+ "requires": {
+ "after": "0.8.2",
+ "arraybuffer.slice": "~0.0.7",
+ "base64-arraybuffer": "0.1.5",
+ "blob": "0.0.5",
+ "has-binary2": "~1.0.2"
+ }
+ },
+ "escape-html": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+ "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="
+ },
+ "escape-string-regexp": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+ "dev": true
+ },
+ "eslint": {
+ "version": "8.57.0",
+ "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz",
+ "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==",
+ "dev": true,
+ "requires": {
+ "@eslint-community/eslint-utils": "^4.2.0",
+ "@eslint-community/regexpp": "^4.6.1",
+ "@eslint/eslintrc": "^2.1.4",
+ "@eslint/js": "8.57.0",
+ "@humanwhocodes/config-array": "^0.11.14",
+ "@humanwhocodes/module-importer": "^1.0.1",
+ "@nodelib/fs.walk": "^1.2.8",
+ "@ungap/structured-clone": "^1.2.0",
+ "ajv": "^6.12.4",
+ "chalk": "^4.0.0",
+ "cross-spawn": "^7.0.2",
+ "debug": "^4.3.2",
+ "doctrine": "^3.0.0",
+ "escape-string-regexp": "^4.0.0",
+ "eslint-scope": "^7.2.2",
+ "eslint-visitor-keys": "^3.4.3",
+ "espree": "^9.6.1",
+ "esquery": "^1.4.2",
+ "esutils": "^2.0.2",
+ "fast-deep-equal": "^3.1.3",
+ "file-entry-cache": "^6.0.1",
+ "find-up": "^5.0.0",
+ "glob-parent": "^6.0.2",
+ "globals": "^13.19.0",
+ "graphemer": "^1.4.0",
+ "ignore": "^5.2.0",
+ "imurmurhash": "^0.1.4",
+ "is-glob": "^4.0.0",
+ "is-path-inside": "^3.0.3",
+ "js-yaml": "^4.1.0",
+ "json-stable-stringify-without-jsonify": "^1.0.1",
+ "levn": "^0.4.1",
+ "lodash.merge": "^4.6.2",
+ "minimatch": "^3.1.2",
+ "natural-compare": "^1.4.0",
+ "optionator": "^0.9.3",
+ "strip-ansi": "^6.0.1",
+ "text-table": "^0.2.0"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "4.3.4",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+ "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+ "dev": true,
+ "requires": {
+ "ms": "2.1.2"
+ }
+ },
+ "ms": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+ "dev": true
+ }
+ }
+ },
+ "eslint-scope": {
+ "version": "7.2.2",
+ "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz",
+ "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==",
+ "dev": true,
+ "requires": {
+ "esrecurse": "^4.3.0",
+ "estraverse": "^5.2.0"
+ }
+ },
+ "eslint-visitor-keys": {
+ "version": "3.4.3",
+ "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz",
+ "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==",
+ "dev": true
+ },
+ "espree": {
+ "version": "9.6.1",
+ "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
+ "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==",
+ "dev": true,
+ "requires": {
+ "acorn": "^8.9.0",
+ "acorn-jsx": "^5.3.2",
+ "eslint-visitor-keys": "^3.4.1"
+ }
+ },
+ "esquery": {
+ "version": "1.5.0",
+ "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz",
+ "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==",
+ "dev": true,
+ "requires": {
+ "estraverse": "^5.1.0"
+ }
+ },
+ "esrecurse": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+ "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+ "dev": true,
+ "requires": {
+ "estraverse": "^5.2.0"
+ }
+ },
+ "estraverse": {
+ "version": "5.3.0",
+ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+ "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+ "dev": true
+ },
+ "esutils": {
+ "version": "2.0.3",
+ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
+ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+ "dev": true
+ },
+ "etag": {
+ "version": "1.8.1",
+ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
+ "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="
+ },
+ "express": {
+ "version": "4.18.2",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
+ "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
+ "requires": {
+ "accepts": "~1.3.8",
+ "array-flatten": "1.1.1",
+ "body-parser": "1.20.1",
+ "content-disposition": "0.5.4",
+ "content-type": "~1.0.4",
+ "cookie": "0.5.0",
+ "cookie-signature": "1.0.6",
+ "debug": "2.6.9",
+ "depd": "2.0.0",
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "etag": "~1.8.1",
+ "finalhandler": "1.2.0",
+ "fresh": "0.5.2",
+ "http-errors": "2.0.0",
+ "merge-descriptors": "1.0.1",
+ "methods": "~1.1.2",
+ "on-finished": "2.4.1",
+ "parseurl": "~1.3.3",
+ "path-to-regexp": "0.1.7",
+ "proxy-addr": "~2.0.7",
+ "qs": "6.11.0",
+ "range-parser": "~1.2.1",
+ "safe-buffer": "5.2.1",
+ "send": "0.18.0",
+ "serve-static": "1.15.0",
+ "setprototypeof": "1.2.0",
+ "statuses": "2.0.1",
+ "type-is": "~1.6.18",
+ "utils-merge": "1.0.1",
+ "vary": "~1.1.2"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ }
+ }
+ },
+ "express-ws": {
+ "version": "5.0.2",
+ "resolved": "https://registry.npmjs.org/express-ws/-/express-ws-5.0.2.tgz",
+ "integrity": "sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ==",
+ "requires": {
+ "ws": "^7.4.6"
+ }
+ },
+ "fast-deep-equal": {
+ "version": "3.1.3",
+ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+ "dev": true
+ },
+ "fast-json-stable-stringify": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+ "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+ "dev": true
+ },
+ "fast-levenshtein": {
+ "version": "2.0.6",
+ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
+ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
+ "dev": true
+ },
+ "fastq": {
+ "version": "1.17.1",
+ "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz",
+ "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==",
+ "dev": true,
+ "requires": {
+ "reusify": "^1.0.4"
+ }
+ },
+ "file-entry-cache": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
+ "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==",
+ "dev": true,
+ "requires": {
+ "flat-cache": "^3.0.4"
+ }
+ },
+ "finalhandler": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
+ "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
+ "requires": {
+ "debug": "2.6.9",
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "on-finished": "2.4.1",
+ "parseurl": "~1.3.3",
+ "statuses": "2.0.1",
+ "unpipe": "~1.0.0"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ }
+ }
+ },
+ "find-up": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
+ "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
+ "dev": true,
+ "requires": {
+ "locate-path": "^6.0.0",
+ "path-exists": "^4.0.0"
+ }
+ },
+ "flat-cache": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz",
+ "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==",
+ "dev": true,
+ "requires": {
+ "flatted": "^3.2.9",
+ "keyv": "^4.5.3",
+ "rimraf": "^3.0.2"
+ }
+ },
+ "flatted": {
+ "version": "3.3.1",
+ "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz",
+ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==",
+ "dev": true
+ },
+ "forwarded": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+ "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="
+ },
+ "fresh": {
+ "version": "0.5.2",
+ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
+ "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q=="
+ },
+ "fs.realpath": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
+ "dev": true
+ },
+ "function-bind": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+ "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="
+ },
+ "get-intrinsic": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz",
+ "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==",
+ "requires": {
+ "function-bind": "^1.1.2",
+ "has-proto": "^1.0.1",
+ "has-symbols": "^1.0.3",
+ "hasown": "^2.0.0"
+ }
+ },
+ "glob": {
+ "version": "7.2.3",
+ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+ "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+ "dev": true,
+ "requires": {
+ "fs.realpath": "^1.0.0",
+ "inflight": "^1.0.4",
+ "inherits": "2",
+ "minimatch": "^3.1.1",
+ "once": "^1.3.0",
+ "path-is-absolute": "^1.0.0"
+ }
+ },
+ "glob-parent": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
+ "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==",
+ "dev": true,
+ "requires": {
+ "is-glob": "^4.0.3"
+ }
+ },
+ "globals": {
+ "version": "13.24.0",
+ "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz",
+ "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==",
+ "dev": true,
+ "requires": {
+ "type-fest": "^0.20.2"
+ }
+ },
+ "gopd": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+ "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+ "requires": {
+ "get-intrinsic": "^1.1.3"
+ }
+ },
+ "graphemer": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz",
+ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==",
+ "dev": true
+ },
+ "has-binary2": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-binary2/-/has-binary2-1.0.3.tgz",
+ "integrity": "sha512-G1LWKhDSvhGeAQ8mPVQlqNcOB2sJdwATtZKl2pDKKHfpf/rYj24lkinxf69blJbnsvtqqNU+L3SL50vzZhXOnw==",
+ "requires": {
+ "isarray": "2.0.1"
+ }
+ },
+ "has-cors": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz",
+ "integrity": "sha512-g5VNKdkFuUuVCP9gYfDJHjK2nqdQJ7aDLTnycnc2+RvsOQbuLdF5pm7vuE5J76SEBIQjs4kQY/BWq74JUmjbXA=="
+ },
+ "has-flag": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+ "dev": true
+ },
+ "has-property-descriptors": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz",
+ "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==",
+ "requires": {
+ "get-intrinsic": "^1.2.2"
+ }
+ },
+ "has-proto": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
+ "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg=="
+ },
+ "has-symbols": {
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+ "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
+ },
+ "hasown": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
+ "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
+ "requires": {
+ "function-bind": "^1.1.2"
+ }
+ },
+ "http-errors": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
+ "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
+ "requires": {
+ "depd": "2.0.0",
+ "inherits": "2.0.4",
+ "setprototypeof": "1.2.0",
+ "statuses": "2.0.1",
+ "toidentifier": "1.0.1"
+ }
+ },
+ "iconv-lite": {
+ "version": "0.4.24",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+ "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+ "requires": {
+ "safer-buffer": ">= 2.1.2 < 3"
+ }
+ },
+ "ignore": {
+ "version": "5.3.1",
+ "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
+ "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==",
+ "dev": true
+ },
+ "import-fresh": {
+ "version": "3.3.0",
+ "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+ "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
+ "dev": true,
+ "requires": {
+ "parent-module": "^1.0.0",
+ "resolve-from": "^4.0.0"
+ }
+ },
+ "imurmurhash": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
+ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
+ "dev": true
+ },
+ "indexof": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz",
+ "integrity": "sha512-i0G7hLJ1z0DE8dsqJa2rycj9dBmNKgXBvotXtZYXakU9oivfB9Uj2ZBC27qqef2U58/ZLwalxa1X/RDCdkHtVg=="
+ },
+ "inflight": {
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+ "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+ "dev": true,
+ "requires": {
+ "once": "^1.3.0",
+ "wrappy": "1"
+ }
+ },
+ "inherits": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+ },
+ "ipaddr.js": {
+ "version": "1.9.1",
+ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
+ "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
+ },
+ "is-extglob": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+ "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+ "dev": true
+ },
+ "is-glob": {
+ "version": "4.0.3",
+ "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+ "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
+ "dev": true,
+ "requires": {
+ "is-extglob": "^2.1.1"
+ }
+ },
+ "is-path-inside": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
+ "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
+ "dev": true
+ },
+ "isarray": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz",
+ "integrity": "sha512-c2cu3UxbI+b6kR3fy0nRnAhodsvR9dx7U5+znCOzdj6IfP3upFURTr0Xl5BlQZNKZjEtxrmVyfSdeE3O57smoQ=="
+ },
+ "isexe": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
+ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
+ "dev": true
+ },
+ "js-yaml": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+ "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+ "dev": true,
+ "requires": {
+ "argparse": "^2.0.1"
+ }
+ },
+ "json-buffer": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
+ "dev": true
+ },
+ "json-schema-traverse": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+ "dev": true
+ },
+ "json-stable-stringify-without-jsonify": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
+ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
+ "dev": true
+ },
+ "keyv": {
+ "version": "4.5.4",
+ "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
+ "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==",
+ "dev": true,
+ "requires": {
+ "json-buffer": "3.0.1"
+ }
+ },
+ "levn": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
+ "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==",
+ "dev": true,
+ "requires": {
+ "prelude-ls": "^1.2.1",
+ "type-check": "~0.4.0"
+ }
+ },
+ "locate-path": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
+ "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
+ "dev": true,
+ "requires": {
+ "p-locate": "^5.0.0"
+ }
+ },
+ "lodash.merge": {
+ "version": "4.6.2",
+ "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
+ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
+ "dev": true
+ },
+ "media-typer": {
+ "version": "0.3.0",
+ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+ "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ=="
+ },
+ "merge-descriptors": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
+ "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w=="
+ },
+ "methods": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+ "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w=="
+ },
+ "mime": {
+ "version": "1.6.0",
+ "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+ "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
+ },
+ "mime-db": {
+ "version": "1.52.0",
+ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+ "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
+ },
+ "mime-types": {
+ "version": "2.1.35",
+ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+ "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+ "requires": {
+ "mime-db": "1.52.0"
+ }
+ },
+ "minimatch": {
+ "version": "3.1.2",
+ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+ "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+ "dev": true,
+ "requires": {
+ "brace-expansion": "^1.1.7"
+ }
+ },
+ "ms": {
+ "version": "2.1.3",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+ },
+ "natural-compare": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
+ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
+ "dev": true
+ },
+ "negotiator": {
+ "version": "0.6.3",
+ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
+ "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="
+ },
+ "object-assign": {
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+ "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="
+ },
+ "object-component": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/object-component/-/object-component-0.0.3.tgz",
+ "integrity": "sha512-S0sN3agnVh2SZNEIGc0N1X4Z5K0JeFbGBrnuZpsxuUh5XLF0BnvWkMjRXo/zGKLd/eghvNIKcx1pQkmUjXIyrA=="
+ },
+ "object-inspect": {
+ "version": "1.13.1",
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
+ "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ=="
+ },
+ "on-finished": {
+ "version": "2.4.1",
+ "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
+ "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
+ "requires": {
+ "ee-first": "1.1.1"
+ }
+ },
+ "once": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+ "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+ "dev": true,
+ "requires": {
+ "wrappy": "1"
+ }
+ },
+ "optionator": {
+ "version": "0.9.3",
+ "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
+ "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
+ "dev": true,
+ "requires": {
+ "@aashutoshrathi/word-wrap": "^1.2.3",
+ "deep-is": "^0.1.3",
+ "fast-levenshtein": "^2.0.6",
+ "levn": "^0.4.1",
+ "prelude-ls": "^1.2.1",
+ "type-check": "^0.4.0"
+ }
+ },
+ "p-limit": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
+ "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
+ "dev": true,
+ "requires": {
+ "yocto-queue": "^0.1.0"
+ }
+ },
+ "p-locate": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
+ "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
+ "dev": true,
+ "requires": {
+ "p-limit": "^3.0.2"
+ }
+ },
+ "parent-module": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
+ "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
+ "dev": true,
+ "requires": {
+ "callsites": "^3.0.0"
+ }
+ },
+ "parseqs": {
+ "version": "0.0.6",
+ "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.6.tgz",
+ "integrity": "sha512-jeAGzMDbfSHHA091hr0r31eYfTig+29g3GKKE/PPbEQ65X0lmMwlEoqmhzu0iztID5uJpZsFlUPDP8ThPL7M8w=="
+ },
+ "parseuri": {
+ "version": "0.0.6",
+ "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.6.tgz",
+ "integrity": "sha512-AUjen8sAkGgao7UyCX6Ahv0gIK2fABKmYjvP4xmy5JaKvcbTRueIqIPHLAfq30xJddqSE033IOMUSOMCcK3Sow=="
+ },
+ "parseurl": {
+ "version": "1.3.3",
+ "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
+ "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
+ },
+ "path-exists": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+ "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+ "dev": true
+ },
+ "path-is-absolute": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+ "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
+ "dev": true
+ },
+ "path-key": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+ "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+ "dev": true
+ },
+ "path-to-regexp": {
+ "version": "0.1.7",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
+ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
+ },
+ "prelude-ls": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
+ "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
+ "dev": true
+ },
+ "proxy-addr": {
+ "version": "2.0.7",
+ "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
+ "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
+ "requires": {
+ "forwarded": "0.2.0",
+ "ipaddr.js": "1.9.1"
+ }
+ },
+ "punycode": {
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+ "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
+ "dev": true
+ },
+ "qs": {
+ "version": "6.11.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+ "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
+ "requires": {
+ "side-channel": "^1.0.4"
+ }
+ },
+ "queue-microtask": {
+ "version": "1.2.3",
+ "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
+ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+ "dev": true
+ },
+ "range-parser": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
+ "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
+ },
+ "raw-body": {
+ "version": "2.5.1",
+ "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
+ "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
+ "requires": {
+ "bytes": "3.1.2",
+ "http-errors": "2.0.0",
+ "iconv-lite": "0.4.24",
+ "unpipe": "1.0.0"
+ }
+ },
+ "resolve-from": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+ "dev": true
+ },
+ "reusify": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+ "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+ "dev": true
+ },
+ "rimraf": {
+ "version": "3.0.2",
+ "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
+ "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+ "dev": true,
+ "requires": {
+ "glob": "^7.1.3"
+ }
+ },
+ "run-parallel": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
+ "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+ "dev": true,
+ "requires": {
+ "queue-microtask": "^1.2.2"
+ }
+ },
+ "safe-buffer": {
+ "version": "5.2.1",
+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+ },
+ "safer-buffer": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+ },
+ "sdp": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.0.tgz",
+ "integrity": "sha512-d7wDPgDV3DDiqulJjKiV2865wKsJ34YI+NDREbm+FySq6WuKOikwyNQcm+doLAZ1O6ltdO0SeKle2xMpN3Brgw==",
+ "dev": true
+ },
+ "send": {
+ "version": "0.18.0",
+ "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
+ "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
+ "requires": {
+ "debug": "2.6.9",
+ "depd": "2.0.0",
+ "destroy": "1.2.0",
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "etag": "~1.8.1",
+ "fresh": "0.5.2",
+ "http-errors": "2.0.0",
+ "mime": "1.6.0",
+ "ms": "2.1.3",
+ "on-finished": "2.4.1",
+ "range-parser": "~1.2.1",
+ "statuses": "2.0.1"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "2.6.9",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+ "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+ "requires": {
+ "ms": "2.0.0"
+ },
+ "dependencies": {
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ }
+ }
+ }
+ }
+ },
+ "serve-static": {
+ "version": "1.15.0",
+ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
+ "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
+ "requires": {
+ "encodeurl": "~1.0.2",
+ "escape-html": "~1.0.3",
+ "parseurl": "~1.3.3",
+ "send": "0.18.0"
+ }
+ },
+ "set-function-length": {
+ "version": "1.1.1",
+ "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz",
+ "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==",
+ "requires": {
+ "define-data-property": "^1.1.1",
+ "get-intrinsic": "^1.2.1",
+ "gopd": "^1.0.1",
+ "has-property-descriptors": "^1.0.0"
+ }
+ },
+ "setprototypeof": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
+ },
+ "shebang-command": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+ "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+ "dev": true,
+ "requires": {
+ "shebang-regex": "^3.0.0"
+ }
+ },
+ "shebang-regex": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+ "dev": true
+ },
+ "side-channel": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
+ "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+ "requires": {
+ "call-bind": "^1.0.0",
+ "get-intrinsic": "^1.0.2",
+ "object-inspect": "^1.9.0"
+ }
+ },
+ "socket.io": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.2.0.tgz",
+ "integrity": "sha512-wxXrIuZ8AILcn+f1B4ez4hJTPG24iNgxBBDaJfT6MsyOhVYiTXWexGoPkd87ktJG8kQEcL/NBvRi64+9k4Kc0w==",
+ "requires": {
+ "debug": "~4.1.0",
+ "engine.io": "~3.3.1",
+ "has-binary2": "~1.0.2",
+ "socket.io-adapter": "~1.1.0",
+ "socket.io-client": "2.2.0",
+ "socket.io-parser": "~3.3.0"
+ },
+ "dependencies": {
+ "component-emitter": {
+ "version": "1.2.1",
+ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
+ "integrity": "sha512-jPatnhd33viNplKjqXKRkGU345p263OIWzDL2wH3LGIGp5Kojo+uXizHmOADRvhGFFTnJqX3jBAKP6vvmSDKcA=="
+ },
+ "engine.io-client": {
+ "version": "3.3.3",
+ "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.3.3.tgz",
+ "integrity": "sha512-PXIgpzb1brtBzh8Q6vCjzCMeu4nfEPmaDm+L3Qb2sVHwLkxC1qRiBMSjOB0NJNjZ0hbPNUKQa+s8J2XxLOIEeQ==",
+ "requires": {
+ "component-emitter": "1.2.1",
+ "component-inherit": "0.0.3",
+ "debug": "~3.1.0",
+ "engine.io-parser": "~2.1.1",
+ "has-cors": "1.1.0",
+ "indexof": "0.0.1",
+ "parseqs": "0.0.5",
+ "parseuri": "0.0.5",
+ "ws": "~6.1.0",
+ "xmlhttprequest-ssl": "~1.6.3",
+ "yeast": "0.1.2"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ }
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ },
+ "parseqs": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz",
+ "integrity": "sha512-B3Nrjw2aL7aI4TDujOzfA4NsEc4u1lVcIRE0xesutH8kjeWF70uk+W5cBlIQx04zUH9NTBvuN36Y9xLRPK6Jjw==",
+ "requires": {
+ "better-assert": "~1.0.0"
+ }
+ },
+ "parseuri": {
+ "version": "0.0.5",
+ "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz",
+ "integrity": "sha512-ijhdxJu6l5Ru12jF0JvzXVPvsC+VibqeaExlNoMhWN6VQ79PGjkmc7oA4W1lp00sFkNyj0fx6ivPLdV51/UMog==",
+ "requires": {
+ "better-assert": "~1.0.0"
+ }
+ },
+ "socket.io-client": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.2.0.tgz",
+ "integrity": "sha512-56ZrkTDbdTLmBIyfFYesgOxsjcLnwAKoN4CiPyTVkMQj3zTUh0QAx3GbvIvLpFEOvQWu92yyWICxB0u7wkVbYA==",
+ "requires": {
+ "backo2": "1.0.2",
+ "base64-arraybuffer": "0.1.5",
+ "component-bind": "1.0.0",
+ "component-emitter": "1.2.1",
+ "debug": "~3.1.0",
+ "engine.io-client": "~3.3.1",
+ "has-binary2": "~1.0.2",
+ "has-cors": "1.1.0",
+ "indexof": "0.0.1",
+ "object-component": "0.0.3",
+ "parseqs": "0.0.5",
+ "parseuri": "0.0.5",
+ "socket.io-parser": "~3.3.0",
+ "to-array": "0.1.4"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ }
+ }
+ },
+ "ws": {
+ "version": "6.1.4",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
+ "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+ "requires": {
+ "async-limiter": "~1.0.0"
+ }
+ },
+ "xmlhttprequest-ssl": {
+ "version": "1.6.3",
+ "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.6.3.tgz",
+ "integrity": "sha512-3XfeQE/wNkvrIktn2Kf0869fC0BN6UpydVasGIeSm2B1Llihf7/0UfZM+eCkOw3P7bP4+qPgqhm7ZoxuJtFU0Q=="
+ }
+ }
+ },
+ "socket.io-adapter": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.2.tgz",
+ "integrity": "sha512-WzZRUj1kUjrTIrUKpZLEzFZ1OLj5FwLlAFQs9kuZJzJi5DKdU7FsWc36SNmA8iDOtwBQyT8FkrriRM8vXLYz8g=="
+ },
+ "socket.io-client": {
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.3.1.tgz",
+ "integrity": "sha512-YXmXn3pA8abPOY//JtYxou95Ihvzmg8U6kQyolArkIyLd0pgVhrfor/iMsox8cn07WCOOvvuJ6XKegzIucPutQ==",
+ "requires": {
+ "backo2": "1.0.2",
+ "component-bind": "1.0.0",
+ "component-emitter": "~1.3.0",
+ "debug": "~3.1.0",
+ "engine.io-client": "~3.4.0",
+ "has-binary2": "~1.0.2",
+ "indexof": "0.0.1",
+ "parseqs": "0.0.6",
+ "parseuri": "0.0.6",
+ "socket.io-parser": "~3.3.0",
+ "to-array": "0.1.4"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ }
+ }
+ },
+ "socket.io-parser": {
+ "version": "3.3.3",
+ "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.3.3.tgz",
+ "integrity": "sha512-qOg87q1PMWWTeO01768Yh9ogn7chB9zkKtQnya41Y355S0UmpXgpcrFwAgjYJxu9BdKug5r5e9YtVSeWhKBUZg==",
+ "requires": {
+ "component-emitter": "~1.3.0",
+ "debug": "~3.1.0",
+ "isarray": "2.0.1"
+ },
+ "dependencies": {
+ "debug": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
+ "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+ "requires": {
+ "ms": "2.0.0"
+ }
+ },
+ "ms": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
+ }
+ }
+ },
+ "statuses": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+ "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="
+ },
+ "strip-ansi": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+ "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+ "dev": true,
+ "requires": {
+ "ansi-regex": "^5.0.1"
+ }
+ },
+ "strip-json-comments": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz",
+ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
+ "dev": true
+ },
+ "supports-color": {
+ "version": "7.2.0",
+ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+ "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+ "dev": true,
+ "requires": {
+ "has-flag": "^4.0.0"
+ }
+ },
+ "text-table": {
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
+ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
+ "dev": true
+ },
+ "to-array": {
+ "version": "0.1.4",
+ "resolved": "https://registry.npmjs.org/to-array/-/to-array-0.1.4.tgz",
+ "integrity": "sha512-LhVdShQD/4Mk4zXNroIQZJC+Ap3zgLcDuwEdcmLv9CCO73NWockQDwyUnW/m8VX/EElfL6FcYx7EeutN4HJA6A=="
+ },
+ "toidentifier": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+ "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
+ },
+ "toolsocket": {
+ "version": "git+ssh://git@github.com/ptcrealitylab/toolsocket.git#b4271baf7214d1c5810ed8ecd1bddebd0d786384",
+ "from": "toolsocket@github:ptcrealitylab/toolsocket",
+ "requires": {
+ "ws": "^8.2.3"
+ },
+ "dependencies": {
+ "ws": {
+ "version": "8.14.2",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
+ "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
+ "requires": {}
+ }
+ }
+ },
+ "type-check": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
+ "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
+ "dev": true,
+ "requires": {
+ "prelude-ls": "^1.2.1"
+ }
+ },
+ "type-fest": {
+ "version": "0.20.2",
+ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz",
+ "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==",
+ "dev": true
+ },
+ "type-is": {
+ "version": "1.6.18",
+ "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
+ "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
+ "requires": {
+ "media-typer": "0.3.0",
+ "mime-types": "~2.1.24"
+ }
+ },
+ "unpipe": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+ "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="
+ },
+ "uri-js": {
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+ "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+ "dev": true,
+ "requires": {
+ "punycode": "^2.1.0"
+ }
+ },
+ "utils-merge": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
+ "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="
+ },
+ "vary": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
+ "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="
+ },
+ "webrtc-adapter": {
+ "version": "9.0.1",
+ "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.1.tgz",
+ "integrity": "sha512-1AQO+d4ElfVSXyzNVTOewgGT/tAomwwztX/6e3totvyyzXPvXIIuUUjAmyZGbKBKbZOXauuJooZm3g6IuFuiNQ==",
+ "dev": true,
+ "requires": {
+ "sdp": "^3.2.0"
+ }
+ },
+ "which": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+ "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+ "dev": true,
+ "requires": {
+ "isexe": "^2.0.0"
+ }
+ },
+ "wrappy": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+ "dev": true
+ },
+ "ws": {
+ "version": "7.5.9",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz",
+ "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==",
+ "requires": {}
+ },
+ "xmlhttprequest-ssl": {
+ "version": "1.5.5",
+ "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
+ "integrity": "sha512-/bFPLUgJrfGUL10AIv4Y7/CUt6so9CLtB/oFxQSHseSDNNCdC6vwwKEqwLN6wNPBg9YWXAiMu8jkf6RPRS/75Q=="
+ },
+ "yeast": {
+ "version": "0.1.2",
+ "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz",
+ "integrity": "sha512-8HFIh676uyGYP6wP13R/j6OJ/1HwJ46snpvzE7aHAN3Ryqh2yX6Xox2B4CUmTwwOIzlG3Bs7ocsP5dZH/R1Qbg=="
+ },
+ "yocto-queue": {
+ "version": "0.1.0",
+ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
+ "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
+ "dev": true
+ }
+ }
+}
diff --git a/package.json b/package.json
new file mode 100644
index 00000000..5f7d5930
--- /dev/null
+++ b/package.json
@@ -0,0 +1,32 @@
+{
+ "name": "vuforia-spatial-remote-operator-addon",
+ "version": "1.0.0",
+ "description": "Remote Operator Add-on for the Vuforia Spatial Edge Server",
+ "scripts": {
+ "test": "npm run lint",
+ "lint": "eslint ."
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/ptcrealitylab/vuforia-spatial-remote-operator-addon.git"
+ },
+ "author": "Ben Reynolds ",
+ "license": "MPL-2.0",
+ "bugs": {
+ "url": "https://github.com/ptcrealitylab/vuforia-spatial-remote-operator-addon/issues"
+ },
+ "homepage": "https://github.com/ptcrealitylab/vuforia-spatial-remote-operator-addon#readme",
+ "dependencies": {
+ "@ffmpeg-installer/ffmpeg": "^1.1.0",
+ "cors": "^2.8.5",
+ "express": "^4.17.1",
+ "express-ws": "^5.0.2",
+ "socket.io": "~2.2.0",
+ "socket.io-client": "~2.3.1",
+ "toolsocket": "github:ptcrealitylab/toolsocket"
+ },
+ "devDependencies": {
+ "eslint": "^8.57.0",
+ "webrtc-adapter": "^9.0.1"
+ }
+}
diff --git a/tools/.identity/spatialPatch/settings.json b/tools/.identity/spatialPatch/settings.json
new file mode 100644
index 00000000..d0637618
--- /dev/null
+++ b/tools/.identity/spatialPatch/settings.json
@@ -0,0 +1,3 @@
+{
+ "enabled": true
+}
\ No newline at end of file
diff --git a/tools/spatialPatch/icon-foreground.svg b/tools/spatialPatch/icon-foreground.svg
new file mode 100644
index 00000000..ff4fe2fb
--- /dev/null
+++ b/tools/spatialPatch/icon-foreground.svg
@@ -0,0 +1,10 @@
+
+
+ spatialPatch-icon
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tools/spatialPatch/icon.gif b/tools/spatialPatch/icon.gif
new file mode 100644
index 00000000..f8b19ce7
Binary files /dev/null and b/tools/spatialPatch/icon.gif differ
diff --git a/tools/spatialPatch/index.css b/tools/spatialPatch/index.css
new file mode 100644
index 00000000..5661a8fb
--- /dev/null
+++ b/tools/spatialPatch/index.css
@@ -0,0 +1,20 @@
+img {
+ width: 100%;
+ height: 100%;
+}
+.tool-color-gradient {
+ background: black;
+ border-radius: 30px;
+}
+
+.launchButtonExpanded {
+ opacity: 0.3;
+}
+
+.launchButtonCollapsed {
+ opacity: 0.7;
+}
+
+.launchButtonPressed {
+ opacity: 0.9;
+}
diff --git a/tools/spatialPatch/index.html b/tools/spatialPatch/index.html
new file mode 100644
index 00000000..bb023c56
--- /dev/null
+++ b/tools/spatialPatch/index.html
@@ -0,0 +1,18 @@
+
+
+
+
+ Spatial Sensor
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tools/spatialPatch/index.js b/tools/spatialPatch/index.js
new file mode 100644
index 00000000..926ebc37
--- /dev/null
+++ b/tools/spatialPatch/index.js
@@ -0,0 +1,166 @@
+/* global SpatialInterface, EnvelopeContents */
+
+const SYSTEM_PROMPT = `Describe what you can see in the image, and generate the description so it is suitable to be used for alt-text.`;
+const apiKey = '';
+
+const ShaderMode = {
+ HIDDEN: 'HIDDEN',
+ SOLID: 'SOLID',
+ DIFF: 'DIFF',
+ DIFF_DEPTH: 'DIFF_DEPTH',
+};
+
+let shaderMode = 'SOLID';
+
+let spatialInterface;
+let envelopeContents;
+
+if (!spatialInterface) {
+ spatialInterface = new SpatialInterface();
+
+ // allow the tool to be nested inside of envelopes
+ envelopeContents = new EnvelopeContents(spatialInterface, document.body);
+
+ // hide the associated spatial snapshot when the parent envelope containing this tool closes
+ envelopeContents.onClose(() => {
+ spatialInterface.patchSetShaderMode(ShaderMode.HIDDEN);
+ });
+
+ // restore the associated spatial snapshot when the parent envelope containing this tool opens
+ envelopeContents.onOpen(() => {
+ spatialInterface.patchSetShaderMode(shaderMode);
+ });
+
+ // listen for isEditable and expandFrame messages from envelope
+ envelopeContents.onMessageFromEnvelope(function(e) {
+ console.log('spatial patch got message from envelope', e);
+ if (typeof e.toggleVisibility !== 'undefined') {
+ let newShaderMode = e.toggleVisibility ? ShaderMode.SOLID : ShaderMode.HIDDEN;
+ setShaderMode(newShaderMode);
+ spatialInterface.writePublicData('storage', 'shaderMode', shaderMode);
+ }
+ });
+}
+
+const launchButton = document.getElementById('launchButton');
+launchButton.classList.add('launchButtonExpanded');
+
+launchButton.addEventListener('pointerup', function () {
+ launchButton.classList.remove('launchButtonPressed');
+
+ switch (shaderMode) {
+ case ShaderMode.HIDDEN:
+ shaderMode = ShaderMode.SOLID;
+ break;
+ case ShaderMode.SOLID: // skips over DIFF and DIFF_DEPTH for now
+ default:
+ shaderMode = ShaderMode.HIDDEN;
+ break;
+ }
+ setShaderMode(shaderMode);
+
+ if (envelopeContents) {
+ console.log('spatial patch sending new toggle state to envelope');
+ envelopeContents.sendMessageToEnvelope({
+ toggleVisibility: shaderMode === ShaderMode.SOLID
+ });
+ }
+
+ spatialInterface.writePublicData('storage', 'shaderMode', shaderMode);
+}, false);
+
+// add some slight visual feedback when you tap on the button
+launchButton.addEventListener('pointerdown', () => {
+ launchButton.classList.add('launchButtonPressed');
+});
+
+// add random init gradient for the tool icon
+const randomDelay = -Math.floor(Math.random() * 100);
+launchButton.style.animationDelay = `${randomDelay}s`;
+
+function setShaderMode(newShaderMode) {
+ shaderMode = newShaderMode;
+ // add some visual feedback, so you know if it's open or closed
+ if (shaderMode === ShaderMode.HIDDEN) {
+ launchButton.classList.remove('launchButtonExpanded');
+ launchButton.classList.add('launchButtonCollapsed');
+ } else if (shaderMode === ShaderMode.SOLID) {
+ launchButton.classList.remove('launchButtonCollapsed');
+ launchButton.classList.add('launchButtonExpanded');
+ }
+ spatialInterface.patchSetShaderMode(shaderMode);
+}
+
+async function generateDescription(serialization) {
+ if (!apiKey) {
+ return;
+ }
+
+ const messages = [{
+ role: 'system',
+ content: SYSTEM_PROMPT,
+ }, {
+ role: 'user',
+ content: [{
+ type: 'image',
+ image_url: {
+ url: serialization.texture,
+ }
+ }],
+ }];
+
+ const body = {
+ model: 'gpt-4-vision-preview',
+ max_tokens: 4096,
+ temperature: 0,
+ messages,
+ };
+
+ try {
+ const response = await fetch('https://api.openai.com/v1/chat/completions', {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${apiKey}`,
+ 'Content-type': 'application/json',
+ },
+ body: JSON.stringify(body),
+ });
+ const data = await response.json();
+ console.log('got openai', data);
+ serialization.description = data.choices[0].message.content;
+ spatialInterface.writePublicData('storage', 'serialization', serialization);
+ } catch (error) {
+ console.error('not openai', error);
+ }
+}
+
+
+spatialInterface.onSpatialInterfaceLoaded(function() {
+ spatialInterface.setVisibilityDistance(100);
+ spatialInterface.setMoveDelay(300);
+ // spatialInterface.setAlwaysFaceCamera(true);
+
+ spatialInterface.initNode('storage', 'storeData');
+
+ spatialInterface.addReadPublicDataListener('storage', 'serialization', serialization => {
+ if (!serialization.description) {
+ generateDescription(serialization);
+ }
+ spatialInterface.patchHydrate(serialization);
+ });
+
+ spatialInterface.addReadPublicDataListener('storage', 'shaderMode', storedShaderMode => {
+ if (storedShaderMode !== shaderMode) {
+ shaderMode = storedShaderMode;
+
+ if (envelopeContents) {
+ console.log('spatial patch sending stored toggle state to envelope');
+ envelopeContents.sendMessageToEnvelope({
+ toggleVisibility: shaderMode === ShaderMode.SOLID
+ });
+ }
+
+ setShaderMode(shaderMode);
+ }
+ });
+});
diff --git a/tools/spatialPatch/launchIcon.svg b/tools/spatialPatch/launchIcon.svg
new file mode 100644
index 00000000..532f790e
--- /dev/null
+++ b/tools/spatialPatch/launchIcon.svg
@@ -0,0 +1,11 @@
+
+
+ spatialPatch-bounded
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tools/spatialPatch/sprites/empty.png b/tools/spatialPatch/sprites/empty.png
new file mode 100644
index 00000000..aed1f8e6
Binary files /dev/null and b/tools/spatialPatch/sprites/empty.png differ
diff --git a/tools/spatialPatch/sprites/markstep.png b/tools/spatialPatch/sprites/markstep.png
new file mode 100644
index 00000000..2014b03d
Binary files /dev/null and b/tools/spatialPatch/sprites/markstep.png differ
diff --git a/tools/spatialPatch/sprites/recording.png b/tools/spatialPatch/sprites/recording.png
new file mode 100644
index 00000000..e5a96af7
Binary files /dev/null and b/tools/spatialPatch/sprites/recording.png differ