Merge branch 'UEPR-282-face-sensing' of github.com:scratchfoundation/scratch-editor into UEPR-56-UEPR-282

Author: adzhindzhi

package-lock.json

@@ -57,12 +57,15 @@
     "watch": "webpack --watch"
   },
   "dependencies": {
+    "@mediapipe/face_detection": "0.4.1646425229",
     "@microbit/microbit-universal-hex": "0.2.2",
     "@radix-ui/react-context-menu": "^2.2.5",
     "@testing-library/user-event": "^14.6.1",
     "@scratch/scratch-render": "11.6.0",
     "@scratch/scratch-svg-renderer": "11.6.0",
     "@scratch/scratch-vm": "11.6.0",
+    "@tensorflow-models/face-detection": "^1.0.3",
+    "@tensorflow/tfjs": "^4.22.0",
     "arraybuffer-loader": "1.0.8",
     "autoprefixer": "9.8.8",
     "balance-text": "3.3.1",

packages/scratch-gui/package.json

@@ -144,6 +144,12 @@ class Blocks extends React.Component {
         if (this.props.isVisible) {
             this.setLocale();
         }
+
+        window.addEventListener('load-extension', () => {
+            this.props.vm.extensionManager.loadExtensionURL('faceSensing').then(() => {
+                this.handleCategorySelected('faceSensing');
+            });
+        });
     }
     shouldComponentUpdate (nextProps, nextState) {
         return (

packages/scratch-gui/src/containers/blocks.jsx

@@ -340,16 +340,29 @@ class Stage extends React.Component {
     }
     onStartDrag (x, y) {
         if (this.state.dragId) return;
-        const drawableId = this.renderer.pick(x, y);
+        
+        // Targets with no attached drawable cannot be dragged.
+        let draggableTargets = this.props.vm.runtime.targets.filter(
+            target => Number.isFinite(target.drawableID)
+        );
+
+        // Because pick queries can be expensive, only perform them for drawables that are currently draggable.
+        // If we're in the editor, we can drag all targets. Otherwise, filter.
+        if (!this.props.useEditorDragStyle) {
+            draggableTargets = draggableTargets.filter(
+                target => target.draggable
+            );
+        }
+        if (draggableTargets.length === 0) return;
+
+        const draggableIDs = draggableTargets.map(target => target.drawableID);
+        const drawableId = this.renderer.pick(x, y, 1, 1, draggableIDs);
         if (drawableId === null) return;
         const targetId = this.props.vm.getTargetIdForDrawableId(drawableId);
         if (targetId === null) return;
 
         const target = this.props.vm.runtime.getTargetById(targetId);
 
-        // Do not start drag unless in editor drag mode or target is draggable
-        if (!(this.props.useEditorDragStyle || target.draggable)) return;
-
         // Dragging always brings the target to the front
         target.goToFront();
 

packages/scratch-gui/src/containers/stage.jsx

@@ -215,6 +215,20 @@ const alerts = [
         ),
         iconSpinner: true,
         level: AlertLevels.SUCCESS
+    },
+    {
+        alertId: 'loadingExtensionData',
+        alertType: AlertTypes.STANDARD,
+        clearList: [],
+        content: (
+            <FormattedMessage
+                defaultMessage="Loading extension..."
+                description="Message indicating that extension is in process of loading"
+                id="gui.alerts.loadingExtensionData"
+            />
+        ),
+        iconSpinner: true,
+        level: AlertLevels.SUCCESS
     }
 ];
 

packages/scratch-gui/src/lib/alerts/index.jsx

@@ -101,7 +101,11 @@ const cloudManagerHOC = function (WrappedComponent) {
         handleExtensionAdded (categoryInfo) {
             // Note that props.vm.extensionManager.isExtensionLoaded('videoSensing') is still false
             // at the point of this callback, so it is difficult to reuse the canModifyCloudData logic.
-            if (categoryInfo.id === 'videoSensing' && this.isConnected()) {
+            if (
+                (categoryInfo.id === 'videoSensing' ||
+                    categoryInfo.id === 'faceSensing') &&
+                this.isConnected()
+            ) {
                 this.disconnectFromCloud();
             }
         }
@@ -157,9 +161,18 @@ const cloudManagerHOC = function (WrappedComponent) {
             isShowingWithId: getIsShowingWithId(loadingState),
             projectId: state.scratchGui.projectState.projectId,
             // if you're editing someone else's project, you can't modify cloud data
-            canModifyCloudData: (!state.scratchGui.mode.hasEverEnteredEditor || ownProps.canSave) &&
+            canModifyCloudData:
+                (!state.scratchGui.mode.hasEverEnteredEditor ||
+                    ownProps.canSave) &&
                 // possible security concern if the program attempts to encode webcam data over cloud variables
-                !ownProps.vm.extensionManager.isExtensionLoaded('videoSensing')
+                !(
+                    ownProps.vm.extensionManager.isExtensionLoaded(
+                        'videoSensing'
+                    ) ||
+                    ownProps.vm.extensionManager.isExtensionLoaded(
+                        'faceSensing'
+                    )
+                )
         };
     };
 

packages/scratch-gui/src/lib/cloud-manager-hoc.jsx

@@ -46,6 +46,9 @@ import gdxforInsetIconURL from './gdxfor/gdxfor-small.svg';
 import gdxforConnectionIconURL from './gdxfor/gdxfor-illustration.svg';
 import gdxforConnectionSmallIconURL from './gdxfor/gdxfor-small.svg';
 
+import faceSensingIconURL from './faceSensing/faceSensing.png';
+import faceSensingInsetIconURL from './faceSensing/faceSensing-small.svg';
+
 export default [
     {
         name: (
@@ -107,6 +110,26 @@ export default [
         ),
         featured: true
     },
+    {
+        name: (
+            <FormattedMessage
+                defaultMessage="Face Sensing"
+                description="Name for the 'Face Sensing' extension"
+                id="gui.extension.faceSensing.name"
+            />
+        ),
+        extensionId: 'faceSensing',
+        iconURL: faceSensingIconURL,
+        insetIconURL: faceSensingInsetIconURL,
+        description: (
+            <FormattedMessage
+                defaultMessage="Sense faces with the camera."
+                description="Description for the 'Face Sensing' extension"
+                id="gui.extension.faceSensing.description"
+            />
+        ),
+        featured: true
+    },
     {
         name: (
             <FormattedMessage