SkalskiP · SkalskiP · Sep 27, 2022 · Sep 27, 2022 · Sep 27, 2022 · Sep 27, 2022
diff --git a/index.html b/index.html
@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
-
+    <script src="https://cdn.roboflow.com/0.2.20/roboflow.js"></script>
     <script async src="https://www.googletagmanager.com/gtag/js?id=UA-155837750-1"></script>
     <script>
       window.dataLayer = window.dataLayer || [];

diff --git a/public/ico/roboflow-logo.png b/public/ico/roboflow-logo.png
diff --git a/src/App.tsx b/src/App.tsx
@@ -17,11 +17,14 @@ import NotificationsView from './views/NotificationsView/NotificationsView';
 interface IProps {
     projectType: ProjectType;
     windowSize: ISize;
-    ObjectDetectorLoaded: boolean;
-    PoseDetectionLoaded: boolean;
+    objectDetectorLoaded: boolean;
+    poseDetectionLoaded: boolean;
+    roboflowJSObjectDetectorLoaded: boolean;
 }
 
-const App: React.FC<IProps> = ({projectType, windowSize, ObjectDetectorLoaded, PoseDetectionLoaded}) => {
+const App: React.FC<IProps> = (
+    {projectType, windowSize, objectDetectorLoaded, poseDetectionLoaded, roboflowJSObjectDetectorLoaded}
+) => {
     const selectRoute = () => {
         if (!!PlatformModel.mobileDeviceData.manufacturer && !!PlatformModel.mobileDeviceData.os)
             return <MobileMainView/>;
@@ -36,8 +39,10 @@ const App: React.FC<IProps> = ({projectType, windowSize, ObjectDetectorLoaded, P
         }
     };
 
+    const isAILoaded = objectDetectorLoaded || poseDetectionLoaded || roboflowJSObjectDetectorLoaded
+
       return (
-        <div className={classNames('App', {'AI': ObjectDetectorLoaded || PoseDetectionLoaded})}
+        <div className={classNames('App', {'AI': isAILoaded})}
             draggable={false}
         >
             {selectRoute()}
@@ -50,8 +55,9 @@ const App: React.FC<IProps> = ({projectType, windowSize, ObjectDetectorLoaded, P
 const mapStateToProps = (state: AppState) => ({
     projectType: state.general.projectData.type,
     windowSize: state.general.windowSize,
-    ObjectDetectorLoaded: state.ai.isObjectDetectorLoaded,
-    PoseDetectionLoaded: state.ai.isPoseDetectorLoaded
+    objectDetectorLoaded: state.ai.isObjectDetectorLoaded,
+    poseDetectionLoaded: state.ai.isPoseDetectorLoaded,
+    roboflowJSObjectDetectorLoaded: state.ai.isRoboflowJSObjectDetectorLoaded
 });
 
 export default connect(

diff --git a/src/ai/RoboflowObjectDetector.ts b/src/ai/RoboflowObjectDetector.ts
@@ -0,0 +1,61 @@
+import {max} from "lodash";
+
+
+export interface DetectedObject {
+    bbox: [number, number, number, number];
+    class: string;
+    score: number;
+}
+
+
+export class RoboflowObjectDetector {
+    private static model;
+
+    public static loadModel(
+        publishableKey: string,
+        modelId: string,
+        modelVersion: number,
+        onSuccess: () => unknown,
+        onError: (error: Error) => unknown
+    ) {
+        roboflow.auth({
+            publishable_key: publishableKey
+        }).load({
+            model: modelId,
+            version: modelVersion
+        }).then((model) => {
+            RoboflowObjectDetector.model = model
+            onSuccess()
+        }).catch((error) => {
+            onError(error)
+        })
+    }
+
+    public static predict(image: HTMLImageElement, callback?: (predictions: DetectedObject[]) => unknown) {
+        if (!RoboflowObjectDetector.model) return;
+
+        RoboflowObjectDetector.model
+            .detect(image)
+            .then((predictions) => {
+                const processedPredictions: DetectedObject[] = predictions.map((raw) => {
+                    return {
+                        bbox: [
+                            max([raw.bbox.x - raw.bbox.width / 2, 0]),
+                            max([raw.bbox.y - raw.bbox.height / 2, 0]),
+                            raw.bbox.width,
+                            raw.bbox.height
+                        ],
+                        class: raw.class,
+                        score: raw.confidence
+                    }
+                })
+                if (callback) {
+                    callback(processedPredictions)
+                }
+            })
+            .catch((error) => {
+                // TODO
+                throw new Error(error as string);
+            })
+    }
+}
diff --git a/src/data/enums/AIModel.ts b/src/data/enums/AIModel.ts
@@ -1,4 +1,5 @@
 export enum AIModel {
-    OBJECT_DETECTION = "OBJECT_DETECTION",
-    POSE_DETECTION = "POSE_DETECTION"
-}
+    OBJECT_DETECTION = 'OBJECT_DETECTION',
+    POSE_DETECTION = 'POSE_DETECTION',
+    ROBOFLOW_JS_OBJECT_DETECTION = 'ROBOFLOW_JS_OBJECT_DETECTION'
+}
diff --git a/src/data/enums/Notification.ts b/src/data/enums/Notification.ts
@@ -1,4 +1,5 @@
 export enum Notification {
     EMPTY_LABEL_NAME_ERROR = 0,
-    NON_UNIQUE_LABEL_NAMES_ERROR = 1
+    NON_UNIQUE_LABEL_NAMES_ERROR = 1,
+    ROBOFLOW_JS_MODEL_COULD_NOT_BE_LOADED_ERROR = 2
 }
diff --git a/src/data/enums/PopupWindowType.ts b/src/data/enums/PopupWindowType.ts
@@ -4,6 +4,7 @@ export enum PopupWindowType {
     SUGGEST_LABEL_NAMES = 'SUGGEST_LABEL_NAMES',
     IMPORT_IMAGES = 'IMPORT_IMAGES',
     LOAD_AI_MODEL = 'LOAD_AI_MODEL',
+    LOAD_ROBOFLOW_JS_MODEL = 'LOAD_ROBOFLOW_JS_MODEL',
     EXPORT_ANNOTATIONS = 'EXPORT_ANNOTATIONS',
     IMPORT_ANNOTATIONS = 'IMPORT_ANNOTATIONS',
     INSERT_LABEL_NAMES = 'INSERT_LABEL_NAMES',

diff --git a/src/data/info/NotificationsData.ts b/src/data/info/NotificationsData.ts
@@ -17,5 +17,10 @@ export const NotificationsDataMap = {
         header: 'Non unique label names',
         description: 'Looks like not all your label names are unique. Unique names are necessary to guarantee correct' +
             ' data export when you complete your work. Make your names unique and try again.'
+    },
+    [Notification.ROBOFLOW_JS_MODEL_COULD_NOT_BE_LOADED_ERROR]: {
+        header: 'roboflow.js model could not be loaded',
+        description: 'We ware unable to load your roboflow.js model. Please make sure that your publishable key and ' +
+            'model id are correct.'
     }
 }
diff --git a/src/logic/actions/AIObjectDetectionActions.ts b/src/logic/actions/AIObjectDetectionActions.ts
@@ -13,6 +13,7 @@ import {PopupWindowType} from '../../data/enums/PopupWindowType';
 import {updateActivePopupType} from '../../store/general/actionCreators';
 import {AISelector} from '../../store/selectors/AISelector';
 import {AIActions} from './AIActions';
+import {RoboflowObjectDetector} from '../../ai/RoboflowObjectDetector';
 
 export class AIObjectDetectionActions {
     public static detectRectsForActiveImage(): void {
@@ -21,22 +22,36 @@ export class AIObjectDetectionActions {
     }
 
     public static detectRects(imageId: string, image: HTMLImageElement): void {
-        if (LabelsSelector.getImageDataById(imageId).isVisitedByObjectDetector || !AISelector.isAIObjectDetectorModelLoaded())
+        if (LabelsSelector.getImageDataById(imageId).isVisitedByObjectDetector)
             return;
 
-        store.dispatch(updateActivePopupType(PopupWindowType.LOADER));
-        ObjectDetector.predict(image, (predictions: DetectedObject[]) => {
-            const suggestedLabelNames = AIObjectDetectionActions.extractNewSuggestedLabelNames(LabelsSelector.getLabelNames(), predictions);
-            const rejectedLabelNames = AISelector.getRejectedSuggestedLabelList();
-            const newlySuggestedNames = AIActions.excludeRejectedLabelNames(suggestedLabelNames, rejectedLabelNames);
-            if (newlySuggestedNames.length > 0) {
-                store.dispatch(updateSuggestedLabelList(newlySuggestedNames));
-                store.dispatch(updateActivePopupType(PopupWindowType.SUGGEST_LABEL_NAMES));
-            } else {
-                store.dispatch(updateActivePopupType(null));
-            }
-            AIObjectDetectionActions.saveRectPredictions(imageId, predictions);
-        })
+        if (AISelector.isAIObjectDetectorModelLoaded()) {
+            store.dispatch(updateActivePopupType(PopupWindowType.LOADER));
+            ObjectDetector.predict(image, (predictions: DetectedObject[]) => {
+                AIObjectDetectionActions.handleObjectDetectorResults(imageId, predictions);
+            })
+        }
+
+        if (AISelector.isAIRoboflowJSObjectDetectorModelLoaded()) {
+            store.dispatch(updateActivePopupType(PopupWindowType.LOADER));
+            RoboflowObjectDetector.predict(image, (predictions: DetectedObject[]) => {
+                AIObjectDetectionActions.handleObjectDetectorResults(imageId, predictions);
+            })
+        }
+    }
+
+    private static handleObjectDetectorResults = (imageId: string, predictions: DetectedObject[]) => {
+        const suggestedLabelNames = AIObjectDetectionActions
+            .extractNewSuggestedLabelNames(LabelsSelector.getLabelNames(), predictions);
+        const rejectedLabelNames = AISelector.getRejectedSuggestedLabelList();
+        const newlySuggestedNames = AIActions.excludeRejectedLabelNames(suggestedLabelNames, rejectedLabelNames);
+        if (newlySuggestedNames.length > 0) {
+            store.dispatch(updateSuggestedLabelList(newlySuggestedNames));
+            store.dispatch(updateActivePopupType(PopupWindowType.SUGGEST_LABEL_NAMES));
+        } else {
+            store.dispatch(updateActivePopupType(null));
+        }
+        AIObjectDetectionActions.saveRectPredictions(imageId, predictions);
     }
 
     public static saveRectPredictions(imageId: string, predictions: DetectedObject[]) {

diff --git a/src/logic/initializer/AppInitializer.ts b/src/logic/initializer/AppInitializer.ts
@@ -1,11 +1,11 @@
-import {updateWindowSize} from "../../store/general/actionCreators";
-import {ContextManager} from "../context/ContextManager";
-import {store} from "../../index";
-import {PlatformUtil} from "../../utils/PlatformUtil";
-import {PlatformModel} from "../../staticModels/PlatformModel";
-import {EventType} from "../../data/enums/EventType";
-import {GeneralSelector} from "../../store/selectors/GeneralSelector";
-import {EnvironmentUtil} from "../../utils/EnvironmentUtil";
+import {updateWindowSize} from '../../store/general/actionCreators';
+import {ContextManager} from '../context/ContextManager';
+import {store} from '../../index';
+import {PlatformUtil} from '../../utils/PlatformUtil';
+import {PlatformModel} from '../../staticModels/PlatformModel';
+import {EventType} from '../../data/enums/EventType';
+import {GeneralSelector} from '../../store/selectors/GeneralSelector';
+import {EnvironmentUtil} from '../../utils/EnvironmentUtil';
 
 export class AppInitializer {
     public static inti():void {
@@ -37,11 +37,7 @@ export class AppInitializer {
     };
 
     private static disableUnwantedKeyBoardBehaviour = (event: KeyboardEvent) => {
-        if (PlatformModel.isMac && event.metaKey) {
-            event.preventDefault();
-        }
-
-        if (["=", "+", "-"].includes(event.key)) {
+        if (['=', '+', '-'].includes(event.key)) {
             if (event.ctrlKey || (PlatformModel.isMac && event.metaKey)) {
                 event.preventDefault();
             }
@@ -61,4 +57,4 @@ export class AppInitializer {
         PlatformModel.isSafari = PlatformUtil.isSafari(userAgent);
         PlatformModel.isFirefox = PlatformUtil.isFirefox(userAgent);
     };
-}
+}
diff --git a/src/store/Actions.ts b/src/store/Actions.ts
@@ -4,6 +4,7 @@ export enum Action {
     UPDATE_REJECTED_SUGGESTED_LABEL_LIST = '@@UPDATE_REJECTED_SUGGESTED_LABEL_LIST',
     UPDATE_OBJECT_DETECTOR_STATUS = '@@UPDATE_OBJECT_DETECTOR_STATUS',
     UPDATE_POSE_DETECTOR_STATUS = '@@UPDATE_POSE_DETECTOR_STATUS',
+    UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS = '@@UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS',
     UPDATE_DISABLED_AI_FLAG = '@@UPDATE_DISABLED_AI_FLAG',
 
     // GENERAL

diff --git a/src/store/ai/actionCreators.ts b/src/store/ai/actionCreators.ts
@@ -37,11 +37,20 @@ export function updatePoseDetectorStatus(isPoseDetectorLoaded: boolean): AIActio
     }
 }
 
+export function updateRoboflowJSObjectDetectorStatus(isRoboflowJSObjectDetectorLoaded: boolean): AIActionTypes {
+    return {
+        type: Action.UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS,
+        payload: {
+            isRoboflowJSObjectDetectorLoaded,
+        }
+    }
+}
+
 export function updateDisabledAIFlag(isAIDisabled: boolean): AIActionTypes {
     return {
         type: Action.UPDATE_DISABLED_AI_FLAG,
         payload: {
             isAIDisabled,
         }
     }
-}
+}
diff --git a/src/store/ai/reducer.ts b/src/store/ai/reducer.ts
@@ -6,6 +6,7 @@ const initialState: AIState = {
     rejectedSuggestedLabelList: [],
     isObjectDetectorLoaded: false,
     isPoseDetectorLoaded: false,
+    isRoboflowJSObjectDetectorLoaded: false,
     isAIDisabled: false
 };
 
@@ -38,6 +39,12 @@ export function aiReducer(
                 isPoseDetectorLoaded: action.payload.isPoseDetectorLoaded
             }
         }
+        case Action.UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS: {
+            return {
+                ...state,
+                isRoboflowJSObjectDetectorLoaded: action.payload.isRoboflowJSObjectDetectorLoaded
+            }
+        }
         case Action.UPDATE_DISABLED_AI_FLAG: {
             return {
                 ...state,

diff --git a/src/store/ai/types.ts b/src/store/ai/types.ts
@@ -7,6 +7,9 @@ export type AIState = {
     // POSE NET
     isPoseDetectorLoaded: boolean;
 
+    // ROBOFLOW
+    isRoboflowJSObjectDetectorLoaded: boolean;
+
     // GENERAL
     suggestedLabelList: string[];
     rejectedSuggestedLabelList: string[];
@@ -41,6 +44,13 @@ interface UpdatePoseDetectorStatus {
     }
 }
 
+interface UpdateRoboflowJSObjectDetectorStatus {
+    type: typeof Action.UPDATE_ROBOFLOW_JS_OBJECT_DETECTOR_STATUS;
+    payload: {
+        isRoboflowJSObjectDetectorLoaded: boolean;
+    }
+}
+
 interface UpdateDisabledAIFlag {
     type: typeof Action.UPDATE_DISABLED_AI_FLAG;
     payload: {
@@ -52,4 +62,5 @@ export type AIActionTypes = UpdateSuggestedLabelList
     | UpdateRejectedSuggestedLabelList
     | UpdateObjectDetectorStatus
     | UpdatePoseDetectorStatus
+    | UpdateRoboflowJSObjectDetectorStatus
     | UpdateDisabledAIFlag
diff --git a/src/store/selectors/AISelector.ts b/src/store/selectors/AISelector.ts
@@ -13,6 +13,10 @@ export class AISelector {
         return store.getState().ai.isObjectDetectorLoaded;
     }
 
+    public static isAIRoboflowJSObjectDetectorModelLoaded(): boolean {
+        return store.getState().ai.isRoboflowJSObjectDetectorLoaded;
+    }
+
     public static isAIPoseDetectorModelLoaded(): boolean {
         return store.getState().ai.isPoseDetectorLoaded;
     }

diff --git a/src/views/EditorView/EditorTopNavigationBar/EditorTopNavigationBar.tsx b/src/views/EditorView/EditorTopNavigationBar/EditorTopNavigationBar.tsx
@@ -102,6 +102,10 @@ const EditorTopNavigationBar: React.FC<IProps> = (
         updateCrossHairVisibleStatusAction(!crossHairVisible);
     };
 
+    const showAIButtons = (activeLabelType === LabelType.RECT && AISelector.isAIObjectDetectorModelLoaded()) ||
+        (activeLabelType === LabelType.RECT && AISelector.isAIRoboflowJSObjectDetectorModelLoaded()) ||
+        (activeLabelType === LabelType.POINT && AISelector.isAIPoseDetectorModelLoaded())
+
     return (
         <div className={getClassName()}>
             <div className='ButtonWrapper'>
@@ -174,8 +178,7 @@ const EditorTopNavigationBar: React.FC<IProps> = (
                     )
                 }
             </div>
-            {((activeLabelType === LabelType.RECT && AISelector.isAIObjectDetectorModelLoaded()) ||
-                (activeLabelType === LabelType.POINT && AISelector.isAIPoseDetectorModelLoaded())) && <div className='ButtonWrapper'>
+            {showAIButtons && <div className='ButtonWrapper'>
                     {
                         getButtonWithTooltip(
                             'accept-all',