Vision Data Explorer Flyout tests

apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxFridgeImageClassificationModelDebugging/visionDataExplorer.spec.ts

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+  describeVisionDataExplorer,
+  modelAssessmentDatasets
+} from "@responsible-ai/e2e";
+const datasetShape =
+  modelAssessmentDatasets.FridgeImageClassificationModelDebugging;
+describeVisionDataExplorer(
+  datasetShape,
+  "FridgeImageClassificationModelDebugging"
+);

apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxFridgeMultilabelImageClassificationModelDebugging/visionDataExplorer.spec.ts

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+  describeVisionDataExplorer,
+  modelAssessmentDatasets
+} from "@responsible-ai/e2e";
+const datasetShape = modelAssessmentDatasets.FridgeMultilabelModelDebugging;
+describeVisionDataExplorer(datasetShape, "FridgeMultilabelModelDebugging");

apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxFridgeObjectDetectionModelDebugging/visionDataExplorer.spec.ts

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+  describeVisionDataExplorer,
+  modelAssessmentDatasets
+} from "@responsible-ai/e2e";
+const datasetShape =
+  modelAssessmentDatasets.FridgeObjectDetectionModelDebugging;
+describeVisionDataExplorer(datasetShape, "FridgeObjectDetectionModelDebugging");

libs/e2e/src/index.ts

@@ -11,4 +11,5 @@ export * from "./lib/describer/modelAssessment/modelOverview/describeModelOvervi
 export * from "./lib/describer/modelAssessment/whatIfCounterfactuals/describeWhatIf";
 export * from "./lib/describer/modelAssessment/datasets/modelAssessmentDatasets";
 export * from "./lib/describer/modelAssessment/IModelAssessmentData";
+export * from "./lib/describer/modelAssessment/visionDataExplorer/describeVisionDataExplorer";
 export * from "./util/visit";

libs/e2e/src/lib/describer/modelAssessment/Constants.ts

@@ -176,5 +176,20 @@ export enum Locators {
   AggregateBalanceMeasuresHeader = "#aggregateBalanceMeasures #aggregateBalanceMeasuresHeader",
   AggregateBalanceMeasuresTable = "#aggregateBalanceMeasures .ms-DetailsList",
   AggregateBalanceMeasuresTableColumns = "#aggregateBalanceMeasures .ms-DetailsList-headerWrapper div[aria-label]",
-  AggregateBalanceMeasuresTableRows = "#aggregateBalanceMeasures .ms-DetailsRow"
+  AggregateBalanceMeasuresTableRows = "#aggregateBalanceMeasures .ms-DetailsRow",
+  VisionDataAnalysisPivot = "#visionDataAnalysisPivot",
+  VisionDataAnalysisTab = "#visionDataAnalysisPivot .ms-Pivot-link",
+  VisionDataBalancePivotItem = "#visionDataAnalysisPivot button[name='Data balance']",
+  VisionDataExplorer = "#VisionDataExplorer",
+  VisionDataExplorerImageExplorerViewButton = "#VisionDataExplorer button[name='Image explorer view']",
+  VisionDataExplorerImageExplorerViewSuccessImage = "#VisionDataExplorer #visionImage_0",
+  VisionDataExplorerFlyoutPredictionTitle = "#flyoutPredictionTitle",
+  VisionDataExplorerFlyoutPredictionLabel = "#flyoutPredLabel",
+  VisionDataExplorerFlyoutGroundTruthLabel = "#flyoutGtLabel",
+  VisionDataExplorerFlyoutImage = "#flyoutImage_0",
+  VisionDataExplorerFlyoutExplanationImage = "#flyoutExplanationImage_0",
+  VisionDataExplorerFlyoutMetadata = "#flyoutMetadata",
+  VisionDataExplorerFlyoutCloseButton = "button[aria-label='Close']",
+  VisionDataExplorerFlyoutCanvasToolsImageContainer = "#canvasToolsDiv #selectionDiv #editorDiv",
+  VisionDataExplorerFlyoutObjectSelection = "#Object Selection"
 }

libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts

@@ -6,6 +6,7 @@ export interface IModelAssessmentData {
   modelStatisticsData?: IModelStatisticsData;
   modelOverviewData?: IModelOverviewData;
   datasetExplorerData?: IDatasetExplorerData;
+  visionDataExplorerData?: IVisionDataExplorerData;
   featureImportanceData?: IFeatureImportanceData;
   dataBalanceData?: IDataBalanceData;
   causalAnalysisData?: ICausalAnalysisData;
@@ -89,6 +90,12 @@ export interface IDatasetExplorerData {
   cohortDatasetNewValue?: string;
 }
 
+export interface IVisionDataExplorerData {
+  hasVisionDataExplorerComponent?: boolean;
+  errorInstances?: number;
+  successInstances?: number;
+}
+
 export interface IDEWhiskerPlotData {
   featureList?: string[];
   defaultYAxis?: string;

libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeImageClassificationModelDebugging.ts

@@ -50,5 +50,10 @@ export const FridgeImageClassificationModelDebugging = {
       name: "CohortCreateE2E-image-classification",
       sampleSize: "5"
     }
+  },
+  visionDataExplorerData: {
+    errorInstances: 0,
+    hasVisionDataExplorerComponent: true,
+    successInstances: 10
   }
 };

libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeMultilabelModelDebugging.ts

@@ -40,5 +40,10 @@ export const FridgeMultilabelModelDebugging = {
       name: "CohortCreateE2E-multilabel",
       sampleSize: "3"
     }
+  },
+  visionDataExplorerData: {
+    errorInstances: 0,
+    hasVisionDataExplorerComponent: true,
+    successInstances: 10
   }
 };

libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeObjectDetectionModelDebugging.ts

@@ -42,5 +42,10 @@ export const FridgeObjectDetectionModelDebugging = {
       name: "CohortCreateE2E-object-detection",
       sampleSize: "2"
     }
+  },
+  visionDataExplorerData: {
+    errorInstances: 0,
+    hasVisionDataExplorerComponent: true,
+    successInstances: 5
   }
 };

libs/e2e/src/lib/describer/modelAssessment/visionDataExplorer/describeVisionDataExplorer.ts

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { visit } from "../../../../util/visit";
+import { Locators } from "../Constants";
+import { modelAssessmentDatasets } from "../datasets/modelAssessmentDatasets";
+import { IModelAssessmentData } from "../IModelAssessmentData";
+
+import { ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent } from "./ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent";
+
+const testName = "Vision Data Explorer";
+
+export function describeVisionDataExplorer(
+  datasetShape: IModelAssessmentData,
+  name?: keyof typeof modelAssessmentDatasets,
+  isNotebookTest = true
+): void {
+  describe(testName, () => {
+    if (isNotebookTest) {
+      before(() => {
+        visit(name);
+      });
+    } else {
+      before(() => {
+        cy.visit(`#/modelAssessmentVision/${name}/light/english/Version-2`);
+      });
+    }
+
+    if (datasetShape.visionDataExplorerData?.hasVisionDataExplorerComponent) {
+      it("should should Flyout view components when selected", () => {
+        ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent(
+          datasetShape
+        );
+      });
+    } else {
+      it("should not have 'VisionDataExplorer' component", () => {
+        cy.get(Locators.VisionDataExplorer).should("not.exist");
+      });
+    }
+  });
+}

libs/e2e/src/lib/describer/modelAssessment/visionDataExplorer/ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent.ts

@@ -0,0 +1,58 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { expect } from "chai";
+
+import { Locators } from "../Constants";
+import { IModelAssessmentData } from "../IModelAssessmentData";
+
+export function ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent(
+  datasetShape: IModelAssessmentData
+): void {
+  cy.get(Locators.VisionDataExplorerImageExplorerViewButton).click();
+
+  cy.wait(10000);
+
+  cy.get(Locators.VisionDataExplorerImageExplorerViewSuccessImage)
+    .should("be.visible")
+    .and(($image) => {
+      // verifies the image is loaded
+      expect(($image[0] as HTMLImageElement).naturalWidth).to.be.greaterThan(0);
+      expect(($image[0] as HTMLImageElement).naturalHeight).to.be.greaterThan(
+        0
+      );
+    });
+  // before
+  cy.get(Locators.VisionDataExplorerImageExplorerViewSuccessImage).click();
+
+  cy.get(Locators.VisionDataExplorerFlyoutPredictionTitle).should("exist");
+
+  cy.get(Locators.VisionDataExplorerFlyoutPredictionLabel)
+    .invoke("text")
+    .then((predLabel) => {
+      cy.get(Locators.VisionDataExplorerFlyoutGroundTruthLabel)
+        .invoke("text")
+        .should((gtLabel) => {
+          expect(predLabel.split(": ")[1]).to.equal(gtLabel.split(": ")[1]);
+        });
+    });
+
+  cy.get(Locators.VisionDataExplorerFlyoutImage)
+    .should("be.visible")
+    .and(($image) => {
+      // verifies the image is loaded
+      expect(($image[0] as HTMLImageElement).naturalWidth).to.be.greaterThan(0);
+      expect(($image[0] as HTMLImageElement).naturalHeight).to.be.greaterThan(
+        0
+      );
+    });
+
+  if (datasetShape.isObjectDetection) {
+    cy.get(Locators.VisionDataExplorerFlyoutObjectSelection).should("exist");
+  } else {
+    cy.get(Locators.VisionDataExplorerFlyoutExplanationImage).should("exist");
+  }
+  cy.get(Locators.VisionDataExplorerFlyoutMetadata).should("exist");
+
+  cy.get(Locators.VisionDataExplorerFlyoutCloseButton).click();
+}

libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/DataCharacteristics.tsx

@@ -269,9 +269,7 @@ export class DataCharacteristics extends React.Component<
     this.props.selectItem(item);
   };
 
-  private getPageHeight = (): number => {
-    return this.rowHeight;
-  };
+  private getPageHeight = (): number => this.rowHeight;
 
   private getItemCountForPageWrapper = (
     index: number

libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/Flyout.tsx

@@ -149,7 +149,7 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                           }
                         />
                       </Stack.Item>
-                      <Stack.Item>
+                      <Stack.Item id="flyoutPredictionTitle">
                         {predictedY !== trueY ? (
                           <Text
                             variant="large"
@@ -179,13 +179,13 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                         {item?.index}
                       </Text>
                     </Stack.Item>
-                    <Stack.Item>
+                    <Stack.Item id="flyoutPredLabel">
                       <Text variant="large">
                         {localization.InterpretVision.Dashboard.predictedY}
                         {predictedY}
                       </Text>
                     </Stack.Item>
-                    <Stack.Item>
+                    <Stack.Item id="flyoutGtLabel">
                       <Text variant="large">
                         {localization.InterpretVision.Dashboard.trueY}
                         {trueY}
@@ -195,6 +195,7 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                 </Stack.Item>
                 <Stack.Item className={classNames.imageContainer}>
                   <Image
+                    id={`flyoutImage_${item?.index}`}
                     src={`data:image/jpg;base64,${item?.image}`}
                     className={classNames.image}
                     imageFit={ImageFit.contain}
@@ -217,6 +218,7 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
               {!this.props.loadingExplanation[0][index] ? (
                 <Stack.Item>
                   <Image
+                    id={`flyoutExplanationImage_${item?.index}`}
                     src={`data:image/jpg;base64,${this.props.explanations
                       .get(0)
                       ?.get(index)}`}
@@ -244,7 +246,10 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                   {localization.InterpretVision.Dashboard.panelInformation}
                 </Text>
               </Stack.Item>
-              <Stack.Item className={classNames.featureListContainer}>
+              <Stack.Item
+                id="flyoutMetadata"
+                className={classNames.featureListContainer}
+              >
                 <List
                   items={this.state.metadata}
                   onRenderCell={this.onRenderCell}

libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx

@@ -128,7 +128,7 @@ export class FlyoutObjectDetection extends React.Component<
                             }
                           />
                         </FluentUI.Stack.Item>
-                        <FluentUI.Stack.Item>
+                        <FluentUI.Stack.Item id="flyoutPredictionTitle">
                           {predictedY !== trueY ? (
                             <FluentUI.Text
                               variant="large"
@@ -158,13 +158,13 @@ export class FlyoutObjectDetection extends React.Component<
                           {item?.index}
                         </FluentUI.Text>
                       </FluentUI.Stack.Item>
-                      <FluentUI.Stack.Item>
+                      <FluentUI.Stack.Item id="flyoutPredLabel">
                         <FluentUI.Text variant="large">
                           {localization.InterpretVision.Dashboard.predictedY}
                           {predictedY}
                         </FluentUI.Text>
                       </FluentUI.Stack.Item>
-                      <FluentUI.Stack.Item>
+                      <FluentUI.Stack.Item id="flyoutGtLabel">
                         <FluentUI.Text variant="large">
                           {localization.InterpretVision.Dashboard.trueY}
                           {trueY}
@@ -194,6 +194,7 @@ export class FlyoutObjectDetection extends React.Component<
                   </FluentUI.Text>
                 </FluentUI.Stack.Item>
                 <FluentUI.Stack.Item
+                  id="flyoutMetadata"
                   className={classNames.featureListContainer}
                 >
                   <FluentUI.List

libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/ImageList.tsx

@@ -129,6 +129,7 @@ export class ImageList extends React.Component<
             <Image
               {...imageProps}
               alt={alt}
+              id={`visionImage_${item?.index}`}
               src={`data:image/jpg;base64,${item?.image}`}
               onClick={this.callbackWrapper(item)}
               width={this.props.imageDim}

libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboard.tsx

@@ -65,6 +65,7 @@ export class VisionExplanationDashboard extends React.Component<
       <Stack
         horizontal={false}
         grow
+        id="VisionDataExplorer"
         tokens={{ childrenGap: "l1", padding: "m 40px" }}
       >
         <Stack.Item>
@@ -111,6 +112,7 @@ export class VisionExplanationDashboard extends React.Component<
       <Stack
         horizontal={false}
         grow
+        id="VisionDataExplorer"
         tokens={{ childrenGap: "l1", padding: "m 40px" }}
       >
         <VisionExplanationDashboardCommon

notebooks/test_notebooks.py

@@ -270,8 +270,10 @@ def test_responsibleaidashboard_fridge_image_classification_model_debugging():
 @pytest.mark.vision_notebooks
 def test_responsibleaidashboard_fridge_multilabel_ic_model_debugging():
     nb_path = RESPONSIBLEAIDASHBOARD
-    nb_name = "responsibleaidashboard-fridge-multilabel-" + \
-              "image-classification-model-debugging"
+    nb_name = (
+        "responsibleaidashboard-fridge-multilabel-"
+        "image-classification-model-debugging"
+    )
 
     test_values = {}
     assay_one_notebook(nb_path, nb_name, test_values)

package.json

@@ -44,6 +44,7 @@
   "dependencies": {
     "@fluentui/react": "8.58.0",
     "canvas": "^2.11.2",
+    "chai": "^4.3.5",
     "core-js": "^3.6.5",
     "d3-array": "^2.8.0",
     "d3-color": "^2.0.0",