NuClick + Classification model using Consep dataset (#1120)

Signed-off-by: SACHIDANAND ALLE <[email protected]>

Author: SachidanandAlle

monailabel/interfaces/utils/wsi.py

@@ -16,6 +16,7 @@
 from ctypes import cdll
 from math import ceil
 
+import numpy as np
 from monai.utils import optional_import
 
 logger = logging.getLogger(__name__)
@@ -59,6 +60,10 @@ def create_infer_wsi_tasks(request, image):
     infer_tasks = []
     count = 0
     pw, ph = tile_size[0], tile_size[1]
+
+    ignore_small_patches = request.get("ignore_small_patches", False)
+    ignore_non_click_patches = request.get("ignore_non_click_patches", False)
+
     for row in range(rows):
         for col in range(cols):
             tx = col * pw + x
@@ -67,6 +72,22 @@ def create_infer_wsi_tasks(request, image):
             tw = min(pw, x + w - tx)
             th = min(ph, y + h - ty)
 
+            if ignore_small_patches and (tw < pw or th < ph):
+                continue
+
+            if ignore_non_click_patches and (request.get("foreground") or request.get("background")):
+
+                def filter_points(ptype):
+                    pos = request.get(ptype)
+                    pos = (np.array(pos) - (tx, ty)).astype(int).tolist() if pos else []
+                    pos = [p for p in pos if 0 < p[0] < tw and 0 < p[1] < th]
+                    return pos
+
+                fg = filter_points("foreground")
+                bg = filter_points("background")
+                if not fg and not bg:
+                    continue
+
             task = copy.deepcopy(request)
             task.update(
                 {

monailabel/tasks/train/basic_train.py

@@ -173,6 +173,12 @@ def __init__(
         self._labels = [] if labels is None else [labels] if isinstance(labels, str) else labels
         self._disable_tracking = disable_tracking
 
+    def info(self):
+        r = super().info()
+        if self._labels:
+            r["labels"] = self._labels
+        return r
+
     @abstractmethod
     def network(self, context: Context):
         pass
@@ -239,7 +245,12 @@ def train_inferer(self, context: Context):
         return SimpleInferer()
 
     def train_key_metric(self, context: Context):
-        return {self.TRAIN_KEY_METRIC: MeanDice(output_transform=from_engine(["pred", "label"]))}
+        return {
+            self.TRAIN_KEY_METRIC: MeanDice(
+                output_transform=from_engine(["pred", "label"]),
+                include_background=False,
+            )
+        }
 
     def load_path(self, output_dir, pretrained=True):
         load_path = os.path.join(output_dir, self._key_metric_filename)
@@ -297,7 +308,12 @@ def val_handlers(self, context: Context):
         return val_handlers if context.local_rank == 0 else None
 
     def val_key_metric(self, context):
-        return {self.VAL_KEY_METRIC: MeanDice(output_transform=from_engine(["pred", "label"]))}
+        return {
+            self.VAL_KEY_METRIC: MeanDice(
+                output_transform=from_engine(["pred", "label"]),
+                include_background=False,
+            )
+        }
 
     def train_iteration_update(self, context: Context):
         return None

monailabel/utils/others/generic.py

@@ -200,12 +200,16 @@ def device_list():
     return devices
 
 
-def create_dataset_from_path(folder, images="images", labels="labels", img_ext=".jpg", lab_ext=".png"):
-    images = [i for i in os.listdir(os.path.join(folder, images)) if i.endswith(img_ext)]
-    images = sorted(os.path.join(folder, "images", i) for i in images)
+def create_dataset_from_path(folder, image_dir="images", label_dir="labels", img_ext=".jpg", lab_ext=".png"):
+    def _list_files(d, ext):
+        files = [i for i in os.listdir(d) if i.endswith(ext)]
+        return sorted(os.path.join(d, i) for i in files)
 
-    labels = [i for i in os.listdir(os.path.join(folder, labels)) if i.endswith(lab_ext)]
-    labels = sorted(os.path.join(folder, "labels", i) for i in labels)
+    image_dir = os.path.join(folder, image_dir) if image_dir else folder
+    images = _list_files(image_dir, img_ext)
+
+    label_dir = os.path.join(folder, label_dir) if label_dir else folder
+    labels = _list_files(label_dir, img_ext)
 
     for i, l in zip(images, labels):
         if get_basename_no_ext(i) != get_basename_no_ext(l):

plugins/qupath/src/main/java/qupath/lib/extension/monailabel/Extension.java

@@ -45,7 +45,6 @@ public void installExtension(QuPathGUI qupath) {
 
 		var activeLearning = ActionTools.createAction(new NextSample(qupath), "Next Sample/Patch...");
 		activeLearning.setAccelerator(KeyCombination.keyCombination("ctrl+n"));
-		activeLearning.disabledProperty().bind(qupath.imageDataProperty().isNull());
 		MenuTools.addMenuItems(qupath.getMenu("MONAI Label", true), activeLearning);
 
 		MenuTools.addMenuItems(qupath.getMenu("MONAI Label", true), ActionUtils.ACTION_SEPARATOR);

plugins/qupath/src/main/java/qupath/lib/extension/monailabel/MonaiLabelClient.java

@@ -124,11 +124,13 @@ public static class LabelInfo {
 	public static class NextSampleInfo {
 		public String id;
 		public int[] bbox = { 0, 0, 0, 0 };
+		public String path;
 	}
 
 	public static class InferParams {
 		public List<List<Integer>> foreground = new ArrayList<>();
 		public List<List<Integer>> background = new ArrayList<>();
+		public int max_workers = 1;
 
 		public void addClicks(ArrayList<Point2> clicks, boolean f) {
 			List<List<Integer>> t = f ? foreground : background;

plugins/qupath/src/main/java/qupath/lib/extension/monailabel/Settings.java

@@ -14,7 +14,7 @@
 package qupath.lib.extension.monailabel;
 
 import javafx.application.Platform;
-import javafx.beans.property.BooleanProperty;
+import javafx.beans.property.IntegerProperty;
 import javafx.beans.property.StringProperty;
 import qupath.lib.gui.QuPathGUI;
 import qupath.lib.gui.prefs.PathPrefs;
@@ -27,10 +27,10 @@ public static StringProperty serverURLProperty() {
 		return serverURL;
 	}
 
-	private static BooleanProperty wsi = PathPrefs.createPersistentPreference("wsi", Boolean.FALSE);
+	private static IntegerProperty maxWorkers = PathPrefs.createPersistentPreference("max_workers", 1);
 
-	public static BooleanProperty wsiProperty() {
-		return wsi;
+	public static IntegerProperty maxWorkersProperty() {
+		return maxWorkers;
 	}
 
 	void addProperties(QuPathGUI qupath) {
@@ -41,8 +41,8 @@ void addProperties(QuPathGUI qupath) {
 
 		qupath.getPreferencePane().addPropertyPreference(Settings.serverURLProperty(), String.class, "Server URL",
 				"MONAI Label", "Set MONAI Label Server URL (default: http://127.0.0.1:8000)");
-		qupath.getPreferencePane().addPropertyPreference(Settings.wsiProperty(), Boolean.class, "Enable WSI",
-				"MONAI Label", "Allow WSI Inference when ROI is not selected");
+		qupath.getPreferencePane().addPropertyPreference(Settings.maxWorkersProperty(), Integer.class, "Max Workers",
+				"MONAI Label", "Max Workers (WSI Inference)");
 
 	}
 }

plugins/qupath/src/main/java/qupath/lib/extension/monailabel/commands/NextSample.java

@@ -13,6 +13,7 @@
 
 package qupath.lib.extension.monailabel.commands;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -48,7 +49,7 @@ public void run() {
 		try {
 			var viewer = qupath.getViewer();
 			var imageData = viewer.getImageData();
-			String image = Utils.getNameWithoutExtension(imageData.getServerPath());
+			String image = imageData != null ? Utils.getNameWithoutExtension(imageData.getServerPath()) : "";
 
 			ResponseInfo info = MonaiLabelClient.info();
 			List<String> names = new ArrayList<String>();
@@ -60,19 +61,27 @@ public void run() {
 				selectedStrategy = names.isEmpty() ? "" : names.get(0);
 			}
 
+			boolean imageLoaded = qupath.imageDataProperty() == null || qupath.imageDataProperty().isNull().get() ? false : true;
+			boolean nextPatch = imageLoaded;
+			int[] patchSize = {1024, 1024};
+			int[] imageSize = {0, 0};
+
 			ParameterList list = new ParameterList();
 			list.addChoiceParameter("Strategy", "Active Learning Strategy", selectedStrategy, names);
-			list.addBooleanParameter("NextPatch", "Next Patch (from current Image)", true);
-			list.addStringParameter("PatchSize", "PatchSize", Arrays.toString(selectedPatchSize));
+			if (nextPatch) {
+				list.addBooleanParameter("NextPatch", "Next Patch (from current Image)", nextPatch);
+				list.addStringParameter("PatchSize", "PatchSize", Arrays.toString(selectedPatchSize));
+			}
 
 			if (Dialogs.showParameterDialog("MONAILabel", list)) {
 				String strategy = (String) list.getChoiceParameterValue("Strategy");
-				boolean nextPatch = list.getBooleanParameterValue("NextPatch").booleanValue();
-				int[] patchSize = Utils.parseStringArray(list.getStringParameterValue("PatchSize"));
+				if (nextPatch) {
+					nextPatch = list.getBooleanParameterValue("NextPatch").booleanValue();
+					patchSize = Utils.parseStringArray(list.getStringParameterValue("PatchSize"));
+					imageSize = new int[] { imageData.getServer().getWidth(), imageData.getServer().getHeight() };
 
-				var server = imageData.getServer();
-				int[] imageSize = new int[] { server.getWidth(), server.getHeight() };
-				logger.info(String.join(",", imageData.getProperties().keySet()));
+					logger.info(String.join(",", imageData.getProperties().keySet()));
+				}
 
 				selectedStrategy = strategy;
 				selectedPatchSize = patchSize;
@@ -99,6 +108,17 @@ public void run() {
 					var obj = PathObjects.createAnnotationObject(roi);
 					imageData.getHierarchy().addPathObject(obj);
 					imageData.getHierarchy().getSelectionModel().setSelectedObject(obj);
+				} else {
+					String message = "This will close the current image without saving.\nAre you sure to continue?";
+					if (!imageLoaded || Dialogs.showConfirmDialog("MONAILabel", message)) {
+						File f = new File(sample.path);
+						if (f.isFile() && f.exists()) {
+							qupath.openImage(f.getAbsolutePath(), false, false);
+						} else {
+							// TODO:: Download and Open image (wsi) from Remote
+							// String f = "C:\\Dataset\\Pathology\\TCGA-02-0010-01Z-00-DX4.07de2e55-a8fe-40ee-9e98-bcb78050b9f7.svs";
+						}
+					}
 				}
 			}
 

plugins/qupath/src/main/java/qupath/lib/extension/monailabel/commands/RunInference.java

@@ -35,6 +35,7 @@
 import qupath.lib.extension.monailabel.MonaiLabelClient;
 import qupath.lib.extension.monailabel.MonaiLabelClient.RequestInfer;
 import qupath.lib.extension.monailabel.MonaiLabelClient.ResponseInfo;
+import qupath.lib.extension.monailabel.Settings;
 import qupath.lib.extension.monailabel.Utils;
 import qupath.lib.geom.Point2;
 import qupath.lib.gui.QuPathGUI;
@@ -50,6 +51,7 @@
 import qupath.lib.regions.RegionRequest;
 import qupath.lib.roi.PointsROI;
 import qupath.lib.roi.ROIs;
+import qupath.lib.roi.RectangleROI;
 import qupath.lib.roi.interfaces.ROI;
 import qupath.lib.scripting.QP;
 
@@ -72,6 +74,22 @@ public void run() {
 			var imageData = viewer.getImageData();
 			var selected = imageData.getHierarchy().getSelectionModel().getSelectedObject();
 			var roi = selected != null ? selected.getROI() : null;
+
+			// Select first RectangleROI if not selected explicitly
+			if (roi == null || !(roi instanceof RectangleROI)) {
+				List<PathObject> objs = imageData.getHierarchy().getFlattenedObjectList(null);
+				for (int i = 0; i < objs.size(); i++) {
+					var obj = objs.get(i);
+					ROI r = obj.getROI();
+					if (r instanceof RectangleROI) {
+						roi = r;
+						Dialogs.showWarningNotification("MONALabel", "ROI is NOT explicitly selected; using first Rectangle ROI from Hierarchy");
+						imageData.getHierarchy().getSelectionModel().setSelectedObject(obj);
+						break;
+					}
+				}
+			}
+
 			int[] bbox = Utils.getBBOX(roi);
 			int tileSize = selectedTileSize;
 			if (bbox[2] == 0 && bbox[3] == 0 && selectedBBox != null) {
@@ -204,6 +222,7 @@ private void runInference(String model, ResponseInfo info, int[] bbox, int tileS
 			}
 			req.params.addClicks(fg, true);
 			req.params.addClicks(bg, false);
+			req.params.max_workers = Settings.maxWorkersProperty().intValue();
 
 
 			Document dom = MonaiLabelClient.infer(model, image, imageFile, sessionId, req);

sample-apps/pathology/lib/__init__.py

@@ -8,3 +8,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import ctypes.util
+import platform
+from ctypes import cdll
+
+# For windows (preload openslide dll using file_library) https://github.com/openslide/openslide-python/pull/151
+if platform.system() == "Windows":
+    cdll.LoadLibrary(str(ctypes.util.find_library("libopenslide-0.dll")))

sample-apps/pathology/lib/configs/classification_nuclei.py

@@ -32,11 +32,11 @@ def init(self, name: str, model_dir: str, conf: Dict[str, str], planner: Any, **
 
         # Labels
         self.labels = {
-            "Neoplastic cells": 0,
-            "Inflammatory": 1,
-            "Connective/Soft tissue cells": 2,
-            "Dead Cells": 3,
-            "Epithelial": 4,
+            "Neoplastic cells": 1,
+            "Inflammatory": 2,
+            "Connective/Soft tissue cells": 3,
+            "Dead Cells": 4,
+            "Epithelial": 5,
         }
         self.label_colors = {
             "Neoplastic cells": (255, 0, 0),
@@ -46,16 +46,31 @@ def init(self, name: str, model_dir: str, conf: Dict[str, str], planner: Any, **
             "Epithelial": (0, 0, 255),
         }
 
+        consep = strtobool(self.conf.get("consep", "false"))
+        if consep:
+            self.labels = {
+                "Other": 1,
+                "Inflammatory": 2,
+                "Epithelial": 3,
+                "Spindle-Shaped": 4,
+            }
+            self.label_colors = {
+                "Other": (255, 0, 0),
+                "Inflammatory": (255, 255, 0),
+                "Epithelial": (0, 0, 255),
+                "Spindle-Shaped": (0, 255, 0),
+            }
+
         # Model Files
         self.path = [
-            os.path.join(self.model_dir, f"pretrained_{name}.pt"),  # pretrained
-            os.path.join(self.model_dir, f"{name}.pt"),  # published
+            os.path.join(self.model_dir, f"pretrained_{name}{'_consep' if consep else ''}.pt"),  # pretrained
+            os.path.join(self.model_dir, f"{name}{'_consep' if consep else ''}.pt"),  # published
         ]
 
         # Download PreTrained Model
         if strtobool(self.conf.get("use_pretrained_model", "true")):
             url = f"{self.conf.get('pretrained_path', self.PRE_TRAINED_PATH)}"
-            url = f"{url}/pathology_classification_densenet121_nuclei.pt"
+            url = f"{url}/pathology_classification_densenet121_nuclei{'_consep' if consep else ''}.pt"
             download_file(url, self.path[0])
 
         # Network