Skip to content

Commit 21c9f52

Browse files
SachidanandAlleSachidanandAlle
authored
support custom interactor for nuclick without ROI (#1125)
Signed-off-by: SACHIDANAND ALLE <[email protected]> Signed-off-by: SACHIDANAND ALLE <[email protected]>
1 parent ebd6c13 commit 21c9f52

File tree

5 files changed

+178
-18
lines changed

5 files changed

+178
-18
lines changed

‎monailabel/endpoints/datastore.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from monailabel.interfaces.app import MONAILabelApp
2727
from monailabel.interfaces.datastore import Datastore, DefaultLabelTag
2828
from monailabel.interfaces.utils.app import app_instance
29-
from monailabel.utils.others.generic import get_mime_type, remove_file
29+
from monailabel.utils.others.generic import file_checksum, get_mime_type, remove_file
3030

3131
logger = logging.getLogger(__name__)
3232
train_tasks: List = []
@@ -129,13 +129,19 @@ def remove_label(id: str, tag: str, user: Optional[str] = None):
129129
return {}
130130

131131

132-
def download_image(image: str, check_only=False):
132+
def download_image(image: str, check_only=False, check_sum=None):
133133
instance: MONAILabelApp = app_instance()
134134
image = instance.datastore().get_image_uri(image)
135135
if not os.path.isfile(image):
136136
raise HTTPException(status_code=404, detail="Image NOT Found")
137137

138138
if check_only:
139+
if check_sum:
140+
fields = check_sum.split(":")
141+
algo = "SHA256" if len(fields) == 1 else fields[0]
142+
digest = check_sum.lstrip(algo + ":") if len(fields) > 1 else check_sum
143+
if digest != file_checksum(image, algo=algo):
144+
raise HTTPException(status_code=404, detail="Image NOT Found (checksum failed)")
139145
return {}
140146
return FileResponse(image, media_type=get_mime_type(image), filename=os.path.basename(image))
141147

@@ -213,8 +219,8 @@ async def api_remove_image(id: str, user: User = Depends(get_admin_user)):
213219

214220

215221
@router.head("/image", summary="Check If Image Exists")
216-
async def api_check_image(image: str, user: User = Depends(get_basic_user)):
217-
return download_image(image, check_only=True)
222+
async def api_check_image(image: str, check_sum: Optional[str] = None, user: User = Depends(get_basic_user)):
223+
return download_image(image, check_only=True, check_sum=check_sum)
218224

219225

220226
@router.get("/image", summary="Download Image")

‎plugins/qupath/src/main/java/qupath/lib/extension/monailabel/Extension.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,27 @@
1515

1616
import java.net.URL;
1717

18+
import org.bytedeco.openblas.global.openblas;
1819
import org.controlsfx.control.action.ActionUtils;
20+
import org.controlsfx.glyphfont.Glyph;
21+
import org.controlsfx.glyphfont.GlyphFontRegistry;
1922
import org.slf4j.Logger;
2023
import org.slf4j.LoggerFactory;
2124

25+
import javafx.application.Platform;
2226
import javafx.geometry.Orientation;
27+
import javafx.scene.Node;
2328
import javafx.scene.control.Button;
2429
import javafx.scene.control.ContextMenu;
2530
import javafx.scene.control.Separator;
2631
import javafx.scene.control.Tooltip;
2732
import javafx.scene.image.Image;
2833
import javafx.scene.image.ImageView;
34+
import javafx.scene.input.KeyCode;
35+
import javafx.scene.input.KeyCodeCombination;
2936
import javafx.scene.input.KeyCombination;
37+
import javafx.scene.paint.Color;
38+
import qupath.lib.common.GeneralTools;
3039
import qupath.lib.common.Version;
3140
import qupath.lib.extension.monailabel.commands.NextSample;
3241
import qupath.lib.extension.monailabel.commands.RunInference;
@@ -35,11 +44,14 @@
3544
import qupath.lib.gui.ActionTools;
3645
import qupath.lib.gui.QuPathGUI;
3746
import qupath.lib.gui.extensions.QuPathExtension;
47+
import qupath.lib.gui.tools.IconFactory;
3848
import qupath.lib.gui.tools.MenuTools;
49+
import qupath.lib.gui.viewer.tools.PathTools;
3950

4051
public class Extension implements QuPathExtension {
4152
final private static Logger logger = LoggerFactory.getLogger(Extension.class);
4253

54+
4355
@Override
4456
public void installExtension(QuPathGUI qupath) {
4557

@@ -85,11 +97,50 @@ public void installExtension(QuPathGUI qupath) {
8597
});
8698

8799
toolbar.getItems().add(btnAnnotation);
100+
101+
installInteractor(qupath);
88102
} catch (Exception e) {
89103
logger.error("Error adding toolbar buttons", e);
90104
}
91105
}
92106

107+
public static Node createIconNode(char code, Color color) {
108+
try {
109+
GlyphFontRegistry.register("icomoon", IconFactory.class.getClassLoader().getResourceAsStream("fonts/icomoon.ttf") , 12);
110+
var font = GlyphFontRegistry.font("FontAwesome");
111+
112+
Glyph g = font.create(code).size(QuPathGUI.TOOLBAR_ICON_SIZE);
113+
g.setIcon(code);
114+
g.color(color);
115+
g.getStyleClass().add("qupath-icon");
116+
return g;
117+
}
118+
catch (Exception e) {
119+
logger.error("Unable to load icon {}", code, e);
120+
return null;
121+
}
122+
}
123+
124+
125+
private void installInteractor(QuPathGUI qupath) {
126+
var t = new Thread(() -> {
127+
if (!GeneralTools.isWindows()) {
128+
openblas.blas_set_num_threads(1);
129+
}
130+
131+
var interactorTool = PathTools.createTool(new InteractorTool(), "Interactor",
132+
Extension.createIconNode('\uf192', Color.DARKCYAN));
133+
134+
logger.info("Installing Interactor Tool");
135+
Platform.runLater(() -> {
136+
qupath.installTool(interactorTool, new KeyCodeCombination(KeyCode.I));
137+
qupath.getToolAction(interactorTool).setLongText("Click to annotate using MONAILabel Interaction models.");
138+
});
139+
});
140+
t.start();
141+
}
142+
143+
93144
@Override
94145
public String getName() {
95146
return "MONAILabel extension";

‎plugins/qupath/src/main/java/qupath/lib/extension/monailabel/InteractorTool.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package qupath.lib.extension.monailabel;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
6+
import org.slf4j.Logger;
7+
import org.slf4j.LoggerFactory;
8+
9+
import javafx.scene.input.MouseEvent;
10+
import qupath.lib.extension.monailabel.MonaiLabelClient.ResponseInfo;
11+
import qupath.lib.extension.monailabel.commands.RunInference;
12+
import qupath.lib.gui.dialogs.Dialogs;
13+
import qupath.lib.gui.viewer.tools.PointsTool;
14+
import qupath.lib.objects.PathObject;
15+
import qupath.lib.objects.PathROIObject;
16+
import qupath.lib.plugins.parameters.ParameterList;
17+
import qupath.lib.roi.PointsROI;
18+
19+
public class InteractorTool extends PointsTool {
20+
private final static Logger logger = LoggerFactory.getLogger(InteractorTool.class);
21+
22+
private static String selectedModel;
23+
private static int selectedPatchSize = 128;
24+
private static ResponseInfo info;
25+
26+
public void mousePressed(MouseEvent e) {
27+
var viewer = getViewer();
28+
if (viewer == null || viewer.getImageData() == null) {
29+
return;
30+
}
31+
32+
logger.info("+++++++ Interaction Tool... mouse pressed...");
33+
super.mousePressed(e);
34+
35+
PathObject currentObjectTemp = viewer.getSelectedObject();
36+
if (!(currentObjectTemp == null || currentObjectTemp instanceof PathROIObject))
37+
return;
38+
39+
PathROIObject currentObject = (PathROIObject) currentObjectTemp;
40+
if (currentObject == null || !(currentObject.getROI() instanceof PointsROI))
41+
return;
42+
43+
PointsROI roi = (PointsROI) currentObject.getROI();
44+
45+
try {
46+
if (info == null) {
47+
info = MonaiLabelClient.info();
48+
List<String> names = new ArrayList<String>();
49+
for (String n : info.models.keySet()) {
50+
if (info.models.get(n).nuclick) {
51+
names.add(n);
52+
}
53+
}
54+
int patchSize = selectedPatchSize;
55+
if (names.size() == 1) {
56+
selectedModel = names.get(0);
57+
}
58+
59+
if (selectedModel == null || selectedModel.isEmpty()) {
60+
ParameterList list = new ParameterList();
61+
list.addChoiceParameter("Model", "Model Name", names.get(0), names);
62+
list.addIntParameter("PatchSize", "PatchSize", patchSize);
63+
64+
if (!Dialogs.showParameterDialog("MONAILabel", list)) {
65+
return;
66+
}
67+
68+
selectedModel = (String) list.getChoiceParameterValue("Model");
69+
selectedPatchSize = list.getIntParameterValue("PatchSize").intValue();
70+
}
71+
}
72+
73+
int w = Math.max((int) roi.getBoundsWidth() + 20, selectedPatchSize);
74+
int h = Math.max((int) roi.getBoundsHeight() + 20, selectedPatchSize);
75+
int x = Math.max(0, (int) roi.getCentroidX() - w / 2);
76+
int y = Math.max(0, (int) roi.getCentroidY() - h / 2);
77+
int[] bbox = { x, y, w, h };
78+
79+
RunInference.runInference(selectedModel, info, bbox, selectedPatchSize, viewer.getImageData());
80+
//currentObject.setROI(ROIs.createPointsROI(viewer.getImagePlane()));
81+
viewer.getHierarchy().getSelectionModel().clearSelection();
82+
} catch (Exception ex) {
83+
ex.printStackTrace();
84+
Dialogs.showErrorMessage("MONAILabel", ex);
85+
}
86+
}
87+
}

‎plugins/qupath/src/main/java/qupath/lib/extension/monailabel/MonaiLabelClient.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ public static boolean imageExists(String image) {
234234
String uri = "/datastore/image?image=" + URLEncoder.encode(image, "UTF-8");
235235
String res = RequestUtils.request("HEAD", uri, null);
236236

237+
// TODO:: Also verify checksum to make sure it's the same file
237238
logger.info("MONAILabel:: (Image Exists) Response => " + res);
238239
return true;
239240
} catch (IOException | InterruptedException e) {

‎plugins/qupath/src/main/java/qupath/lib/extension/monailabel/commands/RunInference.java‎

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import java.awt.Color;
1717
import java.awt.image.BufferedImage;
18+
import java.io.File;
1819
import java.io.IOException;
1920
import java.nio.file.Path;
2021
import java.util.ArrayList;
@@ -83,7 +84,8 @@ public void run() {
8384
ROI r = obj.getROI();
8485
if (r instanceof RectangleROI) {
8586
roi = r;
86-
Dialogs.showWarningNotification("MONALabel", "ROI is NOT explicitly selected; using first Rectangle ROI from Hierarchy");
87+
Dialogs.showWarningNotification("MONALabel",
88+
"ROI is NOT explicitly selected; using first Rectangle ROI from Hierarchy");
8789
imageData.getHierarchy().getSelectionModel().setSelectedObject(obj);
8890
break;
8991
}
@@ -125,8 +127,8 @@ public void run() {
125127
}
126128
}
127129

128-
ArrayList<Point2> getClicks(String name, ImageData<BufferedImage> imageData, ROI monaiLabelROI, int offsetX,
129-
int offsetY) {
130+
public static ArrayList<Point2> getClicks(String name, ImageData<BufferedImage> imageData, ROI monaiLabelROI,
131+
int offsetX, int offsetY) {
130132
List<PathObject> objs = imageData.getHierarchy().getFlattenedObjectList(null);
131133
ArrayList<Point2> clicks = new ArrayList<Point2>();
132134
for (int i = 0; i < objs.size(); i++) {
@@ -149,7 +151,7 @@ ArrayList<Point2> getClicks(String name, ImageData<BufferedImage> imageData, ROI
149151
return clicks;
150152
}
151153

152-
private void runInference(String model, ResponseInfo info, int[] bbox, int tileSize,
154+
public static void runInference(String model, ResponseInfo info, int[] bbox, int tileSize,
153155
ImageData<BufferedImage> imageData)
154156
throws SAXException, IOException, ParserConfigurationException, InterruptedException {
155157
logger.info("MONAILabel:: Running Inference...");
@@ -173,8 +175,11 @@ private void runInference(String model, ResponseInfo info, int[] bbox, int tileS
173175

174176
ROI roi = ROIs.createRectangleROI(bbox[0], bbox[1], bbox[2], bbox[3], null);
175177
String imageFile = imageData.getServerPath();
176-
if (imageFile.startsWith("file:/"))
177-
imageFile = imageFile.replace("file:/", "");
178+
if (imageFile.indexOf("file:/") >= 0)
179+
imageFile = imageFile.substring(imageFile.indexOf("file:/") + "file:/".length());
180+
181+
int pos = imageFile.indexOf("[");
182+
imageFile = imageFile.substring(0, pos > 0 ? pos : imageFile.length());
178183
logger.info("MONAILabel:: Image File: " + imageFile);
179184

180185
String image = Utils.getNameWithoutExtension(imageFile);
@@ -185,18 +190,30 @@ private void runInference(String model, ResponseInfo info, int[] bbox, int tileS
185190
// check if image exists on server
186191
if (!MonaiLabelClient.imageExists(image) && (sessionId == null || sessionId.isEmpty())) {
187192
logger.info("MONAILabel:: Image does not exist on Server.");
193+
188194
image = null;
189195
offsetX = req.location[0];
190196
offsetY = req.location[1];
191197

192198
req.location[0] = req.location[1] = 0;
193199
req.size[0] = req.size[1] = 0;
194200

201+
String im = imageFile.toLowerCase();
202+
if ((im.endsWith(".png") || im.endsWith(".jpg") || im.endsWith(".jpeg"))
203+
&& new File(imageFile).exists()) {
204+
logger.info("Simple Image.. will directly upload the same");
205+
} else {
206+
if (bbox[2] == 0 && bbox[3] == 0) {
207+
Dialogs.showErrorMessage("MONAILabel",
208+
"Can not run WSI Inference on a remote image (Not exists in Datastore)");
209+
return;
210+
}
195211

196-
imagePatch = java.nio.file.Files.createTempFile("patch", ".png");
197-
imageFile = imagePatch.toString();
198-
var requestROI = RegionRequest.createInstance(imageData.getServer().getPath(), 1, roi);
199-
ImageWriterTools.writeImageRegion(imageData.getServer(), requestROI, imageFile);
212+
imagePatch = java.nio.file.Files.createTempFile("patch", ".png");
213+
imageFile = imagePatch.toString();
214+
var requestROI = RegionRequest.createInstance(imageData.getServer().getPath(), 1, roi);
215+
ImageWriterTools.writeImageRegion(imageData.getServer(), requestROI, imageFile);
216+
}
200217
}
201218

202219
ArrayList<Point2> fg = new ArrayList<>();
@@ -215,16 +232,14 @@ private void runInference(String model, ResponseInfo info, int[] bbox, int tileS
215232
return;
216233
}
217234
if (roi.getBoundsHeight() < 128 || roi.getBoundsWidth() < 128) {
218-
Dialogs.showErrorMessage("MONAILabel",
219-
"Min Height/Width of ROI should be more than 128");
235+
Dialogs.showErrorMessage("MONAILabel", "Min Height/Width of ROI should be more than 128");
220236
return;
221237
}
222238
}
223239
req.params.addClicks(fg, true);
224240
req.params.addClicks(bg, false);
225241
req.params.max_workers = Settings.maxWorkersProperty().intValue();
226242

227-
228243
Document dom = MonaiLabelClient.infer(model, image, imageFile, sessionId, req);
229244
NodeList annotation_list = dom.getElementsByTagName("Annotation");
230245
int count = updateAnnotations(labels, annotation_list, roi, imageData, override, offsetX, offsetY);
@@ -237,7 +252,7 @@ private void runInference(String model, ResponseInfo info, int[] bbox, int tileS
237252
}
238253
}
239254

240-
private int updateAnnotations(Set<String> labels, NodeList annotation_list, ROI roi,
255+
public static int updateAnnotations(Set<String> labels, NodeList annotation_list, ROI roi,
241256
ImageData<BufferedImage> imageData, boolean override, int offsetX, int offsetY) {
242257
if (override) {
243258
List<PathObject> objs = imageData.getHierarchy().getFlattenedObjectList(null);

0 commit comments

Comments
 (0)