From aed597b70302dfff302a9b09d7cb13c4a0be181a Mon Sep 17 00:00:00 2001 From: Alan O'Callaghan Date: Mon, 10 Feb 2025 18:36:58 +0000 Subject: [PATCH 1/2] Revert "Faster Mat to NDArray channels-first conversion" This reverts commit f2e5ab06895baaa7fe3f2ba926010f633fe52511. --- src/main/java/qupath/ext/djl/DjlTools.java | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/main/java/qupath/ext/djl/DjlTools.java b/src/main/java/qupath/ext/djl/DjlTools.java index 1cdb6bf..677a1f2 100644 --- a/src/main/java/qupath/ext/djl/DjlTools.java +++ b/src/main/java/qupath/ext/djl/DjlTools.java @@ -1,5 +1,5 @@ /*- - * Copyright 2022-2024 QuPath developers, University of Edinburgh + * Copyright 2022-2025 QuPath developers, University of Edinburgh * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ import java.util.Set; import ai.djl.Device; +import org.bytedeco.javacpp.Loader; import org.bytedeco.javacpp.PointerScope; import org.bytedeco.javacpp.indexer.BooleanIndexer; import org.bytedeco.javacpp.indexer.ByteIndexer; @@ -40,7 +41,6 @@ import org.bytedeco.javacpp.indexer.UByteIndexer; import org.bytedeco.javacpp.indexer.UShortIndexer; import org.bytedeco.opencv.global.opencv_core; -import org.bytedeco.opencv.global.opencv_dnn; import org.bytedeco.opencv.opencv_core.Mat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -369,16 +369,6 @@ public static Device getOverrideDevice(String engineName) { return defaultDevices.getOrDefault(engineName, null); } -// static ZooModel loadModelCV(URI uri, String ndLayout) throws ModelNotFoundException, MalformedModelException, IOException { -// var criteria = Criteria.builder() -// .setTypes(Mat.class, Mat.class) -// .optModelUrls(uri.toString()) -// .optProgress(new ProgressBar()) -// .optTranslator(new MatTranslator(ndLayout, ndLayout)) -// .build(); -// return ModelZoo.loadModel(criteria); -// } - static Mat predict(Model model, Mat mat) throws TranslateException { try (var predictor = model.newPredictor(new MatTranslator("CHW", "CHW"))) { @@ -389,6 +379,7 @@ static Mat predict(Model model, Mat mat) throws TranslateException { /** * Convert an Opencv {@link Mat} to a Deep Java Library {@link NDArray}. + * Note that this ass * @param manager an {@link NDManager}, required to create the NDArray * @param mat the mat to convert * @param ndLayout a layout string for the NDArray, e.g. "CHW"; currently, HW must appear together (in that order) @@ -407,15 +398,10 @@ public static NDArray matToNDArray(NDManager manager, Mat mat, String ndLayout) // TODO: Check what this order is!!! NDArray array = null; if (indC > indHW || shape.get(indC) == 1) { - // Channels-last, or single-channel var buffer = mat.createBuffer(); array = manager.create(buffer, shape, dataType); - } else if ("NCHW".equals(ndLayout) || "CHW".equals(ndLayout)) { - // Channels-first - an OpenCV blob is defined to have the order NCHW - array = manager.create(opencv_dnn.blobFromImage(mat).createBuffer(), shape, dataType); } else { - // Really awkward strategy to handle channels in an uncommon place (shouldn't actually occur?) - var shapeDims = shape.getShape().clone(); + var shapeDims = shape.getShape(); shapeDims[indC] = 1; var shapeChannel = new Shape(shapeDims, shape.getLayout()); From a0829a50c84023197ca43ee163662bb893d7db37 Mon Sep 17 00:00:00 2001 From: Alan O'Callaghan Date: Tue, 11 Feb 2025 19:18:51 +0000 Subject: [PATCH 2/2] Retain faster code for edge case --- src/main/java/qupath/ext/djl/DjlTools.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/qupath/ext/djl/DjlTools.java b/src/main/java/qupath/ext/djl/DjlTools.java index 677a1f2..9998c5f 100644 --- a/src/main/java/qupath/ext/djl/DjlTools.java +++ b/src/main/java/qupath/ext/djl/DjlTools.java @@ -41,6 +41,7 @@ import org.bytedeco.javacpp.indexer.UByteIndexer; import org.bytedeco.javacpp.indexer.UShortIndexer; import org.bytedeco.opencv.global.opencv_core; +import org.bytedeco.opencv.global.opencv_dnn; import org.bytedeco.opencv.opencv_core.Mat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -394,14 +395,19 @@ public static NDArray matToNDArray(NDManager manager, Mat mat, String ndLayout) int indHW = ndLayout.indexOf("HW"); if (indHW < 0) throw new IllegalArgumentException("Expected layout contains HW, but provided layout is " + ndLayout); + long nChannels = shape.get(indC); // Copy all at once is using the same storage order as OpenCV // TODO: Check what this order is!!! NDArray array = null; if (indC > indHW || shape.get(indC) == 1) { var buffer = mat.createBuffer(); array = manager.create(buffer, shape, dataType); + } else if (("NCHW".equals(ndLayout) || "CHW".equals(ndLayout)) && (nChannels == 3L || nChannels == 4L)) { + // Channels-first - an OpenCV blob is defined to have the order NCHW, but an Image can only have 1, 3 or 4 channels + array = manager.create(opencv_dnn.blobFromImage(mat).createBuffer(), shape, dataType); } else { - var shapeDims = shape.getShape(); + // Really awkward strategy to handle channels in an uncommon place (shouldn't actually occur?) + var shapeDims = shape.getShape().clone(); shapeDims[indC] = 1; var shapeChannel = new Shape(shapeDims, shape.getLayout());