Hack in aruco-nano

Author: mcm001

photon-core/build.gradle

@@ -13,6 +13,10 @@ dependencies {
     implementation 'org.zeroturnaround:zt-zip:1.14'
 
     implementation "org.xerial:sqlite-jdbc:3.41.0.0"
+
+    def arucoVer = "dev-v2024.0.1-11-g5190e06"
+    implementation "org.photonvision:photonmiscjni-jni:$arucoVer:$jniPlatform"
+    implementation "org.photonvision:photonmiscjni-java:$arucoVer"
 }
 
 task writeCurrentVersion {

photon-core/src/main/java/org/photonvision/common/hardware/Platform.java

@@ -17,6 +17,7 @@
 
 package org.photonvision.common.hardware;
 
+import com.jogamp.common.os.Platform.OSType;
 import edu.wpi.first.util.RuntimeDetector;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -27,23 +28,32 @@
 @SuppressWarnings("unused")
 public enum Platform {
     // WPILib Supported (JNI)
-    WINDOWS_64("Windows x64", false, OSType.WINDOWS, true),
-    LINUX_32("Linux x86", false, OSType.LINUX, true),
-    LINUX_64("Linux x64", false, OSType.LINUX, true),
+    WINDOWS_64("Windows x64", "win/x64", false, OSType.WINDOWS, true),
+    LINUX_32("Linux x86", "linux/x64", false, OSType.LINUX, true),
+    LINUX_64("Linux x64", "linux/x86-64", false, OSType.LINUX, true),
     LINUX_RASPBIAN32(
-            "Linux Raspbian 32-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 32-bit image
+            "Linux Raspbian 32-bit",
+            "linuxarm32",
+            true,
+            OSType.LINUX,
+            true), // Raspberry Pi 3/4 with a 32-bit image
     LINUX_RASPBIAN64(
-            "Linux Raspbian 64-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 64-bit image
-    LINUX_AARCH64("Linux AARCH64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
+            "Linux Raspbian 64-bit",
+            "linuxarm64",
+            true,
+            OSType.LINUX,
+            true), // Raspberry Pi 3/4 with a 64-bit image
+    LINUX_AARCH64(
+            "Linux AARCH64", "linuxarm64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
 
     // PhotonVision Supported (Manual build/install)
-    LINUX_ARM32("Linux ARM32", false, OSType.LINUX, true), // ODROID XU4, C1+
-    LINUX_ARM64("Linux ARM64", false, OSType.LINUX, true), // ODROID C2, N2
+    LINUX_ARM32("Linux ARM32", "linuxarm32", false, OSType.LINUX, true), // ODROID XU4, C1+
+    LINUX_ARM64("Linux ARM64", "linuxarm64", false, OSType.LINUX, true), // ODROID C2, N2
 
     // Completely unsupported
-    WINDOWS_32("Windows x86", false, OSType.WINDOWS, false),
-    MACOS("Mac OS", false, OSType.MACOS, false),
-    UNKNOWN("Unsupported Platform", false, OSType.UNKNOWN, false);
+    WINDOWS_32("Windows x86", "windowsx64", false, OSType.WINDOWS, false),
+    MACOS("Mac OS", "osxuniversal", false, OSType.MACOS, false),
+    UNKNOWN("Unsupported Platform", "", false, OSType.UNKNOWN, false);
 
     private enum OSType {
         WINDOWS,
@@ -54,6 +64,7 @@ private enum OSType {
 
     private static final ShellExec shell = new ShellExec(true, false);
     public final String description;
+    public final String nativeLibraryFolderName;
     public final boolean isPi;
     public final OSType osType;
     public final boolean isSupported;
@@ -62,11 +73,17 @@ private enum OSType {
     private static final Platform currentPlatform = getCurrentPlatform();
     private static final boolean isRoot = checkForRoot();
 
-    Platform(String description, boolean isPi, OSType osType, boolean isSupported) {
+    Platform(
+            String description,
+            String nativeLibFolderName,
+            boolean isPi,
+            OSType osType,
+            boolean isSupported) {
         this.description = description;
         this.isPi = isPi;
         this.osType = osType;
         this.isSupported = isSupported;
+        this.nativeLibraryFolderName = nativeLibFolderName;
     }
 
     //////////////////////////////////////////////////////
@@ -89,6 +106,10 @@ public static String getPlatformName() {
         }
     }
 
+    public static final String getNativeLibraryFolderName() {
+        return currentPlatform.nativeLibraryFolderName;
+    }
+
     public static boolean isRoot() {
         return isRoot;
     }
@@ -212,4 +233,9 @@ private static boolean fileHasText(String filename, String text) {
             return false;
         }
     }
+
+    public static boolean isWindows() {
+        var p = getCurrentPlatform();
+        return (p == WINDOWS_32 || p == WINDOWS_64);
+    }
 }

photon-core/src/main/java/org/photonvision/jni/ArucoNanoDetector.java

@@ -0,0 +1,31 @@
+package org.photonvision.jni;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+
+import org.photonvision.ArucoNanoV5Detector;
+import org.photonvision.ArucoNanoV5Detector.DetectionResult;
+import org.photonvision.common.util.TestUtils;
+import org.photonvision.vision.aruco.ArucoDetectionResult;
+import org.photonvision.vision.opencv.CVMat;
+
+public class ArucoNanoDetector extends PhotonJniCommon {
+    public static synchronized void forceLoad() throws IOException {
+        forceLoad(ArucoNanoDetector.class, List.of("photonmiscjnijni"));
+    }
+    
+    public static List<ArucoDetectionResult> detect(CVMat in) {
+        DetectionResult[] ret = ArucoNanoV5Detector.detect(in.getMat().getNativeObjAddr(), 0);
+
+        return List.of(ret).stream().map(it->new ArucoDetectionResult(
+            it.xCorners, it.yCorners, it.id
+        )).collect(Collectors.toList());
+    }
+
+    public static void main(String[] args) throws IOException {
+        TestUtils.loadLibraries();
+        forceLoad();
+    }
+}

photon-core/src/main/java/org/photonvision/jni/PhotonJniCommon.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.jni;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+import org.photonvision.common.hardware.Platform;
+import org.photonvision.common.logging.LogGroup;
+import org.photonvision.common.logging.Logger;
+
+public abstract class PhotonJniCommon {
+    static boolean libraryLoaded = false;
+    protected static Logger logger = null;
+
+    protected static synchronized void forceLoad(Class<?> clazz, List<String> libraries)
+            throws IOException {
+        if (libraryLoaded) return;
+        if (logger == null) logger = new Logger(clazz, LogGroup.Camera);
+
+        for (var libraryName : libraries) {
+            try {
+                // We always extract the shared object (we could hash each so, but that's a lot of work)
+                var arch_name = Platform.getNativeLibraryFolderName();
+                var nativeLibName = System.mapLibraryName(libraryName);
+                var in = clazz.getResourceAsStream("/" + arch_name + "/" + nativeLibName);
+
+                if (in == null) {
+                    libraryLoaded = false;
+                    return;
+                }
+
+                // It's important that we don't mangle the names of these files on Windows at least
+                File temp = new File(System.getProperty("java.io.tmpdir"), nativeLibName);
+                FileOutputStream fos = new FileOutputStream(temp);
+
+                int read = -1;
+                byte[] buffer = new byte[1024];
+                while ((read = in.read(buffer)) != -1) {
+                    fos.write(buffer, 0, read);
+                }
+                fos.close();
+                in.close();
+
+                System.load(temp.getAbsolutePath());
+
+                logger.info("Successfully loaded shared object " + temp.getName());
+
+            } catch (UnsatisfiedLinkError e) {
+                logger.error("Couldn't load shared object " + libraryName, e);
+                e.printStackTrace();
+                // logger.error(System.getProperty("java.library.path"));
+                break;
+            }
+        }
+        libraryLoaded = true;
+    }
+
+    protected static synchronized void forceLoad(Class<?> clazz, String libraryName)
+            throws IOException {
+        forceLoad(clazz, List.of(libraryName));
+    }
+
+    public static boolean isWorking() {
+        return libraryLoaded;
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipe/impl/ArucoDetectionPipe.java

@@ -26,6 +26,7 @@
 import org.opencv.core.TermCriteria;
 import org.opencv.imgproc.Imgproc;
 import org.opencv.objdetect.Objdetect;
+import org.photonvision.jni.ArucoNanoDetector;
 import org.photonvision.vision.aruco.ArucoDetectionResult;
 import org.photonvision.vision.aruco.PhotonArucoDetector;
 import org.photonvision.vision.opencv.CVMat;
@@ -43,8 +44,13 @@ public class ArucoDetectionPipe
 
     @Override
     protected List<ArucoDetectionResult> process(CVMat in) {
+        if (in.getMat().empty()) return List.of();
+
         var imgMat = in.getMat();
-        var detections = photonDetector.detect(imgMat);
+
+        // var detections = photonDetector.detect(imgMat);
+        var detections = ArucoNanoDetector.detect(in);
+
         // manually do corner refinement ourselves
         if (params.useCornerRefinement) {
             for (var detection : detections) {
@@ -86,7 +92,9 @@ protected List<ArucoDetectionResult> process(CVMat in) {
                 }
             }
         }
-        return List.of(detections);
+
+        // return List.of(detections);
+        return (detections);
     }
 
     @Override

photon-server/src/main/java/org/photonvision/Main.java

@@ -37,6 +37,7 @@
 import org.photonvision.common.networking.NetworkManager;
 import org.photonvision.common.util.TestUtils;
 import org.photonvision.common.util.numbers.IntegerCouple;
+import org.photonvision.jni.ArucoNanoDetector;
 import org.photonvision.raspi.LibCameraJNI;
 import org.photonvision.server.Server;
 import org.photonvision.vision.camera.FileVisionSource;
@@ -309,6 +310,13 @@ public static void main(String[] args) {
             logger.error("Failed to load native libraries!", e);
         }
 
+        try {
+            ArucoNanoDetector.forceLoad();
+            logger.info("Native libraries loaded.");
+        } catch (Exception e) {
+            logger.error("Failed to load native libraries!", e);
+        }
+
         try {
             if (Platform.isRaspberryPi()) {
                 LibCameraJNI.forceLoad();