diff --git a/.styleguide b/.styleguide index 7259f0408f..9f48ba17e1 100644 --- a/.styleguide +++ b/.styleguide @@ -39,3 +39,7 @@ includeOtherLibs { ^units/ ^wpi/ } + +licenseUpdateExclude { + photon-core/src/main/java/org/photonvision/jni/GpuDetectorJNI.java +} diff --git a/photon-client/src/components/dashboard/ConfigOptions.vue b/photon-client/src/components/dashboard/ConfigOptions.vue index e995b293df..f052bc43b4 100644 --- a/photon-client/src/components/dashboard/ConfigOptions.vue +++ b/photon-client/src/components/dashboard/ConfigOptions.vue @@ -25,16 +25,46 @@ interface ConfigOption { } const allTabs = Object.freeze({ - inputTab: { tabName: "Input", component: InputTab }, - thresholdTab: { tabName: "Threshold", component: ThresholdTab }, - contoursTab: { tabName: "Contours", component: ContoursTab }, - apriltagTab: { tabName: "AprilTag", component: AprilTagTab }, - arucoTab: { tabName: "Aruco", component: ArucoTab }, - objectDetectionTab: { tabName: "Object Detection", component: ObjectDetectionTab }, - outputTab: { tabName: "Output", component: OutputTab }, - targetsTab: { tabName: "Targets", component: TargetsTab }, - pnpTab: { tabName: "PnP", component: PnPTab }, - map3dTab: { tabName: "3D", component: Map3DTab } + inputTab: { + tabName: "Input", + component: InputTab + }, + thresholdTab: { + tabName: "Threshold", + component: ThresholdTab + }, + contoursTab: { + tabName: "Contours", + component: ContoursTab + }, + apriltagTab: { + tabName: "AprilTag", + component: AprilTagTab + }, + arucoTab: { + tabName: "Aruco", + component: ArucoTab + }, + objectDetectionTab: { + tabName: "Object Detection", + component: ObjectDetectionTab + }, + outputTab: { + tabName: "Output", + component: OutputTab + }, + targetsTab: { + tabName: "Targets", + component: TargetsTab + }, + pnpTab: { + tabName: "PnP", + component: PnPTab + }, + map3dTab: { + tabName: "3D", + component: Map3DTab + } }); const selectedTabs = ref([0, 0, 0, 0]); diff --git a/photon-client/src/components/dashboard/tabs/AprilTagTab.vue b/photon-client/src/components/dashboard/tabs/AprilTagTab.vue index 249f80607e..9b7e98b0dc 100644 --- a/photon-client/src/components/dashboard/tabs/AprilTagTab.vue +++ b/photon-client/src/components/dashboard/tabs/AprilTagTab.vue @@ -88,5 +88,14 @@ const interactiveCols = computed(() => (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ refineEdges: value }, false) " /> + diff --git a/photon-client/src/types/PipelineTypes.ts b/photon-client/src/types/PipelineTypes.ts index 1b56311941..f334a67462 100644 --- a/photon-client/src/types/PipelineTypes.ts +++ b/photon-client/src/types/PipelineTypes.ts @@ -221,6 +221,7 @@ export interface AprilTagPipelineSettings extends PipelineSettings { tagFamily: AprilTagFamily; doMultiTarget: boolean; doSingleTargetAlways: boolean; + cudaAcceleration: boolean; } export type ConfigurableAprilTagPipelineSettings = Partial< Omit @@ -245,7 +246,8 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = { threads: 4, tagFamily: AprilTagFamily.Family36h11, doMultiTarget: false, - doSingleTargetAlways: false + doSingleTargetAlways: false, + cudaAcceleration: false }; export interface ArucoPipelineSettings extends PipelineSettings { diff --git a/photon-core/src/main/java/org/photonvision/jni/GpuDetectorJNI.java b/photon-core/src/main/java/org/photonvision/jni/GpuDetectorJNI.java new file mode 100644 index 0000000000..59b3debc7a --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/jni/GpuDetectorJNI.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy of the License + * at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in + * writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package org.photonvision.jni; + +import edu.wpi.first.apriltag.AprilTagDetection; + +public class GpuDetectorJNI { + static boolean libraryLoaded = false; + + static { + if (!libraryLoaded) System.loadLibrary("971apriltag"); + libraryLoaded = true; + } + + public static native long createGpuDetector(int width, int height); + + public static native void destroyGpuDetector(long handle); + + public static native void setparams( + long handle, + double fx, + double cx, + double fy, + double cy, + double k1, + double k2, + double p1, + double p2, + double k3); + + public static native AprilTagDetection[] processimage(long handle, long p); +} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java b/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java index c500592227..4e7f66a4c9 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java @@ -85,9 +85,26 @@ public class QuirkyCamera { CameraQuirk.ArduCamCamera, CameraQuirk.Gain, CameraQuirk.ArduOV9782Controls), + // Innomaker OV9281 Jetson Orin MIPI + new QuirkyCamera( + -1, -1, "vi-output, ov9281 10-0060", "vi-output,_ov9821_10-0060", CameraQuirk.Gain), + new QuirkyCamera( + -1, -1, "vi-output, ov9281 9-0060", "vi-output,_ov9821_9-0060", CameraQuirk.Gain), // Innomaker OV9281 new QuirkyCamera( - 0x0c45, 0x636d, "USB Camera", "Innomaker OV9281", CameraQuirk.InnoOV9281Controls)); + 0x0c45, 0x636d, "USB Camera", "Innomaker OV9281", CameraQuirk.InnoOV9281Controls), + new QuirkyCamera( + -1, + -1, + "vi-output, ov9281 9-0060", + CameraQuirk.InnoOV9281Controls, + CameraQuirk.StickyFPS), + new QuirkyCamera( + -1, + -1, + "vi-output, ov9281 10-0060", + CameraQuirk.InnoOV9281Controls, + CameraQuirk.StickyFPS)); public static final QuirkyCamera DefaultCamera = new QuirkyCamera(0, 0, ""); public static final QuirkyCamera ZeroCopyPiCamera = diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/GenericUSBCameraSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/GenericUSBCameraSettables.java index 7d956643e4..57c861ff70 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/GenericUSBCameraSettables.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/GenericUSBCameraSettables.java @@ -83,6 +83,7 @@ protected void setUpWhiteBalanceProperties() { } protected void setUpExposureProperties() { + logger.debug("start usb setupexposure"); // Photonvision needs to be able to control absolute exposure. Make sure we can // first. var expProp = @@ -231,7 +232,7 @@ public void setExposureRaw(double exposureRaw) { @Override public void setBrightness(int brightness) { try { - camera.setBrightness(brightness); + softSet("brightness", brightness); this.lastBrightness = brightness; } catch (VideoException e) { logger.error("Failed to set camera brightness!", e); diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/USBCameraSource.java b/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/USBCameraSource.java index b0a4dde81a..77f228c3df 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/USBCameraSource.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/USBCameras/USBCameraSource.java @@ -84,7 +84,7 @@ public USBCameraSource(CameraConfiguration config) { } if (getCameraConfiguration().cameraQuirks.hasQuirks()) { - logger.info("Quirky camera detected: " + getCameraConfiguration().cameraQuirks); + logger.info("Quirky camera detected: " + getCameraConfiguration().cameraQuirks.baseName); } var cameraBroken = getCameraConfiguration().cameraQuirks.hasQuirk(CameraQuirk.CompletelyBroken); @@ -147,6 +147,7 @@ protected GenericUSBCameraSettables createSettables( logger.debug("Using Arducam OV9782 Settables"); settables = new ArduOV9782CameraSettables(config, camera); } else if (quirks.hasQuirk(CameraQuirk.InnoOV9281Controls)) { + logger.debug("Using Innovision OV9782 Settables"); settables = new InnoOV9281CameraSettables(config, camera); } else if (quirks.hasQuirk(CameraQuirk.See3Cam_24CUG)) { settables = new See3Cam24CUGSettables(config, camera); diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java index 01e237c410..1695b76dbe 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/AprilTagDetectionPipe.java @@ -20,7 +20,10 @@ import edu.wpi.first.apriltag.AprilTagDetection; import edu.wpi.first.apriltag.AprilTagDetector; import java.util.List; +import org.opencv.core.Mat; +import org.photonvision.jni.GpuDetectorJNI; import org.photonvision.vision.apriltag.AprilTagFamily; +import org.photonvision.vision.calibration.CameraCalibrationCoefficients; import org.photonvision.vision.opencv.CVMat; import org.photonvision.vision.opencv.Releasable; import org.photonvision.vision.pipe.CVPipe; @@ -29,13 +32,24 @@ public class AprilTagDetectionPipe extends CVPipe< CVMat, List, AprilTagDetectionPipe.AprilTagDetectionPipeParams> implements Releasable { - private AprilTagDetector m_detector = new AprilTagDetector(); + private AprilTagDetector m_detector = null; + private GpuDetectorJNI m_cudadetector = null; + private long pointer = 0; + private boolean cudaAccelerated; - public AprilTagDetectionPipe() { + public AprilTagDetectionPipe(boolean cudaAccelerated) { super(); - m_detector.addFamily("tag16h5"); - m_detector.addFamily("tag36h11"); + this.cudaAccelerated = cudaAccelerated; + + if (cudaAccelerated) { + m_cudadetector = new GpuDetectorJNI(); + pointer = m_cudadetector.createGpuDetector(640, 480); // just a guess + } else { + m_detector = new AprilTagDetector(); + m_detector.addFamily("tag16h5"); + m_detector.addFamily("tag36h11"); + } } @Override @@ -60,11 +74,30 @@ protected List process(CVMat in) { @Override public void setParams(AprilTagDetectionPipeParams newParams) { if (this.params == null || !this.params.equals(newParams)) { - m_detector.setConfig(newParams.detectorParams()); - m_detector.setQuadThresholdParameters(newParams.quadParams()); + if (cudaAccelerated) { + if (newParams.cal == null) return; + + final Mat cameraMatrix = newParams.cal.getCameraIntrinsicsMat(); + final Mat distCoeffs = newParams.cal.getDistCoeffsMat(); + if (cameraMatrix == null || distCoeffs == null) return; + var cx = cameraMatrix.get(0, 2)[0]; + var cy = cameraMatrix.get(1, 2)[0]; + var fx = cameraMatrix.get(0, 0)[0]; + var fy = cameraMatrix.get(1, 1)[0]; + var k1 = distCoeffs.get(0, 0)[0]; + var k2 = distCoeffs.get(0, 1)[0]; + var k3 = distCoeffs.get(0, 4)[0]; + var p1 = distCoeffs.get(0, 2)[0]; + var p2 = distCoeffs.get(0, 3)[0]; - m_detector.clearFamilies(); - m_detector.addFamily(newParams.family().getNativeName()); + m_cudadetector.setparams(pointer, fx, cx, fy, cy, k1, k2, p1, p2, k3); + } else { + m_detector.setConfig(newParams.detectorParams()); + m_detector.setQuadThresholdParameters(newParams.quadParams()); + + m_detector.clearFamilies(); + m_detector.addFamily(newParams.family().getNativeName()); + } } super.setParams(newParams); @@ -72,12 +105,18 @@ public void setParams(AprilTagDetectionPipeParams newParams) { @Override public void release() { - m_detector.close(); - m_detector = null; + if (cudaAccelerated) { + m_cudadetector.destroyGpuDetector(pointer); + m_cudadetector = null; + } else { + m_detector.close(); + m_detector = null; + } } public static record AprilTagDetectionPipeParams( AprilTagFamily family, AprilTagDetector.Config detectorParams, - AprilTagDetector.QuadThresholdParameters quadParams) {} + AprilTagDetector.QuadThresholdParameters quadParams, + CameraCalibrationCoefficients cal) {} } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java index 3e98f7e9e1..19866cffc9 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java @@ -49,7 +49,7 @@ import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters; public class AprilTagPipeline extends CVPipeline { - private final AprilTagDetectionPipe aprilTagDetectionPipe = new AprilTagDetectionPipe(); + private final AprilTagDetectionPipe aprilTagDetectionPipe; private final AprilTagPoseEstimatorPipe singleTagPoseEstimatorPipe = new AprilTagPoseEstimatorPipe(); private final MultiTargetPNPPipe multiTagPNPPipe = new MultiTargetPNPPipe(); @@ -60,11 +60,13 @@ public class AprilTagPipeline extends CVPipeline