From 387e42e0bea780e5ee059d1a9dc95047f1bb6037 Mon Sep 17 00:00:00 2001 From: totaltaxamount Date: Tue, 21 Oct 2025 22:24:09 -0600 Subject: [PATCH 1/7] basler: basic "working" version --- .envrc | 1 + .gitignore | 3 + .vscode/settings.json | 3 +- build.gradle | 9 + photon-core/build.gradle | 3 + .../org/photonvision/jni/BaslerCameraJNI.java | 36 ++++ .../vision/camera/CameraType.java | 1 + .../vision/camera/PVCameraInfo.java | 51 +++++- .../baslerCameras/BaslerCameraSettables.java | 169 ++++++++++++++++++ .../baslerCameras/BaslerCameraSource.java | 61 +++++++ .../frame/consumer/MJPGFrameConsumer.java | 1 + .../frame/provider/BaslerFrameProvider.java | 63 +++++++ .../vision/pipeline/AprilTagPipeline.java | 1 + .../vision/pipeline/DriverModePipeline.java | 1 + .../vision/pipeline/OutputStreamPipeline.java | 1 + .../vision/processes/VisionSourceManager.java | 14 ++ .../src/main/java/org/photonvision/Main.java | 14 ++ shell.nix | 84 +++++++++ 18 files changed, 514 insertions(+), 2 deletions(-) create mode 100644 .envrc create mode 100644 photon-core/src/main/java/org/photonvision/jni/BaslerCameraJNI.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java create mode 100644 shell.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000..1d953f4bd7 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use nix diff --git a/.gitignore b/.gitignore index 6416c2fba8..8178990427 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,6 @@ node_modules dist components.d.ts photon-server/src/main/resources/web/index.html + +# Direnv +.direnv \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 376e6c941c..e9ad1f0f94 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, - "python.testing.cwd": "photon-lib/py" + "python.testing.cwd": "photon-lib/py", + "java.configuration.updateBuildConfiguration": "interactive" } \ No newline at end of file diff --git a/build.gradle b/build.gradle index 1f0d90b7d8..03a8b8930f 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,14 @@ allprojects { maven { url = "https://maven.photonvision.org/releases" } maven { url = "https://maven.photonvision.org/snapshots" } maven { url = "https://jogamp.org/deployment/maven/" } + + maven { + url = uri("https://maven.pkg.github.com/teamdeadbolts/amd_jni") + credentials { + username = project.findProperty("gpr.user") ?: System.getenv("GITHUB_ACTOR") + password = project.findProperty("gpr.token") ?: System.getenv("GITHUB_TOKEN") + } + } } wpilibRepositories.addAllReleaseRepositories(it) wpilibRepositories.addAllDevelopmentRepositories(it) @@ -40,6 +48,7 @@ ext { libcameraDriverVersion = "v2025.0.4" rknnVersion = "dev-v2025.0.0-5-g666c0c6" rubikVersion = "dev-v2025.1.0-8-g067a316" + baslerVersion = "0.0.1-local" frcYear = "2025" mrcalVersion = "v2025.0.0"; diff --git a/photon-core/build.gradle b/photon-core/build.gradle index 684c329b2d..ca71321fa9 100644 --- a/photon-core/build.gradle +++ b/photon-core/build.gradle @@ -45,6 +45,9 @@ dependencies { implementation("org.photonvision:photon-libcamera-gl-driver-jni:$libcameraDriverVersion:linuxarm64") { transitive = false } + implementation("org.teamdeadbolts:basler_jni-linuxx64:$baslerVersion") { + transitive = false + } implementation "org.photonvision:photon-libcamera-gl-driver-java:$libcameraDriverVersion" implementation "org.photonvision:photon-mrcal-java:$mrcalVersion" diff --git a/photon-core/src/main/java/org/photonvision/jni/BaslerCameraJNI.java b/photon-core/src/main/java/org/photonvision/jni/BaslerCameraJNI.java new file mode 100644 index 0000000000..6bff716d6b --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/jni/BaslerCameraJNI.java @@ -0,0 +1,36 @@ +package org.photonvision.jni; + +import java.io.IOException; +import java.util.List; +import org.photonvision.common.util.TestUtils; + +public class BaslerCameraJNI extends PhotonJNICommon { + private boolean isLoaded; + private static BaslerCameraJNI instance = null; + + private BaslerCameraJNI() { + isLoaded = false; + } + + public static BaslerCameraJNI getInstance() { + if (instance == null) instance = new BaslerCameraJNI(); + + return instance; + } + + public static synchronized void forceLoad() throws IOException { + TestUtils.loadLibraries(); + + forceLoad(getInstance(), BaslerCameraJNI.class, List.of("baslerjni")); + } + + @Override + public boolean isLoaded() { + return isLoaded; + } + + @Override + public void setLoaded(boolean state) { + isLoaded = state; + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/CameraType.java b/photon-core/src/main/java/org/photonvision/vision/camera/CameraType.java index c080f366e5..8d186832e0 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/CameraType.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/CameraType.java @@ -20,5 +20,6 @@ public enum CameraType { UsbCamera, ZeroCopyPicam, + BaslerCamera, FileCamera // special case for File-based vision sources } diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java b/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java index d67bdfb34f..0e84ff49ee 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java @@ -26,13 +26,16 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import edu.wpi.first.cscore.UsbCameraInfo; import java.util.Arrays; +import org.teamdeadbolts.basler.BaslerJNI; +import org.teamdeadbolts.basler.BaslerJNI.CameraModel; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) @JsonIgnoreProperties(ignoreUnknown = true) @JsonSubTypes({ @JsonSubTypes.Type(value = PVCameraInfo.PVUsbCameraInfo.class), @JsonSubTypes.Type(value = PVCameraInfo.PVCSICameraInfo.class), - @JsonSubTypes.Type(value = PVCameraInfo.PVFileCameraInfo.class) + @JsonSubTypes.Type(value = PVCameraInfo.PVFileCameraInfo.class), + @JsonSubTypes.Type(value = PVCameraInfo.PVBaslerCameraInfo.class) }) public sealed interface PVCameraInfo { /** @@ -257,6 +260,48 @@ public String toString() { } } + @JsonTypeName("PVBaslerCameraInfo") + public static final class PVBaslerCameraInfo implements PVCameraInfo { + + public final String serial; + public final BaslerJNI.CameraModel model; + + public PVBaslerCameraInfo( + @JsonProperty("serial") String serial, @JsonProperty("model") BaslerJNI.CameraModel model) { + this.serial = serial; + this.model = model; + } + + @Override + public String path() { + return this.serial; + } + + @Override + public String name() { + return this.model.getFriendlyName(); + } + + @Override + public String uniquePath() { + return this.serial; + } + + @Override + public String[] otherPaths() { + return new String[] {}; + } + + @Override + public CameraType type() { + return CameraType.BaslerCamera; + } + + public BaslerJNI.CameraModel getModel() { + return this.model; + } + } + public static PVCameraInfo fromUsbCameraInfo(UsbCameraInfo info) { return new PVUsbCameraInfo(info); } @@ -268,4 +313,8 @@ public static PVCameraInfo fromCSICameraInfo(String path, String baseName) { public static PVCameraInfo fromFileInfo(String path, String baseName) { return new PVFileCameraInfo(path, baseName); } + + public static PVCameraInfo fromBaslerCameraInfo(String serial, CameraModel model) { + return new PVBaslerCameraInfo(serial, model); + } } diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java new file mode 100644 index 0000000000..3418fe76e1 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java @@ -0,0 +1,169 @@ +package org.photonvision.vision.camera.baslerCameras; + +import edu.wpi.first.cscore.VideoMode; +import edu.wpi.first.util.PixelFormat; +import java.util.HashMap; +import org.photonvision.common.configuration.CameraConfiguration; +import org.photonvision.vision.camera.PVCameraInfo.PVBaslerCameraInfo; +import org.photonvision.vision.processes.VisionSourceSettables; +import org.teamdeadbolts.basler.BaslerJNI; + +public class BaslerCameraSettables extends VisionSourceSettables { + public long ptr = 0; + private VideoMode currentVideoMode; + private String serial; + + public final Object LOCK = new Object(); + + protected BaslerCameraSettables(CameraConfiguration configuration) { + // configuration.matchedCameraInfo.path() + super(configuration); + if (!(configuration.matchedCameraInfo instanceof PVBaslerCameraInfo)) { + throw new IllegalArgumentException( + "Cannot create Basler Camera Settables from non basler info"); + } + + PVBaslerCameraInfo info = (PVBaslerCameraInfo) configuration.matchedCameraInfo; + + this.serial = configuration.getDevicePath(); + + switch (info.getModel()) { + case daA1280_54uc: + videoModes.put(0, new VideoMode(PixelFormat.kBGR.getValue(), 1280, 960, 43)); + videoModes.put(1, new VideoMode(PixelFormat.kUYVY.getValue(), 1280, 960, 52)); + default: + break; + } + + this.currentVideoMode = videoModes.get(0); + } + + @Override + public void setExposureRaw(double exposureRaw) { + logger.debug("Setting exposure to " + exposureRaw); + boolean success = BaslerJNI.setExposure(ptr, exposureRaw); + if (!success) { + BaslerCameraSource.logger.warn("Failed to set exposure to " + exposureRaw); + } + } + + @Override + public void setAutoExposure(boolean cameraAutoExposure) { + logger.debug("Setting auto exposure to " + cameraAutoExposure); + logger.debug("PTR: " + ptr); + logger.debug("Supported formats: "); + for (int f : BaslerJNI.getSupportedPixelFormats(ptr)) { + logger.debug(PixelFormat.getFromInt(f).toString()); + } + + boolean success = BaslerJNI.setAutoExposure(ptr, cameraAutoExposure); + if (!success) { + BaslerCameraSource.logger.warn("Failed to set auto exposure to " + cameraAutoExposure); + } + } + + @Override + public void setWhiteBalanceTemp(double temp) { + logger.debug("Setting white balance to " + temp); + boolean success = BaslerJNI.setWhiteBalance(ptr, new double[] {temp, temp, temp}); + if (!success) { + BaslerCameraSource.logger.warn("Failed to set white balance to " + temp); + } + } + + @Override + public void setAutoWhiteBalance(boolean autowb) { + logger.debug("Setting auto white balance to " + autowb); + boolean success = BaslerJNI.setAutoWhiteBalance(ptr, autowb); + if (!success) { + BaslerCameraSource.logger.warn("Failed to set auto white balance to " + autowb); + } + } + + @Override + public void setBrightness(int brightness) { + // TODO + // BaslerJNI.getMin + } + + @Override + public void setGain(int gain) { + logger.debug("Setting gain to " + gain); + boolean success = BaslerJNI.setGain(ptr, gain); + if (!success) { + BaslerCameraSource.logger.warn("Failed to set gain to " + gain); + } + } + + @Override + public VideoMode getCurrentVideoMode() { + return currentVideoMode; + } + + @Override + protected void setVideoModeInternal(VideoMode videoMode) { + synchronized (LOCK) { + if (ptr != 0) { + logger.debug("Stopping camera"); + if (!BaslerJNI.stopCamera(ptr)) { + logger.warn("Failed to stop camera when changing video mode"); + } + + logger.debug("Destroying camera"); + if (!BaslerJNI.destroyCamera(ptr)) { + logger.warn("Failed to destroy camera when changing video mode"); + } + } + + logger.debug("Creating camera"); + + ptr = BaslerJNI.createCamera(serial); + if (!BaslerJNI.setPixelFormat(ptr, videoMode.pixelFormat.getValue())) { + logger.warn("Failed to set pixel format"); + return; + } + + if (ptr == 0) { + logger.error("Failed to create camera when changing video mode"); + return; + } + + logger.debug("Starting camera"); + if (!BaslerJNI.startCamera(ptr)) { + logger.error("Failed to start camera when changing video mode"); + BaslerJNI.destroyCamera(ptr); + ptr = 0; + return; + } + } + + this.currentVideoMode = videoMode; + } + + @Override + public HashMap getAllVideoModes() { + return videoModes; + } + + @Override + public double getMinWhiteBalanceTemp() { + return BaslerJNI.getMinWhiteBalance(ptr); + } + + @Override + public double getMaxWhiteBalanceTemp() { + return BaslerJNI.getMaxWhiteBalance(ptr); + } + + @Override + public double getMinExposureRaw() { + // return BaslerJNI. + return BaslerJNI.getMinExposure(ptr); + } + + @Override + public double getMaxExposureRaw() { + return BaslerJNI.getMaxExposure(ptr); + } + +} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java new file mode 100644 index 0000000000..ac799eef6c --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java @@ -0,0 +1,61 @@ +package org.photonvision.vision.camera.baslerCameras; + +import org.photonvision.common.configuration.CameraConfiguration; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.vision.camera.CameraType; +import org.photonvision.vision.camera.QuirkyCamera; +import org.photonvision.vision.frame.FrameProvider; +import org.photonvision.vision.frame.provider.BaslerFrameProvider; +import org.photonvision.vision.processes.VisionSource; +import org.photonvision.vision.processes.VisionSourceSettables; + +public class BaslerCameraSource extends VisionSource { + static final Logger logger = new Logger(BaslerCameraSource.class, LogGroup.Camera); + + private final BaslerCameraSettables settables; + private BaslerFrameProvider frameProvider; + + public BaslerCameraSource(CameraConfiguration cameraConfiguration) { + super(cameraConfiguration); + if (cameraConfiguration.matchedCameraInfo.type() != CameraType.BaslerCamera) { + throw new IllegalArgumentException( + "BaslerCameraSource only accepts CameraConfigurations with type BaslerCamera"); + } + + this.settables = new BaslerCameraSettables(cameraConfiguration); + this.frameProvider = new BaslerFrameProvider(this.settables); + this.getCameraConfiguration().cameraQuirks = QuirkyCamera.DefaultCamera; + } + + @Override + public void release() { + this.frameProvider.release(); + this.frameProvider = null; + } + + @Override + public FrameProvider getFrameProvider() { + return frameProvider; + } + + @Override + public VisionSourceSettables getSettables() { + return settables; + } + + @Override + public boolean isVendorCamera() { + return false; + } + + @Override + public boolean hasLEDs() { + return false; + } + + @Override + public void remakeSettables() { + // TODO: implement + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java index b99dc23f31..2418b99276 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java @@ -49,6 +49,7 @@ public void accept(CVMat image) { long now = MathUtils.wpiNanoTime(); if (image == null || image.getMat() == null || image.getMat().empty()) { + System.out.println("Empty/nonexistant image"); image.copyFrom(StaticFrames.LOST_MAT); } diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java new file mode 100644 index 0000000000..38f6c39f80 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java @@ -0,0 +1,63 @@ +package org.photonvision.vision.frame.provider; + +import edu.wpi.first.util.PixelFormat; +import edu.wpi.first.util.RawFrame; +import org.opencv.core.Mat; +import org.photonvision.common.logging.LogGroup; +import org.photonvision.common.logging.Logger; +import org.photonvision.vision.camera.baslerCameras.BaslerCameraSettables; +import org.photonvision.vision.opencv.CVMat; +import org.teamdeadbolts.basler.BaslerJNI; + +public class BaslerFrameProvider extends CpuImageProcessor { + private final BaslerCameraSettables settables; + + static final Logger logger = new Logger(BaslerFrameProvider.class, LogGroup.Camera); + + public BaslerFrameProvider(BaslerCameraSettables settables) { + this.settables = settables; + + var vidMode = settables.getCurrentVideoMode(); + settables.setVideoMode(vidMode); + this.cameraPropertiesCached = true; + + BaslerJNI.startCamera(settables.ptr); + } + + @Override + public String getName() { + return "BaslerCameraFrameProvider"; + } + + @Override + public void release() { + BaslerJNI.stopCamera(settables.ptr); + BaslerJNI.destroyCamera(settables.ptr); + } + + @Override + public boolean isConnected() { + return true; // TODO: implement + } + + @Override + public boolean checkCameraConnected() { + return true; // TODO: implement + } + + @Override + CapturedFrame getInputMat() { + + // TODO: Latency + var cameraMode = settables.getCurrentVideoMode(); + var frame = new RawFrame(); + frame.setInfo(cameraMode.width, cameraMode.height, cameraMode.width * 3, PixelFormat.kBGR); + + CVMat ret; + BaslerJNI.awaitNewFrame(settables.ptr); + Mat mat = new Mat(BaslerJNI.takeFrame(settables.ptr)); + ret = new CVMat(mat, frame); + + return new CapturedFrame(ret, settables.getFrameStaticProperties(), 10); + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java index 3e98f7e9e1..ec8f2c296b 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/AprilTagPipeline.java @@ -136,6 +136,7 @@ protected CVPipelineResult process(Frame frame, AprilTagPipelineSettings setting CVPipeResult> tagDetectionPipeResult = aprilTagDetectionPipe.run(frame.processedImage); + sumPipeNanosElapsed += tagDetectionPipeResult.nanosElapsed; List detections = tagDetectionPipeResult.output; diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java index a8c52efc85..f4692c880a 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java @@ -70,6 +70,7 @@ public DriverModePipelineResult process(Frame frame, DriverModePipelineSettings totalNanos += resizeImagePipe.run(inputMat).nanosElapsed; if (settings.crosshair) { + System.out.print("DMP"); var draw2dCrosshairResult = draw2dCrosshairPipe.run(Pair.of(inputMat, List.of())); // calculate elapsed nanoseconds diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java index 366ef1a0a2..32fabdc894 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/OutputStreamPipeline.java @@ -144,6 +144,7 @@ public CVPipelineResult process( } // Draw 2D Crosshair on output + var draw2dCrosshairResultOnInput = draw2dCrosshairPipe.run(Pair.of(inMat, targetsToDraw)); sumPipeNanosElapsed += pipeProfileNanos[3] = draw2dCrosshairResultOnInput.nanosElapsed; diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java index a0c18e128d..85964675aa 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionSourceManager.java @@ -44,7 +44,10 @@ import org.photonvision.vision.camera.FileVisionSource; import org.photonvision.vision.camera.PVCameraInfo; import org.photonvision.vision.camera.USBCameras.USBCameraSource; +import org.photonvision.vision.camera.baslerCameras.BaslerCameraSource; import org.photonvision.vision.camera.csi.LibcameraGpuSource; +import org.teamdeadbolts.basler.BaslerJNI; +import org.teamdeadbolts.basler.BaslerJNI.CameraModel; /** * This class manages starting up VisionModules for serialized devices ({@link @@ -310,6 +313,16 @@ protected List getConnectedCameras() { }) .forEach(cameraInfos::add); } + if (BaslerJNI.isSupported()) { + Stream.of(BaslerJNI.getConnectedCameras()) + .map( + serial -> { + CameraModel model = BaslerJNI.getCameraModel(serial); + return PVCameraInfo.fromBaslerCameraInfo(serial, model); + }) + .forEach(cameraInfos::add); + ; + } // FileVisionSources are a bit quirky. They aren't enumerated by the above, but i still want my // UI to look like it ought to work @@ -400,6 +413,7 @@ protected VisionSource loadVisionSourceFromCamConfig(CameraConfiguration configu case UsbCamera -> new USBCameraSource(configuration); case ZeroCopyPicam -> new LibcameraGpuSource(configuration); case FileCamera -> new FileVisionSource(configuration); + case BaslerCamera -> new BaslerCameraSource(configuration); }; if (source.getFrameProvider() == null) { diff --git a/photon-server/src/main/java/org/photonvision/Main.java b/photon-server/src/main/java/org/photonvision/Main.java index 442fdb6d09..9eb718380b 100644 --- a/photon-server/src/main/java/org/photonvision/Main.java +++ b/photon-server/src/main/java/org/photonvision/Main.java @@ -38,6 +38,7 @@ import org.photonvision.common.logging.PvCSCoreLogger; import org.photonvision.common.networking.NetworkManager; import org.photonvision.common.util.TestUtils; +import org.photonvision.jni.BaslerCameraJNI; import org.photonvision.jni.PhotonTargetingJniLoader; import org.photonvision.jni.RknnDetectorJNI; import org.photonvision.jni.RubikDetectorJNI; @@ -229,6 +230,19 @@ public static void main(String[] args) { } catch (IOException e) { logger.error("Failed to load libcamera-JNI!", e); } + + try { + if (Platform.getCurrentPlatform() == Platform.LINUX_64) { + BaslerCameraJNI.forceLoad(); + if (BaslerCameraJNI.getInstance().isLoaded()) { + logger.info("BaslerCameraJNI loaded successfully."); + } else { + logger.error("Failed to load basler JNI!"); + } + } + } catch (IOException e) { + logger.error("Failed to load basler JNI!", e); + } try { if (Platform.isRK3588()) { RknnDetectorJNI.forceLoad(); diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000..945602b75c --- /dev/null +++ b/shell.nix @@ -0,0 +1,84 @@ +{ pkgs ? import {} }: +let + ade = pkgs.stdenv.mkDerivation rec { + pname = "ade"; + version = "0.1.2e"; + + src = pkgs.fetchFromGitHub { + owner = "opencv"; + repo = "ade"; + rev = "v${version}"; + hash = "sha256-1z5ChmXyanEghBLpopJlRIjOMu+GFAON0X8K2ZhYVlA="; + }; + + nativeBuildInputs = [ pkgs.cmake ]; + + cmakeFlags = [ + "-DBUILD_SHARED_LIBS=OFF" + ]; + }; + + + opencv4100 = pkgs.opencv4.overrideAttrs (oldAttr: rec { + version = "4.10.0"; + src = pkgs.fetchFromGitHub { + owner = "opencv"; + repo = "opencv"; + rev = version; + hash = "sha256-s+KvBrV/BxrxEvPhHzWCVFQdUQwhUdRJyb0wcGDFpeo=" ; + }; + + buildInputs = (oldAttr.buildInputs or []) ++ [ ade ]; + + nativeBuildInputs = + oldAttr.nativeBuildInputs + ++ (with pkgs; [ + ant + openjdk + python3 + python3Packages.numpy + ]); + + cmakeFlags = + oldAttr.cmakeFlags + ++ [ + "-DBUILD_JAVA=ON" + "-DBUILD_opencv_dnn=OFF" + "-DBUILD_opencv_gapi=ON" + "-DWITH_ADE=ON" + "-Dade_DIR=${ade}/lib/cmake/ade" + ]; + + postInstall = (oldAttr.postInstall or "") + '' + cd $out/lib + for lib in libopencv_*.so.4.10.0; do + if [ -f "$lib" ]; then + base=$(basename "$lib" .so.4.10.0) + ln -sf "$lib" "$base.so.4.10" + fi + done + ''; + }); + + buildInputs = with pkgs; [ + openjdk17 + cmake + opencv4100 + clang + lapack + suitesparse + pnpm + re2 + ]; +in +pkgs.mkShell { + buildInputs = buildInputs; + + shellHook = '' + + export LD_LIBRARY_PATH=${pkgs.lib.makeLibraryPath buildInputs}:$LD_LIBRARY_PATH + + export LD_LIBRARY_PATH=${opencv4100}/share/java/opencv4:$LD_LIBRARY_PATH + export JAVA_HOME=${pkgs.openjdk17} + ''; +} \ No newline at end of file From b5213df2e0a49d00d7744de4780fbb4ef11cdcf8 Mon Sep 17 00:00:00 2001 From: totaltaxamount Date: Wed, 22 Oct 2025 22:37:17 -0600 Subject: [PATCH 2/7] basler: Fix gain settings + exposure settings + some white balance stuff + misc small issues --- build.gradle | 2 +- .../components/common/pv-camera-info-card.vue | 4 ++ .../common/pv-camera-match-card.vue | 5 ++ photon-client/src/types/SettingTypes.ts | 14 ++++- .../src/views/CameraMatchingView.vue | 17 ++++-- .../baslerCameras/BaslerCameraSettables.java | 53 +++++++++++-------- .../baslerCameras/BaslerCameraSource.java | 3 ++ .../frame/consumer/MJPGFrameConsumer.java | 1 - .../frame/provider/BaslerFrameProvider.java | 7 ++- .../vision/pipeline/DriverModePipeline.java | 1 - 10 files changed, 76 insertions(+), 31 deletions(-) diff --git a/build.gradle b/build.gradle index 03a8b8930f..7f09d65c3c 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ ext { libcameraDriverVersion = "v2025.0.4" rknnVersion = "dev-v2025.0.0-5-g666c0c6" rubikVersion = "dev-v2025.1.0-8-g067a316" - baslerVersion = "0.0.1-local" + baslerVersion = "local" frcYear = "2025" mrcalVersion = "v2025.0.0"; diff --git a/photon-client/src/components/common/pv-camera-info-card.vue b/photon-client/src/components/common/pv-camera-info-card.vue index f726cc6eb0..b318ef9691 100644 --- a/photon-client/src/components/common/pv-camera-info-card.vue +++ b/photon-client/src/components/common/pv-camera-info-card.vue @@ -18,6 +18,9 @@ const cameraInfoFor: any = (camera: PVCameraInfo) => { if (camera.PVFileCameraInfo) { return camera.PVFileCameraInfo; } + if (camera.PVBaslerCameraInfo) { + return camera.PVBaslerCameraInfo; + } return {}; }; @@ -39,6 +42,7 @@ const cameraInfoFor: any = (camera: PVCameraInfo) => { USB Camera CSI Camera File Camera + Basler Camera Unidentified Camera Type diff --git a/photon-client/src/components/common/pv-camera-match-card.vue b/photon-client/src/components/common/pv-camera-match-card.vue index 644741bd3f..f5c505e955 100644 --- a/photon-client/src/components/common/pv-camera-match-card.vue +++ b/photon-client/src/components/common/pv-camera-match-card.vue @@ -36,6 +36,9 @@ const cameraInfoFor = (camera: PVCameraInfo): any => { if (camera.PVFileCameraInfo) { return camera.PVFileCameraInfo; } + if (camera.PVBaslerCameraInfo) { + return camera.PVBaslerCameraInfo; + } return {}; }; @@ -78,10 +81,12 @@ const cameraInfoFor = (camera: PVCameraInfo): any => { USB Camera CSI Camera File Camera + Basler Camera Unidentified Camera Type USB Camera CSI Camera File Camera + Basler Camera Unidentified Camera Type { camera1.PVFileCameraInfo.uniquePath === camera2.PVFileCameraInfo.uniquePath && camera1.PVFileCameraInfo.name === camera2.PVFileCameraInfo.name ); + else if (camera1.PVBaslerCameraInfo && camera2.PVBaslerCameraInfo) + return ( + camera1.PVBaslerCameraInfo.serial === camera2.PVBaslerCameraInfo.serial && + camera1.PVBaslerCameraInfo.uniquePath === camera2.PVBaslerCameraInfo.uniquePath + ); else return false; }; -const cameraInfoFor = (camera: PVCameraInfo | null): PVUsbCameraInfo | PVCSICameraInfo | PVFileCameraInfo | any => { +const cameraInfoFor = (camera: PVCameraInfo | null): PVUsbCameraInfo | PVCSICameraInfo | PVFileCameraInfo | PVBaslerCameraInfo | any => { if (!camera) return null; if (camera.PVUsbCameraInfo) { return camera.PVUsbCameraInfo; @@ -200,6 +206,9 @@ const cameraInfoFor = (camera: PVCameraInfo | null): PVUsbCameraInfo | PVCSICame if (camera.PVFileCameraInfo) { return camera.PVFileCameraInfo; } + if (camera.PVBaslerCameraInfo) { + return camera.PVBaslerCameraInfo; + } return {}; }; @@ -211,7 +220,8 @@ const getMatchedDevice = (info: PVCameraInfo | undefined): PVCameraInfo => { return { PVFileCameraInfo: undefined, PVCSICameraInfo: undefined, - PVUsbCameraInfo: undefined + PVUsbCameraInfo: undefined, + PVBaslerCameraInfo: undefined }; } return ( @@ -220,7 +230,8 @@ const getMatchedDevice = (info: PVCameraInfo | undefined): PVCameraInfo => { ) || { PVFileCameraInfo: undefined, PVCSICameraInfo: undefined, - PVUsbCameraInfo: undefined + PVUsbCameraInfo: undefined, + PVBaslerCameraInfo: undefined, } ); }; diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java index 3418fe76e1..9f6e59957b 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java @@ -1,9 +1,11 @@ package org.photonvision.vision.camera.baslerCameras; import edu.wpi.first.cscore.VideoMode; +import edu.wpi.first.math.MathUtil; import edu.wpi.first.util.PixelFormat; import java.util.HashMap; import org.photonvision.common.configuration.CameraConfiguration; +import org.photonvision.common.util.math.MathUtils; import org.photonvision.vision.camera.PVCameraInfo.PVBaslerCameraInfo; import org.photonvision.vision.processes.VisionSourceSettables; import org.teamdeadbolts.basler.BaslerJNI; @@ -13,10 +15,11 @@ public class BaslerCameraSettables extends VisionSourceSettables { private VideoMode currentVideoMode; private String serial; + private double[] currentRatios = new double[3]; + public final Object LOCK = new Object(); protected BaslerCameraSettables(CameraConfiguration configuration) { - // configuration.matchedCameraInfo.path() super(configuration); if (!(configuration.matchedCameraInfo instanceof PVBaslerCameraInfo)) { throw new IllegalArgumentException( @@ -29,19 +32,22 @@ protected BaslerCameraSettables(CameraConfiguration configuration) { switch (info.getModel()) { case daA1280_54uc: - videoModes.put(0, new VideoMode(PixelFormat.kBGR.getValue(), 1280, 960, 43)); + videoModes.put(0, new VideoMode(PixelFormat.kBGR.getValue(), 1280, 960, 43)); videoModes.put(1, new VideoMode(PixelFormat.kUYVY.getValue(), 1280, 960, 52)); - default: break; + default: + logger.warn("Unsupported camera model: " + info.getModel().getFriendlyName()); + videoModes.put(0, new VideoMode(PixelFormat.kBGR.getValue(), 1280, 960, 43)); // Just guess } this.currentVideoMode = videoModes.get(0); + this.currentRatios = new double[] {1, 1, 1}; } @Override public void setExposureRaw(double exposureRaw) { logger.debug("Setting exposure to " + exposureRaw); - boolean success = BaslerJNI.setExposure(ptr, exposureRaw); + boolean success = BaslerJNI.setExposure(ptr, exposureRaw * 1000); if (!success) { BaslerCameraSource.logger.warn("Failed to set exposure to " + exposureRaw); } @@ -50,11 +56,6 @@ public void setExposureRaw(double exposureRaw) { @Override public void setAutoExposure(boolean cameraAutoExposure) { logger.debug("Setting auto exposure to " + cameraAutoExposure); - logger.debug("PTR: " + ptr); - logger.debug("Supported formats: "); - for (int f : BaslerJNI.getSupportedPixelFormats(ptr)) { - logger.debug(PixelFormat.getFromInt(f).toString()); - } boolean success = BaslerJNI.setAutoExposure(ptr, cameraAutoExposure); if (!success) { @@ -64,11 +65,7 @@ public void setAutoExposure(boolean cameraAutoExposure) { @Override public void setWhiteBalanceTemp(double temp) { - logger.debug("Setting white balance to " + temp); - boolean success = BaslerJNI.setWhiteBalance(ptr, new double[] {temp, temp, temp}); - if (!success) { - BaslerCameraSource.logger.warn("Failed to set white balance to " + temp); - } + throw new RuntimeException("Dont do this, use ratios insted"); } @Override @@ -89,12 +86,27 @@ public void setBrightness(int brightness) { @Override public void setGain(int gain) { logger.debug("Setting gain to " + gain); - boolean success = BaslerJNI.setGain(ptr, gain); + double min = BaslerJNI.getMinGain(ptr) + 1.0; // No divide by 0 + double max = BaslerJNI.getMaxGain(ptr); + boolean success = + BaslerJNI.setGain(ptr, MathUtil.clamp(MathUtils.map(gain, 0.0, 100.0, min, max), min, max)); if (!success) { BaslerCameraSource.logger.warn("Failed to set gain to " + gain); } } + @Override + public void setRedGain(int red) { + this.currentRatios[0] = MathUtil.clamp(MathUtils.map(red, 0.0, 100, 1.0, 3.0), 1.0, 3.0); + BaslerJNI.setWhiteBalance(ptr, this.currentRatios); + } + + @Override + public void setBlueGain(int blue) { + this.currentRatios[2] = MathUtil.clamp(MathUtils.map(blue, 0.0, 100, 1.0, 3.0), 1.0, 3.0); + BaslerJNI.setWhiteBalance(ptr, this.currentRatios); + } + @Override public VideoMode getCurrentVideoMode() { return currentVideoMode; @@ -147,23 +159,22 @@ public HashMap getAllVideoModes() { @Override public double getMinWhiteBalanceTemp() { - return BaslerJNI.getMinWhiteBalance(ptr); + return 1; } @Override public double getMaxWhiteBalanceTemp() { - return BaslerJNI.getMaxWhiteBalance(ptr); + return 2; } - @Override + @Override public double getMinExposureRaw() { // return BaslerJNI. - return BaslerJNI.getMinExposure(ptr); + return BaslerJNI.getMinExposure(ptr) / 1000; } @Override public double getMaxExposureRaw() { - return BaslerJNI.getMaxExposure(ptr); + return BaslerJNI.getMaxExposure(ptr) / 1000; } - } diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java index ac799eef6c..ec923e3f6f 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java @@ -3,6 +3,7 @@ import org.photonvision.common.configuration.CameraConfiguration; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; +import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.camera.CameraType; import org.photonvision.vision.camera.QuirkyCamera; import org.photonvision.vision.frame.FrameProvider; @@ -26,6 +27,8 @@ public BaslerCameraSource(CameraConfiguration cameraConfiguration) { this.settables = new BaslerCameraSettables(cameraConfiguration); this.frameProvider = new BaslerFrameProvider(this.settables); this.getCameraConfiguration().cameraQuirks = QuirkyCamera.DefaultCamera; + this.getCameraConfiguration().cameraQuirks.quirks.put(CameraQuirk.Gain, true); + this.getCameraConfiguration().cameraQuirks.quirks.put(CameraQuirk.AwbRedBlueGain, true); // Not really correct } @Override diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java index 2418b99276..b99dc23f31 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/consumer/MJPGFrameConsumer.java @@ -49,7 +49,6 @@ public void accept(CVMat image) { long now = MathUtils.wpiNanoTime(); if (image == null || image.getMat() == null || image.getMat().empty()) { - System.out.println("Empty/nonexistant image"); image.copyFrom(StaticFrames.LOST_MAT); } diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java index 38f6c39f80..3ec09541e3 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java @@ -5,6 +5,7 @@ import org.opencv.core.Mat; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; +import org.photonvision.common.util.math.MathUtils; import org.photonvision.vision.camera.baslerCameras.BaslerCameraSettables; import org.photonvision.vision.opencv.CVMat; import org.teamdeadbolts.basler.BaslerJNI; @@ -33,6 +34,7 @@ public String getName() { public void release() { BaslerJNI.stopCamera(settables.ptr); BaslerJNI.destroyCamera(settables.ptr); + BaslerJNI.cleanUp(); } @Override @@ -48,16 +50,17 @@ public boolean checkCameraConnected() { @Override CapturedFrame getInputMat() { - // TODO: Latency var cameraMode = settables.getCurrentVideoMode(); var frame = new RawFrame(); frame.setInfo(cameraMode.width, cameraMode.height, cameraMode.width * 3, PixelFormat.kBGR); CVMat ret; + var start = MathUtils.wpiNanoTime(); BaslerJNI.awaitNewFrame(settables.ptr); Mat mat = new Mat(BaslerJNI.takeFrame(settables.ptr)); ret = new CVMat(mat, frame); - return new CapturedFrame(ret, settables.getFrameStaticProperties(), 10); + return new CapturedFrame( + ret, settables.getFrameStaticProperties(), start); // TODO: Timestamping is kinda off rn } } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java index f4692c880a..a8c52efc85 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/DriverModePipeline.java @@ -70,7 +70,6 @@ public DriverModePipelineResult process(Frame frame, DriverModePipelineSettings totalNanos += resizeImagePipe.run(inputMat).nanosElapsed; if (settings.crosshair) { - System.out.print("DMP"); var draw2dCrosshairResult = draw2dCrosshairPipe.run(Pair.of(inputMat, List.of())); // calculate elapsed nanoseconds From aba5db2e9f38ffa7b1afe4956dd3852021a29395 Mon Sep 17 00:00:00 2001 From: totaltaxamount Date: Wed, 22 Oct 2025 23:05:05 -0600 Subject: [PATCH 3/7] misc: remove useless maven repo --- build.gradle | 8 -------- 1 file changed, 8 deletions(-) diff --git a/build.gradle b/build.gradle index 7f09d65c3c..951871e0f7 100644 --- a/build.gradle +++ b/build.gradle @@ -20,14 +20,6 @@ allprojects { maven { url = "https://maven.photonvision.org/releases" } maven { url = "https://maven.photonvision.org/snapshots" } maven { url = "https://jogamp.org/deployment/maven/" } - - maven { - url = uri("https://maven.pkg.github.com/teamdeadbolts/amd_jni") - credentials { - username = project.findProperty("gpr.user") ?: System.getenv("GITHUB_ACTOR") - password = project.findProperty("gpr.token") ?: System.getenv("GITHUB_TOKEN") - } - } } wpilibRepositories.addAllReleaseRepositories(it) wpilibRepositories.addAllDevelopmentRepositories(it) From 8e84afd019d84f16ea9f5be945613e8d32014e24 Mon Sep 17 00:00:00 2001 From: totaltaxamount Date: Fri, 24 Oct 2025 14:45:31 -0600 Subject: [PATCH 4/7] basler: Add support for different basler camera models and also add binning --- .../components/dashboard/tabs/InputTab.vue | 10 +- photon-client/src/lib/PhotonUtils.ts | 5 +- .../stores/settings/CameraSettingsStore.ts | 13 +- photon-client/src/types/SettingTypes.ts | 16 ++- photon-client/src/types/WebsocketDataTypes.ts | 3 + .../vision/camera/CameraQuirk.java | 3 + .../vision/camera/PVCameraInfo.java | 29 ++++- .../vision/camera/QuirkyCamera.java | 5 +- .../baslerCameras/BaslerCameraSource.java | 73 ++++++++++- .../BaslerDaA1280CameraSettables.java | 48 +++++++ ...java => GenericBaslerCameraSettables.java} | 119 ++++++++++++------ .../frame/provider/BaslerFrameProvider.java | 57 +++++++-- .../vision/pipe/impl/PixelBinPipe.java | 32 +++++ .../vision/processes/VisionModule.java | 9 +- 14 files changed, 352 insertions(+), 70 deletions(-) create mode 100644 photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java rename photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/{BaslerCameraSettables.java => GenericBaslerCameraSettables.java} (57%) create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipe/impl/PixelBinPipe.java diff --git a/photon-client/src/components/dashboard/tabs/InputTab.vue b/photon-client/src/components/dashboard/tabs/InputTab.vue index fe0dc9f343..2ea56160af 100644 --- a/photon-client/src/components/dashboard/tabs/InputTab.vue +++ b/photon-client/src/components/dashboard/tabs/InputTab.vue @@ -6,7 +6,7 @@ import PvSelect from "@/components/common/pv-select.vue"; import { computed } from "vue"; import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore"; import { useStateStore } from "@/stores/StateStore"; -import { getResolutionString } from "@/lib/PhotonUtils"; +import { getBinningString, getResolutionString } from "@/lib/PhotonUtils"; import { useDisplay } from "vuetify"; // Due to something with libcamera or something else IDK much about, the 90° rotations need to be disabled if the libcamera drivers are being used. @@ -31,9 +31,11 @@ const getFilteredStreamDivisors = (): number[] => { const getNumberOfSkippedDivisors = () => streamDivisors.length - getFilteredStreamDivisors().length; const cameraResolutions = computed(() => - useCameraSettingsStore().currentCameraSettings.validVideoFormats.map( - (f) => `${getResolutionString(f.resolution)} at ${f.fps} FPS, ${f.pixelFormat}` - ) + useCameraSettingsStore().currentCameraSettings.validVideoFormats.map((f) => { + console.log("hello?"); + console.log(f.binning?.mode + " " + f.binning?.vert); + return `${getResolutionString(f.resolution)} at ${f.fps} FPS, ${f.pixelFormat}${getBinningString(f.binning)}`; + }) ); const handleResolutionChange = (value: number) => { useCameraSettingsStore().changeCurrentPipelineSetting({ cameraVideoModeIndex: value }, false); diff --git a/photon-client/src/lib/PhotonUtils.ts b/photon-client/src/lib/PhotonUtils.ts index 9b31aa08b2..0852ce6d0d 100644 --- a/photon-client/src/lib/PhotonUtils.ts +++ b/photon-client/src/lib/PhotonUtils.ts @@ -1,4 +1,4 @@ -import type { Resolution } from "@/types/SettingTypes"; +import type { BinningConfig, Resolution } from "@/types/SettingTypes"; export const resolutionsAreEqual = (a: Resolution, b: Resolution) => { return a.height === b.height && a.width === b.width; @@ -6,6 +6,9 @@ export const resolutionsAreEqual = (a: Resolution, b: Resolution) => { export const getResolutionString = (resolution: Resolution): string => `${resolution.width}x${resolution.height}`; +export const getBinningString = (binning?: BinningConfig): string => + binning != null ? ` ${binning.mode} ${binning.horz}x${binning.vert}` : ""; + export const parseJsonFile = async >(file: File): Promise => { return new Promise((resolve, reject) => { const fileReader = new FileReader(); diff --git a/photon-client/src/stores/settings/CameraSettingsStore.ts b/photon-client/src/stores/settings/CameraSettingsStore.ts index 32c3f94edf..5a78039c90 100644 --- a/photon-client/src/stores/settings/CameraSettingsStore.ts +++ b/photon-client/src/stores/settings/CameraSettingsStore.ts @@ -101,6 +101,7 @@ export const useCameraSettingsStore = defineStore("cameraSettings", { actions: { updateCameraSettingsFromWebsocket(data: WebsocketCameraSettingsUpdate[]) { const configuredCameras = data.reduce<{ [key: string]: UiCameraConfiguration }>((acc, d) => { + console.log("Bmode: " + d.videoFormatList[0].binningMode); acc[d.uniqueName] = { cameraPath: d.cameraPath, nickname: d.nickname, @@ -128,7 +129,16 @@ export const useCameraSettingsStore = defineStore("cameraSettings", { horizontalFOV: v.horizontalFOV, verticalFOV: v.verticalFOV, standardDeviation: v.standardDeviation, - mean: v.mean + mean: v.mean, + ...(v.binningMode !== undefined && + v.binningHorz !== undefined && + v.binningVert !== undefined && { + binning: { + mode: v.binningMode, + horz: v.binningHorz, + vert: v.binningVert + } + }) })), completeCalibrations: d.calibrations, isCSICamera: d.isCSICamera, @@ -145,6 +155,7 @@ export const useCameraSettingsStore = defineStore("cameraSettings", { hasConnected: d.hasConnected, mismatch: d.mismatch }; + return acc; }, {}); this.cameras = diff --git a/photon-client/src/types/SettingTypes.ts b/photon-client/src/types/SettingTypes.ts index 7063c21ec2..11eeaf1d5c 100644 --- a/photon-client/src/types/SettingTypes.ts +++ b/photon-client/src/types/SettingTypes.ts @@ -108,7 +108,6 @@ export interface PVBaslerCameraInfo { model: string; uniquePath: string; - } // This camera info will only ever hold one of its members - the others should be undefined. @@ -148,10 +147,17 @@ export interface Resolution { height: number; } +export interface BinningConfig { + mode: String; + horz: number; + vert: number; +} + export interface VideoFormat { resolution: Resolution; fps: number; pixelFormat: string; + binning?: BinningConfig; index?: number; diagonalFOV?: number; horizontalFOV?: number; @@ -229,7 +235,8 @@ export enum ValidQuirks { PiCam = "PiCam", StickyFPS = "StickyFPS", LifeCamControls = "LifeCamControls", - PsEyeControls = "PsEyeControls" + PsEyeControls = "PsEyeControls", + BaslerDaA1280Controls = "BaslerDaA1280Controls" } export interface QuirkyCamera { @@ -380,7 +387,8 @@ export const PlaceholderCameraSettings: UiCameraConfiguration = { StickyFPS: false, InnoOV9281Controls: false, LifeCamControls: false, - PsEyeControls: false + PsEyeControls: false, + BaslerDaA1280Controls: false } }, isCSICamera: false, @@ -396,7 +404,7 @@ export const PlaceholderCameraSettings: UiCameraConfiguration = { }, PVCSICameraInfo: undefined, PVUsbCameraInfo: undefined, - PVBaslerCameraInfo: undefined, + PVBaslerCameraInfo: undefined }, isConnected: true, hasConnected: true, diff --git a/photon-client/src/types/WebsocketDataTypes.ts b/photon-client/src/types/WebsocketDataTypes.ts index 4ca05d7895..361e6786ca 100644 --- a/photon-client/src/types/WebsocketDataTypes.ts +++ b/photon-client/src/types/WebsocketDataTypes.ts @@ -43,6 +43,9 @@ export type WebsocketVideoFormat = Record< verticalFOV?: number; standardDeviation?: number; mean?: number; + binningMode?: String; + binningHorz?: number; + binningVert?: number; } >; diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java b/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java index a3dace6549..89f9005183 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java @@ -58,4 +58,7 @@ public enum CameraQuirk { ArduOV9782, /** Camera has odd exposure range, and supports gain control */ See3Cam_24CUG, + + /** Quirks to tell the difference between basler cameras */ + BaslerDaA1280Controls, } diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java b/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java index 0dce0daf48..3b2a3705c4 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java @@ -26,9 +26,9 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import edu.wpi.first.cscore.UsbCameraInfo; import java.util.Arrays; +import java.util.Objects; import org.teamdeadbolts.basler.BaslerJNI; import org.teamdeadbolts.basler.BaslerJNI.CameraModel; -import java.util.Objects; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT) @JsonIgnoreProperties(ignoreUnknown = true) @@ -316,7 +316,7 @@ public String name() { @Override public String uniquePath() { - return this.serial; + return path(); } @Override @@ -329,6 +329,31 @@ public CameraType type() { return CameraType.BaslerCamera; } + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (!(obj instanceof PVBaslerCameraInfo info)) return false; + + return this.model.equals(info.model) && serial.equals(info.serial); + } + + @Override + public int hashCode() { + return Objects.hash(model, serial); + } + + @Override + public String toString() { + return "PVBaslerCameraInfo[type=" + + type() + + ", model=" + + model.toString() + + ", serial='" + + serial + + "']"; + } + public BaslerJNI.CameraModel getModel() { return this.model; } diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java b/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java index c500592227..1b78e93d2c 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java @@ -87,7 +87,10 @@ public class QuirkyCamera { CameraQuirk.ArduOV9782Controls), // Innomaker OV9281 new QuirkyCamera( - 0x0c45, 0x636d, "USB Camera", "Innomaker OV9281", CameraQuirk.InnoOV9281Controls)); + 0x0c45, 0x636d, "USB Camera", "Innomaker OV9281", CameraQuirk.InnoOV9281Controls), + + // Basler daA1280-54uc + new QuirkyCamera(-1, -1, "Basler daA1280-54uc", CameraQuirk.BaslerDaA1280Controls)); public static final QuirkyCamera DefaultCamera = new QuirkyCamera(0, 0, ""); public static final QuirkyCamera ZeroCopyPiCamera = diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java index ec923e3f6f..8057080695 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSource.java @@ -1,5 +1,6 @@ package org.photonvision.vision.camera.baslerCameras; +import edu.wpi.first.cscore.VideoMode; import org.photonvision.common.configuration.CameraConfiguration; import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; @@ -8,15 +9,20 @@ import org.photonvision.vision.camera.QuirkyCamera; import org.photonvision.vision.frame.FrameProvider; import org.photonvision.vision.frame.provider.BaslerFrameProvider; +import org.photonvision.vision.pipe.impl.PixelBinPipe; import org.photonvision.vision.processes.VisionSource; import org.photonvision.vision.processes.VisionSourceSettables; public class BaslerCameraSource extends VisionSource { static final Logger logger = new Logger(BaslerCameraSource.class, LogGroup.Camera); - private final BaslerCameraSettables settables; + private final GenericBaslerCameraSettables settables; private BaslerFrameProvider frameProvider; + private void onCameraConnected() { + settables.onCameraConnected(); + } + public BaslerCameraSource(CameraConfiguration cameraConfiguration) { super(cameraConfiguration); if (cameraConfiguration.matchedCameraInfo.type() != CameraType.BaslerCamera) { @@ -24,11 +30,33 @@ public BaslerCameraSource(CameraConfiguration cameraConfiguration) { "BaslerCameraSource only accepts CameraConfigurations with type BaslerCamera"); } - this.settables = new BaslerCameraSettables(cameraConfiguration); - this.frameProvider = new BaslerFrameProvider(this.settables); - this.getCameraConfiguration().cameraQuirks = QuirkyCamera.DefaultCamera; - this.getCameraConfiguration().cameraQuirks.quirks.put(CameraQuirk.Gain, true); - this.getCameraConfiguration().cameraQuirks.quirks.put(CameraQuirk.AwbRedBlueGain, true); // Not really correct + this.getCameraConfiguration().cameraQuirks = + QuirkyCamera.getQuirkyCamera(-1, -1, cameraConfiguration.matchedCameraInfo.name()); + + this.settables = createSettables(cameraConfiguration); + // this.settables.setupVideoModes(); + this.frameProvider = new BaslerFrameProvider(this.settables, this::onCameraConnected); + // logger.debug(QuirkyCamera.getQuirkyCamera(-1, -1, + // cameraConfiguration.matchedCameraInfo.name()).toString()); + // this.getCameraConfiguration().cameraQuirks.quirks.put(CameraQuirk.Gain, true); + // this.getCameraConfiguration().cameraQuirks.quirks.put(CameraQuirk.AwbRedBlueGain, true); // + // Not really correct + } + + protected GenericBaslerCameraSettables createSettables(CameraConfiguration config) { + var quirks = getCameraConfiguration().cameraQuirks; + + GenericBaslerCameraSettables settables; + + if (quirks.hasQuirk(CameraQuirk.BaslerDaA1280Controls)) { + logger.info("Using Basler DaA1280 Settables"); + settables = new BaslerDaA1280CameraSettables(config); + } else { + logger.debug("Using generic basler settables"); + settables = new GenericBaslerCameraSettables(config); + } + + return settables; } @Override @@ -61,4 +89,37 @@ public boolean hasLEDs() { public void remakeSettables() { // TODO: implement } + + public static class BaslerVideoMode extends VideoMode { + public static class BinningConfig { + public final PixelBinPipe.PixelBinParams.BinMode mode; + public final int horz; + public final int vert; + + public BinningConfig(PixelBinPipe.PixelBinParams.BinMode mode, int horz, int vert) { + this.mode = mode; + this.horz = horz; + this.vert = vert; + } + + @Override + public String toString() { + return "BinningConfig[mode=" + + this.mode + + ",horz=" + + this.horz + + ",vert=" + + this.vert + + "]"; + } + } + + public final BinningConfig binningConfig; + + public BaslerVideoMode( + int pixelFormat, int width, int height, int fps, BinningConfig binningConfig) { + super(pixelFormat, width, height, fps); + this.binningConfig = binningConfig; + } + } } diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java new file mode 100644 index 0000000000..c5d6da8473 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java @@ -0,0 +1,48 @@ +package org.photonvision.vision.camera.baslerCameras; + +import edu.wpi.first.util.PixelFormat; +import org.photonvision.common.configuration.CameraConfiguration; +import org.photonvision.vision.camera.CameraQuirk; +import org.photonvision.vision.camera.baslerCameras.BaslerCameraSource.BaslerVideoMode; +import org.photonvision.vision.camera.baslerCameras.BaslerCameraSource.BaslerVideoMode.BinningConfig; +import org.photonvision.vision.pipe.impl.PixelBinPipe.PixelBinParams.BinMode; + +public class BaslerDaA1280CameraSettables extends GenericBaslerCameraSettables { + + protected BaslerDaA1280CameraSettables(CameraConfiguration configuration) { + super(configuration); + + this.maxExposure = 1000; + this.maxGain = 18; + + this.getConfiguration().cameraQuirks.quirks.put(CameraQuirk.Gain, true); + } + + @Override + protected void setupVideoModes() { + videoModes.put( + 0, + new BaslerVideoMode( + PixelFormat.kBGR.getValue(), 1280, 960, 43, new BinningConfig(BinMode.NONE, 0, 0))); + videoModes.put( + 1, + new BaslerVideoMode( + PixelFormat.kUYVY.getValue(), 1280, 960, 52, new BinningConfig(BinMode.NONE, 0, 0))); + videoModes.put( + 2, + new BaslerVideoMode( + PixelFormat.kBGR.getValue(), + 1280 / 2, + 960 / 2, + 43, + new BinningConfig(BinMode.SUM, 2, 2))); + videoModes.put( + 3, + new BaslerVideoMode( + PixelFormat.kBGR.getValue(), + 1280 / 2, + 960 / 2, + 43, + new BinningConfig(BinMode.AVERAGE, 2, 2))); + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/GenericBaslerCameraSettables.java similarity index 57% rename from photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java rename to photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/GenericBaslerCameraSettables.java index 9f6e59957b..df6cbaf5ba 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerCameraSettables.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/GenericBaslerCameraSettables.java @@ -2,50 +2,47 @@ import edu.wpi.first.cscore.VideoMode; import edu.wpi.first.math.MathUtil; -import edu.wpi.first.util.PixelFormat; import java.util.HashMap; import org.photonvision.common.configuration.CameraConfiguration; import org.photonvision.common.util.math.MathUtils; import org.photonvision.vision.camera.PVCameraInfo.PVBaslerCameraInfo; +import org.photonvision.vision.camera.baslerCameras.BaslerCameraSource.BaslerVideoMode; import org.photonvision.vision.processes.VisionSourceSettables; import org.teamdeadbolts.basler.BaslerJNI; -public class BaslerCameraSettables extends VisionSourceSettables { +public class GenericBaslerCameraSettables extends VisionSourceSettables { public long ptr = 0; - private VideoMode currentVideoMode; - private String serial; - - private double[] currentRatios = new double[3]; + public String serial; public final Object LOCK = new Object(); - protected BaslerCameraSettables(CameraConfiguration configuration) { + protected BaslerVideoMode currentVideoMode; + + protected double minExposure = -1; + protected double maxExposure = 1000; + + protected double minGain = 1; + protected double maxGain = 1; + + protected double lastExposure = -1; + protected int lastGain = -1; + + protected PVBaslerCameraInfo info; + + protected GenericBaslerCameraSettables(CameraConfiguration configuration) { super(configuration); if (!(configuration.matchedCameraInfo instanceof PVBaslerCameraInfo)) { throw new IllegalArgumentException( "Cannot create Basler Camera Settables from non basler info"); } - PVBaslerCameraInfo info = (PVBaslerCameraInfo) configuration.matchedCameraInfo; - + this.info = (PVBaslerCameraInfo) configuration.matchedCameraInfo; this.serial = configuration.getDevicePath(); - - switch (info.getModel()) { - case daA1280_54uc: - videoModes.put(0, new VideoMode(PixelFormat.kBGR.getValue(), 1280, 960, 43)); - videoModes.put(1, new VideoMode(PixelFormat.kUYVY.getValue(), 1280, 960, 52)); - break; - default: - logger.warn("Unsupported camera model: " + info.getModel().getFriendlyName()); - videoModes.put(0, new VideoMode(PixelFormat.kBGR.getValue(), 1280, 960, 43)); // Just guess - } - - this.currentVideoMode = videoModes.get(0); - this.currentRatios = new double[] {1, 1, 1}; } @Override public void setExposureRaw(double exposureRaw) { + this.lastExposure = exposureRaw; logger.debug("Setting exposure to " + exposureRaw); boolean success = BaslerJNI.setExposure(ptr, exposureRaw * 1000); if (!success) { @@ -61,6 +58,7 @@ public void setAutoExposure(boolean cameraAutoExposure) { if (!success) { BaslerCameraSource.logger.warn("Failed to set auto exposure to " + cameraAutoExposure); } + if (!cameraAutoExposure) setExposureRaw(this.lastExposure); } @Override @@ -86,34 +84,57 @@ public void setBrightness(int brightness) { @Override public void setGain(int gain) { logger.debug("Setting gain to " + gain); - double min = BaslerJNI.getMinGain(ptr) + 1.0; // No divide by 0 - double max = BaslerJNI.getMaxGain(ptr); + + // double min = BaslerJNI.getMinGain(ptr) + 1.0; // No divide by 0 + // double max = BaslerJNI.getMaxGain(ptr); + this.lastGain = gain; boolean success = - BaslerJNI.setGain(ptr, MathUtil.clamp(MathUtils.map(gain, 0.0, 100.0, min, max), min, max)); + BaslerJNI.setGain( + ptr, + MathUtil.clamp(MathUtils.map(gain, 0.0, 100.0, minGain, maxGain), minGain, maxGain)); if (!success) { BaslerCameraSource.logger.warn("Failed to set gain to " + gain); } } - @Override - public void setRedGain(int red) { - this.currentRatios[0] = MathUtil.clamp(MathUtils.map(red, 0.0, 100, 1.0, 3.0), 1.0, 3.0); - BaslerJNI.setWhiteBalance(ptr, this.currentRatios); - } + // @Override + // public void setRedGain(int red) { + // this.currentRatios[0] = MathUtil.clamp(MathUtils.map(red, 0.0, 100, 1.0, 3.0), 1.0, 3.0); + // BaslerJNI.setWhiteBalance(ptr, this.currentRatios); + // } - @Override - public void setBlueGain(int blue) { - this.currentRatios[2] = MathUtil.clamp(MathUtils.map(blue, 0.0, 100, 1.0, 3.0), 1.0, 3.0); - BaslerJNI.setWhiteBalance(ptr, this.currentRatios); - } + // @Override + // public void setBlueGain(int blue) { + // this.currentRatios[2] = MathUtil.clamp(MathUtils.map(blue, 0.0, 100, 1.0, 3.0), 1.0, 3.0); + // BaslerJNI.setWhiteBalance(ptr, this.currentRatios); + // } @Override - public VideoMode getCurrentVideoMode() { + public BaslerVideoMode getCurrentVideoMode() { return currentVideoMode; } + public void setVideoMode(VideoMode mode) { + var bMode = (BaslerVideoMode) mode; + logger.info( + "Setting video mode to " + + "FPS: " + + mode.fps + + " Width: " + + mode.width + + " Height: " + + mode.height + + " Pixel Format: " + + mode.pixelFormat + + " Binning config: " + + bMode.binningConfig); + setVideoModeInternal(mode); + calculateFrameStaticProps(); + } + @Override protected void setVideoModeInternal(VideoMode videoMode) { + var mode = (BaslerVideoMode) videoMode; synchronized (LOCK) { if (ptr != 0) { logger.debug("Stopping camera"); @@ -130,10 +151,16 @@ protected void setVideoModeInternal(VideoMode videoMode) { logger.debug("Creating camera"); ptr = BaslerJNI.createCamera(serial); - if (!BaslerJNI.setPixelFormat(ptr, videoMode.pixelFormat.getValue())) { + if (!BaslerJNI.setPixelFormat(ptr, mode.pixelFormat.getValue())) { logger.warn("Failed to set pixel format"); return; } + if (this.lastGain != -1) { + this.setGain(this.lastGain); + } + if (this.lastExposure != -1) { + this.setExposureRaw(this.lastExposure); + } if (ptr == 0) { logger.error("Failed to create camera when changing video mode"); @@ -149,7 +176,7 @@ protected void setVideoModeInternal(VideoMode videoMode) { } } - this.currentVideoMode = videoMode; + this.currentVideoMode = mode; } @Override @@ -170,11 +197,23 @@ public double getMaxWhiteBalanceTemp() { @Override public double getMinExposureRaw() { // return BaslerJNI. - return BaslerJNI.getMinExposure(ptr) / 1000; + return minExposure; } @Override public double getMaxExposureRaw() { - return BaslerJNI.getMaxExposure(ptr) / 1000; + return maxExposure; + } + + @Override + public void onCameraConnected() { + super.onCameraConnected(); + setupVideoModes(); + if (!videoModes.isEmpty()) this.currentVideoMode = (BaslerVideoMode) videoModes.get(0); + else logger.warn("Video modes empty"); + + calculateFrameStaticProps(); } + + protected void setupVideoModes() {} } diff --git a/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java b/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java index 3ec09541e3..61f9ffb530 100644 --- a/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java +++ b/photon-core/src/main/java/org/photonvision/vision/frame/provider/BaslerFrameProvider.java @@ -6,28 +6,36 @@ import org.photonvision.common.logging.LogGroup; import org.photonvision.common.logging.Logger; import org.photonvision.common.util.math.MathUtils; -import org.photonvision.vision.camera.baslerCameras.BaslerCameraSettables; +import org.photonvision.vision.camera.baslerCameras.BaslerCameraSource.BaslerVideoMode; +import org.photonvision.vision.camera.baslerCameras.GenericBaslerCameraSettables; import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.pipe.impl.PixelBinPipe; +import org.photonvision.vision.pipe.impl.PixelBinPipe.PixelBinParams; +import org.photonvision.vision.pipe.impl.PixelBinPipe.PixelBinParams.BinMode; import org.teamdeadbolts.basler.BaslerJNI; public class BaslerFrameProvider extends CpuImageProcessor { - private final BaslerCameraSettables settables; + private final GenericBaslerCameraSettables settables; static final Logger logger = new Logger(BaslerFrameProvider.class, LogGroup.Camera); - public BaslerFrameProvider(BaslerCameraSettables settables) { + private PixelBinPipe pixelBinPipe = new PixelBinPipe(); + + private Runnable connectedCallback; + + public BaslerFrameProvider(GenericBaslerCameraSettables settables, Runnable connectedCallback) { this.settables = settables; + this.connectedCallback = connectedCallback; - var vidMode = settables.getCurrentVideoMode(); - settables.setVideoMode(vidMode); - this.cameraPropertiesCached = true; + // var vidMode = settables.getCurrentVideoMode(); + // settables.setVideoMode(vidMode); BaslerJNI.startCamera(settables.ptr); } @Override public String getName() { - return "BaslerCameraFrameProvider"; + return "BaslerCameraFrameProvider-" + this.settables.serial; } @Override @@ -39,17 +47,32 @@ public void release() { @Override public boolean isConnected() { - return true; // TODO: implement + var serials = BaslerJNI.getConnectedCameras(); + for (String serial : serials) { + if (serial.equals(settables.serial)) { + return true; + } + } + + return false; } @Override public boolean checkCameraConnected() { - return true; // TODO: implement + boolean connected = isConnected(); + if (connected && !cameraPropertiesCached) { + logger.info("Camera connected! running callback"); + onCameraConnected(); + } + + return connected; } @Override CapturedFrame getInputMat() { - + if (!cameraPropertiesCached && isConnected()) { + onCameraConnected(); + } var cameraMode = settables.getCurrentVideoMode(); var frame = new RawFrame(); frame.setInfo(cameraMode.width, cameraMode.height, cameraMode.width * 3, PixelFormat.kBGR); @@ -58,9 +81,23 @@ CapturedFrame getInputMat() { var start = MathUtils.wpiNanoTime(); BaslerJNI.awaitNewFrame(settables.ptr); Mat mat = new Mat(BaslerJNI.takeFrame(settables.ptr)); + BaslerVideoMode.BinningConfig binningConfig = + this.settables.getCurrentVideoMode().binningConfig; + if (binningConfig.mode != BinMode.NONE) { + pixelBinPipe.setParams( + new PixelBinParams(binningConfig.mode, binningConfig.horz, binningConfig.vert)); + pixelBinPipe.run(mat); + } + ret = new CVMat(mat, frame); return new CapturedFrame( ret, settables.getFrameStaticProperties(), start); // TODO: Timestamping is kinda off rn } + + @Override + public void onCameraConnected() { + super.onCameraConnected(); + this.connectedCallback.run(); + } } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/PixelBinPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/PixelBinPipe.java new file mode 100644 index 0000000000..902bdcb307 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/PixelBinPipe.java @@ -0,0 +1,32 @@ +package org.photonvision.vision.pipe.impl; + +import org.opencv.core.Mat; +import org.photonvision.vision.pipe.MutatingPipe; +import org.teamdeadbolts.basler.BaslerJNI; + +public class PixelBinPipe extends MutatingPipe { + + @Override + protected Void process(Mat in) { + switch (params.mode) { + case AVERAGE: + BaslerJNI.avgBin(in, params.binHorz(), params.binVert()); + break; + case SUM: + BaslerJNI.sumBin(in, params.binHorz(), params.binVert()); + break; + case NONE: + break; + } + + return null; + } + + public static record PixelBinParams(BinMode mode, int binHorz, int binVert) { + public enum BinMode { + SUM, + AVERAGE, + NONE, + } + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java index f3ebab44b8..3e9aa1a2b0 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java @@ -46,6 +46,7 @@ import org.photonvision.vision.camera.CameraQuirk; import org.photonvision.vision.camera.CameraType; import org.photonvision.vision.camera.QuirkyCamera; +import org.photonvision.vision.camera.baslerCameras.BaslerCameraSource.BaslerVideoMode; import org.photonvision.vision.camera.csi.LibcameraGpuSource; import org.photonvision.vision.frame.Frame; import org.photonvision.vision.frame.consumer.FileSaveFrameConsumer; @@ -590,7 +591,13 @@ public UICameraConfiguration toUICameraConfig() { ? "kPicam" : k.getValue().pixelFormat.toString()) .substring(1)); // Remove the k prefix - + if (k.getValue() instanceof BaslerVideoMode) { + var bMode = (BaslerVideoMode) k.getValue(); + logger.debug("Have basler binning: " + bMode.binningConfig.mode); + internalMap.put("binningMode", bMode.binningConfig.mode.toString()); + internalMap.put("binningHorz", bMode.binningConfig.horz); + internalMap.put("binningVert", bMode.binningConfig.vert); + } temp.put(k.getKey(), internalMap); } From a184e36b55d5ed5a21d1fd2bd2e33b3cbeaa747e Mon Sep 17 00:00:00 2001 From: totaltaxamount Date: Fri, 24 Oct 2025 21:17:20 -0600 Subject: [PATCH 5/7] basler: add brightness control --- .../components/dashboard/tabs/InputTab.vue | 1 + .../GenericBaslerCameraSettables.java | 20 +++++++++++++++---- .../vision/processes/VisionModule.java | 8 ++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/photon-client/src/components/dashboard/tabs/InputTab.vue b/photon-client/src/components/dashboard/tabs/InputTab.vue index 2ea56160af..9dfae2fbd9 100644 --- a/photon-client/src/components/dashboard/tabs/InputTab.vue +++ b/photon-client/src/components/dashboard/tabs/InputTab.vue @@ -150,6 +150,7 @@ const interactiveCols = computed(() => " /> { + it.cameraWhiteBalanceTemp = -1; + }); + } + this.pipelineManager = pipelineManager; this.visionSource = visionSource; changeSubscriber = new VisionModuleChangeSubscriber(this); From d21e9922347fdaddaf088587efe329189b5d4641 Mon Sep 17 00:00:00 2001 From: totaltaxamount Date: Fri, 24 Oct 2025 22:32:47 -0600 Subject: [PATCH 6/7] basler: some cleanup and inital manual wb control --- .../components/dashboard/tabs/InputTab.vue | 24 ++++++++--- photon-client/src/lib/PhotonUtils.ts | 2 +- photon-client/src/types/SettingTypes.ts | 6 ++- .../vision/camera/CameraQuirk.java | 3 ++ .../vision/camera/QuirkyCamera.java | 2 +- .../BaslerDaA1280CameraSettables.java | 1 + .../GenericBaslerCameraSettables.java | 40 +++++++++++++------ .../vision/pipeline/CVPipelineSettings.java | 3 +- .../vision/processes/VisionModule.java | 4 +- 9 files changed, 60 insertions(+), 25 deletions(-) diff --git a/photon-client/src/components/dashboard/tabs/InputTab.vue b/photon-client/src/components/dashboard/tabs/InputTab.vue index 9dfae2fbd9..4afaf4d4a7 100644 --- a/photon-client/src/components/dashboard/tabs/InputTab.vue +++ b/photon-client/src/components/dashboard/tabs/InputTab.vue @@ -32,8 +32,6 @@ const getNumberOfSkippedDivisors = () => streamDivisors.length - getFilteredStre const cameraResolutions = computed(() => useCameraSettingsStore().currentCameraSettings.validVideoFormats.map((f) => { - console.log("hello?"); - console.log(f.binning?.mode + " " + f.binning?.vert); return `${getResolutionString(f.resolution)} at ${f.fps} FPS, ${f.pixelFormat}${getBinningString(f.binning)}`; }) ); @@ -118,24 +116,38 @@ const interactiveCols = computed(() => /> { export const getResolutionString = (resolution: Resolution): string => `${resolution.width}x${resolution.height}`; export const getBinningString = (binning?: BinningConfig): string => - binning != null ? ` ${binning.mode} ${binning.horz}x${binning.vert}` : ""; + binning != null && binning.mode !== "NONE" ? ` ${binning.mode} ${binning.horz}x${binning.vert}` : ""; export const parseJsonFile = async >(file: File): Promise => { return new Promise((resolve, reject) => { diff --git a/photon-client/src/types/SettingTypes.ts b/photon-client/src/types/SettingTypes.ts index 11eeaf1d5c..490604de87 100644 --- a/photon-client/src/types/SettingTypes.ts +++ b/photon-client/src/types/SettingTypes.ts @@ -236,7 +236,8 @@ export enum ValidQuirks { StickyFPS = "StickyFPS", LifeCamControls = "LifeCamControls", PsEyeControls = "PsEyeControls", - BaslerDaA1280Controls = "BaslerDaA1280Controls" + BaslerDaA1280Controls = "BaslerDaA1280Controls", + ManualWB = "ManualWB" } export interface QuirkyCamera { @@ -388,7 +389,8 @@ export const PlaceholderCameraSettings: UiCameraConfiguration = { InnoOV9281Controls: false, LifeCamControls: false, PsEyeControls: false, - BaslerDaA1280Controls: false + BaslerDaA1280Controls: false, + ManualWB: false } }, isCSICamera: false, diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java b/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java index 89f9005183..96b2f7b1ae 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/CameraQuirk.java @@ -61,4 +61,7 @@ public enum CameraQuirk { /** Quirks to tell the difference between basler cameras */ BaslerDaA1280Controls, + + /** Other Basler Quirks */ + ManualWB, } diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java b/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java index 1b78e93d2c..6a3602c6b8 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/QuirkyCamera.java @@ -90,7 +90,7 @@ public class QuirkyCamera { 0x0c45, 0x636d, "USB Camera", "Innomaker OV9281", CameraQuirk.InnoOV9281Controls), // Basler daA1280-54uc - new QuirkyCamera(-1, -1, "Basler daA1280-54uc", CameraQuirk.BaslerDaA1280Controls)); + new QuirkyCamera(-1, -1, "Basler daA1280-54uc", CameraQuirk.BaslerDaA1280Controls, CameraQuirk.ManualWB)); public static final QuirkyCamera DefaultCamera = new QuirkyCamera(0, 0, ""); public static final QuirkyCamera ZeroCopyPiCamera = diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java index c5d6da8473..e8397f8e1b 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/BaslerDaA1280CameraSettables.java @@ -16,6 +16,7 @@ protected BaslerDaA1280CameraSettables(CameraConfiguration configuration) { this.maxGain = 18; this.getConfiguration().cameraQuirks.quirks.put(CameraQuirk.Gain, true); + this.getConfiguration().cameraQuirks.quirks.put(CameraQuirk.AwbRedBlueGain, true); } @Override diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/GenericBaslerCameraSettables.java b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/GenericBaslerCameraSettables.java index 9399827a25..5c340f8bfb 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/GenericBaslerCameraSettables.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/baslerCameras/GenericBaslerCameraSettables.java @@ -2,6 +2,8 @@ import edu.wpi.first.cscore.VideoMode; import edu.wpi.first.math.MathUtil; + +import java.util.Arrays; import java.util.HashMap; import org.photonvision.common.configuration.CameraConfiguration; import org.photonvision.common.util.math.MathUtils; @@ -27,8 +29,12 @@ public class GenericBaslerCameraSettables extends VisionSourceSettables { protected double minBrightness = -1; protected double maxBrightness = 1; + protected double minWBGain = 0; + protected double maxWBGain = 6; + protected double lastExposure = -1; protected int lastGain = -1; + protected double[] lastWBValues = new double[] {-1, 1, -1}; protected PVBaslerCameraInfo info; @@ -76,6 +82,10 @@ public void setAutoWhiteBalance(boolean autowb) { if (!success) { BaslerCameraSource.logger.warn("Failed to set auto white balance to " + autowb); } + + if (!autowb && lastWBValues[0] != -1 && lastWBValues[2] != -1) { + BaslerJNI.setWhiteBalance(ptr, lastWBValues); + } } @Override @@ -84,7 +94,7 @@ public void setBrightness(int brightness) { double scaledBrightness = MathUtil.clamp( - MathUtils.map(brightness, 0, 100, minBrightness, maxBrightness), + MathUtils.map(brightness, 0, 101, minBrightness, maxBrightness), minBrightness, maxBrightness); boolean success = BaslerJNI.setBrightness(ptr, scaledBrightness); @@ -103,23 +113,28 @@ public void setGain(int gain) { boolean success = BaslerJNI.setGain( ptr, - MathUtil.clamp(MathUtils.map(gain, 0.0, 100.0, minGain, maxGain), minGain, maxGain)); + MathUtil.clamp(MathUtils.map(gain, 0.0, 101.0, minGain, maxGain), minGain, maxGain)); if (!success) { logger.warn("Failed to set gain to " + gain); } } - // @Override - // public void setRedGain(int red) { - // this.currentRatios[0] = MathUtil.clamp(MathUtils.map(red, 0.0, 100, 1.0, 3.0), 1.0, 3.0); - // BaslerJNI.setWhiteBalance(ptr, this.currentRatios); - // } + @Override + public void setRedGain(int red) { + double scaledRed = + MathUtil.clamp(MathUtils.map(red, 0.0, 101, minWBGain, maxWBGain), minWBGain, maxWBGain); + lastWBValues[0] = scaledRed; + BaslerJNI.setWhiteBalance(ptr, lastWBValues); + } - // @Override - // public void setBlueGain(int blue) { - // this.currentRatios[2] = MathUtil.clamp(MathUtils.map(blue, 0.0, 100, 1.0, 3.0), 1.0, 3.0); - // BaslerJNI.setWhiteBalance(ptr, this.currentRatios); - // } + @Override + public void setBlueGain(int blue) { + double scaledBlue = + MathUtil.clamp(MathUtils.map(blue, 0.0, 101, minWBGain, maxWBGain), minWBGain, maxWBGain); + lastWBValues[2] = scaledBlue; + logger.debug("WB: " + Arrays.toString(lastWBValues)); + BaslerJNI.setWhiteBalance(ptr, lastWBValues); + } @Override public BaslerVideoMode getCurrentVideoMode() { @@ -208,7 +223,6 @@ public double getMaxWhiteBalanceTemp() { @Override public double getMinExposureRaw() { - // return BaslerJNI. return minExposure; } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipelineSettings.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipelineSettings.java index a512e1fcf9..a93a0ca86c 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipelineSettings.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/CVPipelineSettings.java @@ -49,7 +49,7 @@ public class CVPipelineSettings implements Cloneable { // Currently only used by a few cameras (notably the zero-copy Pi Camera driver) with the Gain // quirk public int cameraGain = 75; - // Currently only used by the zero-copy Pi Camera driver + // Currently only used by the zero-copy Pi Camera driver and some Basler Cameras public int cameraRedGain = 11; public int cameraBlueGain = 20; public int cameraVideoModeIndex = 0; @@ -61,6 +61,7 @@ public class CVPipelineSettings implements Cloneable { public boolean cameraAutoWhiteBalance = false; public double cameraWhiteBalanceTemp = 4000; + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java index 1aa7737f68..4e15eabb2f 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java @@ -127,10 +127,12 @@ public VisionModule(PipelineManager pipelineManager, VisionSource visionSource) } // Basler stuff - if (cameraQuirks.hasQuirk(CameraQuirk.BaslerDaA1280Controls)) { + if (cameraQuirks.hasQuirk(CameraQuirk.ManualWB)) { pipelineManager.userPipelineSettings.forEach( it -> { it.cameraWhiteBalanceTemp = -1; + if (it.cameraRedGain == -1) it.cameraRedGain = 11; + if (it.cameraBlueGain == -1) it.cameraBlueGain = 20; }); } From 0bb3f02a33e560f9d273dc4d02a31fe14f8ca3e5 Mon Sep 17 00:00:00 2001 From: totaltaxamount Date: Sat, 25 Oct 2025 13:29:10 -0600 Subject: [PATCH 7/7] basler: fix multi camera --- photon-client/src/views/CameraMatchingView.vue | 12 +++++++++--- .../org/photonvision/vision/camera/PVCameraInfo.java | 2 +- .../photonvision/vision/processes/VisionModule.java | 8 +++++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/photon-client/src/views/CameraMatchingView.vue b/photon-client/src/views/CameraMatchingView.vue index 6dabc3d92c..ccda5768c4 100644 --- a/photon-client/src/views/CameraMatchingView.vue +++ b/photon-client/src/views/CameraMatchingView.vue @@ -222,7 +222,7 @@ const yesDeleteMySettingsText = ref(""); /** * Get the connection-type-specific camera info from the given PVCameraInfo object. */ -const cameraInfoFor = (camera: PVCameraInfo | null): PVUsbCameraInfo | PVCSICameraInfo | PVFileCameraInfo | any => { +const cameraInfoFor = (camera: PVCameraInfo | null): PVUsbCameraInfo | PVCSICameraInfo | PVFileCameraInfo | PVBaslerCameraInfo | any => { if (!camera) return null; if (camera.PVUsbCameraInfo) { return camera.PVUsbCameraInfo; @@ -233,6 +233,9 @@ const cameraInfoFor = (camera: PVCameraInfo | null): PVUsbCameraInfo | PVCSICame if (camera.PVFileCameraInfo) { return camera.PVFileCameraInfo; } + if (camera.PVBaslerCameraInfo) { + return camera.PVBaslerCameraInfo; + } return {}; }; @@ -244,7 +247,8 @@ const getMatchedDevice = (info: PVCameraInfo | undefined): PVCameraInfo => { return { PVFileCameraInfo: undefined, PVCSICameraInfo: undefined, - PVUsbCameraInfo: undefined + PVUsbCameraInfo: undefined, + PVBaslerCameraInfo: undefined, }; } return ( @@ -253,7 +257,8 @@ const getMatchedDevice = (info: PVCameraInfo | undefined): PVCameraInfo => { ) || { PVFileCameraInfo: undefined, PVCSICameraInfo: undefined, - PVUsbCameraInfo: undefined + PVUsbCameraInfo: undefined, + PVBaslerCameraInfo: undefined, } ); }; @@ -478,6 +483,7 @@ const getMatchedDevice = (info: PVCameraInfo | undefined): PVCameraInfo => { USB Camera: CSI Camera: File Camera: + Basler Camera: Unknown Camera:  {{ cameraInfoFor(camera)?.name ?? cameraInfoFor(camera)?.baseName }} diff --git a/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java b/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java index 3b2a3705c4..5c58187988 100644 --- a/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java +++ b/photon-core/src/main/java/org/photonvision/vision/camera/PVCameraInfo.java @@ -335,7 +335,7 @@ public boolean equals(Object obj) { if (obj == null) return false; if (!(obj instanceof PVBaslerCameraInfo info)) return false; - return this.model.equals(info.model) && serial.equals(info.serial); + return this.model.equals(info.model) && this.serial.equals(info.serial); } @Override diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java index 4e15eabb2f..379081ffa1 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/VisionModule.java @@ -492,13 +492,19 @@ boolean setPipeline(int index) { if (pipelineSettings.cameraBlueGain == -1) pipelineSettings.cameraBlueGain = 20; settables.setRedGain(Math.max(0, pipelineSettings.cameraRedGain)); settables.setBlueGain(Math.max(0, pipelineSettings.cameraBlueGain)); + + if (cameraQuirks.hasQuirk(CameraQuirk.ManualWB)) { + settables.setAutoWhiteBalance(pipelineSettings.cameraAutoWhiteBalance); + } } else { pipelineSettings.cameraRedGain = -1; pipelineSettings.cameraBlueGain = -1; // All other cameras (than picams) should support AWB temp settables.setWhiteBalanceTemp(pipelineSettings.cameraWhiteBalanceTemp); - settables.setAutoWhiteBalance(pipelineSettings.cameraAutoWhiteBalance); + + if (!cameraQuirks.hasQuirk(CameraQuirk.BaslerDaA1280Controls)) + settables.setAutoWhiteBalance(pipelineSettings.cameraAutoWhiteBalance); } setVisionLEDs(pipelineSettings.ledMode);