Skip to content

Commit d16e05d

Browse files
SamCarlbergSamCarlberg
committed
Use cscore for PublishVideoOperation
Updates JavaCV to 1.3.3 and javacpp-opencv to 3.2.0
1 parent 316458a commit d16e05d

File tree

5 files changed

+94
-124
lines changed

5 files changed

+94
-124
lines changed

‎build.gradle

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -225,10 +225,10 @@ project(":core") {
225225

226226
dependencies {
227227
compile group: 'com.google.code.findbugs', name: 'jsr305', version: '3.0.1'
228-
compile group: 'org.bytedeco', name: 'javacv', version: '1.1'
229-
compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.0.0-1.1'
230-
compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.0.0-1.1', classifier: os
231-
compile group: 'org.bytedeco.javacpp-presets', name: 'opencv-3.0.0-1.1', classifier: 'linux-frc'
228+
compile group: 'org.bytedeco', name: 'javacv', version: '1.3.3'
229+
compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.2.0-1.3'
230+
compile group: 'org.bytedeco.javacpp-presets', name: 'opencv', version: '3.2.0-1.3', classifier: os
231+
//compile group: 'org.bytedeco.javacpp-presets', name: 'opencv-3.0.0-1.1', classifier: 'linux-frc'
232232
compile group: 'org.bytedeco.javacpp-presets', name: 'videoinput', version: '0.200-1.1', classifier: os
233233
compile group: 'org.bytedeco.javacpp-presets', name: 'ffmpeg', version: '0.200-1.1', classifier: os
234234
compile group: 'org.python', name: 'jython', version: '2.7.0'
@@ -253,6 +253,13 @@ project(":core") {
253253
compile group: 'org.ros.rosjava_messages', name: 'grip_msgs', version: '0.0.1'
254254
compile group: 'edu.wpi.first.wpilib.networktables.java', name: 'NetworkTables', version: '3.1.2', classifier: 'desktop'
255255
compile group: 'edu.wpi.first.wpilib.networktables.java', name: 'NetworkTables', version: '3.1.2', classifier: 'arm'
256+
257+
// cscore dependencies
258+
compile group: 'edu.wpi.first.cscore', name: 'cscore-java', version: '1.1.0-beta-2'
259+
compile group: 'edu.wpi.first.cscore', name: 'cscore-jni', version: '1.1.0-beta-2', classifier: 'all'
260+
runtime group: 'edu.wpi.first.wpiutil', name: 'wpiutil-java', version: '3.+'
261+
compile group: 'org.opencv', name: 'opencv-java', version: '3.2.0'
262+
compile group: 'org.opencv', name: 'opencv-jni', version: '3.2.0', classifier: 'all'
256263
}
257264

258265
mainClassName = 'edu.wpi.grip.core.Main'

‎core/src/main/java/edu/wpi/grip/core/operations/composite/PublishVideoOperation.java

Lines changed: 75 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -10,34 +10,42 @@
1010
import com.google.common.collect.ImmutableList;
1111

1212
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
13+
import edu.wpi.cscore.CameraServerJNI;
14+
import edu.wpi.cscore.CvSource;
15+
import edu.wpi.cscore.MjpegServer;
16+
import edu.wpi.cscore.VideoMode;
17+
import edu.wpi.first.wpilibj.networktables.NetworkTable;
1318

14-
import org.bytedeco.javacpp.BytePointer;
15-
import org.bytedeco.javacpp.IntPointer;
19+
import org.bytedeco.javacpp.opencv_core;
20+
import org.opencv.core.Mat;
1621

17-
import java.io.DataInputStream;
18-
import java.io.DataOutputStream;
19-
import java.io.IOException;
20-
import java.net.ServerSocket;
21-
import java.net.Socket;
22+
import java.lang.reflect.Field;
2223
import java.util.List;
2324
import java.util.logging.Level;
2425
import java.util.logging.Logger;
2526

26-
import static org.bytedeco.javacpp.opencv_core.Mat;
27-
import static org.bytedeco.javacpp.opencv_imgcodecs.CV_IMWRITE_JPEG_QUALITY;
28-
import static org.bytedeco.javacpp.opencv_imgcodecs.imencode;
27+
import static org.bytedeco.javacpp.opencv_core.CV_8S;
28+
import static org.bytedeco.javacpp.opencv_core.CV_8U;
2929

3030
/**
3131
* Publish an M-JPEG stream with the protocol used by SmartDashboard and the FRC Dashboard. This
3232
* allows FRC teams to view video streams on their dashboard during competition even when GRIP has
33-
* exclusive access to the camera. In addition, an intermediate processed image in the pipeline
34-
* could be published instead. Based on WPILib's CameraServer class:
35-
* https://github.com/robotpy/allwpilib/blob/master/wpilibj/src/athena/java/edu/wpi/first/wpilibj
36-
* /CameraServer.java
33+
* exclusive access to the camera. Uses cscore to host the image streaming server.
3734
*/
3835
public class PublishVideoOperation implements Operation {
3936

4037
private static final Logger logger = Logger.getLogger(PublishVideoOperation.class.getName());
38+
39+
static {
40+
try {
41+
// Loading the CameraServerJNI class will load the appropriate platform-specific OpenCV JNI
42+
CameraServerJNI.getHostname();
43+
} catch (Throwable e) {
44+
logger.log(Level.SEVERE, "CameraServerJNI load failed! Exiting", e);
45+
System.exit(31);
46+
}
47+
}
48+
4149
public static final OperationDescription DESCRIPTION =
4250
OperationDescription.builder()
4351
.name("Publish Video")
@@ -46,110 +54,40 @@ public class PublishVideoOperation implements Operation {
4654
.icon(Icon.iconStream("publish-video"))
4755
.build();
4856
private static final int PORT = 1180;
49-
private static final byte[] MAGIC_NUMBER = {0x01, 0x00, 0x00, 0x00};
5057

5158
@SuppressWarnings("PMD.AssignmentToNonFinalStatic")
5259
private static int numSteps;
53-
private final Object imageLock = new Object();
54-
private final BytePointer imagePointer = new BytePointer();
55-
private final Thread serverThread;
56-
private final InputSocket<Mat> inputSocket;
60+
private static final int MAX_STEP_COUNT = 10;
61+
62+
private final InputSocket<opencv_core.Mat> inputSocket;
5763
private final InputSocket<Number> qualitySocket;
58-
@SuppressWarnings("PMD.SingularField")
59-
private volatile boolean connected = false;
60-
/**
61-
* Listens for incoming connections on port 1180 and writes JPEG data whenever there's a new
62-
* frame.
63-
*/
64-
private final Runnable runServer = () -> {
65-
// Loop forever (or at least until the thread is interrupted). This lets us recover from the
66-
// dashboard
67-
// disconnecting or the network connection going away temporarily.
68-
while (!Thread.currentThread().isInterrupted()) {
69-
try (ServerSocket serverSocket = new ServerSocket(PORT)) {
70-
logger.info("Starting camera server");
71-
72-
try (Socket socket = serverSocket.accept()) {
73-
logger.info("Got connection from " + socket.getInetAddress());
74-
connected = true;
75-
76-
DataOutputStream socketOutputStream = new DataOutputStream(socket.getOutputStream());
77-
DataInputStream socketInputStream = new DataInputStream(socket.getInputStream());
78-
79-
byte[] buffer = new byte[128 * 1024];
80-
int bufferSize;
81-
82-
final int fps = socketInputStream.readInt();
83-
final int compression = socketInputStream.readInt();
84-
final int size = socketInputStream.readInt();
85-
86-
if (compression != -1) {
87-
logger.warning("Dashboard video should be in HW mode");
88-
}
89-
90-
final long frameDuration = 1000000000L / fps;
91-
long startTime = System.nanoTime();
92-
93-
while (!socket.isClosed() && !Thread.currentThread().isInterrupted()) {
94-
// Wait for the main thread to put a new image. This happens whenever perform() is
95-
// called with
96-
// a new input.
97-
synchronized (imageLock) {
98-
imageLock.wait();
99-
100-
// Copy the image data into a pre-allocated buffer, growing it if necessary
101-
bufferSize = imagePointer.limit();
102-
if (bufferSize > buffer.length) {
103-
buffer = new byte[imagePointer.limit()];
104-
}
105-
imagePointer.get(buffer, 0, bufferSize);
106-
}
107-
108-
// The FRC dashboard image protocol consists of a magic number, the size of the image
109-
// data,
110-
// and the image data itself.
111-
socketOutputStream.write(MAGIC_NUMBER);
112-
socketOutputStream.writeInt(bufferSize);
113-
socketOutputStream.write(buffer, 0, bufferSize);
114-
115-
// Limit the FPS to whatever the dashboard requested
116-
int remainingTime = (int) (frameDuration - (System.nanoTime() - startTime));
117-
if (remainingTime > 0) {
118-
Thread.sleep(remainingTime / 1000000, remainingTime % 1000000);
119-
}
120-
121-
startTime = System.nanoTime();
122-
}
123-
}
124-
} catch (IOException e) {
125-
logger.log(Level.WARNING, e.getMessage(), e);
126-
} catch (InterruptedException e) {
127-
Thread.currentThread().interrupt(); // This is really unnecessary since the thread is
128-
// about to exit
129-
logger.info("Shutting down camera server");
130-
return;
131-
} finally {
132-
connected = false;
133-
}
134-
}
135-
};
64+
private final MjpegServer server;
65+
private final CvSource serverSource;
66+
private static final NetworkTable cameraPublisherTable =
67+
NetworkTable.getTable("/CameraPublisher");
68+
private final Mat publishMat = new Mat();
69+
private long lastFrame = -1;
13670

13771
@SuppressWarnings("JavadocMethod")
13872
@SuppressFBWarnings(value = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD",
13973
justification = "Do not need to synchronize inside of a constructor")
14074
public PublishVideoOperation(InputSocket.Factory inputSocketFactory) {
141-
if (numSteps != 0) {
142-
throw new IllegalStateException("Only one instance of PublishVideoOperation may exist");
75+
if (numSteps >= MAX_STEP_COUNT) {
76+
throw new IllegalStateException(
77+
"Only " + MAX_STEP_COUNT + " instances of PublishVideoOperation may exist");
14378
}
14479
this.inputSocket = inputSocketFactory.create(SocketHints.Inputs.createMatSocketHint("Image",
14580
false));
14681
this.qualitySocket = inputSocketFactory.create(SocketHints.Inputs
14782
.createNumberSliderSocketHint("Quality", 80, 0, 100));
148-
numSteps++;
14983

150-
serverThread = new Thread(runServer, "Camera Server");
151-
serverThread.setDaemon(true);
152-
serverThread.start();
84+
server = new MjpegServer("GRIP server " + numSteps, PORT + numSteps);
85+
serverSource = new CvSource("GRIP CvSource" + numSteps, VideoMode.PixelFormat.kMJPEG, 0, 0, 0);
86+
server.setSource(serverSource);
87+
cameraPublisherTable.putStringArray("streams",
88+
new String[]{CameraServerJNI.getHostname() + ":" + server.getPort()});
89+
90+
numSteps++;
15391
}
15492

15593
@Override
@@ -167,25 +105,49 @@ public List<OutputSocket> getOutputSockets() {
167105

168106
@Override
169107
public void perform() {
170-
if (!connected) {
171-
return; // Don't waste any time converting images if there's no dashboard connected
172-
}
173-
174-
if (inputSocket.getValue().get().empty()) {
108+
final long now = System.nanoTime();
109+
opencv_core.Mat input = inputSocket.getValue().get();
110+
if (input.empty() || input.isNull()) {
175111
throw new IllegalArgumentException("Input image must not be empty");
176112
}
177113

178-
synchronized (imageLock) {
179-
imencode(".jpeg", inputSocket.getValue().get(), imagePointer,
180-
new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue()));
181-
imageLock.notifyAll();
114+
copyJavaCvToOpenCvMat(input, publishMat);
115+
serverSource.putFrame(publishMat);
116+
if (lastFrame != -1) {
117+
long dt = now - lastFrame;
118+
serverSource.setFPS((int) (1e9 / dt));
182119
}
120+
lastFrame = now;
121+
server.setSource(serverSource);
183122
}
184123

185124
@Override
186125
public synchronized void cleanUp() {
187126
// Stop the video server if there are no Publish Video steps left
188-
serverThread.interrupt();
189127
numSteps--;
190128
}
129+
130+
private void copyJavaCvToOpenCvMat(opencv_core.Mat javaCvMat, Mat openCvMat) {
131+
if (javaCvMat.depth() != CV_8U && javaCvMat.depth() != CV_8S) {
132+
throw new IllegalArgumentException("Only 8-bit depth images are supported");
133+
}
134+
135+
final opencv_core.Size size = javaCvMat.size();
136+
137+
// Make sure the output resolution is up to date
138+
serverSource.setResolution(size.width(), size.height());
139+
140+
// Make the OpenCV Mat object point to the same block of memory as the JavaCV object.
141+
// This requires no data transfers or copies and is O(1) instead of O(n)
142+
if (javaCvMat.address() != openCvMat.nativeObj) {
143+
try {
144+
Field nativeObjField = Mat.class.getField("nativeObj");
145+
nativeObjField.setAccessible(true);
146+
nativeObjField.setLong(openCvMat, javaCvMat.address());
147+
} catch (ReflectiveOperationException e) {
148+
logger.log(Level.WARNING, "Could not set native object pointer", e);
149+
}
150+
}
151+
}
152+
191153
}

‎core/src/main/java/edu/wpi/grip/core/operations/composite/SaveImageOperation.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,9 @@ public void perform() {
124124
imencode("." + fileTypesSocket.getValue().get(), inputSocket.getValue().get(), imagePointer,
125125
new IntPointer(CV_IMWRITE_JPEG_QUALITY, qualitySocket.getValue().get().intValue()));
126126
byte[] buffer = new byte[128 * 1024];
127-
int bufferSize = imagePointer.limit();
127+
int bufferSize = (int) imagePointer.limit();
128128
if (bufferSize > buffer.length) {
129-
buffer = new byte[imagePointer.limit()];
129+
buffer = new byte[bufferSize];
130130
}
131131
imagePointer.get(buffer, 0, bufferSize);
132132

‎core/src/main/java/edu/wpi/grip/core/operations/opencv/MinMaxLoc.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import com.google.common.collect.ImmutableList;
1010

11+
import org.bytedeco.javacpp.DoublePointer;
1112
import org.bytedeco.javacpp.opencv_core;
1213
import org.bytedeco.javacpp.opencv_core.Mat;
1314
import org.bytedeco.javacpp.opencv_core.Point;
@@ -85,14 +86,14 @@ public void perform() {
8586
if (mask.empty()) {
8687
mask = null;
8788
}
88-
final double[] minVal = new double[1];
89-
final double[] maxVal = new double[1];
89+
DoublePointer minVal = new DoublePointer(0.0);
90+
DoublePointer maxVal = new DoublePointer(0.0);
9091
final Point minLoc = minLocSocket.getValue().get();
9192
final Point maxLoc = maxLocSocket.getValue().get();
9293

9394
opencv_core.minMaxLoc(src, minVal, maxVal, minLoc, maxLoc, mask);
94-
minValSocket.setValue(minVal[0]);
95-
maxValSocket.setValue(maxVal[0]);
95+
minValSocket.setValue(minVal.get(0));
96+
maxValSocket.setValue(maxVal.get(0));
9697
minLocSocket.setValue(minLocSocket.getValue().get());
9798
maxLocSocket.setValue(maxLocSocket.getValue().get());
9899
}

‎core/src/test/java/edu/wpi/grip/core/sources/CameraSourceTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ static class MockFrameGrabber extends FrameGrabber {
281281
for (int y = 0; y < frameIdx.rows(); y++) {
282282
for (int x = 0; x < frameIdx.cols(); x++) {
283283
for (int z = 0; z < frameIdx.channels(); z++) {
284-
frameIdx.putDouble(new int[]{y, x, z}, y + x + z);
284+
frameIdx.putDouble(new long[]{y, x, z}, y + x + z);
285285
}
286286
}
287287
}

0 commit comments

Comments
 (0)