How to send my video captured by Gocv to the Browser with WebRTC #2716
-
|
I want to capture my video using Gocv and send it to the Browser using WebRTC with VP8, similar to what was done in the example using mediadevices but I need it to be with Gocv, I tried using the code below but without success, there is no error but the video is there in the Browser does not play, I tested with other browsers but without success.
Your environment.
What did you do?Backendpackage main
import (
"fmt"
"time"
signal "github.com/Sup3r-Us3r/test-app/internal"
"github.com/pion/mediadevices"
"github.com/pion/mediadevices/pkg/codec/vpx"
"github.com/pion/webrtc/v3"
"github.com/pion/webrtc/v3/pkg/media"
"gocv.io/x/gocv"
)
func main() {
webcam, _ := gocv.OpenVideoCapture(0)
defer webcam.Close()
webRTCConfig := webrtc.Configuration{
ICEServers: []webrtc.ICEServer{
{
URLs: []string{"stun:stun.l.google.com:19302"},
},
},
}
// Wait for the offer to be pasted
offer := webrtc.SessionDescription{}
signal.Decode(signal.MustReadStdin(), &offer)
// Create a new RTCPeerConnection
vp8Params, err := vpx.NewVP8Params()
if err != nil {
panic(err)
}
vp8Params.BitRate = 500_000 // 500kbps
codecSelector := mediadevices.NewCodecSelector(
mediadevices.WithVideoEncoders(&vp8Params),
)
// Configure WebRTC Connection
mediaEngine := webrtc.MediaEngine{}
mediaEngine.RegisterDefaultCodecs()
codecSelector.Populate(&mediaEngine)
api := webrtc.NewAPI(webrtc.WithMediaEngine(&mediaEngine))
peerConnection, _ := api.NewPeerConnection(webRTCConfig)
// Set the handler for ICE connection state
// This will notify you when the peer has connected/disconnected
peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
fmt.Printf("Connection State has changed %s \n", connectionState.String())
})
// Create track video
videoTrack, _ := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8}, "video", "pion")
// Add track to PeerConnection
_, err = peerConnection.AddTrack(videoTrack)
if err != nil {
fmt.Println("\nERROR ADD TRACK: ", err)
}
// Set the remote SessionDescription
err = peerConnection.SetRemoteDescription(offer)
if err != nil {
panic(err)
}
// Create an answer
answer, err := peerConnection.CreateAnswer(nil)
if err != nil {
panic(err)
}
// Create channel that is blocked until ICE Gathering is complete
gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
// Sets the LocalDescription, and starts our UDP listeners
err = peerConnection.SetLocalDescription(answer)
if err != nil {
panic(err)
}
// Block until ICE Gathering is complete, disabling trickle ICE
// we do this because we only can exchange one signaling message
// in a production application you should exchange ICE Candidates via OnICECandidate
<-gatherComplete
// Output the answer in base64 so we can paste it in browser
fmt.Println(signal.Encode(*peerConnection.LocalDescription()))
img := gocv.NewMat()
defer img.Close()
ticker := time.NewTicker(time.Second * 3)
defer ticker.Stop()
for {
if ok := webcam.Read(&img); !ok {
fmt.Printf("Device closed: %v\n", 0)
return
}
if img.Empty() {
continue
}
// Encode the frame in VP8
frame, _ := gocv.IMEncode(gocv.JPEGFileExt, img)
err = videoTrack.WriteSample(media.Sample{Data: frame.GetBytes(), Duration: time.Second})
if err != nil {
fmt.Println("\nERROR: ", err)
}
// select {
// case <-ticker.C:
// table := crc32.MakeTable(crc32.IEEE)
// checksum := crc32.Checksum([]byte(videoTrack.StreamID()), table)
// if rtcpSendErr := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: checksum}}); rtcpSendErr != nil {
// fmt.Println(rtcpSendErr)
// }
// default:
// }
}
}Clientconst pc = new RTCPeerConnection({
iceServers: [
{
urls: 'stun:stun.l.google.com:19302',
},
],
});
const log = (msg) => {
document.getElementById('logs').innerHTML += msg + '<br>';
};
pc.ontrack = (event) => {
const el = document.createElement(event.track.kind);
el.srcObject = event.streams[0];
el.autoplay = true;
el.controls = true;
document.getElementById('remoteVideos').appendChild(el);
};
pc.oniceconnectionstatechange = (e) => log(pc.iceConnectionState);
pc.onicecandidate = (event) => {
if (event.candidate === null) {
document.getElementById('localSessionDescription').value = btoa(
JSON.stringify(pc.localDescription)
);
}
};
// Offer to receive 1 audio, and 1 video tracks
pc.addTransceiver('video', {
direction: 'recvonly',
});
pc.createOffer()
.then((d) => pc.setLocalDescription(d))
.catch(log);
window.startSession = () => {
const sd = document.getElementById('remoteSessionDescription').value;
if (sd === '') {
return alert('Session Description must not be empty');
}
try {
pc.setRemoteDescription(new RTCSessionDescription(JSON.parse(atob(sd))));
} catch (e) {
alert(e);
}
}; |
Beta Was this translation helpful? Give feedback.
Replies: 3 comments 11 replies
-
|
Hi @maydersonmellops. Sorry I didn't answer this month backs. Just getting around to finally triaging things. Looking at the code my assumption was that you weren't properly encoding to VP8. Hopefully you figured it out! If you are still struggling I would love to help though! |
Beta Was this translation helpful? Give feedback.
-
|
May I ask if you have solved this problem? I have also encountered this problem now. I have established a connection and the front-end web also shows that the stream has been received, but the video has not been played |
Beta Was this translation helpful? Give feedback.
-
|
Hello @qidailove I couldn't solve this either, I still have the same problem, I tried to encode to VP8 but without success, @Sean-Der do you have any example for this process? |
Beta Was this translation helpful? Give feedback.
-
|
Hello, I have also tried using ffmpeg to convert JPEG image data into VP8 format encoded data, but the front-end page still does not display it |
Beta Was this translation helpful? Give feedback.
-
|
Got it, let's see if @Sean-Der can support us with this. |
Beta Was this translation helpful? Give feedback.
-
|
yeah |
Beta Was this translation helpful? Give feedback.
-
|
I have been struggling with the same issue I have also implemented it but the gocv frames doesn't seem to convert it to the webrtc supported codecs, can someone help me with this ? |
Beta Was this translation helpful? Give feedback.
-
|
That would be really cool, thanks in advance |
Beta Was this translation helpful? Give feedback.
-
|
Hey so here is the simple way I was able to achieve this, well I wouldn't recommend this way for production as it increases the computation overhead package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/pion/webrtc/v4/pkg/media/ivfreader"
"io"
"log"
"net/http"
"os/exec"
"time"
"github.com/pion/webrtc/v4"
"github.com/pion/webrtc/v4/pkg/media"
gocv "gocv.io/x/gocv"
)
func main() {
// Serve the static/ folder on http://localhost:8080
http.Handle("/", http.FileServer(http.Dir("static")))
// POST /offer will handle the browser's WebRTC offer
http.HandleFunc("/offer", handleOffer)
fmt.Println("Listening on http://localhost:8080")
log.Fatal(http.ListenAndServe(":8080", nil))
}
func handleOffer(w http.ResponseWriter, r *http.Request) {
// 1) Read the Offer from the browser
var offer webrtc.SessionDescription
if err := json.NewDecoder(r.Body).Decode(&offer); err != nil {
http.Error(w, "invalid offer", http.StatusBadRequest)
return
}
// 2) Create a new PeerConnection
peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{
ICEServers: []webrtc.ICEServer{
{URLs: []string{"stun:stun.l.google.com:19302"}},
},
})
if err != nil {
http.Error(w, "failed to create PeerConnection", http.StatusInternalServerError)
return
}
// 3) Create a video track for VP8
videoTrack, err := webrtc.NewTrackLocalStaticSample(
webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8},
"video", // track id
"gocv", // stream id
)
if err != nil {
http.Error(w, "failed to create video track", http.StatusInternalServerError)
return
}
// Add the track to the PeerConnection
rtpSender, err := peerConnection.AddTrack(videoTrack)
if err != nil {
http.Error(w, "failed to add track", http.StatusInternalServerError)
return
}
// Read RTCP (for NACK, etc.) in a separate goroutine
go func() {
rtcpBuf := make([]byte, 1500)
for {
if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
return
}
}
}()
// 4) Watch for ICE connection state
iceConnectedCtx, iceConnectedCancel := context.WithCancel(context.Background())
peerConnection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) {
log.Printf("ICE state: %s\n", state)
if state == webrtc.ICEConnectionStateConnected {
iceConnectedCancel()
}
})
// 5) Set the remote description (the browser's Offer)
if err := peerConnection.SetRemoteDescription(offer); err != nil {
http.Error(w, "failed to set remote desc", http.StatusInternalServerError)
return
}
// 6) Create an Answer
answer, err := peerConnection.CreateAnswer(nil)
if err != nil {
http.Error(w, "failed to create answer", http.StatusInternalServerError)
return
}
// 7) Gather ICE candidates
gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
if err := peerConnection.SetLocalDescription(answer); err != nil {
http.Error(w, "failed to set local desc", http.StatusInternalServerError)
return
}
<-gatherComplete
// 8) Write the Answer back to the browser
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(peerConnection.LocalDescription())
// 9) Once ICE is connected, start reading frames from the camera via GoCV,
// pipe them into FFmpeg for VP8 encoding, and push the IVF frames into the track.
go func() {
<-iceConnectedCtx.Done()
if err := startCameraAndStream(videoTrack); err != nil {
log.Printf("camera streaming error: %v\n", err)
}
}()
}
// startCameraAndStream opens the webcam with GoCV, sends raw frames to FFmpeg (via stdin),
// reads IVF from FFmpeg (via stdout), and writes them into the WebRTC video track.
func startCameraAndStream(videoTrack *webrtc.TrackLocalStaticSample) error {
// Open default camera with GoCV
webcam, err := gocv.OpenVideoCapture(2)
if err != nil {
return fmt.Errorf("cannot open camera: %w", err)
}
defer webcam.Close()
// Set some camera settings if needed
// e.g. webcam.Set(gocv.VideoCaptureFrameWidth, 640)
// webcam.Set(gocv.VideoCaptureFrameHeight, 480)
// Or rely on defaults
// Prepare FFmpeg cmd:
// -f rawvideo: We feed raw frames
// -pixel_format bgr24: Our GoCV frames come in BGR format
// -video_size 640x480: must match your actual capture size
// -i pipe:0 : read from stdin
// Then encode with libvpx -> IVF on stdout
ffmpeg := exec.Command(
"ffmpeg",
"-y",
"-f", "rawvideo",
"-pixel_format", "bgr24",
"-video_size", "640x480",
"-framerate", "30", // assume ~30fps
"-i", "pipe:0",
"-c:v", "libvpx",
"-b:v", "1M",
"-f", "ivf",
"pipe:1",
)
stdin, err := ffmpeg.StdinPipe()
if err != nil {
return fmt.Errorf("ffmpeg stdin error: %w", err)
}
stdout, err := ffmpeg.StdoutPipe()
if err != nil {
return fmt.Errorf("ffmpeg stdout error: %w", err)
}
// Start FFmpeg
if err := ffmpeg.Start(); err != nil {
return fmt.Errorf("failed to start ffmpeg: %w", err)
}
// Goroutine to write raw frames to FFmpeg stdin
go func() {
defer stdin.Close()
frame := gocv.NewMat()
defer frame.Close()
ticker := time.NewTicker(time.Millisecond * 33) // ~30fps
defer ticker.Stop()
for range ticker.C {
if ok := webcam.Read(&frame); !ok {
log.Println("cannot read frame from camera")
continue
}
if frame.Empty() {
continue
}
// (Optional) do any OpenCV processing on `frame` here
// Write raw BGR bytes to FFmpeg
// frame.DataPtrUint8() points to the underlying byte array
_, _ = stdin.Write(frame.ToBytes())
}
}()
// Read IVF from FFmpeg stdout; parse frames with ivfreader
ivf, _, err := ivfreader.NewWith(stdout)
if err != nil {
return fmt.Errorf("ivfreader init error: %w", err)
}
// Loop reading IVF frames; push them to the video track
for {
frame, _, err := ivf.ParseNextFrame()
if errors.Is(err, io.EOF) {
log.Println("ffmpeg ended (EOF)")
break
}
if err != nil {
return fmt.Errorf("ivf parse error: %w", err)
}
// Deliver the VP8 frame
writeErr := videoTrack.WriteSample(media.Sample{
Data: frame,
Duration: time.Second / 30,
})
if writeErr != nil {
return fmt.Errorf("write sample error: %w", writeErr)
}
}
// Wait for ffmpeg to exit
if err := ffmpeg.Wait(); err != nil {
return fmt.Errorf("ffmpeg wait error: %w", err)
}
return nil
}adjust your video port accordingly this implemented is a server which it waits for the offer from the frontend make static folder and in that index.html static/index.html in the root <!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>GoCV + FFmpeg + Pion WebRTC</title>
</head>
<body>
<h2>GoCV + WebRTC Demo</h2>
<button id="btnStart">Start Session</button>
<h3>Remote Video</h3>
<div id="remoteVideos"></div>
<h3>Logs</h3>
<div id="log"></div>
<script>
const logDiv = document.getElementById('log');
function log(msg) {
logDiv.innerHTML += msg + '<br>';
}
document.getElementById('btnStart').onclick = async () => {
const pc = new RTCPeerConnection({
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
});
pc.oniceconnectionstatechange = () => {
log('ICE state: ' + pc.iceConnectionState);
};
// When the server sends a video track, display it
pc.ontrack = (event) => {
const el = document.createElement(event.track.kind);
el.srcObject = event.streams[0];
el.autoplay = true;
el.controls = true;
document.getElementById('remoteVideos').appendChild(el);
};
// We want to receive (only) a video track from the server
pc.addTransceiver('video', { direction: 'recvonly' });
// 1) Create an offer
log('Creating offer...');
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);
// 2) Wait for ICE gathering to finish
await new Promise((resolve) => {
if (pc.iceGatheringState === 'complete') {
resolve();
} else {
pc.onicegatheringstatechange = () => {
if (pc.iceGatheringState === 'complete') {
resolve();
}
};
}
});
// 3) POST the offer to the server
log('Sending offer to server...');
const resp = await fetch('/offer', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(pc.localDescription)
});
if (!resp.ok) {
const errText = await resp.text();
throw new Error('Error from server: ' + errText);
}
// 4) Get the server's answer and set it
const answer = await resp.json();
log('Received answer, setting remote description...');
await pc.setRemoteDescription(answer);
log('WebRTC negotiation complete! Waiting for remote video...');
};
</script>
</body>
</html>So hopefully this will work for you guys there was another thing media devices in pion that could potentially get gocv frames but not able work that out |
Beta Was this translation helpful? Give feedback.
-
|
Oh damn @FaranMustafa that's amazing! Would you be interested in making a PR to GitHub.com/pion/example-webrtc-applications, or I can do it! |
Beta Was this translation helpful? Give feedback.
-
I’d be glad to do that. Just give me a bit — I’ll make a PR for it. |
Beta Was this translation helpful? Give feedback.
-
Made the PR for it - pion/example-webrtc-applications#318, Let me know if it needs any changes. |
Beta Was this translation helpful? Give feedback.
Hi @maydersonmellops. Sorry I didn't answer this month backs. Just getting around to finally triaging things.
Looking at the code my assumption was that you weren't properly encoding to VP8.
frame, _ := gocv.IMEncode(gocv.JPEGFileExt, img)is taking it to jpeg?Hopefully you figured it out! If you are still struggling I would love to help though!