Gstreamer AppSource Implementation for AV playback on the desktop (#36)

* extract video pipeline into separate method, run go fmt

* added an example of a failure

* use a waveparser, works with an oggmux, just not hearing anything with an audiosink

* added vorbis hack, and now it works

* added queues to the video pipeline

* works

* use vtdec to bring down cpu usage

* add comments

* add fix for NSThread mainloop problem

* minor clean ups, add mac os only compile flag

* minor clean ups, add mac os only compile flag

* add linux adapter

* set sync to false for faster playback

* minor fixes

Author: danielpaulus

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815
 	github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750
 	github.com/lijo-jose/glib v0.0.0-20191012030101-93ee72d7d646
+	github.com/lijo-jose/gst v0.0.0-20191012030143-e3a5557394c7
 	github.com/pion/rtp v1.1.4
 	github.com/pion/webrtc/v2 v2.1.11
 	github.com/sirupsen/logrus v1.4.2

go.sum

@@ -23,6 +23,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lijo-jose/glib v0.0.0-20191012030101-93ee72d7d646 h1:7I8sylThkL59rDoHMANuIxtB490DvrLAIZosNY9fDMM=
 github.com/lijo-jose/glib v0.0.0-20191012030101-93ee72d7d646/go.mod h1:wzypjnJX+g/LKnKDVvJni/u0gNlQestTwv6Kt/Qf3fk=
+github.com/lijo-jose/gst v0.0.0-20191012030143-e3a5557394c7 h1:ZrDIsE60UtVmeYnTVzcarL+pCmk9KFoQebBDzyMm71o=
+github.com/lijo-jose/gst v0.0.0-20191012030143-e3a5557394c7/go.mod h1:CAqd8DUUbXc086paYmyPzUF0K42o6tFpIzoelwMioEI=
 github.com/lucas-clemente/quic-go v0.7.1-0.20190401152353-907071221cf9/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

qvh

@@ -113,25 +113,34 @@ func writeWavDataSubChunkHeader(target *bytes.Buffer, dataLength int) error {
 //WriteWavHeader creates a wave file header using the given length and writes it at the BEGINNING of the wavFile.
 //Please make sure that the file has enough zero bytes before the audio data.
 func WriteWavHeader(length int, wavFile *os.File) error {
+	headerBytes, err := GetWavHeaderBytes(length)
+	if err != nil {
+		return err
+	}
+	_, err = wavFile.WriteAt(headerBytes, 0)
+	return err
+}
+
+//GetWavHeaderBytes creates a byte slice containing a valid wav header using the supplied length.
+func GetWavHeaderBytes(length int) ([]byte, error) {
 	buffer := bytes.NewBuffer(make([]byte, 100))
 	buffer.Reset()
 
 	riffHeader := newRiffHeader(length)
 	err := riffHeader.serialize(buffer)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	fmtSubChunk := newFmtSubChunk()
 	err = fmtSubChunk.serialize(buffer)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	err = writeWavDataSubChunkHeader(buffer, length)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	_, err = wavFile.WriteAt(buffer.Bytes(), 0)
-	return err
+	return buffer.Bytes(), nil
 }

screencapture/coremedia/wav_format.go

@@ -32,8 +32,17 @@ func (fe *LengthFieldBasedFrameExtractor) ExtractFrame(bytes []byte) ([]byte, bo
 		return fe.handleNewFrame(bytes)
 	}
 	if fe.readyForNextFrame && fe.frameBuffer.Len() != 0 {
-		if fe.frameBuffer.Len()<4{
-			log.Fatal("wtf:"+hex.Dump(fe.frameBuffer.Bytes()))
+		if fe.frameBuffer.Len() < 4 {
+			/* examples:
+			FATA[0032] wtf:00000000  ac 10                                             |..|
+			*/
+			/*
+				{"level":"fatal","msg":"wtf:00000000  ac                                                |.|\n","time":"2019-11-18T12:09:10+01:00"}
+			*/
+			/*
+				{"level":"fatal","msg":"wtf:00000000  ac 10 00                                          |...|\n","time":"2019-11-18T12:12:11+01:00"}
+			*/
+			log.Fatal("wtf:" + hex.Dump(fe.frameBuffer.Bytes()))
 		}
 		fe.nextFrameSize = int(binary.LittleEndian.Uint32(fe.frameBuffer.Next(4))) - 4
 		fe.readyForNextFrame = false