Add support for custom gst pipelines (#40)

* add custom pipeline parsing

* add custom pipeline support to CLI interface

* add appsrces to pipeline, add example

* add support for custom pipelines, add EOS signal support, add first example of mp4 pipeline (audio does not work here, will figure out why)

* add constants to linux code

* move fixes to linux code

* move fixes to linux code

* add working example for writing to mp4 file

* add working mac os example

* add hack to wait long enough for MOOV box to be written, add working linux example

* wait for EOS signal instead of sleeping a second

* add Stop to interface, increase timeout

* increase timeout

* fix test

* bump version, update readme

Author: danielpaulus

README.md

@@ -24,13 +24,13 @@ Currently you can use it to dump a h264 file and a wave file or mirror your devi
 
 ## 3. Usage
 ```
-Q.uickTime V.ideo H.ack (qvh) v0.2-beta
+Q.uickTime V.ideo H.ack (qvh) v0.3-beta
 
 Usage:
   qvh devices [-v]
   qvh activate [--udid=<udid>] [-v]
   qvh record <h264file> <wavfile> [-v] [--udid=<udid>]
-  qvh gstreamer [-v]
+  qvh gstreamer [--pipeline=<pipeline>] [--examples] [-v]
   qvh --version | version
 
 
@@ -41,12 +41,15 @@ Options:
   --udid=<udid>   UDID of the device. If not specified, the first found device will be used automatically.
 
 The commands work as following:
-	devices		lists iOS devices attached to this host and tells you if video streaming was activated for them
-	activate	enables the video streaming config for the device specified by --udid
-	record		will start video&audio recording. Video will be saved in a raw h264 file playable by VLC.
-				Audio will be saved in a uncompressed wav file.
-				Run like: "qvh record /home/yourname/out.h264 /home/yourname/out.wav"
-	gstreamer   qvh will open a new window and push AV data to gstreamer.
+        devices         lists iOS devices attached to this host and tells you if video streaming was activated for them
+        activate        enables the video streaming config for the device specified by --udid
+        record          will start video&audio recording. Video will be saved in a raw h264 file playable by VLC. 
+                     Audio will be saved in a uncompressed wav file. Run like: "qvh record /home/yourname/out.h264 /home/yourname/out.wav"
+
+        gstreamer   If no additional param is provided, qvh will open a new window and push AV data to gstreamer.
+                                If "qvh gstreamer --examples" is provided, qvh will print some common gstreamer pipeline examples.
+                                If --pipeline is provided, qvh will use the provided gstreamer pipeline instead of 
+                                displaying audio and video in a window. 
 ```
 
 ## 3. Technical Docs/ Roll your own implementation

go.mod

@@ -4,7 +4,7 @@ go 1.13
 
 require (
 	github.com/danielpaulus/go-ios v0.0.0-20190926190740-cc977db05eea
-	github.com/danielpaulus/gst v0.0.0-20191031112308-dfcba780ec66
+	github.com/danielpaulus/gst v0.0.0-20200201205042-e6d2974fceb8
 	github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815
 	github.com/google/gousb v0.0.0-20190812193832-18f4c1d8a750
 	github.com/lijo-jose/glib v0.0.0-20191012030101-93ee72d7d646

go.sum

@@ -4,6 +4,8 @@ github.com/danielpaulus/gst v0.0.0-20191031102428-eb100cb83442 h1:AKA48ahee1DRBV
 github.com/danielpaulus/gst v0.0.0-20191031102428-eb100cb83442/go.mod h1:JbhjLST5AaUXpKQK65g9144BK8QHftbpuFoYuhDuONw=
 github.com/danielpaulus/gst v0.0.0-20191031112308-dfcba780ec66 h1:UDoHlePd6+rNFfJ/s2ASd2MTRE0W1sWzCeujJVTjVKU=
 github.com/danielpaulus/gst v0.0.0-20191031112308-dfcba780ec66/go.mod h1:JbhjLST5AaUXpKQK65g9144BK8QHftbpuFoYuhDuONw=
+github.com/danielpaulus/gst v0.0.0-20200201205042-e6d2974fceb8 h1:+XiTgRoo1bCA3paC4e/0WYWI7+J2O7hR/IYuSikANMw=
+github.com/danielpaulus/gst v0.0.0-20200201205042-e6d2974fceb8/go.mod h1:JbhjLST5AaUXpKQK65g9144BK8QHftbpuFoYuhDuONw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

main.go

@@ -14,7 +14,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-const version = "v0.2-beta"
+const version = "v0.3-beta"
 
 func main() {
 	usage := fmt.Sprintf(`Q.uickTime V.ideo H.ack (qvh) %s
@@ -23,7 +23,7 @@ Usage:
   qvh devices [-v]
   qvh activate [--udid=<udid>] [-v]
   qvh record <h264file> <wavfile> [-v] [--udid=<udid>]
-  qvh gstreamer [-v]
+  qvh gstreamer [--pipeline=<pipeline>] [--examples] [-v]
   qvh --version | version
 
 
@@ -36,10 +36,13 @@ Options:
 The commands work as following:
 	devices		lists iOS devices attached to this host and tells you if video streaming was activated for them
 	activate	enables the video streaming config for the device specified by --udid
-	record		will start video&audio recording. Video will be saved in a raw h264 file playable by VLC.
-				Audio will be saved in a uncompressed wav file.
-				Run like: "qvh record /home/yourname/out.h264 /home/yourname/out.wav"
-	gstreamer   qvh will open a new window and push AV data to gstreamer.
+	record		will start video&audio recording. Video will be saved in a raw h264 file playable by VLC. 
+	             Audio will be saved in a uncompressed wav file. Run like: "qvh record /home/yourname/out.h264 /home/yourname/out.wav"
+
+	gstreamer   If no additional param is provided, qvh will open a new window and push AV data to gstreamer.
+				If "qvh gstreamer --examples" is provided, qvh will print some common gstreamer pipeline examples.
+				If --pipeline is provided, qvh will use the provided gstreamer pipeline instead of 
+				displaying audio and video in a window. 
   `, version)
 	arguments, _ := docopt.ParseDoc(usage)
 	log.SetFormatter(&log.JSONFormatter{})
@@ -87,7 +90,17 @@ The commands work as following:
 	}
 	gstreamerCommand, _ := arguments.Bool("gstreamer")
 	if gstreamerCommand {
-		startGStreamer(udid)
+		shouldPrintExamples, _ := arguments.Bool("--examples")
+		if shouldPrintExamples {
+			printExamples()
+			return
+		}
+		gstPipeline, _ := arguments.String("--pipeline")
+		if gstPipeline == "" {
+			startGStreamer(udid)
+			return
+		}
+		startGStreamerWithCustomPipeline(udid, gstPipeline)
 	}
 }
 
@@ -98,6 +111,42 @@ func printVersion() {
 	printJSON(versionMap)
 }
 
+func printExamples() {
+
+	examples := `Examples:
+	
+	Writing an MP4 file
+	This pipeline will save the recording in video.mp4 with h264 and aac format. The default settings 
+	of this pipeline will create a compressed video that takes up way less space than raw h264.
+	Note that you need to set "ignore-length" on the wavparse because we are streaming and do not know the length in advance.
+
+	Write MP4 file Mac OSX: 
+	vtdec is the hardware accelerated decoder on the mac. 
+
+	qvh gstreamer --pipeline "mp4mux name=mux ! filesink location=video.mp4 \
+	queue name=audio_target ! wavparse ignore-length=true ! audioconvert ! faac ! aacparse ! mux. \
+	queue name=video_target ! h264parse ! vtdec ! videoconvert ! x264enc  tune=zerolatency !  mux."
+	
+	Write MP4 file Linux:
+    note that I am using software en and decoding, if you have intel VAAPI available, maybe use those. 
+
+	gstreamer --pipeline "mp4mux name=mux ! filesink location=video.mp4 \
+    queue name=audio_target ! wavparse ignore-length=true ! audioconvert ! avenc_aac ! aacparse ! mux. \
+    queue name=video_target ! h264parse ! avdec_h264 ! videoconvert ! x264enc tune=zerolatency ! mux."
+	`
+	fmt.Print(examples)
+}
+
+func startGStreamerWithCustomPipeline(udid string, pipelineString string) {
+	log.Debug("Starting Gstreamer with custom pipeline")
+	gStreamer, err := gstadapter.NewWithCustomPipeline(pipelineString)
+	if err != nil {
+		printErrJSON(err, "Failed creating custom pipeline")
+		return
+	}
+	startWithConsumer(gStreamer, udid)
+}
+
 func startGStreamer(udid string) {
 	log.Debug("Starting Gstreamer")
 	gStreamer := gstadapter.New()
@@ -194,9 +243,11 @@ func startWithConsumer(consumer screencapture.CmSampleBufConsumer, udid string)
 	adapter := screencapture.UsbAdapter{}
 	stopSignal := make(chan interface{})
 	waitForSigInt(stopSignal)
+
 	mp := screencapture.NewMessageProcessor(&adapter, stopSignal, consumer)
 
 	err = adapter.StartReading(device, &mp, stopSignal)
+	consumer.Stop()
 	if err != nil {
 		printErrJSON(err, "failed connecting to usb")
 	}

screencapture/coremedia/avfilewriter.go

@@ -30,6 +30,9 @@ func (avfw AVFileWriter) Consume(buf CMSampleBuffer) error {
 	return avfw.consumeVideo(buf)
 }
 
+//Nothing currently
+func (avfw AVFileWriter) Stop() {}
+
 func (avfw AVFileWriter) consumeVideo(buf CMSampleBuffer) error {
 	if buf.HasFormatDescription {
 		err := avfw.writeNalu(buf.FormatDescription.PPS)

screencapture/gstadapter/gst_adapter_linux.go

@@ -18,9 +18,13 @@ import (
 type GstAdapter struct {
 	videoAppSrc      *gst.AppSrc
 	audioAppSrc      *gst.AppSrc
+	pipeline         *gst.Pipeline
 	firstAudioSample bool
 }
 
+const audioAppSrcTargetElementName = "audio_target"
+const videoAppSrcTargetElementName = "video_target"
+
 //New creates a new MAC OSX compatible gstreamer pipeline that will play device video and audio
 //in a nice little window :-D
 func New() *GstAdapter {
@@ -39,6 +43,79 @@ func New() *GstAdapter {
 	return &gsta
 }
 
+//NewWithCustomPipeline will parse the given pipelineString, connect the videoAppSrc to whatever element has the name "video_target" and the audioAppSrc to "audio_target"
+//see also: https://gstreamer.freedesktop.org/documentation/application-development/appendix/programs.html?gi-language=c
+func NewWithCustomPipeline(pipelineString string) (*GstAdapter, error) {
+	log.Info("Starting Gstreamer..")
+	log.WithFields(log.Fields{"custom_pipeline": pipelineString}).Debug("Starting Gstreamer with custom pipeline")
+	pipeline, err := gst.ParseLaunch(pipelineString)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid Pipeline, checkout --examples for help. Gstreamer parsing error was: %s", err)
+	}
+
+	audioAppSrcTargetElement := pipeline.AsBin().GetByName(audioAppSrcTargetElementName)
+	if audioAppSrcTargetElement == nil {
+		return nil, fmt.Errorf("The pipeline needs an element with a property 'name=%s' so I can link the audio source to it. run with --examples for details.", audioAppSrcTargetElementName)
+	}
+
+	videoAppSrcTargetElement := pipeline.AsBin().GetByName(videoAppSrcTargetElementName)
+	if videoAppSrcTargetElement == nil {
+		return nil, fmt.Errorf("The pipeline needs an element with a property 'name=%s' so I can link the video source to it. run with --examples for details.", videoAppSrcTargetElementName)
+	}
+
+	videoAppSrc := gst.NewAppSrc("my-video-src")
+	videoAppSrc.SetProperty("is-live", true)
+
+	audioAppSrc := gst.NewAppSrc("my-audio-src")
+	audioAppSrc.SetProperty("is-live", true)
+
+	pipeline.Add(videoAppSrc.AsElement())
+	pipeline.Add(audioAppSrc.AsElement())
+
+	audioAppSrc.Link(audioAppSrcTargetElement)
+	videoAppSrc.Link(videoAppSrcTargetElement)
+
+	pipeline.SetState(gst.STATE_PLAYING)
+	//runGlibMainLoop()
+
+	log.Info("Gstreamer is running!")
+	gsta := GstAdapter{videoAppSrc: videoAppSrc, audioAppSrc: audioAppSrc, firstAudioSample: true, pipeline: pipeline}
+
+	return &gsta, nil
+}
+
+//Stop sends an EOS (end of stream) event downstream the gstreamer pipeline.
+//Some Elements need this to correctly finish. F.ex. writing mp4 video without
+//sending EOS will result in a broken mp4 file
+func (gsta GstAdapter) Stop() {
+	log.Info("Stopping Gstreamer..")
+	success := gsta.audioAppSrc.SendEvent(gst.Eos())
+	if !success {
+		log.Warn("Failed sending EOS signal for audio app source")
+	}
+	success = gsta.videoAppSrc.SendEvent(gst.Eos())
+	if !success {
+		log.Warn("Failed sending EOS signal for video app source")
+	}
+
+	bus := gsta.pipeline.GetBus()
+
+	//I hope those are 60 seconds
+	msg := bus.TimedPopFiltered(1000000000*1000*60, gst.MESSAGE_EOS|gst.MESSAGE_ERROR)
+
+	if msg == nil {
+		log.Warn("No EOS received, video files might be broken")
+		return
+	}
+	if msg.GetType() == gst.MESSAGE_ERROR {
+		log.Warn("Error received, video files might be broken")
+		return
+	}
+	log.Info("EOS received")
+	gsta.pipeline.SetState(gst.STATE_NULL)
+	log.Info("Gstreamer finished")
+}
+
 //runGlibMainLoop starts the glib Mainloop necessary for the video player to work on MAC OS X.
 func runGlibMainLoop() {
 	go func() {

screencapture/gstadapter/gst_adapter_macos.go

@@ -18,9 +18,13 @@ import (
 type GstAdapter struct {
 	videoAppSrc      *gst.AppSrc
 	audioAppSrc      *gst.AppSrc
+	pipeline         *gst.Pipeline
 	firstAudioSample bool
 }
 
+const audioAppSrcTargetElementName = "audio_target"
+const videoAppSrcTargetElementName = "video_target"
+
 //New creates a new MAC OSX compatible gstreamer pipeline that will play device video and audio
 //in a nice little window :-D
 func New() *GstAdapter {
@@ -39,6 +43,78 @@ func New() *GstAdapter {
 	return &gsta
 }
 
+//NewWithCustomPipeline will parse the given pipelineString, connect the videoAppSrc to whatever element has the name "video_target" and the audioAppSrc to "audio_target"
+//see also: https://gstreamer.freedesktop.org/documentation/application-development/appendix/programs.html?gi-language=c
+func NewWithCustomPipeline(pipelineString string) (*GstAdapter, error) {
+	log.Info("Starting Gstreamer..")
+	log.WithFields(log.Fields{"custom_pipeline": pipelineString}).Debug("Starting Gstreamer with custom pipeline")
+	pipeline, err := gst.ParseLaunch(pipelineString)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid Pipeline, checkout --examples for help. Gstreamer parsing error was: %s", err)
+	}
+
+	audioAppSrcTargetElement := pipeline.AsBin().GetByName(audioAppSrcTargetElementName)
+	if audioAppSrcTargetElement == nil {
+		return nil, fmt.Errorf("The pipeline needs an element with a property 'name=%s' so I can link the audio source to it. run with --examples for details.", audioAppSrcTargetElementName)
+	}
+
+	videoAppSrcTargetElement := pipeline.AsBin().GetByName(videoAppSrcTargetElementName)
+	if videoAppSrcTargetElement == nil {
+		return nil, fmt.Errorf("The pipeline needs an element with a property 'name=%s' so I can link the video source to it. run with --examples for details.", videoAppSrcTargetElementName)
+	}
+
+	videoAppSrc := gst.NewAppSrc("my-video-src")
+	videoAppSrc.SetProperty("is-live", true)
+
+	audioAppSrc := gst.NewAppSrc("my-audio-src")
+	audioAppSrc.SetProperty("is-live", true)
+
+	pipeline.Add(videoAppSrc.AsElement())
+	pipeline.Add(audioAppSrc.AsElement())
+
+	audioAppSrc.Link(audioAppSrcTargetElement)
+	videoAppSrc.Link(videoAppSrcTargetElement)
+
+	pipeline.SetState(gst.STATE_PLAYING)
+	//runGlibMainLoop()
+
+	log.Info("Gstreamer is running!")
+	gsta := GstAdapter{videoAppSrc: videoAppSrc, audioAppSrc: audioAppSrc, firstAudioSample: true, pipeline: pipeline}
+
+	return &gsta, nil
+}
+
+//Stop sends an EOS (end of stream) event downstream the gstreamer pipeline.
+//Some Elements need this to correctly finish. F.ex. writing mp4 video without
+//sending EOS will result in a broken mp4 file
+func (gsta GstAdapter) Stop() {
+	log.Info("Stopping Gstreamer..")
+	success := gsta.audioAppSrc.SendEvent(gst.Eos())
+	if !success {
+		log.Warn("Failed sending EOS signal for audio app source")
+	}
+	success = gsta.videoAppSrc.SendEvent(gst.Eos())
+	if !success {
+		log.Warn("Failed sending EOS signal for video app source")
+	}
+
+	bus := gsta.pipeline.GetBus()
+
+	//I hope those are 60 seconds
+	msg := bus.TimedPopFiltered(1000000000*1000*60, gst.MESSAGE_EOS|gst.MESSAGE_ERROR)
+	if msg == nil {
+		log.Warn("No EOS received, video files might be broken")
+		return
+	}
+	if msg.GetType() == gst.MESSAGE_ERROR {
+		log.Warn("Error received, video files might be broken")
+		return
+	}
+	log.Info("EOS received")
+	gsta.pipeline.SetState(gst.STATE_NULL)
+	log.Info("Gstreamer finished")
+}
+
 //runGlibMainLoop starts the glib Mainloop necessary for the video player to work on MAC OS X.
 func runGlibMainLoop() {
 	go func() {

screencapture/gstadapter/gst_adapter_test.go

@@ -0,0 +1,24 @@
+package gstadapter_test
+
+import (
+	"testing"
+
+	"github.com/danielpaulus/quicktime_video_hack/screencapture/gstadapter"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCustomPipelineParsing(t *testing.T) {
+
+	_, err := gstadapter.NewWithCustomPipeline("daniel")
+	assert.Error(t, err)
+
+	_, err = gstadapter.NewWithCustomPipeline("queue name=my_filesrc ! fakesink")
+	assert.Error(t, err)
+
+	_, err = gstadapter.NewWithCustomPipeline("queue name=audio_target ! fakesink")
+	assert.Error(t, err)
+
+	gsta, err := gstadapter.NewWithCustomPipeline("rtpmux name=mux ! fakesink \n queue name=audio_target ! mux.sink_0 \n queue name=video_target ! mux.sink_1")
+	assert.NoError(t, err)
+	assert.NotNil(t, gsta)
+}

screencapture/interfaces.go

@@ -5,6 +5,7 @@ import "github.com/danielpaulus/quicktime_video_hack/screencapture/coremedia"
 //CmSampleBufConsumer is a simple interface with one function that consumes CMSampleBuffers
 type CmSampleBufConsumer interface {
 	Consume(buf coremedia.CMSampleBuffer) error
+	Stop()
 }
 
 //UsbDataReceiver should receive USB SYNC, ASYN and PING packets with the correct length and with the 4 bytes length removed.