Skip to content

Commit 166e8b9

Browse files
danielpaulusdanielpaulus
authored
Merge pull request #30 from danielpaulus/implementSkewResponses
reduce logging verbosity, send skew values
2 parents 158af38 + e15e766 commit 166e8b9

File tree

9 files changed

+240
-32
lines changed

9 files changed

+240
-32
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
quicktime_video_hack
33
# Binaries for programs and plugins
44
main
5+
*.h264
6+
out.wav
57
*.exe
68
*.exe~
79
*.dll

‎doc/technical_documentation.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -260,8 +260,13 @@ This packet requests from us to send a RPLY with the current CMTime for the Cloc
260260

261261
#### 3.2.7. SKEW Packet
262262
##### General Description
263-
(WIP) It seems like this packet requires us to reply with the difference of our local clock in 48khz ticks to the device clock, which
264-
we can get from the audio samples.
263+
This packet tells the device about the clock skew of the audio clock (clockRef used in EAT! packets, which we sent as response to cwpa). As denoted in this [wikipedia](https://en.wikipedia.org/wiki/Clock_skew#On_a_network) article, clock skew means the difference in frequency of both clocks. In other words, both clocks supposedly
264+
run at 48khz, and the device wants to know how many ticks per second our clock executed during the time the device clock had one tick.
265+
So we have to respond with:
266+
- 48000 if the clocks were aligned
267+
- some value above 48000 if our clock was slower
268+
- and some value below 48000 if our clock was faster than the device clock
269+
If implemented correctly, we should see that the skew responses converge towards 48000 with small deviations sometimes `(48000+x where -1 < x <1)`
265270

266271
##### Request Format Description
267272

‎screencapture/coremedia/cmclock.go‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,30 @@ import (
88
type CMClock struct {
99
ID uint64
1010
TimeScale uint32
11+
factor float64
1112
startTime time.Time
1213
}
1314

15+
//NanoSecondScale is the default system clock scale where 1/NanoSecondScale == 1 Nanosecond.
16+
const NanoSecondScale = 1000000000
17+
1418
//NewCMClockWithHostTime creates a new Clock with the given ID with a nanosecond scale.
1519
//Calls to GetTime will measure the time difference since the clock was created.
1620
func NewCMClockWithHostTime(ID uint64) CMClock {
1721
return CMClock{
18-
ID: ID,
19-
//NanoSecond Scale
20-
TimeScale: 1000000000,
22+
ID: ID,
23+
TimeScale: NanoSecondScale,
24+
factor: 1,
25+
startTime: time.Now(),
26+
}
27+
}
28+
29+
//NewCMClockWithHostTimeAndScale creates a new CMClock with given ID and a custom timeScale
30+
func NewCMClockWithHostTimeAndScale(ID uint64, timeScale uint32) CMClock {
31+
return CMClock{
32+
ID: ID,
33+
TimeScale: timeScale,
34+
factor: float64(timeScale) / float64(NanoSecondScale),
2135
startTime: time.Now(),
2236
}
2337
}
@@ -26,9 +40,28 @@ func NewCMClockWithHostTime(ID uint64) CMClock {
2640
//This is monotonic and does NOT use wallclock time.
2741
func (c CMClock) GetTime() CMTime {
2842
return CMTime{
29-
CMTimeValue: uint64(time.Since(c.startTime).Nanoseconds()),
43+
CMTimeValue: c.calcValue(time.Since(c.startTime).Nanoseconds()),
3044
CMTimeScale: c.TimeScale,
3145
CMTimeFlags: KCMTimeFlagsHasBeenRounded,
3246
CMTimeEpoch: 0,
3347
}
3448
}
49+
50+
func (c CMClock) calcValue(val int64) uint64 {
51+
if NanoSecondScale == c.TimeScale {
52+
return uint64(val)
53+
}
54+
return uint64(c.factor * float64(val))
55+
}
56+
57+
//CalculateSkew calculates the deviation between the frequencies of two given clocks by using time diffs and returns a skew value float64
58+
//scaled to match the second clock.
59+
func CalculateSkew(startTimeClock1 CMTime, endTimeClock1 CMTime, startTimeClock2 CMTime, endTimeClock2 CMTime) float64 {
60+
timeDiffClock1 := endTimeClock1.CMTimeValue - startTimeClock1.CMTimeValue
61+
timeDiffClock2 := endTimeClock2.CMTimeValue - startTimeClock2.CMTimeValue
62+
63+
diffTime := CMTime{CMTimeValue: timeDiffClock1, CMTimeScale: startTimeClock1.CMTimeScale}
64+
scaledDiff := diffTime.GetTimeForScale(startTimeClock2)
65+
//println("scaleddiff:" + scaledDiff)
66+
return float64(startTimeClock2.CMTimeScale) * scaledDiff / float64(timeDiffClock2)
67+
}

‎screencapture/coremedia/cmclock_test.go‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,67 @@
11
package coremedia_test
22

33
import (
4+
"testing"
5+
46
"github.com/danielpaulus/quicktime_video_hack/screencapture/coremedia"
57
"github.com/stretchr/testify/assert"
6-
"testing"
78
)
89

910
func TestCMClock_GetTime(t *testing.T) {
1011
cmclock := coremedia.NewCMClockWithHostTime(uint64(5))
11-
assert.Equal(t, uint32(1000000000), cmclock.TimeScale)
12+
assert.Equal(t, uint32(coremedia.NanoSecondScale), cmclock.TimeScale)
1213
time1 := cmclock.GetTime()
1314
time2 := cmclock.GetTime()
1415
assert.Equal(t, cmclock.TimeScale, time1.CMTimeScale)
1516
//The clock is monotonic with nanosecond precision, so this should always be true
1617
assert.True(t, true, time2.CMTimeValue > time1.CMTimeValue)
18+
19+
cmclock = coremedia.NewCMClockWithHostTimeAndScale(0, 1)
20+
assert.Equal(t, uint64(0), cmclock.GetTime().CMTimeValue)
21+
}
22+
23+
func TestCalculateSkew(t *testing.T) {
24+
testCases := map[string]struct {
25+
startTimeClock1 coremedia.CMTime
26+
endTimeClock1 coremedia.CMTime
27+
startTimeClock2 coremedia.CMTime
28+
endTimeClock2 coremedia.CMTime
29+
expectedValue float64
30+
}{
31+
"check simple case, no skew": {
32+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
33+
coremedia.CMTime{CMTimeValue: 1, CMTimeScale: 48000},
34+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
35+
coremedia.CMTime{CMTimeValue: 1, CMTimeScale: 48000},
36+
float64(48000.0)},
37+
"check simple case, positive skew": {
38+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
39+
coremedia.CMTime{CMTimeValue: 2, CMTimeScale: 48000},
40+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
41+
coremedia.CMTime{CMTimeValue: 1, CMTimeScale: 48000},
42+
float64(96000.0)},
43+
"check simple case, negative skew": {
44+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
45+
coremedia.CMTime{CMTimeValue: 2000, CMTimeScale: 48000},
46+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
47+
coremedia.CMTime{CMTimeValue: 2001, CMTimeScale: 48000},
48+
float64(47976.011994003)},
49+
"check different scales, negative skew": {
50+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: coremedia.NanoSecondScale},
51+
coremedia.CMTime{CMTimeValue: 20833 * 5, CMTimeScale: coremedia.NanoSecondScale},
52+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
53+
coremedia.CMTime{CMTimeValue: 5, CMTimeScale: 48000},
54+
float64(47999.232)},
55+
"check different scales, positive skew": {
56+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: coremedia.NanoSecondScale},
57+
coremedia.CMTime{CMTimeValue: 20833 * 5001, CMTimeScale: coremedia.NanoSecondScale},
58+
coremedia.CMTime{CMTimeValue: 0, CMTimeScale: 48000},
59+
coremedia.CMTime{CMTimeValue: 5000, CMTimeScale: 48000},
60+
float64(48008.8318464)},
61+
}
62+
63+
for s, tc := range testCases {
64+
calculatedSkew := coremedia.CalculateSkew(tc.startTimeClock1, tc.endTimeClock1, tc.startTimeClock2, tc.endTimeClock2)
65+
assert.Equal(t, tc.expectedValue, calculatedSkew, s)
66+
}
1767
}

‎screencapture/coremedia/cmtime.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ type CMTime struct {
2929
however, since epoch length may be unknown/variable. */
3030
}
3131

32+
//GetTimeForScale calculates a float64 TimeValue by rescaling this CMTime to the CMTimeScale of the given CMTime
33+
func (time CMTime) GetTimeForScale(newScaleToUse CMTime) float64 {
34+
scalingFactor := float64(newScaleToUse.CMTimeScale) / float64(time.CMTimeScale)
35+
return (float64(time.CMTimeValue) * scalingFactor)
36+
}
37+
3238
//Seconds returns CMTimeValue/CMTimeScale and 0 when all values are 0
3339
func (time CMTime) Seconds() uint64 {
3440
//prevent division by 0

‎screencapture/coremedia/cmtime_test.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,35 @@
11
package coremedia_test
22

33
import (
4+
"testing"
5+
46
"github.com/danielpaulus/quicktime_video_hack/screencapture/coremedia"
57
"github.com/stretchr/testify/assert"
6-
"testing"
78
)
89

10+
func TestScaleConversion(t *testing.T) {
11+
testCases := map[string]struct {
12+
originalTime coremedia.CMTime
13+
destinationScaleTime coremedia.CMTime
14+
expectedTime float64
15+
}{
16+
"check zero valued CMTime works": {coremedia.CMTime{CMTimeValue: 0, CMTimeScale: coremedia.NanoSecondScale},
17+
coremedia.CMTime{CMTimeValue: 1, CMTimeScale: 2 * coremedia.NanoSecondScale},
18+
0},
19+
"doubling scale": {coremedia.CMTime{CMTimeValue: 100, CMTimeScale: coremedia.NanoSecondScale},
20+
coremedia.CMTime{CMTimeValue: 1, CMTimeScale: 2 * coremedia.NanoSecondScale},
21+
float64(0xC8)},
22+
"smaller scale": {coremedia.CMTime{CMTimeValue: 100, CMTimeScale: 1},
23+
coremedia.CMTime{CMTimeValue: 1, CMTimeScale: 48000},
24+
float64(0x493e00)},
25+
}
26+
27+
for s, tc := range testCases {
28+
actualTime := tc.originalTime.GetTimeForScale(tc.destinationScaleTime)
29+
assert.Equal(t, tc.expectedTime, actualTime, s)
30+
}
31+
}
32+
933
func TestSeconds(t *testing.T) {
1034
time := createCmTime()
1135
assert.Equal(t, uint64(2), time.Seconds())

‎screencapture/messageprocessor.go‎

Lines changed: 41 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,25 @@ import (
1313
//It receives readily split byte frames, parses them, responds to them and passes on
1414
//extracted CMSampleBuffers to a consumer
1515
type MessageProcessor struct {
16-
connectionState int
17-
usbWriter UsbWriter
18-
stopSignal chan interface{}
19-
clock coremedia.CMClock
20-
localAudioClock coremedia.CMClock
21-
totalBytesReceived int
22-
needClockRef packet.CFTypeID
23-
needMessage []byte
24-
audioSamplesReceived int
25-
cmSampleBufConsumer CmSampleBufConsumer
26-
clockBuilder func(uint64) coremedia.CMClock
27-
deviceAudioClockRef packet.CFTypeID
28-
releaseWaiter chan interface{}
16+
connectionState int
17+
usbWriter UsbWriter
18+
stopSignal chan interface{}
19+
clock coremedia.CMClock
20+
localAudioClock coremedia.CMClock
21+
totalBytesReceived int
22+
needClockRef packet.CFTypeID
23+
needMessage []byte
24+
audioSamplesReceived int
25+
videoSamplesReceived int
26+
cmSampleBufConsumer CmSampleBufConsumer
27+
clockBuilder func(uint64) coremedia.CMClock
28+
deviceAudioClockRef packet.CFTypeID
29+
releaseWaiter chan interface{}
30+
firstAudioTimeTaken bool
31+
startTimeDeviceAudioClock coremedia.CMTime
32+
startTimeLocalAudioClock coremedia.CMTime
33+
lastEatFrameReceivedDeviceAudioClockTime coremedia.CMTime
34+
lastEatFrameReceivedLocalAudioClockTime coremedia.CMTime
2935
}
3036

3137
//NewMessageProcessor creates a new MessageProcessor that will write answers to the given UsbWriter,
@@ -37,7 +43,7 @@ func NewMessageProcessor(usbWriter UsbWriter, stopSignal chan interface{}, consu
3743

3844
//NewMessageProcessorWithClockBuilder lets you inject a clockBuilder for the sake of testability.
3945
func NewMessageProcessorWithClockBuilder(usbWriter UsbWriter, stopSignal chan interface{}, consumer CmSampleBufConsumer, clockBuilder func(uint64) coremedia.CMClock) MessageProcessor {
40-
var mp = MessageProcessor{usbWriter: usbWriter, stopSignal: stopSignal, cmSampleBufConsumer: consumer, clockBuilder: clockBuilder, releaseWaiter: make(chan interface{})}
46+
var mp = MessageProcessor{usbWriter: usbWriter, stopSignal: stopSignal, cmSampleBufConsumer: consumer, clockBuilder: clockBuilder, releaseWaiter: make(chan interface{}), firstAudioTimeTaken: false}
4147
return mp
4248
}
4349

@@ -149,7 +155,9 @@ func (mp *MessageProcessor) handleSyncPacket(data []byte) {
149155
if err != nil {
150156
log.Error("Error parsing SYNC SKEW packet", err)
151157
}
152-
log.Debugf("Rcv and ignore:%s", skewPacket.String())
158+
skewValue := coremedia.CalculateSkew(mp.startTimeLocalAudioClock, mp.lastEatFrameReceivedLocalAudioClockTime, mp.startTimeDeviceAudioClock, mp.lastEatFrameReceivedDeviceAudioClockTime)
159+
log.Debugf("Rcv:%s Reply:%f", skewPacket.String(), skewValue)
160+
mp.usbWriter.WriteDataToUsb(skewPacket.NewReply(skewValue))
153161
case packet.STOP:
154162
stopPacket, err := packet.NewSyncStopPacketFromBytes(data)
155163
if err != nil {
@@ -172,6 +180,17 @@ func (mp *MessageProcessor) handleAsyncPacket(data []byte) {
172180
log.Warn("unknown eat")
173181
return
174182
}
183+
if !mp.firstAudioTimeTaken {
184+
mp.startTimeDeviceAudioClock = eatPacket.CMSampleBuf.OutputPresentationTimestamp
185+
mp.startTimeLocalAudioClock = mp.localAudioClock.GetTime()
186+
mp.lastEatFrameReceivedDeviceAudioClockTime = eatPacket.CMSampleBuf.OutputPresentationTimestamp
187+
mp.lastEatFrameReceivedLocalAudioClockTime = mp.startTimeLocalAudioClock
188+
mp.firstAudioTimeTaken = true
189+
} else {
190+
mp.lastEatFrameReceivedDeviceAudioClockTime = eatPacket.CMSampleBuf.OutputPresentationTimestamp
191+
mp.lastEatFrameReceivedLocalAudioClockTime = mp.localAudioClock.GetTime()
192+
}
193+
175194
err = mp.cmSampleBufConsumer.Consume(eatPacket.CMSampleBuf)
176195
if err != nil {
177196
log.Warn("failed consuming audio buf", err)
@@ -183,16 +202,20 @@ func (mp *MessageProcessor) handleAsyncPacket(data []byte) {
183202
case packet.FEED:
184203
feedPacket, err := packet.NewAsynCmSampleBufPacketFromBytes(data)
185204
if err != nil {
186-
//log.Errorf("Error parsing FEED packet: %x %s", data, err)
187-
log.Warn("unknown feed")
205+
log.Errorf("Error parsing FEED packet: %x %s", data, err)
188206
mp.usbWriter.WriteDataToUsb(mp.needMessage)
189207
return
190208
}
209+
mp.videoSamplesReceived++
191210
err = mp.cmSampleBufConsumer.Consume(feedPacket.CMSampleBuf)
192211
if err != nil {
193212
log.Fatal("Failed writing sample data to Consumer", err)
194213
}
195-
log.Debugf("Rcv:%s", feedPacket.String())
214+
if mp.videoSamplesReceived%500 == 0 {
215+
log.Debugf("Rcv'd(%d) last:%s", mp.videoSamplesReceived, feedPacket.String())
216+
mp.videoSamplesReceived = 0
217+
}
218+
196219
mp.usbWriter.WriteDataToUsb(mp.needMessage)
197220
case packet.SPRP:
198221
sprpPacket, err := packet.NewAsynSprpPacketFromBytes(data)

0 commit comments

Comments
 (0)