Skip to content

Commit 2a5a1f3

Browse files
ns6089ns6089
authored
Add support for LTR ACK control messages (#122)
* Add support for LTR ACK control messages
1 parent 435bc6a commit 2a5a1f3

File tree

5 files changed

+121
-44
lines changed

5 files changed

+121
-44
lines changed

‎src/ControlStream.c‎

Lines changed: 94 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,12 @@ typedef struct _NVCTL_ENCRYPTED_PACKET_HEADER {
3131
// encrypted NVCTL_ENET_PACKET_HEADER_V2 and payload data follow
3232
} NVCTL_ENCRYPTED_PACKET_HEADER, *PNVCTL_ENCRYPTED_PACKET_HEADER;
3333

34-
typedef struct _QUEUED_FRAME_INVALIDATION_TUPLE {
34+
typedef struct _QUEUED_REFERENCE_FRAME_CONTROL {
3535
uint32_t startFrame;
3636
uint32_t endFrame;
37+
bool invalidate; // true: RFI(startFrame, endFrame); false: LTR_ACK(startFrame)
3738
LINKED_BLOCKING_QUEUE_ENTRY entry;
38-
} QUEUED_FRAME_INVALIDATION_TUPLE, *PQUEUED_FRAME_INVALIDATION_TUPLE;
39+
} QUEUED_REFERENCE_FRAME_CONTROL, *PQUEUED_REFERENCE_FRAME_CONTROL;
3940

4041
typedef struct _QUEUED_FRAME_FEC_STATUS {
4142
SS_FRAME_FEC_STATUS fecStatus;
@@ -113,7 +114,7 @@ static int lastConnectionStatusUpdate;
113114
static uint32_t currentEnetSequenceNumber;
114115
static uint64_t firstFrameTimeMs;
115116

116-
static LINKED_BLOCKING_QUEUE invalidReferenceFrameTuples;
117+
static LINKED_BLOCKING_QUEUE referenceFrameControlQueue;
117118
static LINKED_BLOCKING_QUEUE frameFecStatusQueue;
118119
static LINKED_BLOCKING_QUEUE asyncCallbackQueue;
119120
static PLT_EVENT idrFrameRequiredEvent;
@@ -300,7 +301,7 @@ static bool supportsIdrFrameRequest;
300301
int initializeControlStream(void) {
301302
stopping = false;
302303
PltCreateEvent(&idrFrameRequiredEvent);
303-
LbqInitializeLinkedBlockingQueue(&invalidReferenceFrameTuples, 20);
304+
LbqInitializeLinkedBlockingQueue(&referenceFrameControlQueue, 20);
304305
LbqInitializeLinkedBlockingQueue(&frameFecStatusQueue, 8); // Limits number of frame status reports per periodic ping interval
305306
LbqInitializeLinkedBlockingQueue(&asyncCallbackQueue, 30);
306307
PltCreateMutex(&enetMutex);
@@ -375,7 +376,7 @@ void destroyControlStream(void) {
375376
PltDestroyCryptoContext(encryptionCtx);
376377
PltDestroyCryptoContext(decryptionCtx);
377378
PltCloseEvent(&idrFrameRequiredEvent);
378-
freeBasicLbqList(LbqDestroyLinkedBlockingQueue(&invalidReferenceFrameTuples));
379+
freeBasicLbqList(LbqDestroyLinkedBlockingQueue(&referenceFrameControlQueue));
379380
freeBasicLbqList(LbqDestroyLinkedBlockingQueue(&frameFecStatusQueue));
380381
freeBasicLbqList(LbqDestroyLinkedBlockingQueue(&asyncCallbackQueue));
381382

@@ -386,12 +387,15 @@ static void queueFrameInvalidationTuple(uint32_t startFrame, uint32_t endFrame)
386387
LC_ASSERT(startFrame <= endFrame);
387388

388389
if (isReferenceFrameInvalidationEnabled()) {
389-
PQUEUED_FRAME_INVALIDATION_TUPLE qfit;
390+
PQUEUED_REFERENCE_FRAME_CONTROL qfit;
390391
qfit = malloc(sizeof(*qfit));
391392
if (qfit != NULL) {
392-
qfit->startFrame = startFrame;
393-
qfit->endFrame = endFrame;
394-
if (LbqOfferQueueItem(&invalidReferenceFrameTuples, qfit, &qfit->entry) == LBQ_BOUND_EXCEEDED) {
393+
*qfit = (QUEUED_REFERENCE_FRAME_CONTROL){
394+
.startFrame = startFrame,
395+
.endFrame = endFrame,
396+
.invalidate = true,
397+
};
398+
if (LbqOfferQueueItem(&referenceFrameControlQueue, qfit, &qfit->entry) == LBQ_BOUND_EXCEEDED) {
395399
// Too many invalidation tuples, so we need an IDR frame now
396400
Limelog("RFI range list reached maximum size limit\n");
397401
free(qfit);
@@ -411,7 +415,7 @@ static void queueFrameInvalidationTuple(uint32_t startFrame, uint32_t endFrame)
411415
void LiRequestIdrFrame(void) {
412416
// Any reference frame invalidation requests should be dropped now.
413417
// We require a full IDR frame to recover.
414-
freeBasicLbqList(LbqFlushQueueItems(&invalidReferenceFrameTuples));
418+
freeBasicLbqList(LbqFlushQueueItems(&referenceFrameControlQueue));
415419

416420
// Request the IDR frame
417421
PltSetEvent(&idrFrameRequiredEvent);
@@ -423,9 +427,29 @@ void connectionDetectedFrameLoss(uint32_t startFrame, uint32_t endFrame) {
423427
}
424428

425429
// When we receive a frame, update the number of our current frame
426-
void connectionReceivedCompleteFrame(uint32_t frameIndex) {
430+
// and send ACK control message if the frame is LTR
431+
void connectionReceivedCompleteFrame(uint32_t frameIndex, bool frameIsLTR) {
427432
lastGoodFrame = frameIndex;
428433
intervalGoodFrameCount++;
434+
435+
if (frameIsLTR && IS_SUNSHINE() && isReferenceFrameInvalidationEnabled()) {
436+
// Queue LTR frame ACK control message
437+
PQUEUED_REFERENCE_FRAME_CONTROL qfit;
438+
qfit = malloc(sizeof(*qfit));
439+
if (qfit != NULL) {
440+
*qfit = (QUEUED_REFERENCE_FRAME_CONTROL){
441+
.startFrame = frameIndex,
442+
.invalidate = false,
443+
};
444+
if (LbqOfferQueueItem(&referenceFrameControlQueue, qfit, &qfit->entry) == LBQ_BOUND_EXCEEDED) {
445+
// This shouldn't happen and indicates that something has gone wrong with the queue
446+
LC_ASSERT(false);
447+
Limelog("Couldn't queue LTR ACK because the list has reached maximum size limit\n");
448+
free(qfit);
449+
LiRequestIdrFrame();
450+
}
451+
}
452+
}
429453
}
430454

431455
void connectionSendFrameFecStatus(PSS_FRAME_FEC_STATUS fecStatus) {
@@ -1512,55 +1536,88 @@ static void requestIdrFrame(void) {
15121536
}
15131537

15141538
static void requestInvalidateReferenceFrames(uint32_t startFrame, uint32_t endFrame) {
1515-
int64_t payload[3];
1516-
15171539
LC_ASSERT(startFrame <= endFrame);
15181540
LC_ASSERT(isReferenceFrameInvalidationEnabled());
15191541

1520-
payload[0] = LE64(startFrame);
1521-
payload[1] = LE64(endFrame);
1522-
payload[2] = 0;
1542+
SS_RFI_REQUEST payload = {
1543+
.firstFrameIndex = LE32(startFrame),
1544+
.lastFrameIndex = LE32(endFrame),
1545+
};
15231546

15241547
// Send the reference frame invalidation request and read the response
15251548
if (!sendMessageAndDiscardReply(packetTypes[IDX_INVALIDATE_REF_FRAMES],
15261549
sizeof(payload),
1527-
payload, CTRL_CHANNEL_URGENT,
1550+
&payload,
1551+
CTRL_CHANNEL_URGENT,
15281552
ENET_PACKET_FLAG_RELIABLE,
15291553
false)) {
1530-
Limelog("Request Invaldiate Reference Frames: Transaction failed: %d\n", (int)LastSocketError());
1554+
Limelog("Request Invalidate Reference Frames: Transaction failed: %d\n", (int)LastSocketError());
15311555
ListenerCallbacks.connectionTerminated(LastSocketFail());
15321556
return;
15331557
}
15341558

15351559
Limelog("Invalidate reference frame request sent (%d to %d)\n", startFrame, endFrame);
15361560
}
15371561

1538-
static void invalidateRefFramesFunc(void* context) {
1562+
static void confirmLongtermReferenceFrame(uint32_t frameIndex) {
1563+
LC_ASSERT(isReferenceFrameInvalidationEnabled());
1564+
1565+
SS_LTR_FRAME_ACK payload = {
1566+
.frameIndex = LE32(frameIndex),
1567+
};
1568+
1569+
// Send LTR frame ACK and don't wait for response
1570+
if (!sendMessageAndForget(SS_LTR_FRAME_ACK_PTYPE,
1571+
sizeof(payload),
1572+
&payload,
1573+
CTRL_CHANNEL_URGENT,
1574+
ENET_PACKET_FLAG_RELIABLE,
1575+
false)) {
1576+
Limelog("LTR frame ACK: Transaction failed: %d\n", (int)LastSocketError());
1577+
ListenerCallbacks.connectionTerminated(LastSocketFail());
1578+
return;
1579+
}
1580+
}
1581+
1582+
static void referenceFrameControlFunc(void* context) {
15391583
LC_ASSERT(isReferenceFrameInvalidationEnabled());
15401584

15411585
while (!PltIsThreadInterrupted(&invalidateRefFramesThread)) {
1542-
PQUEUED_FRAME_INVALIDATION_TUPLE qfit;
1543-
uint32_t startFrame;
1544-
uint32_t endFrame;
1586+
PQUEUED_REFERENCE_FRAME_CONTROL qfit;
1587+
uint32_t invalidateStartFrame;
1588+
uint32_t invalidateEndFrame;
1589+
bool invalidate = false;
15451590

1546-
// Wait for a reference frame invalidation request or a request to shutdown
1547-
if (LbqWaitForQueueElement(&invalidReferenceFrameTuples, (void**)&qfit) != LBQ_SUCCESS) {
1591+
// Wait for a reference frame control message or a request to shutdown
1592+
if (LbqWaitForQueueElement(&referenceFrameControlQueue, (void**)&qfit) != LBQ_SUCCESS) {
15481593
// Bail if we're stopping
15491594
return;
15501595
}
15511596

1552-
startFrame = qfit->startFrame;
1553-
endFrame = qfit->endFrame;
1554-
1555-
// Aggregate all lost frames into one range
15561597
do {
1557-
LC_ASSERT(qfit->endFrame >= endFrame);
1558-
endFrame = qfit->endFrame;
1598+
if (qfit->invalidate) {
1599+
if (!invalidate) {
1600+
invalidateStartFrame = qfit->startFrame;
1601+
invalidateEndFrame = qfit->endFrame;
1602+
invalidate = true;
1603+
}
1604+
else {
1605+
// Aggregate all lost frames into one range
1606+
LC_ASSERT(qfit->endFrame >= invalidateEndFrame);
1607+
invalidateEndFrame = qfit->endFrame;
1608+
}
1609+
}
1610+
else {
1611+
// Send LTR frame ACK
1612+
confirmLongtermReferenceFrame(qfit->startFrame);
1613+
}
15591614
free(qfit);
1560-
} while (LbqPollQueueElement(&invalidReferenceFrameTuples, (void**)&qfit) == LBQ_SUCCESS);
1615+
} while (LbqPollQueueElement(&referenceFrameControlQueue, (void**)&qfit) == LBQ_SUCCESS);
15611616

1562-
// Send the reference frame invalidation request
1563-
requestInvalidateReferenceFrames(startFrame, endFrame);
1617+
if (invalidate) {
1618+
// Send the reference frame invalidation request
1619+
requestInvalidateReferenceFrames(invalidateStartFrame, invalidateEndFrame);
1620+
}
15641621
}
15651622
}
15661623

@@ -1574,8 +1631,8 @@ static void requestIdrFrameFunc(void* context) {
15741631
return;
15751632
}
15761633

1577-
// Any pending reference frame invalidation requests are now redundant
1578-
freeBasicLbqList(LbqFlushQueueItems(&invalidReferenceFrameTuples));
1634+
// Any pending RFI requests and LTR frame ACK messages are now redundant
1635+
freeBasicLbqList(LbqFlushQueueItems(&referenceFrameControlQueue));
15791636

15801637
// Request the IDR frame
15811638
requestIdrFrame();
@@ -1585,7 +1642,7 @@ static void requestIdrFrameFunc(void* context) {
15851642
// Stops the control stream
15861643
int stopControlStream(void) {
15871644
stopping = true;
1588-
LbqSignalQueueShutdown(&invalidReferenceFrameTuples);
1645+
LbqSignalQueueShutdown(&referenceFrameControlQueue);
15891646
LbqSignalQueueShutdown(&frameFecStatusQueue);
15901647
LbqSignalQueueDrain(&asyncCallbackQueue);
15911648
PltSetEvent(&idrFrameRequiredEvent);
@@ -1972,7 +2029,7 @@ int startControlStream(void) {
19722029

19732030
// Only create the reference frame invalidation thread if RFI is enabled
19742031
if (isReferenceFrameInvalidationEnabled()) {
1975-
err = PltCreateThread("InvRefFrames", invalidateRefFramesFunc, NULL, &invalidateRefFramesThread);
2032+
err = PltCreateThread("InvRefFrames", referenceFrameControlFunc, NULL, &invalidateRefFramesThread);
19762033
if (err != 0) {
19772034
stopping = true;
19782035
PltSetEvent(&idrFrameRequiredEvent);

‎src/Limelight-internal.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ extern uint32_t EncryptionFeaturesEnabled;
5555

5656
// ENet channel ID values
5757
#define CTRL_CHANNEL_GENERIC 0x00
58-
#define CTRL_CHANNEL_URGENT 0x01 // IDR and reference frame invalidation requests
58+
#define CTRL_CHANNEL_URGENT 0x01 // IDR, LTR ACK and RFI
5959
#define CTRL_CHANNEL_KEYBOARD 0x02
6060
#define CTRL_CHANNEL_MOUSE 0x03
6161
#define CTRL_CHANNEL_PEN 0x04
@@ -119,7 +119,7 @@ int startControlStream(void);
119119
int stopControlStream(void);
120120
void destroyControlStream(void);
121121
void connectionDetectedFrameLoss(uint32_t startFrame, uint32_t endFrame);
122-
void connectionReceivedCompleteFrame(uint32_t frameIndex);
122+
void connectionReceivedCompleteFrame(uint32_t frameIndex, bool frameIsLTR);
123123
void connectionSawFrame(uint32_t frameIndex);
124124
void connectionSendFrameFecStatus(PSS_FRAME_FEC_STATUS fecStatus);
125125
int sendInputPacketOnControlStream(unsigned char* data, int length, uint8_t channelId, uint32_t flags, bool moreData);

‎src/RtpVideoQueue.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -382,9 +382,9 @@ static int reconstructFrame(PRTP_VIDEO_QUEUE queue) {
382382

383383
// Check all NV_VIDEO_PACKET fields except FEC stuff which differs in the recovered packet
384384
LC_ASSERT_VT(nvPacket->flags == droppedNvPacket->flags);
385+
LC_ASSERT_VT(nvPacket->extraFlags == droppedNvPacket->extraFlags);
385386
LC_ASSERT_VT(nvPacket->frameIndex == droppedNvPacket->frameIndex);
386387
LC_ASSERT_VT(nvPacket->streamPacketIndex == droppedNvPacket->streamPacketIndex);
387-
LC_ASSERT_VT(nvPacket->reserved == droppedNvPacket->reserved);
388388
LC_ASSERT_VT(!queue->multiFecCapable || nvPacket->multiFecBlocks == droppedNvPacket->multiFecBlocks);
389389

390390
// Check the data itself - use memcmp() and only loop if an error is detected

‎src/Video.h‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,13 @@ typedef struct _ENC_VIDEO_HEADER {
2222
#define FLAG_EOF 0x2
2323
#define FLAG_SOF 0x4
2424

25+
#define NV_VIDEO_PACKET_EXTRA_FLAG_LTR_FRAME 0x1
26+
2527
typedef struct _NV_VIDEO_PACKET {
2628
uint32_t streamPacketIndex;
2729
uint32_t frameIndex;
2830
uint8_t flags;
29-
uint8_t reserved;
31+
uint8_t extraFlags;
3032
uint8_t multiFecFlags;
3133
uint8_t multiFecBlocks;
3234
uint32_t fecInfo;
@@ -67,4 +69,20 @@ typedef struct _SS_FRAME_FEC_STATUS {
6769
uint8_t multiFecBlockCount;
6870
} SS_FRAME_FEC_STATUS, *PSS_FRAME_FEC_STATUS;
6971

72+
// Fields are little-endian
73+
#define SS_LTR_FRAME_ACK_PTYPE 0x0350
74+
typedef struct _SS_LTR_FRAME_ACK {
75+
uint32_t frameIndex;
76+
uint32_t reserved;
77+
} SS_LTR_FRAME_ACK, *PSS_LTR_FRAME_ACK;
78+
79+
// Fields are little-endian
80+
#define SS_RFI_REQUEST_PTYPE 0x0301
81+
typedef struct _SS_RFI_REQUEST {
82+
uint32_t firstFrameIndex;
83+
uint32_t reserved1;
84+
uint32_t lastFrameIndex;
85+
uint32_t reserved2[3];
86+
} SS_RFI_REQUEST, *PSS_RFI_REQUEST;
87+
7088
#pragma pack(pop)

‎src/VideoDepacketizer.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,7 @@ static bool isIdrFrameStart(PBUFFER_DESC buffer) {
466466
}
467467

468468
// Reassemble the frame with the given frame number
469-
static void reassembleFrame(int frameNumber) {
469+
static void reassembleFrame(int frameNumber, bool frameIsLTR) {
470470
if (nalChainHead != NULL) {
471471
QUEUED_DECODE_UNIT qduDS;
472472
PQUEUED_DECODE_UNIT qdu;
@@ -539,7 +539,7 @@ static void reassembleFrame(int frameNumber) {
539539
}
540540

541541
// Notify the control connection
542-
connectionReceivedCompleteFrame(frameNumber);
542+
connectionReceivedCompleteFrame(frameNumber, frameIsLTR);
543543

544544
// Clear frame drops
545545
consecutiveFrameDrops = 0;
@@ -748,6 +748,7 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
748748
BUFFER_DESC currentPos;
749749
uint32_t frameIndex;
750750
uint8_t flags;
751+
uint8_t extraFlags;
751752
bool firstPacket, lastPacket;
752753
uint32_t streamPacketIndex;
753754
uint8_t fecCurrentBlockNumber;
@@ -765,6 +766,7 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
765766
fecLastBlockNumber = (videoPacket->multiFecBlocks >> 6) & 0x3;
766767
frameIndex = videoPacket->frameIndex;
767768
flags = videoPacket->flags;
769+
extraFlags = videoPacket->extraFlags;
768770
firstPacket = isFirstPacket(flags, fecCurrentBlockNumber);
769771
lastPacket = (flags & FLAG_EOF) && fecCurrentBlockNumber == fecLastBlockNumber;
770772

@@ -1119,7 +1121,7 @@ static void processRtpPayload(PNV_VIDEO_PACKET videoPacket, int length,
11191121
}
11201122
}
11211123

1122-
reassembleFrame(frameIndex);
1124+
reassembleFrame(frameIndex, extraFlags & NV_VIDEO_PACKET_EXTRA_FLAG_LTR_FRAME);
11231125
}
11241126
}
11251127

0 commit comments

Comments
 (0)