https://github.com/Microsoft/HoloLensCompanionKit/issues/113

Update spectator view to the latest HoloToolkit (1.5.7)
Update color frame caching to increase stability of composited holograms.

- With this change, a few actions are required:
1. Updated spectator view code will need to be merged into existing projects.
2. The compositor DLLs will need to be rebuilt and copied into existing projects.

Author: fieldsJacksonG

SpectatorView/Calibration/Calibration/CalibrationApp.cpp

@@ -100,7 +100,7 @@ void CalibrationApp::Initialize(HWND window, int width, int height)
     frameProvider = new DeckLinkManager(true, true);
 #endif
 #if USE_OPENCV
-    frameProvider = new OpenCVFrameProvider();
+    frameProvider = new OpenCVFrameProvider(false);
 #endif
 
     frameProvider->Initialize(srv, nullptr);

SpectatorView/Compositor/CompositorDLL/CompositorInterface.cpp

@@ -50,7 +50,7 @@ bool CompositorInterface::Initialize(ID3D11Device* device, ID3D11ShaderResourceV
 
     _device = device;
 
-    hologramQueue = new HologramQueue(device);
+    hologramQueue = new HologramQueue();
 
     return SUCCEEDED(frameProvider->Initialize(colorSRV, outputTexture));
 }
@@ -124,7 +124,7 @@ void CompositorInterface::StopFrameProvider()
 #endif
 }
 
-LONGLONG CompositorInterface::GetColorTime()
+LONGLONG CompositorInterface::GetTimestamp()
 {
     if (frameProvider != nullptr)
     {

SpectatorView/Compositor/CompositorDLL/CompositorInterface.h

@@ -57,7 +57,8 @@ class CompositorInterface
     DLLEXPORT void Update();
     DLLEXPORT void StopFrameProvider();
 
-    DLLEXPORT LONGLONG GetColorTime();
+    DLLEXPORT LONGLONG GetTimestamp();
+
     DLLEXPORT LONGLONG GetColorDuration();
 
     DLLEXPORT void TakePicture(ID3D11Device* device, int width, int height, int bpp, 

SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.cpp

@@ -79,9 +79,10 @@ DeckLinkDevice::~DeckLinkDevice()
     DeleteCriticalSection(&m_outputCriticalSection);
     DeleteCriticalSection(&m_frameAccessCriticalSection);
 
-    delete[] cachedBuffer;
-    delete[] stagingBuffer;
+    delete[] thirdCachedBuffer;
+    delete[] secondCachedBuffer;
     delete[] latestBuffer;
+    delete[] stagingBuffer;
     delete[] outputBuffer;
     delete[] outputBufferRaw;
 }
@@ -148,7 +149,8 @@ bool DeckLinkDevice::Init(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* o
     BSTR                            deviceNameBSTR = NULL;
 
     ZeroMemory(rawBuffer, FRAME_BUFSIZE_RAW);
-    ZeroMemory(cachedBuffer, FRAME_BUFSIZE);
+    ZeroMemory(thirdCachedBuffer, FRAME_BUFSIZE);
+    ZeroMemory(secondCachedBuffer, FRAME_BUFSIZE);
     ZeroMemory(latestBuffer, FRAME_BUFSIZE);
     ZeroMemory(outputBuffer, FRAME_BUFSIZE);
     ZeroMemory(outputBufferRaw, FRAME_BUFSIZE_RAW);
@@ -372,13 +374,16 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
     {
         if (frame->GetBytes((void**)&rawBuffer) == S_OK)
         {
+            // Always return the latest buffer when using the CPU.
             if (_useCPU)
             {
-                DirectXHelper::ConvertYUVtoBGRA(rawBuffer, cachedBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
+                DirectXHelper::ConvertYUVtoBGRA(rawBuffer, latestBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
             }
             else
             {
-                memcpy(cachedBuffer, rawBuffer, FRAME_BUFSIZE_RAW);
+                memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE_RAW);
+                memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE_RAW);
+                memcpy(latestBuffer, rawBuffer, FRAME_BUFSIZE_RAW);
             }
         }
     }
@@ -391,11 +396,14 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
                 //TODO: Remove this block if R and B components are swapped in color feed.
                 memcpy(stagingBuffer, localFrameBuffer, FRAME_BUFSIZE);
                 DirectXHelper::ConvertBGRAtoRGBA(stagingBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
-                memcpy(cachedBuffer, stagingBuffer, FRAME_BUFSIZE);
+                // Do not cache frames when using the CPU
+                memcpy(latestBuffer, stagingBuffer, FRAME_BUFSIZE);
             }
             else
             {
-                memcpy(cachedBuffer, localFrameBuffer, FRAME_BUFSIZE);
+                memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
+                memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
+                memcpy(latestBuffer, localFrameBuffer, FRAME_BUFSIZE);
             }
         }
     }
@@ -404,7 +412,9 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
     frame->GetStreamTime(&t, &frameDuration, QPC_MULTIPLIER);
 
     // Get frame time.
-    cachedTimeStamp = time.QuadPart;
+    thirdTimeStamp = secondTimeStamp;
+    secondTimeStamp = latestTimeStamp;
+    latestTimeStamp = time.QuadPart;
 
     dirtyFrame = false;
 
@@ -445,13 +455,20 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
 void DeckLinkDevice::Update()
 {
     if (_colorSRV != nullptr &&
-        cachedBuffer != nullptr &&
         device != nullptr)
     {
         if (!dirtyFrame)
         {
             dirtyFrame = true;
-            DirectXHelper::UpdateSRV(device, _colorSRV, cachedBuffer, FRAME_WIDTH * FRAME_BPP);
+            if (_useCPU && latestBuffer != nullptr)
+            {
+                // Do not cache when using CPU.
+                DirectXHelper::UpdateSRV(device, _colorSRV, latestBuffer, FRAME_WIDTH * FRAME_BPP);
+            }
+            else if (!_useCPU && thirdCachedBuffer != nullptr)
+            {
+                DirectXHelper::UpdateSRV(device, _colorSRV, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
+            }
 
             EnterCriticalSection(&m_frameAccessCriticalSection);
             isVideoFrameReady = true;

SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.h

@@ -57,18 +57,20 @@ class DeckLinkDevice : public IDeckLinkInputCallback
     CRITICAL_SECTION          m_outputCriticalSection;
 
     BYTE* localFrameBuffer;
-    BYTE* rawBuffer =         new BYTE[FRAME_BUFSIZE_RAW];
+    BYTE* rawBuffer =           new BYTE[FRAME_BUFSIZE_RAW];
 
-    BYTE* cachedBuffer =      new BYTE[FRAME_BUFSIZE];
-    BYTE* stagingBuffer =     new BYTE[FRAME_BUFSIZE];
-    BYTE* latestBuffer =      new BYTE[FRAME_BUFSIZE];
-    BYTE* outputBuffer =      new BYTE[FRAME_BUFSIZE];
-    BYTE* outputBufferRaw =   new BYTE[FRAME_BUFSIZE_RAW];
+    BYTE* thirdCachedBuffer =   new BYTE[FRAME_BUFSIZE];
+    BYTE* secondCachedBuffer =  new BYTE[FRAME_BUFSIZE];
+    BYTE* latestBuffer =        new BYTE[FRAME_BUFSIZE];
+    BYTE* stagingBuffer =       new BYTE[FRAME_BUFSIZE];
+    BYTE* outputBuffer =        new BYTE[FRAME_BUFSIZE];
+    BYTE* outputBufferRaw =     new BYTE[FRAME_BUFSIZE_RAW];
 
     BMDTimeValue frameDuration = 0;
 
     LONGLONG latestTimeStamp = 0;
-    LONGLONG cachedTimeStamp = 0;
+    LONGLONG secondTimeStamp = 0;
+    LONGLONG thirdTimeStamp = 0;
 
     bool dirtyFrame = true;
     bool isVideoFrameReady = false;
@@ -104,9 +106,9 @@ class DeckLinkDevice : public IDeckLinkInputCallback
     virtual HRESULT  STDMETHODCALLTYPE    VideoInputFormatChanged (/* in */ BMDVideoInputFormatChangedEvents notificationEvents, /* in */ IDeckLinkDisplayMode *newDisplayMode, /* in */ BMDDetectedVideoInputFormatFlags detectedSignalFlags);
     virtual HRESULT  STDMETHODCALLTYPE    VideoInputFrameArrived (/* in */ IDeckLinkVideoInputFrame* frame, /* in */ IDeckLinkAudioInputPacket* audioPacket);
 
-    LONGLONG GetTimeStamp()
+    LONGLONG GetTimestamp()
     {
-        return cachedTimeStamp;
+        return thirdTimeStamp;
     }
 
     LONGLONG GetDurationHNS()

SpectatorView/Compositor/CompositorDLL/DeckLinkManager.cpp

@@ -123,7 +123,7 @@ LONGLONG DeckLinkManager::GetTimestamp()
 {
     if (IsEnabled())
     {
-        return deckLinkDevice->GetTimeStamp();
+        return deckLinkDevice->GetTimestamp();
     }
 
     return 0;

SpectatorView/Compositor/CompositorDLL/DeckLinkManager.h

@@ -16,7 +16,10 @@ class DeckLinkManager : public IFrameProvider
 
     HRESULT Initialize(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* outputTexture);
 
+    // 3 frames are caches for reliable hologram stability:
+    // Get the timestamp of the earliest (and currently rendered) cached frame.
     LONGLONG GetTimestamp();
+
     LONGLONG GetDurationHNS();
 
     void Update();

SpectatorView/Compositor/CompositorDLL/ElgatoFrameProvider.h

@@ -106,8 +106,6 @@ class ElgatoFrameProvider : public IFrameProvider
     IBaseFilter *pNullF = NULL;
     ElgatoSampleCallback *frameCallback = NULL;
     IElgatoVideoCaptureFilter6 *filter = NULL;
-
-    BYTE* cachedBuffer = new BYTE[FRAME_BUFSIZE];
 };
 
 #endif

SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.cpp

@@ -20,7 +20,9 @@ STDMETHODIMP ElgatoSampleCallback::BufferCB(double time, BYTE *pBuffer, long len
     // Get frame time.
     LARGE_INTEGER t;
     QueryPerformanceCounter(&t);
-    cachedTimestamp = t.QuadPart;
+    thirdTimeStamp = secondTimeStamp;
+    secondTimeStamp = latestTimeStamp;
+    latestTimeStamp = t.QuadPart;
 
     int copyLength = length;
     if (copyLength > FRAME_BUFSIZE)
@@ -29,7 +31,10 @@ STDMETHODIMP ElgatoSampleCallback::BufferCB(double time, BYTE *pBuffer, long len
         copyLength = FRAME_BUFSIZE;
     }
 
-    memcpy(cachedBytes, pBuffer, copyLength);
+    memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
+    memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
+    memcpy(latestBuffer, pBuffer, copyLength);
+
     EnterCriticalSection(&frameAccessCriticalSection);
     isVideoFrameReady = true;
     LeaveCriticalSection(&frameAccessCriticalSection);
@@ -42,13 +47,14 @@ void ElgatoSampleCallback::UpdateSRV(ID3D11ShaderResourceView* srv, bool useCPU)
     if (useCPU)
     {
         BYTE* stagingBytes = new BYTE[FRAME_BUFSIZE];
-        DirectXHelper::ConvertYUVtoBGRA(cachedBytes, stagingBytes, FRAME_WIDTH, FRAME_HEIGHT, true);
+        // Do not cache when using the CPU
+        DirectXHelper::ConvertYUVtoBGRA(latestBuffer, stagingBytes, FRAME_WIDTH, FRAME_HEIGHT, true);
         DirectXHelper::UpdateSRV(_device, srv, stagingBytes, FRAME_WIDTH * FRAME_BPP);
         delete[] stagingBytes;
     }
     else
     {
-        DirectXHelper::UpdateSRV(_device, srv, cachedBytes, FRAME_WIDTH * FRAME_BPP);
+        DirectXHelper::UpdateSRV(_device, srv, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
     }
 }
 

SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.h

@@ -56,7 +56,7 @@ class ElgatoSampleCallback : public ISampleGrabberCB
 
     LONGLONG GetTimestamp()
     {
-        return cachedTimestamp;
+        return thirdTimeStamp;
     }
 
     bool IsVideoFrameReady();
@@ -65,8 +65,13 @@ class ElgatoSampleCallback : public ISampleGrabberCB
     ULONG m_cRef = 0;
 
     ID3D11Device* _device;
-    BYTE* cachedBytes = new BYTE[FRAME_BUFSIZE];
-    LONGLONG cachedTimestamp = -1;
+    BYTE* thirdCachedBuffer = new BYTE[FRAME_BUFSIZE];
+    BYTE* secondCachedBuffer = new BYTE[FRAME_BUFSIZE];
+    BYTE* latestBuffer = new BYTE[FRAME_BUFSIZE];
+
+    LONGLONG latestTimeStamp = 0;
+    LONGLONG secondTimeStamp = 0;
+    LONGLONG thirdTimeStamp = 0;
 
     CRITICAL_SECTION frameAccessCriticalSection;
     bool isVideoFrameReady = false;