CoreAudio: add UI to enable/disable spatial audio and head-tracking

Author: andygrundman

app/gui/SettingsView.qml

@@ -881,6 +881,78 @@ Flickable {
                     }
                 }
 
+                Label {
+                    width: parent.width
+                    id: resSpatialAudioTitle
+                    text: qsTr("Spatial audio")
+                    font.pointSize: 12
+                    wrapMode: Text.Wrap
+                    visible: Qt.platform.os == "osx"
+                }
+
+                Row {
+                    spacing: 5
+                    width: parent.width
+                    visible: Qt.platform.os == "osx"
+
+                    AutoResizingComboBox {
+                        // ignore setting the index at first, and actually set it when the component is loaded
+                        Component.onCompleted: {
+                            var saved_sac = StreamingPreferences.spatialAudioConfig
+                            currentIndex = 0
+                            for (var i = 0; i < spatialAudioListModel.count; i++) {
+                                var el_audio = spatialAudioListModel.get(i).val;
+                                if (saved_sac === el_audio) {
+                                    currentIndex = i
+                                    break
+                                }
+                            }
+                            activated(currentIndex)
+                        }
+
+                        id: spatialAudioComboBox
+                        enabled: StreamingPreferences.audioConfig != StreamingPreferences.AC_STEREO
+                        textRole: "text"
+                        model: ListModel {
+                            id: spatialAudioListModel
+                            ListElement {
+                                text: qsTr("Enabled")
+                                val: StreamingPreferences.SAC_AUTO
+                            }
+                            ListElement {
+                                text: qsTr("Disabled")
+                                val: StreamingPreferences.SAC_DISABLED
+                            }
+                        }
+
+                        // ::onActivated must be used, as it only listens for when the index is changed by a human
+                        onActivated : {
+                            StreamingPreferences.spatialAudioConfig = spatialAudioListModel.get(currentIndex).val
+                        }
+
+                        ToolTip.delay: 1000
+                        ToolTip.timeout: 5000
+                        ToolTip.visible: hovered
+                        ToolTip.text: qsTr("Spatial audio will be used when using any type of headphones, built-in Macbook speakers, and 2-channel USB devices.")
+                    }
+
+                    CheckBox {
+                        id: spatialHeadTracking
+                        enabled: StreamingPreferences.audioConfig != StreamingPreferences.AC_STEREO && StreamingPreferences.spatialAudioConfig != StreamingPreferences.SAC_DISABLED
+                        width: parent.width
+                        text: qsTr("Enable head-tracking")
+                        font.pointSize: 12
+                        checked: StreamingPreferences.spatialHeadTracking
+                        onCheckedChanged: {
+                            StreamingPreferences.spatialHeadTracking = checked
+                        }
+
+                        ToolTip.delay: 1000
+                        ToolTip.timeout: 5000
+                        ToolTip.visible: hovered
+                        ToolTip.text: qsTr("Requires supported Apple or Beats headphones")
+                    }
+                }
 
                 CheckBox {
                     id: audioPcCheck
@@ -1176,7 +1248,7 @@ Flickable {
                         ListElement {
                             text: qsTr("Maximized")
                             val: StreamingPreferences.UI_MAXIMIZED
-                        }   
+                        }
                         ListElement {
                             text: qsTr("Fullscreen")
                             val: StreamingPreferences.UI_FULLSCREEN

app/settings/streamingpreferences.cpp

@@ -19,9 +19,11 @@
 #define SER_FULLSCREEN "fullscreen"
 #define SER_VSYNC "vsync"
 #define SER_GAMEOPTS "gameopts"
+#define SER_HEADTRACKING "headtracking"
 #define SER_HOSTAUDIO "hostaudio"
 #define SER_MULTICONT "multicontroller"
 #define SER_AUDIOCFG "audiocfg"
+#define SER_SPATIALAUDIOCFG "spatialaudiocfg"
 #define SER_VIDEOCFG "videocfg"
 #define SER_HDR "hdr"
 #define SER_YUV444 "yuv444"
@@ -124,6 +126,7 @@ void StreamingPreferences::reload()
     unlockBitrate = settings.value(SER_UNLOCK_BITRATE, false).toBool();
     enableVsync = settings.value(SER_VSYNC, true).toBool();
     gameOptimizations = settings.value(SER_GAMEOPTS, true).toBool();
+    spatialHeadTracking = settings.value(SER_HEADTRACKING, false).toBool();
     playAudioOnHost = settings.value(SER_HOSTAUDIO, false).toBool();
     multiController = settings.value(SER_MULTICONT, true).toBool();
     enableMdns = settings.value(SER_MDNS, true).toBool();
@@ -148,6 +151,8 @@ void StreamingPreferences::reload()
                                                          static_cast<int>(CaptureSysKeysMode::CSK_OFF)).toInt());
     audioConfig = static_cast<AudioConfig>(settings.value(SER_AUDIOCFG,
                                                   static_cast<int>(AudioConfig::AC_STEREO)).toInt());
+    spatialAudioConfig = static_cast<SpatialAudioConfig>(settings.value(SER_SPATIALAUDIOCFG,
+                                                  static_cast<int>(SpatialAudioConfig::SAC_AUTO)).toInt());
     videoCodecConfig = static_cast<VideoCodecConfig>(settings.value(SER_VIDEOCFG,
                                                   static_cast<int>(VideoCodecConfig::VCC_AUTO)).toInt());
     videoDecoderSelection = static_cast<VideoDecoderSelection>(settings.value(SER_VIDEODEC,
@@ -314,6 +319,7 @@ void StreamingPreferences::save()
     settings.setValue(SER_UNLOCK_BITRATE, unlockBitrate);
     settings.setValue(SER_VSYNC, enableVsync);
     settings.setValue(SER_GAMEOPTS, gameOptimizations);
+    settings.setValue(SER_HEADTRACKING, spatialHeadTracking);
     settings.setValue(SER_HOSTAUDIO, playAudioOnHost);
     settings.setValue(SER_MULTICONT, multiController);
     settings.setValue(SER_MDNS, enableMdns);
@@ -328,6 +334,7 @@ void StreamingPreferences::save()
     settings.setValue(SER_DETECTNETBLOCKING, detectNetworkBlocking);
     settings.setValue(SER_SHOWPERFOVERLAY, showPerformanceOverlay);
     settings.setValue(SER_AUDIOCFG, static_cast<int>(audioConfig));
+    settings.setValue(SER_SPATIALAUDIOCFG, static_cast<int>(spatialAudioConfig));
     settings.setValue(SER_HDR, enableHdr);
     settings.setValue(SER_YUV444, enableYUV444);
     settings.setValue(SER_VIDEOCFG, static_cast<int>(videoCodecConfig));

app/settings/streamingpreferences.h

@@ -26,6 +26,13 @@ class StreamingPreferences : public QObject
     };
     Q_ENUM(AudioConfig)
 
+    enum SpatialAudioConfig
+    {
+        SAC_AUTO,
+        SAC_DISABLED
+    };
+    Q_ENUM(SpatialAudioConfig)
+
     enum VideoCodecConfig
     {
         VCC_AUTO,
@@ -112,6 +119,7 @@ class StreamingPreferences : public QObject
     Q_PROPERTY(bool unlockBitrate MEMBER unlockBitrate NOTIFY unlockBitrateChanged)
     Q_PROPERTY(bool enableVsync MEMBER enableVsync NOTIFY enableVsyncChanged)
     Q_PROPERTY(bool gameOptimizations MEMBER gameOptimizations NOTIFY gameOptimizationsChanged)
+    Q_PROPERTY(bool spatialHeadTracking MEMBER spatialHeadTracking NOTIFY spatialHeadTrackingChanged)
     Q_PROPERTY(bool playAudioOnHost MEMBER playAudioOnHost NOTIFY playAudioOnHostChanged)
     Q_PROPERTY(bool multiController MEMBER multiController NOTIFY multiControllerChanged)
     Q_PROPERTY(bool enableMdns MEMBER enableMdns NOTIFY enableMdnsChanged)
@@ -125,6 +133,7 @@ class StreamingPreferences : public QObject
     Q_PROPERTY(bool detectNetworkBlocking MEMBER detectNetworkBlocking NOTIFY detectNetworkBlockingChanged)
     Q_PROPERTY(bool showPerformanceOverlay MEMBER showPerformanceOverlay NOTIFY showPerformanceOverlayChanged)
     Q_PROPERTY(AudioConfig audioConfig MEMBER audioConfig NOTIFY audioConfigChanged)
+    Q_PROPERTY(SpatialAudioConfig spatialAudioConfig MEMBER spatialAudioConfig NOTIFY spatialAudioConfigChanged)
     Q_PROPERTY(VideoCodecConfig videoCodecConfig MEMBER videoCodecConfig NOTIFY videoCodecConfigChanged)
     Q_PROPERTY(bool enableHdr MEMBER enableHdr NOTIFY enableHdrChanged)
     Q_PROPERTY(bool enableYUV444 MEMBER enableYUV444 NOTIFY enableYUV444Changed)
@@ -151,6 +160,7 @@ class StreamingPreferences : public QObject
     bool unlockBitrate;
     bool enableVsync;
     bool gameOptimizations;
+    bool spatialHeadTracking;
     bool playAudioOnHost;
     bool multiController;
     bool enableMdns;
@@ -171,6 +181,7 @@ class StreamingPreferences : public QObject
     bool keepAwake;
     int packetSize;
     AudioConfig audioConfig;
+    SpatialAudioConfig spatialAudioConfig;
     VideoCodecConfig videoCodecConfig;
     bool enableHdr;
     bool enableYUV444;
@@ -187,6 +198,7 @@ class StreamingPreferences : public QObject
     void unlockBitrateChanged();
     void enableVsyncChanged();
     void gameOptimizationsChanged();
+    void spatialHeadTrackingChanged();
     void playAudioOnHostChanged();
     void multiControllerChanged();
     void unsupportedFpsChanged();
@@ -195,6 +207,7 @@ class StreamingPreferences : public QObject
     void absoluteMouseModeChanged();
     void absoluteTouchModeChanged();
     void audioConfigChanged();
+    void spatialAudioConfigChanged();
     void videoCodecConfigChanged();
     void enableHdrChanged();
     void enableYUV444Changed();

app/streaming/audio/renderers/coreaudio/au_spatial_renderer.mm

@@ -1,6 +1,7 @@
 #import "au_spatial_renderer.h"
 #import "coreaudio_helpers.h"
 #import "AllocatedAudioBufferList.h"
+#include "settings/streamingpreferences.h"
 
 #import <AVFoundation/AVFoundation.h>
 #import <QtGlobal>
@@ -211,12 +212,14 @@ OSStatus inputCallback(void *inRefCon,
         if (outputType == kSpatialMixerOutputType_Headphones) {
             // XXX: Both of these might require the builder to have a paid Apple developer account due to the use of entitlements.
 
-            // XXX Head-tracking doesn't work well currently, with audio glitches on my system. It's off by default pending a config option.
             // For devices that support it, enable head-tracking.
             // Apps that use low-latency head-tracking in iOS/tvOS need to set
             // the audio session category to ambient or run in Game Mode.
             // Head tracking requires the entitlement com.apple.developer.coremotion.head-pose.
-            if (0) {
+
+            // XXX Head-tracking may cause audio glitches. It's off by default.
+            StreamingPreferences *prefs = StreamingPreferences::get();
+            if (prefs->spatialHeadTracking) {
                 uint32_t ht = 1;
                 status = AudioUnitSetProperty(m_Mixer, kAudioUnitProperty_SpatialMixerEnableHeadTracking, kAudioUnitScope_Global, 0, &ht, sizeof(uint32_t));
                 if (status != noErr) {

app/streaming/audio/renderers/coreaudio/coreaudio.cpp

@@ -1,5 +1,6 @@
 #include "coreaudio.h"
 #include "coreaudio_helpers.h"
+#include "settings/streamingpreferences.h"
 
 #if TARGET_OS_OSX
 #include <IOKit/audio/IOAudioTypes.h>
@@ -151,8 +152,8 @@ void CoreAudioRenderer::stringifyAudioStats(AUDIO_STATS& stats, char *output, in
     snprintf(
         output, length,
         "Audio stream: %s-channel Opus low-delay @ 48 kHz\n"
-        "Output device: %s @ %.1f kHz, %u-channel\n"
-        "Render mode: %s, device type: %s %s, %s\n"
+        "Output device: %s @ %.1f kHz, %u-channel, %s %s\n"
+        "Render mode: %s %s\n"
         "Opus config: %s, frame size: %.1fms, bitrate: %dkbps\n"
         "Buffer: length: %0.1f packets (%.1fms), current %.1f%%\n"
         "Packet loss from network: %.2f%%, packets dropped due to resync: %.2f%%\n"
@@ -163,8 +164,6 @@ void CoreAudioRenderer::stringifyAudioStats(AUDIO_STATS& stats, char *output, in
         m_OutputDeviceName,
         m_OutputASBD.mSampleRate / 1000.0,
         m_OutputASBD.mChannelsPerFrame,
-
-        m_Spatial ? (m_SpatialAU.m_PersonalizedHRTF ? "personalized spatial" : "spatial") : "passthrough",
         !strcmp(m_OutputTransportType, "blue") ? "Bluetooth"
             : !strcmp(m_OutputTransportType, "bltn") ? "built-in"
             : !strcmp(m_OutputTransportType, "usb ") ? "USB"
@@ -175,7 +174,9 @@ void CoreAudioRenderer::stringifyAudioStats(AUDIO_STATS& stats, char *output, in
             : !strcmp(m_OutputDataSource, "ispk") ? "internal speakers"
             : !strcmp(m_OutputDataSource, "espk") ? "external speakers"
             : m_OutputDataSource,
-        m_Spatial && m_SpatialAU.m_HeadTracking ? "head-tracking: yes" : "",
+
+        m_Spatial ? (m_SpatialAU.m_PersonalizedHRTF ? "personalized spatial" : "spatial") : "passthrough",
+        m_Spatial && m_SpatialAU.m_HeadTracking ? "with head-tracking" : "",
 
         // Work out if we're getting high or low quality from Sunshine. coupled surround is designed for physical speakers
         ((m_opusConfig->channelCount == 2 && stats.opusBitsPerSec > 128) || !m_opusConfig->coupledStreams)
@@ -300,13 +301,19 @@ bool CoreAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION*
 
     DEBUG_TRACE("CoreAudioRenderer getSpatialMixerOutputType = %d", outputType);
 
-    // XXX: let the user choose this if they want
     if (opusConfig->channelCount > 2) {
         if (outputType != kSpatialMixerOutputType_ExternalSpeakers) {
             m_Spatial = true;
         }
     }
 
+    StreamingPreferences *prefs = StreamingPreferences::get();
+    if (prefs->spatialAudioConfig == StreamingPreferences::SAC_DISABLED) {
+        // User has disabled spatial audio
+        DEBUG_TRACE("CoreAudioRenderer user has disabled spatial audio");
+        m_Spatial = false;
+    }
+
     // indicate the format our callback will provide samples in
     // If necessary, the OS takes care of resampling (but not downmixing, hmm)
     AudioStreamBasicDescription streamDesc;