From 6c9600987364ebec52115c165df8672a01d017f1 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Mon, 29 Oct 2018 16:48:39 -0700 Subject: [PATCH 01/94] Add new project with empty template. --- CoViewingExample.xcodeproj/project.pbxproj | 341 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcshareddata/IDEWorkspaceChecks.plist | 8 + CoViewingExample/AppDelegate.swift | 45 +++ .../AppIcon.appiconset/Contents.json | 98 +++++ .../Assets.xcassets/Contents.json | 6 + .../Base.lproj/LaunchScreen.storyboard | 25 ++ CoViewingExample/Base.lproj/Main.storyboard | 24 ++ CoViewingExample/Info.plist | 45 +++ CoViewingExample/ViewController.swift | 19 + 10 files changed, 618 insertions(+) create mode 100644 CoViewingExample.xcodeproj/project.pbxproj create mode 100644 CoViewingExample.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 CoViewingExample.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 CoViewingExample/AppDelegate.swift create mode 100644 CoViewingExample/Assets.xcassets/AppIcon.appiconset/Contents.json create mode 100644 CoViewingExample/Assets.xcassets/Contents.json create mode 100644 CoViewingExample/Base.lproj/LaunchScreen.storyboard create mode 100644 CoViewingExample/Base.lproj/Main.storyboard create mode 100644 CoViewingExample/Info.plist create mode 100644 CoViewingExample/ViewController.swift diff --git a/CoViewingExample.xcodeproj/project.pbxproj b/CoViewingExample.xcodeproj/project.pbxproj new file mode 100644 index 00000000..5b86b6db --- /dev/null +++ b/CoViewingExample.xcodeproj/project.pbxproj @@ -0,0 +1,341 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 50; + objects = { + +/* Begin PBXBuildFile section */ + 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E422187D2B200437980 /* AppDelegate.swift */; }; + 8A395E452187D2B200437980 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E442187D2B200437980 /* ViewController.swift */; }; + 8A395E482187D2B200437980 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E462187D2B200437980 /* Main.storyboard */; }; + 8A395E4A2187D2B300437980 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E492187D2B300437980 /* Assets.xcassets */; }; + 8A395E4D2187D2B300437980 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E4B2187D2B300437980 /* LaunchScreen.storyboard */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 8A395E3F2187D2B200437980 /* CoViewingExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CoViewingExample.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 8A395E422187D2B200437980 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + 8A395E442187D2B200437980 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; + 8A395E472187D2B200437980 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; + 8A395E492187D2B300437980 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 8A395E4C2187D2B300437980 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; + 8A395E4E2187D2B300437980 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 8A395E3C2187D2B200437980 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 8A395E362187D2B200437980 = { + isa = PBXGroup; + children = ( + 8A395E412187D2B200437980 /* CoViewingExample */, + 8A395E402187D2B200437980 /* Products */, + ); + sourceTree = ""; + }; + 8A395E402187D2B200437980 /* Products */ = { + isa = PBXGroup; + children = ( + 8A395E3F2187D2B200437980 /* CoViewingExample.app */, + ); + name = Products; + sourceTree = ""; + }; + 8A395E412187D2B200437980 /* CoViewingExample */ = { + isa = PBXGroup; + children = ( + 8A395E422187D2B200437980 /* AppDelegate.swift */, + 8A395E442187D2B200437980 /* ViewController.swift */, + 8A395E462187D2B200437980 /* Main.storyboard */, + 8A395E492187D2B300437980 /* Assets.xcassets */, + 8A395E4B2187D2B300437980 /* LaunchScreen.storyboard */, + 8A395E4E2187D2B300437980 /* Info.plist */, + ); + path = CoViewingExample; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 8A395E3E2187D2B200437980 /* CoViewingExample */ = { + isa = PBXNativeTarget; + buildConfigurationList = 8A395E512187D2B300437980 /* Build configuration list for PBXNativeTarget "CoViewingExample" */; + buildPhases = ( + 8A395E3B2187D2B200437980 /* Sources */, + 8A395E3C2187D2B200437980 /* Frameworks */, + 8A395E3D2187D2B200437980 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = CoViewingExample; + productName = CoViewingExample; + productReference = 8A395E3F2187D2B200437980 /* CoViewingExample.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 8A395E372187D2B200437980 /* Project object */ = { + isa = PBXProject; + attributes = { + LastSwiftUpdateCheck = 1000; + LastUpgradeCheck = 1000; + ORGANIZATIONNAME = "Twilio Inc."; + TargetAttributes = { + 8A395E3E2187D2B200437980 = { + CreatedOnToolsVersion = 10.0; + }; + }; + }; + buildConfigurationList = 8A395E3A2187D2B200437980 /* Build configuration list for PBXProject "CoViewingExample" */; + compatibilityVersion = "Xcode 9.3"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 8A395E362187D2B200437980; + productRefGroup = 8A395E402187D2B200437980 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 8A395E3E2187D2B200437980 /* CoViewingExample */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 8A395E3D2187D2B200437980 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 8A395E4D2187D2B300437980 /* LaunchScreen.storyboard in Resources */, + 8A395E4A2187D2B300437980 /* Assets.xcassets in Resources */, + 8A395E482187D2B200437980 /* Main.storyboard in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 8A395E3B2187D2B200437980 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 8A395E452187D2B200437980 /* ViewController.swift in Sources */, + 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXVariantGroup section */ + 8A395E462187D2B200437980 /* Main.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 8A395E472187D2B200437980 /* Base */, + ); + name = Main.storyboard; + sourceTree = ""; + }; + 8A395E4B2187D2B300437980 /* LaunchScreen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 8A395E4C2187D2B300437980 /* Base */, + ); + name = LaunchScreen.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 8A395E4F2187D2B300437980 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_IDENTITY = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + 8A395E502187D2B300437980 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_IDENTITY = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 8A395E522187D2B300437980 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = SX5J6BN2KX; + INFOPLIST_FILE = CoViewingExample/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.twilio.CoViewingExample; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 4.2; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 8A395E532187D2B300437980 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = SX5J6BN2KX; + INFOPLIST_FILE = CoViewingExample/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = com.twilio.CoViewingExample; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 4.2; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 8A395E3A2187D2B200437980 /* Build configuration list for PBXProject "CoViewingExample" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 8A395E4F2187D2B300437980 /* Debug */, + 8A395E502187D2B300437980 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 8A395E512187D2B300437980 /* Build configuration list for PBXNativeTarget "CoViewingExample" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 8A395E522187D2B300437980 /* Debug */, + 8A395E532187D2B300437980 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 8A395E372187D2B200437980 /* Project object */; +} diff --git a/CoViewingExample.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/CoViewingExample.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 00000000..77a38387 --- /dev/null +++ b/CoViewingExample.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/CoViewingExample.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/CoViewingExample.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 00000000..18d98100 --- /dev/null +++ b/CoViewingExample.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/CoViewingExample/AppDelegate.swift b/CoViewingExample/AppDelegate.swift new file mode 100644 index 00000000..8e7ad10a --- /dev/null +++ b/CoViewingExample/AppDelegate.swift @@ -0,0 +1,45 @@ +// +// AppDelegate.swift +// CoViewingExample +// +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +import UIKit + +@UIApplicationMain +class AppDelegate: UIResponder, UIApplicationDelegate { + + var window: UIWindow? + + + func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { + // Override point for customization after application launch. + return true + } + + func applicationWillResignActive(_ application: UIApplication) { + // Sent when the application is about to move from active to inactive state. This can occur for certain types of temporary interruptions (such as an incoming phone call or SMS message) or when the user quits the application and it begins the transition to the background state. + // Use this method to pause ongoing tasks, disable timers, and invalidate graphics rendering callbacks. Games should use this method to pause the game. + } + + func applicationDidEnterBackground(_ application: UIApplication) { + // Use this method to release shared resources, save user data, invalidate timers, and store enough application state information to restore your application to its current state in case it is terminated later. + // If your application supports background execution, this method is called instead of applicationWillTerminate: when the user quits. + } + + func applicationWillEnterForeground(_ application: UIApplication) { + // Called as part of the transition from the background to the active state; here you can undo many of the changes made on entering the background. + } + + func applicationDidBecomeActive(_ application: UIApplication) { + // Restart any tasks that were paused (or not yet started) while the application was inactive. If the application was previously in the background, optionally refresh the user interface. + } + + func applicationWillTerminate(_ application: UIApplication) { + // Called when the application is about to terminate. Save data if appropriate. See also applicationDidEnterBackground:. + } + + +} + diff --git a/CoViewingExample/Assets.xcassets/AppIcon.appiconset/Contents.json b/CoViewingExample/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000..d8db8d65 --- /dev/null +++ b/CoViewingExample/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,98 @@ +{ + "images" : [ + { + "idiom" : "iphone", + "size" : "20x20", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "20x20", + "scale" : "3x" + }, + { + "idiom" : "iphone", + "size" : "29x29", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "29x29", + "scale" : "3x" + }, + { + "idiom" : "iphone", + "size" : "40x40", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "40x40", + "scale" : "3x" + }, + { + "idiom" : "iphone", + "size" : "60x60", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "60x60", + "scale" : "3x" + }, + { + "idiom" : "ipad", + "size" : "20x20", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "20x20", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "29x29", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "29x29", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "40x40", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "40x40", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "76x76", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "76x76", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "83.5x83.5", + "scale" : "2x" + }, + { + "idiom" : "ios-marketing", + "size" : "1024x1024", + "scale" : "1x" + } + ], + "info" : { + "version" : 1, + "author" : "xcode" + } +} \ No newline at end of file diff --git a/CoViewingExample/Assets.xcassets/Contents.json b/CoViewingExample/Assets.xcassets/Contents.json new file mode 100644 index 00000000..da4a164c --- /dev/null +++ b/CoViewingExample/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "version" : 1, + "author" : "xcode" + } +} \ No newline at end of file diff --git a/CoViewingExample/Base.lproj/LaunchScreen.storyboard b/CoViewingExample/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 00000000..bfa36129 --- /dev/null +++ b/CoViewingExample/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/CoViewingExample/Base.lproj/Main.storyboard b/CoViewingExample/Base.lproj/Main.storyboard new file mode 100644 index 00000000..f1bcf384 --- /dev/null +++ b/CoViewingExample/Base.lproj/Main.storyboard @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/CoViewingExample/Info.plist b/CoViewingExample/Info.plist new file mode 100644 index 00000000..16be3b68 --- /dev/null +++ b/CoViewingExample/Info.plist @@ -0,0 +1,45 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift new file mode 100644 index 00000000..227437d7 --- /dev/null +++ b/CoViewingExample/ViewController.swift @@ -0,0 +1,19 @@ +// +// ViewController.swift +// CoViewingExample +// +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +import UIKit + +class ViewController: UIViewController { + + override func viewDidLoad() { + super.viewDidLoad() + // Do any additional setup after loading the view, typically from a nib. + } + + +} + From 06c933cf1cb45b79b5845fc7e29ef6192e01f21c Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Mon, 29 Oct 2018 18:26:19 -0700 Subject: [PATCH 02/94] Add ExampleAVPlayerView. --- CoViewingExample.xcodeproj/project.pbxproj | 4 +++ CoViewingExample/ExampleAVPlayerView.swift | 35 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 CoViewingExample/ExampleAVPlayerView.swift diff --git a/CoViewingExample.xcodeproj/project.pbxproj b/CoViewingExample.xcodeproj/project.pbxproj index 5b86b6db..94cf0b37 100644 --- a/CoViewingExample.xcodeproj/project.pbxproj +++ b/CoViewingExample.xcodeproj/project.pbxproj @@ -12,6 +12,7 @@ 8A395E482187D2B200437980 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E462187D2B200437980 /* Main.storyboard */; }; 8A395E4A2187D2B300437980 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E492187D2B300437980 /* Assets.xcassets */; }; 8A395E4D2187D2B300437980 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E4B2187D2B300437980 /* LaunchScreen.storyboard */; }; + 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -22,6 +23,7 @@ 8A395E492187D2B300437980 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 8A395E4C2187D2B300437980 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 8A395E4E2187D2B300437980 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerView.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -55,6 +57,7 @@ isa = PBXGroup; children = ( 8A395E422187D2B200437980 /* AppDelegate.swift */, + 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */, 8A395E442187D2B200437980 /* ViewController.swift */, 8A395E462187D2B200437980 /* Main.storyboard */, 8A395E492187D2B300437980 /* Assets.xcassets */, @@ -137,6 +140,7 @@ files = ( 8A395E452187D2B200437980 /* ViewController.swift in Sources */, 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */, + 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/CoViewingExample/ExampleAVPlayerView.swift b/CoViewingExample/ExampleAVPlayerView.swift new file mode 100644 index 00000000..6e7b95fc --- /dev/null +++ b/CoViewingExample/ExampleAVPlayerView.swift @@ -0,0 +1,35 @@ +// +// ExampleAVPlayerView.swift +// CoViewingExample +// +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +import AVFoundation +import UIKit + +class ExampleAVPlayerView: UIView { + + init(frame: CGRect, player: AVPlayer) { + super.init(frame: frame) + self.playerLayer.player = player + self.playerLayer.videoGravity = AVLayerVideoGravity.resizeAspect + } + + required init?(coder aDecoder: NSCoder) { + super.init(coder: aDecoder) + // It won't be possible to hookup an AVPlayer yet. + self.playerLayer.videoGravity = AVLayerVideoGravity.resizeAspect + } + + var playerLayer : AVPlayerLayer { + get { + return self.layer as! AVPlayerLayer + } + } + + override class var layerClass : AnyClass { + return AVPlayerLayer.self + } + +} From b65f914e228c73a38b0d9ab42169e76e23762fd6 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Mon, 29 Oct 2018 18:26:50 -0700 Subject: [PATCH 03/94] WIP - Play and pause video. * Need to add KVO. --- CoViewingExample/ViewController.swift | 47 ++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 227437d7..767206c5 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -5,15 +5,60 @@ // Copyright © 2018 Twilio Inc. All rights reserved. // +import AVFoundation import UIKit class ViewController: UIViewController { + var videoPlayer: AVPlayer? = nil + var videoPlayerView: ExampleAVPlayerView? = nil + + static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! + override func viewDidLoad() { super.viewDidLoad() // Do any additional setup after loading the view, typically from a nib. } + override func viewWillAppear(_ animated: Bool) { + super.viewWillAppear(animated) + if videoPlayer == nil { + startVideoPlayer() + } + } -} + override func viewWillLayoutSubviews() { + super.viewWillLayoutSubviews() + + if let playerView = videoPlayerView { + playerView.frame = CGRect(origin: CGPoint.zero, size: self.view.bounds.size) + } + } + + func startVideoPlayer() { + if let player = self.videoPlayer { + player.play() + return + } + // TODO: Add KVO observer? + let player = AVPlayer(url: ViewController.kRemoteContentURL) + videoPlayer = player + + let playerView = ExampleAVPlayerView(frame: CGRect.zero, player: player) + videoPlayerView = playerView + + // We will rely on frame based layout to size and position `self.videoPlayerView`. + self.view.insertSubview(playerView, at: 0) + self.view.setNeedsLayout() + } + + func stopVideoPlayer() { + videoPlayer?.pause() + videoPlayer = nil + + // Remove player UI + videoPlayerView?.removeFromSuperview() + videoPlayerView = nil + } +} From d74becfa5b8880020ada3b3851148258e51bef7e Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Mon, 29 Oct 2018 18:32:18 -0700 Subject: [PATCH 04/94] Play automatically for now. --- CoViewingExample/ViewController.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 767206c5..151a82e5 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -51,6 +51,8 @@ class ViewController: UIViewController { // We will rely on frame based layout to size and position `self.videoPlayerView`. self.view.insertSubview(playerView, at: 0) self.view.setNeedsLayout() + + player.play() } func stopVideoPlayer() { From e62c97a73062eca023e3ec58c4fc283506082f13 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Mon, 29 Oct 2018 19:53:25 -0700 Subject: [PATCH 05/94] Added ExampleAVPlayerSource which captures from an AVPlayerItem. * Use AVPlayerItemVideoOutput, and CADisplayLink to pull frames. --- CoViewingExample.xcodeproj/project.pbxproj | 4 + CoViewingExample/ExampleAVPlayerSource.swift | 77 ++++++++++++++++++++ CoViewingExample/ViewController.swift | 7 +- 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 CoViewingExample/ExampleAVPlayerSource.swift diff --git a/CoViewingExample.xcodeproj/project.pbxproj b/CoViewingExample.xcodeproj/project.pbxproj index 94cf0b37..085d9e1e 100644 --- a/CoViewingExample.xcodeproj/project.pbxproj +++ b/CoViewingExample.xcodeproj/project.pbxproj @@ -13,6 +13,7 @@ 8A395E4A2187D2B300437980 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E492187D2B300437980 /* Assets.xcassets */; }; 8A395E4D2187D2B300437980 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E4B2187D2B300437980 /* LaunchScreen.storyboard */; }; 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */; }; + 8A395E572187F04C00437980 /* ExampleAVPlayerSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E562187F04C00437980 /* ExampleAVPlayerSource.swift */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -24,6 +25,7 @@ 8A395E4C2187D2B300437980 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 8A395E4E2187D2B300437980 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerView.swift; sourceTree = ""; }; + 8A395E562187F04C00437980 /* ExampleAVPlayerSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerSource.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -57,6 +59,7 @@ isa = PBXGroup; children = ( 8A395E422187D2B200437980 /* AppDelegate.swift */, + 8A395E562187F04C00437980 /* ExampleAVPlayerSource.swift */, 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */, 8A395E442187D2B200437980 /* ViewController.swift */, 8A395E462187D2B200437980 /* Main.storyboard */, @@ -141,6 +144,7 @@ 8A395E452187D2B200437980 /* ViewController.swift in Sources */, 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */, 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */, + 8A395E572187F04C00437980 /* ExampleAVPlayerSource.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift new file mode 100644 index 00000000..22d7e627 --- /dev/null +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -0,0 +1,77 @@ +// +// ExampleAVPlayerSource.swift +// CoViewingExample +// +// Created by Chris Eagleston on 10/29/18. +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +import AVFoundation + +class ExampleAVPlayerSource: NSObject { + private let sampleQueue: DispatchQueue + private var outputTimer: CADisplayLink? = nil + private var videoOutput: AVPlayerItemVideoOutput? = nil + + static private var frameCounter = UInt32(0) + + init(item: AVPlayerItem) { + sampleQueue = DispatchQueue(label: "", qos: DispatchQoS.userInteractive, + attributes: DispatchQueue.Attributes(rawValue: 0), + autoreleaseFrequency: DispatchQueue.AutoreleaseFrequency.workItem, + target: nil) + + super.init() + + let timer = CADisplayLink(target: self, + selector: #selector(ExampleAVPlayerSource.displayLinkDidFire(displayLink:))) + timer.preferredFramesPerSecond = 30 + timer.isPaused = true + timer.add(to: RunLoop.current, forMode: RunLoop.Mode.common) + outputTimer = timer + + let attributes = [kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] + videoOutput = AVPlayerItemVideoOutput(pixelBufferAttributes: attributes) + videoOutput?.setDelegate(self, queue: sampleQueue) + videoOutput?.requestNotificationOfMediaDataChange(withAdvanceInterval: 0.1) + + item.add(videoOutput!) + } + + @objc func displayLinkDidFire(displayLink: CADisplayLink) { + guard let output = videoOutput else { + return + } + + let targetHostTime = displayLink.targetTimestamp + let targetItemTime = output.itemTime(forHostTime: targetHostTime) + + if output.hasNewPixelBuffer(forItemTime: targetItemTime) { + var presentationTime = CMTime.zero + let pixelBuffer = output.copyPixelBuffer(forItemTime: targetItemTime, itemTimeForDisplay: &presentationTime) + + ExampleAVPlayerSource.frameCounter += 1 + if ExampleAVPlayerSource.frameCounter % 30 == 0 { + print("Copied new pixel buffer: ", pixelBuffer as Any) + } + } else { + // TODO: Consider suspending the timer and requesting a notification when media becomes available. + } + } + + @objc func stopTimer() { + outputTimer?.invalidate() + } +} + +extension ExampleAVPlayerSource: AVPlayerItemOutputPullDelegate { + func outputMediaDataWillChange(_ sender: AVPlayerItemOutput) { + print(#function) + // Begin to receive video frames. + outputTimer?.isPaused = false + } + + func outputSequenceWasFlushed(_ output: AVPlayerItemOutput) { + // TODO: Flush and output a black frame while we wait. + } +} diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 151a82e5..2b2e30ab 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -12,6 +12,7 @@ class ViewController: UIViewController { var videoPlayer: AVPlayer? = nil var videoPlayerView: ExampleAVPlayerView? = nil + var videoPlayerSource: ExampleAVPlayerSource? = nil static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! @@ -42,7 +43,8 @@ class ViewController: UIViewController { } // TODO: Add KVO observer? - let player = AVPlayer(url: ViewController.kRemoteContentURL) + let playerItem = AVPlayerItem(url: ViewController.kRemoteContentURL) + let player = AVPlayer(playerItem: playerItem) videoPlayer = player let playerView = ExampleAVPlayerView(frame: CGRect.zero, player: player) @@ -53,6 +55,9 @@ class ViewController: UIViewController { self.view.setNeedsLayout() player.play() + + // Configure our capturer to receive output from the AVPlayerItem. + videoPlayerSource = ExampleAVPlayerSource(item: playerItem) } func stopVideoPlayer() { From e5ff609d7cd78b4a2d85ab245f24612786a64f99 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 30 Oct 2018 17:34:08 -0700 Subject: [PATCH 06/94] WIP - Add an AVAudioMix to the asset. * TODO - Set the MTAudioTapProcessor. --- CoViewingExample/ExampleAVPlayerSource.swift | 1 - CoViewingExample/ViewController.swift | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 22d7e627..26af0d44 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -2,7 +2,6 @@ // ExampleAVPlayerSource.swift // CoViewingExample // -// Created by Chris Eagleston on 10/29/18. // Copyright © 2018 Twilio Inc. All rights reserved. // diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 2b2e30ab..fabea71c 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -42,7 +42,6 @@ class ViewController: UIViewController { return } - // TODO: Add KVO observer? let playerItem = AVPlayerItem(url: ViewController.kRemoteContentURL) let player = AVPlayer(playerItem: playerItem) videoPlayer = player @@ -54,10 +53,25 @@ class ViewController: UIViewController { self.view.insertSubview(playerView, at: 0) self.view.setNeedsLayout() + // TODO: Add KVO observer instead? player.play() - // Configure our capturer to receive output from the AVPlayerItem. + // Configure our video capturer to receive video samples from the AVPlayerItem. videoPlayerSource = ExampleAVPlayerSource(item: playerItem) + + // Configure our audio capturer to receive audio samples from the AVPlayerItem. + let audioMix = AVMutableAudioMix() + let itemAsset = playerItem.asset + print("Created asset with tracks: ", itemAsset.tracks as Any) + + if let assetAudioTrack = itemAsset.tracks(withMediaType: AVMediaType.audio).first { + let inputParameters = AVMutableAudioMixInputParameters(track: assetAudioTrack) +// inputParameters.audioTapProcessor = self + audioMix.inputParameters = [inputParameters] + playerItem.audioMix = audioMix + } else { + // Abort, retry, fail? + } } func stopVideoPlayer() { From f3204b491b787a2ca7e3c10a18eb7aac0aa7e4e0 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 30 Oct 2018 18:15:50 -0700 Subject: [PATCH 07/94] Attempt to request an IOSurface. --- CoViewingExample/ExampleAVPlayerSource.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 26af0d44..dd99ad8f 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -29,7 +29,12 @@ class ExampleAVPlayerSource: NSObject { timer.add(to: RunLoop.current, forMode: RunLoop.Mode.common) outputTimer = timer - let attributes = [kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] + // Note: It appears that we don't get an IOSurface backed buffer even after requesting it. + let attributes = [ + kCVPixelBufferIOSurfacePropertiesKey as String : [], + kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + ] as [String : Any] + videoOutput = AVPlayerItemVideoOutput(pixelBufferAttributes: attributes) videoOutput?.setDelegate(self, queue: sampleQueue) videoOutput?.requestNotificationOfMediaDataChange(withAdvanceInterval: 0.1) From d18f28ca4a39ca752254850b30ae4655d147f041 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 30 Oct 2018 18:54:45 -0700 Subject: [PATCH 08/94] Add ExampleAVPlayerAudioTap. --- CoViewingExample.xcodeproj/project.pbxproj | 4 ++ .../ExampleAVPlayerAudioTap.swift | 64 +++++++++++++++++++ CoViewingExample/ViewController.swift | 9 ++- 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 CoViewingExample/ExampleAVPlayerAudioTap.swift diff --git a/CoViewingExample.xcodeproj/project.pbxproj b/CoViewingExample.xcodeproj/project.pbxproj index 085d9e1e..f1302a13 100644 --- a/CoViewingExample.xcodeproj/project.pbxproj +++ b/CoViewingExample.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + 8A34C1D52189333400F22BE9 /* ExampleAVPlayerAudioTap.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */; }; 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E422187D2B200437980 /* AppDelegate.swift */; }; 8A395E452187D2B200437980 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E442187D2B200437980 /* ViewController.swift */; }; 8A395E482187D2B200437980 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E462187D2B200437980 /* Main.storyboard */; }; @@ -17,6 +18,7 @@ /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerAudioTap.swift; sourceTree = ""; }; 8A395E3F2187D2B200437980 /* CoViewingExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CoViewingExample.app; sourceTree = BUILT_PRODUCTS_DIR; }; 8A395E422187D2B200437980 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 8A395E442187D2B200437980 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; @@ -59,6 +61,7 @@ isa = PBXGroup; children = ( 8A395E422187D2B200437980 /* AppDelegate.swift */, + 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */, 8A395E562187F04C00437980 /* ExampleAVPlayerSource.swift */, 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */, 8A395E442187D2B200437980 /* ViewController.swift */, @@ -144,6 +147,7 @@ 8A395E452187D2B200437980 /* ViewController.swift in Sources */, 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */, 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */, + 8A34C1D52189333400F22BE9 /* ExampleAVPlayerAudioTap.swift in Sources */, 8A395E572187F04C00437980 /* ExampleAVPlayerSource.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/CoViewingExample/ExampleAVPlayerAudioTap.swift b/CoViewingExample/ExampleAVPlayerAudioTap.swift new file mode 100644 index 00000000..3e3342e1 --- /dev/null +++ b/CoViewingExample/ExampleAVPlayerAudioTap.swift @@ -0,0 +1,64 @@ +// +// ExampleAVPlayerAudioTap.swift +// CoViewingExample +// +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +import Foundation +import MediaToolbox + +class ExampleAVPlayerAudioTap { + + static func mediaToolboxAudioProcessingTapCreate(audioTap: ExampleAVPlayerAudioTap) -> MTAudioProcessingTap? { + var callbacks = MTAudioProcessingTapCallbacks( + version: kMTAudioProcessingTapCallbacksVersion_0, + clientInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(audioTap).toOpaque()), + init: audioTap.tapInit, + finalize: audioTap.tapFinalize, + prepare: audioTap.tapPrepare, + unprepare: audioTap.tapUnprepare, + process: audioTap.tapProcess + ) + + var tap: Unmanaged? + let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, + &callbacks, + kMTAudioProcessingTapCreationFlag_PostEffects, + &tap) + + if status == kCVReturnSuccess { + return tap!.takeUnretainedValue() + } else { + return nil + } + } + + let tapInit: MTAudioProcessingTapInitCallback = { (tap, clientInfo, tapStorageOut) in + let nonOptionalSelf = clientInfo!.assumingMemoryBound(to: ExampleAVPlayerAudioTap.self).pointee + print("init:", tap, clientInfo as Any, tapStorageOut, nonOptionalSelf) + } + + let tapFinalize: MTAudioProcessingTapFinalizeCallback = { + (tap) in + print(#function) + } + + let tapPrepare: MTAudioProcessingTapPrepareCallback = {(tap, b, c) in + print("Prepare:", tap, b, c) + } + + let tapUnprepare: MTAudioProcessingTapUnprepareCallback = {(tap) in + print("Unprepare:", tap) + } + + let tapProcess: MTAudioProcessingTapProcessCallback = { + (tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in + print("Process callback:", tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) + + let status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut) + if status != kCVReturnSuccess { + print("Failed to get source audio: ", status) + } + } +} diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index fabea71c..0ab8dc5a 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -11,8 +11,9 @@ import UIKit class ViewController: UIViewController { var videoPlayer: AVPlayer? = nil - var videoPlayerView: ExampleAVPlayerView? = nil + var videoPlayerAudioTap: ExampleAVPlayerAudioTap? = nil var videoPlayerSource: ExampleAVPlayerSource? = nil + var videoPlayerView: ExampleAVPlayerView? = nil static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! @@ -66,7 +67,11 @@ class ViewController: UIViewController { if let assetAudioTrack = itemAsset.tracks(withMediaType: AVMediaType.audio).first { let inputParameters = AVMutableAudioMixInputParameters(track: assetAudioTrack) -// inputParameters.audioTapProcessor = self + let processor = ExampleAVPlayerAudioTap() + videoPlayerAudioTap = processor + + // TODO: Memory management of the MTAudioProcessingTap. + inputParameters.audioTapProcessor = ExampleAVPlayerAudioTap.mediaToolboxAudioProcessingTapCreate(audioTap: processor) audioMix.inputParameters = [inputParameters] playerItem.audioMix = audioMix } else { From 65a7d5ed5ce94221ec6576ccb4b5e41f4be91316 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 30 Oct 2018 18:58:18 -0700 Subject: [PATCH 09/94] Comment out IOSurface request, it crashes on device. --- CoViewingExample/ExampleAVPlayerSource.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index dd99ad8f..03170ad1 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -29,9 +29,9 @@ class ExampleAVPlayerSource: NSObject { timer.add(to: RunLoop.current, forMode: RunLoop.Mode.common) outputTimer = timer - // Note: It appears that we don't get an IOSurface backed buffer even after requesting it. + // Note: It appears requesting IOSurface backing causes a crash on iPhone X / iOS 12.0.1? let attributes = [ - kCVPixelBufferIOSurfacePropertiesKey as String : [], +// kCVPixelBufferIOSurfacePropertiesKey as String : [], kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ] as [String : Any] From 2e8486ba409b2853ae23e1fc80feef941b89d04b Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 30 Oct 2018 19:24:23 -0700 Subject: [PATCH 10/94] Drop in a copy of ExampleCoreAudioDevice, and a bridging header. --- CoViewingExample.xcodeproj/project.pbxproj | 18 + .../AudioDevices-Bridging-Header.h | 8 + .../AudioDevices/ExampleCoreAudioDevice.h | 18 + .../AudioDevices/ExampleCoreAudioDevice.m | 516 ++++++++++++++++++ 4 files changed, 560 insertions(+) create mode 100644 CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h create mode 100644 CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h create mode 100644 CoViewingExample/AudioDevices/ExampleCoreAudioDevice.m diff --git a/CoViewingExample.xcodeproj/project.pbxproj b/CoViewingExample.xcodeproj/project.pbxproj index f1302a13..8faa919d 100644 --- a/CoViewingExample.xcodeproj/project.pbxproj +++ b/CoViewingExample.xcodeproj/project.pbxproj @@ -8,6 +8,7 @@ /* Begin PBXBuildFile section */ 8A34C1D52189333400F22BE9 /* ExampleAVPlayerAudioTap.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */; }; + 8A34C1DA2189496A00F22BE9 /* ExampleCoreAudioDevice.m in Sources */ = {isa = PBXBuildFile; fileRef = 8A34C1D92189496A00F22BE9 /* ExampleCoreAudioDevice.m */; }; 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E422187D2B200437980 /* AppDelegate.swift */; }; 8A395E452187D2B200437980 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E442187D2B200437980 /* ViewController.swift */; }; 8A395E482187D2B200437980 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E462187D2B200437980 /* Main.storyboard */; }; @@ -19,6 +20,9 @@ /* Begin PBXFileReference section */ 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerAudioTap.swift; sourceTree = ""; }; + 8A34C1D72189496A00F22BE9 /* ExampleCoreAudioDevice.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ExampleCoreAudioDevice.h; sourceTree = ""; }; + 8A34C1D82189496A00F22BE9 /* AudioDevices-Bridging-Header.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "AudioDevices-Bridging-Header.h"; sourceTree = ""; }; + 8A34C1D92189496A00F22BE9 /* ExampleCoreAudioDevice.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExampleCoreAudioDevice.m; sourceTree = ""; }; 8A395E3F2187D2B200437980 /* CoViewingExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CoViewingExample.app; sourceTree = BUILT_PRODUCTS_DIR; }; 8A395E422187D2B200437980 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 8A395E442187D2B200437980 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; @@ -41,6 +45,16 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 8A34C1D62189496A00F22BE9 /* AudioDevices */ = { + isa = PBXGroup; + children = ( + 8A34C1D82189496A00F22BE9 /* AudioDevices-Bridging-Header.h */, + 8A34C1D72189496A00F22BE9 /* ExampleCoreAudioDevice.h */, + 8A34C1D92189496A00F22BE9 /* ExampleCoreAudioDevice.m */, + ); + path = AudioDevices; + sourceTree = ""; + }; 8A395E362187D2B200437980 = { isa = PBXGroup; children = ( @@ -61,6 +75,7 @@ isa = PBXGroup; children = ( 8A395E422187D2B200437980 /* AppDelegate.swift */, + 8A34C1D62189496A00F22BE9 /* AudioDevices */, 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */, 8A395E562187F04C00437980 /* ExampleAVPlayerSource.swift */, 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */, @@ -148,6 +163,7 @@ 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */, 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */, 8A34C1D52189333400F22BE9 /* ExampleAVPlayerAudioTap.swift in Sources */, + 8A34C1DA2189496A00F22BE9 /* ExampleCoreAudioDevice.m in Sources */, 8A395E572187F04C00437980 /* ExampleAVPlayerSource.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -303,6 +319,7 @@ ); PRODUCT_BUNDLE_IDENTIFIER = com.twilio.CoViewingExample; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h"; SWIFT_VERSION = 4.2; TARGETED_DEVICE_FAMILY = "1,2"; }; @@ -321,6 +338,7 @@ ); PRODUCT_BUNDLE_IDENTIFIER = com.twilio.CoViewingExample; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h"; SWIFT_VERSION = 4.2; TARGETED_DEVICE_FAMILY = "1,2"; }; diff --git a/CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h b/CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h new file mode 100644 index 00000000..7a4623c1 --- /dev/null +++ b/CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h @@ -0,0 +1,8 @@ +// +// AudioDevices-Bridging-Header.h +// CoViewingExample +// +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +#import "ExampleCoreAudioDevice.h" diff --git a/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h b/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h new file mode 100644 index 00000000..76c6c817 --- /dev/null +++ b/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h @@ -0,0 +1,18 @@ +// +// ExampleCoreAudioDevice.h +// AudioDeviceExample +// +// Copyright © 2018 Twilio, Inc. All rights reserved. +// + +#import + +/* + * ExampleCoreAudioDevice uses a RemoteIO audio unit to playback stereo audio at up to 48 kHz. + * In contrast to `TVIDefaultAudioDevice`, this class does not record audio and is intended for high quality playback. + * Since full duplex audio is not needed this device does not use the built in echo cancellation provided by + * CoreAudio's VoiceProcessingIO audio unit. + */ +@interface ExampleCoreAudioDevice : NSObject + +@end diff --git a/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.m b/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.m new file mode 100644 index 00000000..b96a83ed --- /dev/null +++ b/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.m @@ -0,0 +1,516 @@ +// +// ExampleCoreAudioDevice.m +// AudioDeviceExample +// +// Copyright © 2018 Twilio, Inc. All rights reserved. +// + +#import "ExampleCoreAudioDevice.h" + +// We want to get as close to 10 msec buffers as possible because this is what the media engine prefers. +static double const kPreferredIOBufferDuration = 0.01; +// We will use stereo playback where available. Some audio routes may be restricted to mono only. +static size_t const kPreferredNumberOfChannels = 2; +// An audio sample is a signed 16-bit integer. +static size_t const kAudioSampleSize = 2; +static uint32_t const kPreferredSampleRate = 48000; + +typedef struct ExampleCoreAudioContext { + TVIAudioDeviceContext deviceContext; + size_t expectedFramesPerBuffer; + size_t maxFramesPerBuffer; +} ExampleCoreAudioContext; + +// The RemoteIO audio unit uses bus 0 for ouptut, and bus 1 for input. +static int kOutputBus = 0; +static int kInputBus = 1; +// This is the maximum slice size for RemoteIO (as observed in the field). We will double check at initialization time. +static size_t kMaximumFramesPerBuffer = 1156; + +@interface ExampleCoreAudioDevice() + +@property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; +@property (nonatomic, assign) AudioUnit audioUnit; + +@property (nonatomic, strong, nullable) TVIAudioFormat *renderingFormat; +@property (atomic, assign) ExampleCoreAudioContext *renderingContext; + +@end + +@implementation ExampleCoreAudioDevice + +#pragma mark - Init & Dealloc + +- (id)init { + self = [super init]; + if (self) { + } + return self; +} + +- (void)dealloc { + [self unregisterAVAudioSessionObservers]; +} + ++ (NSString *)description { + return @"ExampleCoreAudioDevice (stereo playback)"; +} + +/* + * Determine at runtime the maximum slice size used by RemoteIO. Setting the stream format and sample rate doesn't + * appear to impact the maximum size so we prefer to read this value once at initialization time. + */ ++ (void)initialize { + AudioComponentDescription audioUnitDescription = [self audioUnitDescription]; + AudioComponent audioComponent = AudioComponentFindNext(NULL, &audioUnitDescription); + AudioUnit audioUnit; + OSStatus status = AudioComponentInstanceNew(audioComponent, &audioUnit); + if (status != 0) { + NSLog(@"Could not find RemoteIO AudioComponent instance!"); + return; + } + + UInt32 framesPerSlice = 0; + UInt32 propertySize = sizeof(framesPerSlice); + status = AudioUnitGetProperty(audioUnit, kAudioUnitProperty_MaximumFramesPerSlice, + kAudioUnitScope_Global, kOutputBus, + &framesPerSlice, &propertySize); + if (status != 0) { + NSLog(@"Could not read RemoteIO AudioComponent instance!"); + AudioComponentInstanceDispose(audioUnit); + return; + } + + NSLog(@"This device uses a maximum slice size of %d frames.", (unsigned int)framesPerSlice); + kMaximumFramesPerBuffer = (size_t)framesPerSlice; + AudioComponentInstanceDispose(audioUnit); +} + +#pragma mark - TVIAudioDeviceRenderer + +- (nullable TVIAudioFormat *)renderFormat { + if (!_renderingFormat) { + // Setup the AVAudioSession early. You could also defer to `startRendering:` and `stopRendering:`. + [self setupAVAudioSession]; + + _renderingFormat = [[self class] activeRenderingFormat]; + } + + return _renderingFormat; +} + +- (BOOL)initializeRenderer { + /* + * In this example we don't need any fixed size buffers or other pre-allocated resources. We will simply write + * directly to the AudioBufferList provided in the AudioUnit's rendering callback. + */ + return YES; +} + +- (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { + @synchronized(self) { + NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); + + self.renderingContext = malloc(sizeof(ExampleCoreAudioContext)); + self.renderingContext->deviceContext = context; + self.renderingContext->maxFramesPerBuffer = _renderingFormat.framesPerBuffer; + + const NSTimeInterval sessionBufferDuration = [AVAudioSession sharedInstance].IOBufferDuration; + const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; + const size_t sessionFramesPerBuffer = (size_t)(sessionSampleRate * sessionBufferDuration + .5); + self.renderingContext->expectedFramesPerBuffer = sessionFramesPerBuffer; + + NSAssert(self.audioUnit == NULL, @"The audio unit should not be created yet."); + if (![self setupAudioUnit:self.renderingContext]) { + free(self.renderingContext); + self.renderingContext = NULL; + return NO; + } + } + + BOOL success = [self startAudioUnit]; + if (success) { + TVIAudioSessionActivated(context); + } + return success; +} + +- (BOOL)stopRendering { + [self stopAudioUnit]; + + @synchronized(self) { + NSAssert(self.renderingContext != NULL, @"Should have a rendering context."); + TVIAudioSessionDeactivated(self.renderingContext->deviceContext); + + [self teardownAudioUnit]; + + free(self.renderingContext); + self.renderingContext = NULL; + } + + return YES; +} + +#pragma mark - TVIAudioDeviceCapturer + +- (nullable TVIAudioFormat *)captureFormat { + /* + * We don't support capturing and return a nil format to indicate this. The other TVIAudioDeviceCapturer methods + * are simply stubs. + */ + return nil; +} + +- (BOOL)initializeCapturer { + return NO; +} + +- (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { + return NO; +} + +- (BOOL)stopCapturing { + return NO; +} + +#pragma mark - Private (AudioUnit callbacks) + +static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, + AudioUnitRenderActionFlags *actionFlags, + const AudioTimeStamp *timestamp, + UInt32 busNumber, + UInt32 numFrames, + AudioBufferList *bufferList) { + assert(bufferList->mNumberBuffers == 1); + assert(bufferList->mBuffers[0].mNumberChannels <= 2); + assert(bufferList->mBuffers[0].mNumberChannels > 0); + + ExampleCoreAudioContext *context = (ExampleCoreAudioContext *)refCon; + int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; + UInt32 audioBufferSizeInBytes = bufferList->mBuffers[0].mDataByteSize; + + // Render silence if there are temporary mismatches between CoreAudio and our rendering format. + if (numFrames > context->maxFramesPerBuffer) { + NSLog(@"Can handle a max of %u frames but got %u.", (unsigned int)context->maxFramesPerBuffer, (unsigned int)numFrames); + *actionFlags |= kAudioUnitRenderAction_OutputIsSilence; + memset(audioBuffer, 0, audioBufferSizeInBytes); + return noErr; + } + + // Pull decoded, mixed audio data from the media engine into the AudioUnit's AudioBufferList. + assert(numFrames <= context->maxFramesPerBuffer); + assert(audioBufferSizeInBytes == (bufferList->mBuffers[0].mNumberChannels * kAudioSampleSize * numFrames)); + TVIAudioDeviceReadRenderData(context->deviceContext, audioBuffer, audioBufferSizeInBytes); + return noErr; +} + +#pragma mark - Private (AVAudioSession and CoreAudio) + ++ (nullable TVIAudioFormat *)activeRenderingFormat { + /* + * Use the pre-determined maximum frame size. AudioUnit callbacks are variable, and in most sitations will be close + * to the `AVAudioSession.preferredIOBufferDuration` that we've requested. + */ + const size_t sessionFramesPerBuffer = kMaximumFramesPerBuffer; + const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; + const NSInteger sessionOutputChannels = [AVAudioSession sharedInstance].outputNumberOfChannels; + size_t rendererChannels = sessionOutputChannels >= TVIAudioChannelsStereo ? TVIAudioChannelsStereo : TVIAudioChannelsMono; + + return [[TVIAudioFormat alloc] initWithChannels:rendererChannels + sampleRate:sessionSampleRate + framesPerBuffer:sessionFramesPerBuffer]; +} + ++ (AudioComponentDescription)audioUnitDescription { + AudioComponentDescription audioUnitDescription; + audioUnitDescription.componentType = kAudioUnitType_Output; + audioUnitDescription.componentSubType = kAudioUnitSubType_RemoteIO; + audioUnitDescription.componentManufacturer = kAudioUnitManufacturer_Apple; + audioUnitDescription.componentFlags = 0; + audioUnitDescription.componentFlagsMask = 0; + return audioUnitDescription; +} + +- (void)setupAVAudioSession { + AVAudioSession *session = [AVAudioSession sharedInstance]; + NSError *error = nil; + + if (![session setPreferredSampleRate:kPreferredSampleRate error:&error]) { + NSLog(@"Error setting sample rate: %@", error); + } + + NSInteger preferredOutputChannels = session.outputNumberOfChannels >= kPreferredNumberOfChannels ? kPreferredNumberOfChannels : session.outputNumberOfChannels; + if (![session setPreferredOutputNumberOfChannels:preferredOutputChannels error:&error]) { + NSLog(@"Error setting number of output channels: %@", error); + } + + /* + * We want to be as close as possible to the 10 millisecond buffer size that the media engine needs. If there is + * a mismatch then TwilioVideo will ensure that appropriately sized audio buffers are delivered. + */ + if (![session setPreferredIOBufferDuration:kPreferredIOBufferDuration error:&error]) { + NSLog(@"Error setting IOBuffer duration: %@", error); + } + + if (![session setCategory:AVAudioSessionCategoryPlayback error:&error]) { + NSLog(@"Error setting session category: %@", error); + } + + [self registerAVAudioSessionObservers]; + + if (![session setActive:YES error:&error]) { + NSLog(@"Error activating AVAudioSession: %@", error); + } +} + +- (BOOL)setupAudioUnit:(ExampleCoreAudioContext *)context { + // Find and instantiate the RemoteIO audio unit. + AudioComponentDescription audioUnitDescription = [[self class] audioUnitDescription]; + AudioComponent audioComponent = AudioComponentFindNext(NULL, &audioUnitDescription); + + OSStatus status = AudioComponentInstanceNew(audioComponent, &_audioUnit); + if (status != 0) { + NSLog(@"Could not find RemoteIO AudioComponent instance!"); + return NO; + } + + /* + * Configure the RemoteIO audio unit. Our rendering format attempts to match what AVAudioSession requires to + * prevent any additional format conversions after the media engine has mixed our playout audio. + */ + AudioStreamBasicDescription streamDescription = self.renderingFormat.streamDescription; + + UInt32 enableOutput = 1; + status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Output, kOutputBus, + &enableOutput, sizeof(enableOutput)); + if (status != 0) { + NSLog(@"Could not enable output bus!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, kOutputBus, + &streamDescription, sizeof(streamDescription)); + if (status != 0) { + NSLog(@"Could not enable output bus!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Disable input, we don't want it. + UInt32 enableInput = 0; + status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &enableInput, + sizeof(enableInput)); + + if (status != 0) { + NSLog(@"Could not disable input bus!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Setup the rendering callback. + AURenderCallbackStruct renderCallback; + renderCallback.inputProc = ExampleCoreAudioDevicePlayoutCallback; + renderCallback.inputProcRefCon = (void *)(context); + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Output, kOutputBus, &renderCallback, + sizeof(renderCallback)); + if (status != 0) { + NSLog(@"Could not set rendering callback!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Finally, initialize and start the RemoteIO audio unit. + status = AudioUnitInitialize(_audioUnit); + if (status != 0) { + NSLog(@"Could not initialize the audio unit!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + return YES; +} + +- (BOOL)startAudioUnit { + OSStatus status = AudioOutputUnitStart(_audioUnit); + if (status != 0) { + NSLog(@"Could not start the audio unit!"); + return NO; + } + return YES; +} + +- (BOOL)stopAudioUnit { + OSStatus status = AudioOutputUnitStop(_audioUnit); + if (status != 0) { + NSLog(@"Could not stop the audio unit!"); + return NO; + } + return YES; +} + +- (void)teardownAudioUnit { + if (_audioUnit) { + AudioUnitUninitialize(_audioUnit); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + } +} + +#pragma mark - NSNotification Observers + +- (void)registerAVAudioSessionObservers { + // An audio device that interacts with AVAudioSession should handle events like interruptions and route changes. + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + + [center addObserver:self selector:@selector(handleAudioInterruption:) name:AVAudioSessionInterruptionNotification object:nil]; + /* + * Interruption handling is different on iOS 9.x. If your application becomes interrupted while it is in the + * background then you will not get a corresponding notification when the interruption ends. We workaround this + * by handling UIApplicationDidBecomeActiveNotification and treating it as an interruption end. + */ + if (![[NSProcessInfo processInfo] isOperatingSystemAtLeastVersion:(NSOperatingSystemVersion){10, 0, 0}]) { + [center addObserver:self selector:@selector(handleApplicationDidBecomeActive:) name:UIApplicationDidBecomeActiveNotification object:nil]; + } + + [center addObserver:self selector:@selector(handleRouteChange:) name:AVAudioSessionRouteChangeNotification object:nil]; + [center addObserver:self selector:@selector(handleMediaServiceLost:) name:AVAudioSessionMediaServicesWereLostNotification object:nil]; + [center addObserver:self selector:@selector(handleMediaServiceRestored:) name:AVAudioSessionMediaServicesWereResetNotification object:nil]; +} + +- (void)handleAudioInterruption:(NSNotification *)notification { + AVAudioSessionInterruptionType type = [notification.userInfo[AVAudioSessionInterruptionTypeKey] unsignedIntegerValue]; + + @synchronized(self) { + // If the worker block is executed, then context is guaranteed to be valid. + TVIAudioDeviceContext context = self.renderingContext ? self.renderingContext->deviceContext : NULL; + if (context) { + TVIAudioDeviceExecuteWorkerBlock(context, ^{ + if (type == AVAudioSessionInterruptionTypeBegan) { + NSLog(@"Interruption began."); + self.interrupted = YES; + [self stopAudioUnit]; + TVIAudioSessionDeactivated(context); + } else { + NSLog(@"Interruption ended."); + self.interrupted = NO; + if ([self startAudioUnit]) { + TVIAudioSessionActivated(context); + } + } + }); + } + } +} + +- (void)handleApplicationDidBecomeActive:(NSNotification *)notification { + @synchronized(self) { + // If the worker block is executed, then context is guaranteed to be valid. + TVIAudioDeviceContext context = self.renderingContext ? self.renderingContext->deviceContext : NULL; + if (context) { + TVIAudioDeviceExecuteWorkerBlock(context, ^{ + if (self.isInterrupted) { + NSLog(@"Synthesizing an interruption ended event for iOS 9.x devices."); + self.interrupted = NO; + if ([self startAudioUnit]) { + TVIAudioSessionActivated(context); + } + } + }); + } + } +} + +- (void)handleRouteChange:(NSNotification *)notification { + // Check if the sample rate, or channels changed and trigger a format change if it did. + AVAudioSessionRouteChangeReason reason = [notification.userInfo[AVAudioSessionRouteChangeReasonKey] unsignedIntegerValue]; + + switch (reason) { + case AVAudioSessionRouteChangeReasonUnknown: + case AVAudioSessionRouteChangeReasonNewDeviceAvailable: + case AVAudioSessionRouteChangeReasonOldDeviceUnavailable: + // Each device change might cause the actual sample rate or channel configuration of the session to change. + case AVAudioSessionRouteChangeReasonCategoryChange: + // In iOS 9.2+ switching routes from a BT device in control center may cause a category change. + case AVAudioSessionRouteChangeReasonOverride: + case AVAudioSessionRouteChangeReasonWakeFromSleep: + case AVAudioSessionRouteChangeReasonNoSuitableRouteForCategory: + case AVAudioSessionRouteChangeReasonRouteConfigurationChange: + // With CallKit, AVAudioSession may change the sample rate during a configuration change. + // If a valid route change occurs we may want to update our audio graph to reflect the new output device. + @synchronized(self) { + if (self.renderingContext) { + TVIAudioDeviceExecuteWorkerBlock(self.renderingContext->deviceContext, ^{ + [self handleValidRouteChange]; + }); + } + } + break; + } +} + +- (void)handleValidRouteChange { + // Nothing to process while we are interrupted. We will interrogate the AVAudioSession once the interruption ends. + if (self.isInterrupted) { + return; + } else if (_audioUnit == NULL) { + return; + } + + NSLog(@"A route change ocurred while the AudioUnit was started. Checking the active audio format."); + + // Determine if the format actually changed. We only care about sample rate and number of channels. + TVIAudioFormat *activeFormat = [[self class] activeRenderingFormat]; + + if (![activeFormat isEqual:_renderingFormat]) { + NSLog(@"The rendering format changed. Restarting with %@", activeFormat); + // Signal a change by clearing our cached format, and allowing TVIAudioDevice to drive the process. + _renderingFormat = nil; + + @synchronized(self) { + if (self.renderingContext) { + TVIAudioDeviceFormatChanged(self.renderingContext->deviceContext); + } + } + } +} + +- (void)handleMediaServiceLost:(NSNotification *)notification { + @synchronized(self) { + if (self.renderingContext) { + TVIAudioDeviceExecuteWorkerBlock(self.renderingContext->deviceContext, ^{ + [self stopAudioUnit]; + TVIAudioSessionDeactivated(self.renderingContext->deviceContext); + }); + } + } +} + +- (void)handleMediaServiceRestored:(NSNotification *)notification { + @synchronized(self) { + // If the worker block is executed, then context is guaranteed to be valid. + TVIAudioDeviceContext context = self.renderingContext ? self.renderingContext->deviceContext : NULL; + if (context) { + TVIAudioDeviceExecuteWorkerBlock(context, ^{ + if ([self startAudioUnit]) { + TVIAudioSessionActivated(context); + } + }); + } + } +} + +- (void)unregisterAVAudioSessionObservers { + [[NSNotificationCenter defaultCenter] removeObserver:self]; +} + +@end From 8113a5158bc30ccf191e8746a42c76eb15448456 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 30 Oct 2018 19:37:51 -0700 Subject: [PATCH 11/94] Spacing. --- CoViewingExample/ExampleAVPlayerSource.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 03170ad1..ac369808 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -8,6 +8,7 @@ import AVFoundation class ExampleAVPlayerSource: NSObject { + private let sampleQueue: DispatchQueue private var outputTimer: CADisplayLink? = nil private var videoOutput: AVPlayerItemVideoOutput? = nil @@ -29,7 +30,7 @@ class ExampleAVPlayerSource: NSObject { timer.add(to: RunLoop.current, forMode: RunLoop.Mode.common) outputTimer = timer - // Note: It appears requesting IOSurface backing causes a crash on iPhone X / iOS 12.0.1? + // Note: It appears requesting IOSurface backing causes a crash on iPhone X / iOS 12.0.1. let attributes = [ // kCVPixelBufferIOSurfacePropertiesKey as String : [], kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange From c45ce61ad4be0ad8e8e21d9ad9bc6b037edcb53b Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 30 Oct 2018 19:43:31 -0700 Subject: [PATCH 12/94] Add project to Podfile, and consume TPCircularBuffer. --- Podfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Podfile b/Podfile index 55961cac..8922c462 100644 --- a/Podfile +++ b/Podfile @@ -20,6 +20,13 @@ abstract_target 'TwilioVideo' do project 'AudioSinkExample.xcproject' end + target 'CoViewingExample' do + platform :ios, '12.0' + project 'CoViewingExample.xcproject' + + pod 'TPCircularBuffer', '~> 1.6' + end + target 'VideoQuickStart' do platform :ios, '9.0' project 'VideoQuickStart.xcproject' From 5d29ca43bad7af7346e01baea5214d4aedc77ea1 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 31 Oct 2018 17:22:38 -0700 Subject: [PATCH 13/94] Rename to AVPlayerAudioDevice, WIP - AudioTap. --- CoViewingExample.xcodeproj/project.pbxproj | 12 +-- .../AudioDevices-Bridging-Header.h | 2 +- .../AudioDevices/ExampleAVPlayerAudioDevice.h | 17 +++++ ...oDevice.m => ExampleAVPlayerAudioDevice.m} | 76 ++++++++++++++++--- .../AudioDevices/ExampleCoreAudioDevice.h | 18 ----- 5 files changed, 89 insertions(+), 36 deletions(-) create mode 100644 CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h rename CoViewingExample/AudioDevices/{ExampleCoreAudioDevice.m => ExampleAVPlayerAudioDevice.m} (89%) delete mode 100644 CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h diff --git a/CoViewingExample.xcodeproj/project.pbxproj b/CoViewingExample.xcodeproj/project.pbxproj index 8faa919d..cc68f239 100644 --- a/CoViewingExample.xcodeproj/project.pbxproj +++ b/CoViewingExample.xcodeproj/project.pbxproj @@ -8,7 +8,7 @@ /* Begin PBXBuildFile section */ 8A34C1D52189333400F22BE9 /* ExampleAVPlayerAudioTap.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */; }; - 8A34C1DA2189496A00F22BE9 /* ExampleCoreAudioDevice.m in Sources */ = {isa = PBXBuildFile; fileRef = 8A34C1D92189496A00F22BE9 /* ExampleCoreAudioDevice.m */; }; + 8A34C1DA2189496A00F22BE9 /* ExampleAVPlayerAudioDevice.m in Sources */ = {isa = PBXBuildFile; fileRef = 8A34C1D92189496A00F22BE9 /* ExampleAVPlayerAudioDevice.m */; }; 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E422187D2B200437980 /* AppDelegate.swift */; }; 8A395E452187D2B200437980 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E442187D2B200437980 /* ViewController.swift */; }; 8A395E482187D2B200437980 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E462187D2B200437980 /* Main.storyboard */; }; @@ -20,9 +20,9 @@ /* Begin PBXFileReference section */ 8A34C1D42189333400F22BE9 /* ExampleAVPlayerAudioTap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerAudioTap.swift; sourceTree = ""; }; - 8A34C1D72189496A00F22BE9 /* ExampleCoreAudioDevice.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ExampleCoreAudioDevice.h; sourceTree = ""; }; + 8A34C1D72189496A00F22BE9 /* ExampleAVPlayerAudioDevice.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ExampleAVPlayerAudioDevice.h; sourceTree = ""; }; 8A34C1D82189496A00F22BE9 /* AudioDevices-Bridging-Header.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "AudioDevices-Bridging-Header.h"; sourceTree = ""; }; - 8A34C1D92189496A00F22BE9 /* ExampleCoreAudioDevice.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExampleCoreAudioDevice.m; sourceTree = ""; }; + 8A34C1D92189496A00F22BE9 /* ExampleAVPlayerAudioDevice.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExampleAVPlayerAudioDevice.m; sourceTree = ""; }; 8A395E3F2187D2B200437980 /* CoViewingExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CoViewingExample.app; sourceTree = BUILT_PRODUCTS_DIR; }; 8A395E422187D2B200437980 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 8A395E442187D2B200437980 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; @@ -49,8 +49,8 @@ isa = PBXGroup; children = ( 8A34C1D82189496A00F22BE9 /* AudioDevices-Bridging-Header.h */, - 8A34C1D72189496A00F22BE9 /* ExampleCoreAudioDevice.h */, - 8A34C1D92189496A00F22BE9 /* ExampleCoreAudioDevice.m */, + 8A34C1D72189496A00F22BE9 /* ExampleAVPlayerAudioDevice.h */, + 8A34C1D92189496A00F22BE9 /* ExampleAVPlayerAudioDevice.m */, ); path = AudioDevices; sourceTree = ""; @@ -163,7 +163,7 @@ 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */, 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */, 8A34C1D52189333400F22BE9 /* ExampleAVPlayerAudioTap.swift in Sources */, - 8A34C1DA2189496A00F22BE9 /* ExampleCoreAudioDevice.m in Sources */, + 8A34C1DA2189496A00F22BE9 /* ExampleAVPlayerAudioDevice.m in Sources */, 8A395E572187F04C00437980 /* ExampleAVPlayerSource.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h b/CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h index 7a4623c1..e759f7ce 100644 --- a/CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h +++ b/CoViewingExample/AudioDevices/AudioDevices-Bridging-Header.h @@ -5,4 +5,4 @@ // Copyright © 2018 Twilio Inc. All rights reserved. // -#import "ExampleCoreAudioDevice.h" +#import "ExampleAVPlayerAudioDevice.h" diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h new file mode 100644 index 00000000..09295f74 --- /dev/null +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h @@ -0,0 +1,17 @@ +// +// ExampleAVPlayerAudioDevice.h +// AudioDeviceExample +// +// Copyright © 2018 Twilio, Inc. All rights reserved. +// + +#import + +/* + * ExampleAVPlayerAudioDevice uses a VoiceProcessingIO audio unit to play audio from an MTAudioProcessingTap + * attached to an AVPlayerItem. The AVPlayer audio is mixed with Room audio provided by Twilio. + * The microphone input, and MTAudioProcessingTap output are mixed into a single recorded stream. + */ +@interface ExampleAVPlayerAudioDevice : NSObject + +@end diff --git a/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m similarity index 89% rename from CoViewingExample/AudioDevices/ExampleCoreAudioDevice.m rename to CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index b96a83ed..55acaf39 100644 --- a/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -1,11 +1,13 @@ // -// ExampleCoreAudioDevice.m -// AudioDeviceExample +// ExampleAVPlayerAudioDevice.m +// CoViewingExample // // Copyright © 2018 Twilio, Inc. All rights reserved. // -#import "ExampleCoreAudioDevice.h" +#import "ExampleAVPlayerAudioDevice.h" + +#import "TPCircularBuffer+AudioBufferList.h" // We want to get as close to 10 msec buffers as possible because this is what the media engine prefers. static double const kPreferredIOBufferDuration = 0.01; @@ -15,11 +17,11 @@ static size_t const kAudioSampleSize = 2; static uint32_t const kPreferredSampleRate = 48000; -typedef struct ExampleCoreAudioContext { +typedef struct ExampleAVPlayerContext { TVIAudioDeviceContext deviceContext; size_t expectedFramesPerBuffer; size_t maxFramesPerBuffer; -} ExampleCoreAudioContext; +} ExampleAVPlayerContext; // The RemoteIO audio unit uses bus 0 for ouptut, and bus 1 for input. static int kOutputBus = 0; @@ -27,23 +29,73 @@ // This is the maximum slice size for RemoteIO (as observed in the field). We will double check at initialization time. static size_t kMaximumFramesPerBuffer = 1156; -@interface ExampleCoreAudioDevice() +@interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; @property (nonatomic, assign) AudioUnit audioUnit; +@property (nonatomic, assign, nullable) TPCircularBuffer *audioTapBuffer; @property (nonatomic, strong, nullable) TVIAudioFormat *renderingFormat; -@property (atomic, assign) ExampleCoreAudioContext *renderingContext; +@property (atomic, assign) ExampleAVPlayerContext *renderingContext; @end -@implementation ExampleCoreAudioDevice +#pragma mark - MTAudioProcessingTap + +void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { + // Provide access to our device in the Callbacks. + *tapStorageOut = clientInfo; +} + +void finalize(MTAudioProcessingTapRef tap) { + // TODO +} + +void prepare(MTAudioProcessingTapRef tap, + CMItemCount maxFrames, + const AudioStreamBasicDescription *processingFormat) { + NSLog(@"Preparing the Audio Tap Processor"); + + // Defer creation of the ring buffer until we understand the processing format. + ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); + +} + +void unprepare(MTAudioProcessingTapRef tap) { + // TODO: Destroy the ring buffer? +} + +void process(MTAudioProcessingTapRef tap, + CMItemCount numberFrames, + MTAudioProcessingTapFlags flags, + AudioBufferList *bufferListInOut, + CMItemCount *numberFramesOut, + MTAudioProcessingTapFlags *flagsOut) { + ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); + + OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, + numberFrames, + bufferListInOut, + flagsOut, + NULL, + numberFramesOut); + + if (status != kCVReturnSuccess) { + // TODO + return; + } + + // Fill the ring buffer with content. +} + +@implementation ExampleAVPlayerAudioDevice #pragma mark - Init & Dealloc - (id)init { self = [super init]; if (self) { + _audioTapBuffer = NULL; } return self; } @@ -111,7 +163,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { @synchronized(self) { NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); - self.renderingContext = malloc(sizeof(ExampleCoreAudioContext)); + self.renderingContext = malloc(sizeof(ExampleAVPlayerContext)); self.renderingContext->deviceContext = context; self.renderingContext->maxFramesPerBuffer = _renderingFormat.framesPerBuffer; @@ -173,6 +225,8 @@ - (BOOL)stopCapturing { return NO; } +#pragma mark - Private (MTAudioProcessingTap) + #pragma mark - Private (AudioUnit callbacks) static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, @@ -185,7 +239,7 @@ static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, assert(bufferList->mBuffers[0].mNumberChannels <= 2); assert(bufferList->mBuffers[0].mNumberChannels > 0); - ExampleCoreAudioContext *context = (ExampleCoreAudioContext *)refCon; + ExampleAVPlayerContext *context = (ExampleAVPlayerContext *)refCon; int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; UInt32 audioBufferSizeInBytes = bufferList->mBuffers[0].mDataByteSize; @@ -263,7 +317,7 @@ - (void)setupAVAudioSession { } } -- (BOOL)setupAudioUnit:(ExampleCoreAudioContext *)context { +- (BOOL)setupAudioUnit:(ExampleAVPlayerContext *)context { // Find and instantiate the RemoteIO audio unit. AudioComponentDescription audioUnitDescription = [[self class] audioUnitDescription]; AudioComponent audioComponent = AudioComponentFindNext(NULL, &audioUnitDescription); diff --git a/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h b/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h deleted file mode 100644 index 76c6c817..00000000 --- a/CoViewingExample/AudioDevices/ExampleCoreAudioDevice.h +++ /dev/null @@ -1,18 +0,0 @@ -// -// ExampleCoreAudioDevice.h -// AudioDeviceExample -// -// Copyright © 2018 Twilio, Inc. All rights reserved. -// - -#import - -/* - * ExampleCoreAudioDevice uses a RemoteIO audio unit to playback stereo audio at up to 48 kHz. - * In contrast to `TVIDefaultAudioDevice`, this class does not record audio and is intended for high quality playback. - * Since full duplex audio is not needed this device does not use the built in echo cancellation provided by - * CoreAudio's VoiceProcessingIO audio unit. - */ -@interface ExampleCoreAudioDevice : NSObject - -@end From 20345d2096f2fc5118eb69a72c5b4c227c4757aa Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 31 Oct 2018 17:58:46 -0700 Subject: [PATCH 14/94] WIP - Produce audio using the ring buffer. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 55acaf39..3e591c08 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -42,13 +42,19 @@ @interface ExampleAVPlayerAudioDevice() #pragma mark - MTAudioProcessingTap +// TODO: Bad robot. +static const AudioStreamBasicDescription *audioFormat = NULL; + void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { // Provide access to our device in the Callbacks. *tapStorageOut = clientInfo; } void finalize(MTAudioProcessingTapRef tap) { - // TODO + ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = NULL; + + TPCircularBufferCleanup(buffer); } void prepare(MTAudioProcessingTapRef tap, @@ -58,11 +64,26 @@ void prepare(MTAudioProcessingTapRef tap, // Defer creation of the ring buffer until we understand the processing format. ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = NULL; + size_t bufferSize = processingFormat->mBytesPerFrame * maxFrames; + // We need to add some overhead for the AudioBufferList data structures. + bufferSize += 2048; + // TODO: Size the buffer appropriately, as we may need to accumulate more than maxFrames. + bufferSize *= 12; + + // TODO: If we are re-allocating then check the size? + TPCircularBufferInit(buffer, bufferSize); + audioFormat = processingFormat; } void unprepare(MTAudioProcessingTapRef tap) { - // TODO: Destroy the ring buffer? + // Prevent any more frames from being consumed. Note that this might end audio playback early. + ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = NULL; + + TPCircularBufferClear(buffer); + audioFormat = NULL; } void process(MTAudioProcessingTapRef tap, @@ -72,6 +93,7 @@ void process(MTAudioProcessingTapRef tap, CMItemCount *numberFramesOut, MTAudioProcessingTapFlags *flagsOut) { ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = NULL; OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, @@ -85,7 +107,15 @@ void process(MTAudioProcessingTapRef tap, return; } - // Fill the ring buffer with content. + UInt32 framesToCopy = (UInt32)*numberFramesOut; + bool success = TPCircularBufferCopyAudioBufferList(buffer, + bufferListInOut, + NULL, + framesToCopy, + audioFormat); + if (!success) { + // TODO + } } @implementation ExampleAVPlayerAudioDevice @@ -95,13 +125,18 @@ @implementation ExampleAVPlayerAudioDevice - (id)init { self = [super init]; if (self) { - _audioTapBuffer = NULL; + _audioTapBuffer = malloc(sizeof(TPCircularBuffer)); } return self; } - (void)dealloc { [self unregisterAVAudioSessionObservers]; + + if (_audioTapBuffer != NULL) { + free(_audioTapBuffer); + _audioTapBuffer = NULL; + } } + (NSString *)description { From 27891e306898a6caf868005e76db79e62d793df9 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 31 Oct 2018 18:40:27 -0700 Subject: [PATCH 15/94] WIP - Playback of MTAudioProcessingTap. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 3e591c08..2c39e16c 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -21,6 +21,7 @@ TVIAudioDeviceContext deviceContext; size_t expectedFramesPerBuffer; size_t maxFramesPerBuffer; + TPCircularBuffer *playoutBuffer; } ExampleAVPlayerContext; // The RemoteIO audio unit uses bus 0 for ouptut, and bus 1 for input. @@ -43,7 +44,7 @@ @interface ExampleAVPlayerAudioDevice() #pragma mark - MTAudioProcessingTap // TODO: Bad robot. -static const AudioStreamBasicDescription *audioFormat = NULL; +static AudioStreamBasicDescription *audioFormat = NULL; void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { // Provide access to our device in the Callbacks. @@ -74,7 +75,8 @@ void prepare(MTAudioProcessingTapRef tap, // TODO: If we are re-allocating then check the size? TPCircularBufferInit(buffer, bufferSize); - audioFormat = processingFormat; + audioFormat = malloc(sizeof(AudioStreamBasicDescription)); + memcpy(audioFormat, processingFormat, sizeof(AudioStreamBasicDescription)); } void unprepare(MTAudioProcessingTapRef tap) { @@ -83,7 +85,10 @@ void unprepare(MTAudioProcessingTapRef tap) { TPCircularBuffer *buffer = NULL; TPCircularBufferClear(buffer); - audioFormat = NULL; + if (audioFormat) { + free(audioFormat); + audioFormat = NULL; + } } void process(MTAudioProcessingTapRef tap, @@ -201,6 +206,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { self.renderingContext = malloc(sizeof(ExampleAVPlayerContext)); self.renderingContext->deviceContext = context; self.renderingContext->maxFramesPerBuffer = _renderingFormat.framesPerBuffer; + self.renderingContext->playoutBuffer = _audioTapBuffer; const NSTimeInterval sessionBufferDuration = [AVAudioSession sharedInstance].IOBufferDuration; const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; @@ -275,6 +281,7 @@ static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, assert(bufferList->mBuffers[0].mNumberChannels > 0); ExampleAVPlayerContext *context = (ExampleAVPlayerContext *)refCon; + TPCircularBuffer *buffer = context->playoutBuffer; int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; UInt32 audioBufferSizeInBytes = bufferList->mBuffers[0].mDataByteSize; @@ -286,10 +293,32 @@ static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, return noErr; } - // Pull decoded, mixed audio data from the media engine into the AudioUnit's AudioBufferList. assert(numFrames <= context->maxFramesPerBuffer); assert(audioBufferSizeInBytes == (bufferList->mBuffers[0].mNumberChannels * kAudioSampleSize * numFrames)); - TVIAudioDeviceReadRenderData(context->deviceContext, audioBuffer, audioBufferSizeInBytes); + + // TODO: Include the format in the context? What if the formats are somehow not matched? + AudioStreamBasicDescription format; + format.mBitsPerChannel = 16; + format.mChannelsPerFrame = bufferList->mBuffers[0].mNumberChannels; + format.mBytesPerFrame = format.mChannelsPerFrame * format.mBitsPerChannel; + format.mFormatID = kAudioFormatLinearPCM; + format.mFormatFlags = kAudioFormatFlagIsPacked | kAudioFormatFlagIsSignedInteger; + format.mSampleRate = 44100; + + UInt32 framesInOut = numFrames; + TPCircularBufferDequeueBufferListFrames(buffer, &framesInOut, bufferList, NULL, &format); + + if (framesInOut != numFrames) { + // Render silence for the remaining frames. + UInt32 framesRemaining = numFrames - framesInOut; + UInt32 bytesRemaining = framesRemaining * format.mBytesPerFrame; + audioBuffer += bytesRemaining; + + memset(audioBuffer, 0, bytesRemaining); + } + + // TODO: Pull decoded, mixed audio data from the media engine into the AudioUnit's AudioBufferList. +// TVIAudioDeviceReadRenderData(context->deviceContext, audioBuffer, audioBufferSizeInBytes); return noErr; } From d8eede20d895838b91ed171359123f8a1560467e Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 31 Oct 2018 18:49:39 -0700 Subject: [PATCH 16/94] Create an MTAudioProcessingTap. --- .../AudioDevices/ExampleAVPlayerAudioDevice.h | 8 ++++++ .../AudioDevices/ExampleAVPlayerAudioDevice.m | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h index 09295f74..33b0daf2 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h @@ -14,4 +14,12 @@ */ @interface ExampleAVPlayerAudioDevice : NSObject +/* + * Creates a processing tap bound to the device instance. + * + * @return An `MTAudioProcessingTap` which is bound to the device, or NULL if there is an error. The caller + * assumes all ownership of the tap, and should call CFRelease when they are finished with it. + */ +- (MTAudioProcessingTapRef)createProcessingTap; + @end diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 2c39e16c..c411cb35 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -178,6 +178,31 @@ + (void)initialize { AudioComponentInstanceDispose(audioUnit); } +#pragma mark - Public + +- (MTAudioProcessingTapRef)createProcessingTap { + MTAudioProcessingTapRef processingTap; + + MTAudioProcessingTapCallbacks callbacks; + callbacks.version = kMTAudioProcessingTapCallbacksVersion_0; + callbacks.clientInfo = (__bridge void *)(self); + callbacks.init = init; + callbacks.prepare = prepare; + callbacks.process = process; + callbacks.unprepare = unprepare; + callbacks.finalize = finalize; + + OSStatus status = MTAudioProcessingTapCreate(kCFAllocatorDefault, + &callbacks, + kMTAudioProcessingTapCreationFlag_PostEffects, + &processingTap); + if (status == kCVReturnSuccess) { + return processingTap; + } else { + return NULL; + } +} + #pragma mark - TVIAudioDeviceRenderer - (nullable TVIAudioFormat *)renderFormat { From 2b90a9d8a56fe1562b6609e4280a929cb5f78e68 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 31 Oct 2018 22:44:52 -0700 Subject: [PATCH 17/94] Hook up the audio device, almost there. * Still need to figure out how to consume frames properly. --- .../AudioDevices/ExampleAVPlayerAudioDevice.h | 2 +- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 56 +++++++++---------- CoViewingExample/ViewController.swift | 21 ++++++- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h index 33b0daf2..62700619 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h @@ -20,6 +20,6 @@ * @return An `MTAudioProcessingTap` which is bound to the device, or NULL if there is an error. The caller * assumes all ownership of the tap, and should call CFRelease when they are finished with it. */ -- (MTAudioProcessingTapRef)createProcessingTap; +- (nullable MTAudioProcessingTapRef)createProcessingTap; @end diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index c411cb35..f0b08b7c 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -9,8 +9,8 @@ #import "TPCircularBuffer+AudioBufferList.h" -// We want to get as close to 10 msec buffers as possible because this is what the media engine prefers. -static double const kPreferredIOBufferDuration = 0.01; +// We want to get as close to 20 msec buffers as possible, to match the behavior of TVIDefaultAudioDevice. +static double const kPreferredIOBufferDuration = 0.02; // We will use stereo playback where available. Some audio routes may be restricted to mono only. static size_t const kPreferredNumberOfChannels = 2; // An audio sample is a signed 16-bit integer. @@ -52,25 +52,23 @@ void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { } void finalize(MTAudioProcessingTapRef tap) { - ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = NULL; - + TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); TPCircularBufferCleanup(buffer); } void prepare(MTAudioProcessingTapRef tap, CMItemCount maxFrames, const AudioStreamBasicDescription *processingFormat) { - NSLog(@"Preparing the Audio Tap Processor"); + NSLog(@"Preparing with frames: %d, channels: %d, sample rate: %0.1f", + (int)maxFrames, processingFormat->mChannelsPerFrame, processingFormat->mSampleRate); - // Defer creation of the ring buffer until we understand the processing format. - ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = NULL; + // Defer init of the ring buffer memory until we understand the processing format. + TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); size_t bufferSize = processingFormat->mBytesPerFrame * maxFrames; // We need to add some overhead for the AudioBufferList data structures. bufferSize += 2048; - // TODO: Size the buffer appropriately, as we may need to accumulate more than maxFrames. + // TODO: Size the buffer appropriately, as we may need to accumulate more than maxFrames due to bursty processing. bufferSize *= 12; // TODO: If we are re-allocating then check the size? @@ -81,14 +79,11 @@ void prepare(MTAudioProcessingTapRef tap, void unprepare(MTAudioProcessingTapRef tap) { // Prevent any more frames from being consumed. Note that this might end audio playback early. - ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = NULL; + TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); TPCircularBufferClear(buffer); - if (audioFormat) { - free(audioFormat); - audioFormat = NULL; - } + free(audioFormat); + audioFormat = NULL; } void process(MTAudioProcessingTapRef tap, @@ -97,14 +92,14 @@ void process(MTAudioProcessingTapRef tap, AudioBufferList *bufferListInOut, CMItemCount *numberFramesOut, MTAudioProcessingTapFlags *flagsOut) { - ExampleAVPlayerAudioDevice *device = (__bridge ExampleAVPlayerAudioDevice *) MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = NULL; + TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); + CMTimeRange sourceRange; OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, flagsOut, - NULL, + &sourceRange, numberFramesOut); if (status != kCVReturnSuccess) { @@ -121,10 +116,15 @@ void process(MTAudioProcessingTapRef tap, if (!success) { // TODO } + + // TODO: Silence the audio returned to AVPlayer just in case? +// memset(NULL, 0, numberFramesOut * ) } @implementation ExampleAVPlayerAudioDevice +@synthesize audioTapBuffer = _audioTapBuffer; + #pragma mark - Init & Dealloc - (id)init { @@ -138,10 +138,7 @@ - (id)init { - (void)dealloc { [self unregisterAVAudioSessionObservers]; - if (_audioTapBuffer != NULL) { - free(_audioTapBuffer); - _audioTapBuffer = NULL; - } + free(_audioTapBuffer); } + (NSString *)description { @@ -185,7 +182,7 @@ - (MTAudioProcessingTapRef)createProcessingTap { MTAudioProcessingTapCallbacks callbacks; callbacks.version = kMTAudioProcessingTapCallbacksVersion_0; - callbacks.clientInfo = (__bridge void *)(self); + callbacks.clientInfo = (void *)(_audioTapBuffer); callbacks.init = init; callbacks.prepare = prepare; callbacks.process = process; @@ -227,10 +224,13 @@ - (BOOL)initializeRenderer { - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { @synchronized(self) { NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); + NSAssert(self.audioUnit == NULL, @"The audio unit should not be created yet."); self.renderingContext = malloc(sizeof(ExampleAVPlayerContext)); self.renderingContext->deviceContext = context; self.renderingContext->maxFramesPerBuffer = _renderingFormat.framesPerBuffer; + + // TODO: Do we need to synchronize with the tap being started at this point? self.renderingContext->playoutBuffer = _audioTapBuffer; const NSTimeInterval sessionBufferDuration = [AVAudioSession sharedInstance].IOBufferDuration; @@ -238,7 +238,6 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { const size_t sessionFramesPerBuffer = (size_t)(sessionSampleRate * sessionBufferDuration + .5); self.renderingContext->expectedFramesPerBuffer = sessionFramesPerBuffer; - NSAssert(self.audioUnit == NULL, @"The audio unit should not be created yet."); if (![self setupAudioUnit:self.renderingContext]) { free(self.renderingContext); self.renderingContext = NULL; @@ -248,7 +247,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { BOOL success = [self startAudioUnit]; if (success) { - TVIAudioSessionActivated(context); +// TVIAudioSessionActivated(context); } return success; } @@ -321,7 +320,7 @@ static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, assert(numFrames <= context->maxFramesPerBuffer); assert(audioBufferSizeInBytes == (bufferList->mBuffers[0].mNumberChannels * kAudioSampleSize * numFrames)); - // TODO: Include the format in the context? What if the formats are somehow not matched? + // TODO: Include this format in the context? What if the formats are somehow not matched? AudioStreamBasicDescription format; format.mBitsPerChannel = 16; format.mChannelsPerFrame = bufferList->mBuffers[0].mNumberChannels; @@ -355,7 +354,8 @@ + (nullable TVIAudioFormat *)activeRenderingFormat { * to the `AVAudioSession.preferredIOBufferDuration` that we've requested. */ const size_t sessionFramesPerBuffer = kMaximumFramesPerBuffer; - const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; +// const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; + const double sessionSampleRate = 44100.; const NSInteger sessionOutputChannels = [AVAudioSession sharedInstance].outputNumberOfChannels; size_t rendererChannels = sessionOutputChannels >= TVIAudioChannelsStereo ? TVIAudioChannelsStereo : TVIAudioChannelsMono; diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 0ab8dc5a..dd0c8ac7 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -10,11 +10,13 @@ import UIKit class ViewController: UIViewController { + var audioDevice: ExampleAVPlayerAudioDevice = ExampleAVPlayerAudioDevice() var videoPlayer: AVPlayer? = nil var videoPlayerAudioTap: ExampleAVPlayerAudioTap? = nil var videoPlayerSource: ExampleAVPlayerSource? = nil var videoPlayerView: ExampleAVPlayerView? = nil + static var useAudioDevice = false static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! override func viewDidLoad() { @@ -67,13 +69,26 @@ class ViewController: UIViewController { if let assetAudioTrack = itemAsset.tracks(withMediaType: AVMediaType.audio).first { let inputParameters = AVMutableAudioMixInputParameters(track: assetAudioTrack) - let processor = ExampleAVPlayerAudioTap() - videoPlayerAudioTap = processor // TODO: Memory management of the MTAudioProcessingTap. - inputParameters.audioTapProcessor = ExampleAVPlayerAudioTap.mediaToolboxAudioProcessingTapCreate(audioTap: processor) + if ViewController.useAudioDevice { + inputParameters.audioTapProcessor = audioDevice.createProcessingTap()?.takeUnretainedValue() + player.volume = Float(0) + } else { + let processor = ExampleAVPlayerAudioTap() + videoPlayerAudioTap = processor + inputParameters.audioTapProcessor = ExampleAVPlayerAudioTap.mediaToolboxAudioProcessingTapCreate(audioTap: processor) + } + audioMix.inputParameters = [inputParameters] playerItem.audioMix = audioMix + + if ViewController.useAudioDevice { + // Fake start the device...? + let format = audioDevice.renderFormat() + print("Starting rendering with format:", format as Any) + audioDevice.startRendering(UnsafeMutableRawPointer(bitPattern: 1)!) + } } else { // Abort, retry, fail? } From eee3408c5f53bc2cf2d00dcab186e41f3b204e52 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 31 Oct 2018 22:59:22 -0700 Subject: [PATCH 18/94] Fix the crash consuming buffers. * TODO - Audio is heavily distorted. --- CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index f0b08b7c..09da9d08 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -324,7 +324,7 @@ static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, AudioStreamBasicDescription format; format.mBitsPerChannel = 16; format.mChannelsPerFrame = bufferList->mBuffers[0].mNumberChannels; - format.mBytesPerFrame = format.mChannelsPerFrame * format.mBitsPerChannel; + format.mBytesPerFrame = format.mChannelsPerFrame * format.mBitsPerChannel / 8; format.mFormatID = kAudioFormatLinearPCM; format.mFormatFlags = kAudioFormatFlagIsPacked | kAudioFormatFlagIsSignedInteger; format.mSampleRate = 44100; From de4ee11cb6eacc80b4bb01066d9a91a6dba06b89 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 00:00:10 -0700 Subject: [PATCH 19/94] Use a format converter. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 46 +++++++++++++++---- CoViewingExample/ViewController.swift | 2 +- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 09da9d08..daaf1684 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -45,6 +45,7 @@ @interface ExampleAVPlayerAudioDevice() // TODO: Bad robot. static AudioStreamBasicDescription *audioFormat = NULL; +static AudioConverterRef formatConverter = NULL; void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { // Provide access to our device in the Callbacks. @@ -59,8 +60,9 @@ void finalize(MTAudioProcessingTapRef tap) { void prepare(MTAudioProcessingTapRef tap, CMItemCount maxFrames, const AudioStreamBasicDescription *processingFormat) { - NSLog(@"Preparing with frames: %d, channels: %d, sample rate: %0.1f", - (int)maxFrames, processingFormat->mChannelsPerFrame, processingFormat->mSampleRate); + NSLog(@"Preparing with frames: %d, channels: %d, bits/channel: %d, sample rate: %0.1f", + (int)maxFrames, processingFormat->mChannelsPerFrame, processingFormat->mBitsPerChannel, processingFormat->mSampleRate); + assert(processingFormat->mFormatID == kAudioFormatLinearPCM); // Defer init of the ring buffer memory until we understand the processing format. TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); @@ -75,6 +77,20 @@ void prepare(MTAudioProcessingTapRef tap, TPCircularBufferInit(buffer, bufferSize); audioFormat = malloc(sizeof(AudioStreamBasicDescription)); memcpy(audioFormat, processingFormat, sizeof(AudioStreamBasicDescription)); + + TVIAudioFormat *preferredFormat = [[TVIAudioFormat alloc] initWithChannels:processingFormat->mChannelsPerFrame + sampleRate:processingFormat->mSampleRate + framesPerBuffer:maxFrames]; + AudioStreamBasicDescription preferredDescription = [preferredFormat streamDescription]; + BOOL requiresFormatConversion = preferredDescription.mFormatFlags != processingFormat->mFormatFlags; + + if (requiresFormatConversion) { + OSStatus status = AudioConverterNew(processingFormat, &preferredDescription, &formatConverter); + if (status != 0) { + NSLog(@"Failed to create AudioConverter: %d", (int)status); + return; + } + } } void unprepare(MTAudioProcessingTapRef tap) { @@ -84,6 +100,11 @@ void unprepare(MTAudioProcessingTapRef tap) { TPCircularBufferClear(buffer); free(audioFormat); audioFormat = NULL; + + if (formatConverter != NULL) { + AudioConverterDispose(formatConverter); + formatConverter = NULL; + } } void process(MTAudioProcessingTapRef tap, @@ -108,15 +129,22 @@ void process(MTAudioProcessingTapRef tap, } UInt32 framesToCopy = (UInt32)*numberFramesOut; - bool success = TPCircularBufferCopyAudioBufferList(buffer, - bufferListInOut, - NULL, - framesToCopy, - audioFormat); - if (!success) { - // TODO + // TODO: Assumptions about our producer's format. + UInt32 bytesToCopy = framesToCopy * 4; + AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesToCopy, NULL); + if (producerBufferList == NULL) { + // TODO: + return; + } + + status = AudioConverterConvertComplexBuffer(formatConverter, framesToCopy, bufferListInOut, producerBufferList); + if (status != kCVReturnSuccess) { + // TODO: Do we still produce the buffer list? + return; } + TPCircularBufferProduceAudioBufferList(buffer, NULL); + // TODO: Silence the audio returned to AVPlayer just in case? // memset(NULL, 0, numberFramesOut * ) } diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index dd0c8ac7..ae587bf6 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -16,7 +16,7 @@ class ViewController: UIViewController { var videoPlayerSource: ExampleAVPlayerSource? = nil var videoPlayerView: ExampleAVPlayerView? = nil - static var useAudioDevice = false + static var useAudioDevice = true static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! override func viewDidLoad() { From cb64ff6c22f04ae0cd2d1cd6080a8ee03e9b9b25 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 00:07:59 -0700 Subject: [PATCH 20/94] Workaround for device crashes. --- CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m | 1 + 1 file changed, 1 insertion(+) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index daaf1684..c8eda7c3 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -260,6 +260,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { // TODO: Do we need to synchronize with the tap being started at this point? self.renderingContext->playoutBuffer = _audioTapBuffer; + [NSThread sleepForTimeInterval:0.2]; const NSTimeInterval sessionBufferDuration = [AVAudioSession sharedInstance].IOBufferDuration; const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; From d0079a9468b5c37dedab579eb164de4bafdec274 Mon Sep 17 00:00:00 2001 From: Piyush Tank Date: Thu, 1 Nov 2018 14:50:21 -0700 Subject: [PATCH 21/94] Co-viewing app ui (#2) --- CoViewingExample/Base.lproj/Main.storyboard | 76 ++++- CoViewingExample/Info.plist | 4 + CoViewingExample/ViewController.swift | 299 +++++++++++++++++++- 3 files changed, 365 insertions(+), 14 deletions(-) diff --git a/CoViewingExample/Base.lproj/Main.storyboard b/CoViewingExample/Base.lproj/Main.storyboard index f1bcf384..bea94353 100644 --- a/CoViewingExample/Base.lproj/Main.storyboard +++ b/CoViewingExample/Base.lproj/Main.storyboard @@ -1,7 +1,10 @@ - + + + + - + @@ -9,16 +12,83 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/CoViewingExample/Info.plist b/CoViewingExample/Info.plist index 16be3b68..8a0cc2cb 100644 --- a/CoViewingExample/Info.plist +++ b/CoViewingExample/Info.plist @@ -2,6 +2,10 @@ + NSCameraUsageDescription + ${PRODUCT_NAME} uses your camera to capture video which is shared with other Room Participants. + NSMicrophoneUsageDescription + ${PRODUCT_NAME} uses your microphone to capture audio which is shared with other Room Participants. CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleExecutable diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index ae587bf6..6cfc4772 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -10,25 +10,49 @@ import UIKit class ViewController: UIViewController { + // MARK: View Controller Members + + // Configure access token manually for testing, if desired! Create one manually in the console + // at https://www.twilio.com/console/video/runtime/testing-tools + var accessToken = "TWILIO_ACCESS_TOKEN" + + // Configure remote URL to fetch token from + var tokenUrl = "https://username:password@simple-signaling.appspot.com/access-token" + + // Video SDK components + var room: TVIRoom? + var camera: TVICameraCapturer? + var localVideoTrack: TVILocalVideoTrack! + var localAudioTrack: TVILocalAudioTrack! + var audioDevice: ExampleAVPlayerAudioDevice = ExampleAVPlayerAudioDevice() var videoPlayer: AVPlayer? = nil var videoPlayerAudioTap: ExampleAVPlayerAudioTap? = nil var videoPlayerSource: ExampleAVPlayerSource? = nil var videoPlayerView: ExampleAVPlayerView? = nil + var isPresenter:Bool? + + @IBOutlet weak var presenterButton: UIButton! + @IBOutlet weak var viewerButton: UIButton! + + @IBOutlet weak var remoteView: TVIVideoView! + @IBOutlet weak var localView: TVIVideoView! + static var useAudioDevice = true static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! override func viewDidLoad() { super.viewDidLoad() - // Do any additional setup after loading the view, typically from a nib. + + presenterButton.backgroundColor = UIColor.red + presenterButton.titleLabel?.textColor = UIColor.white + viewerButton.backgroundColor = UIColor.red + viewerButton.titleLabel?.textColor = UIColor.white } override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) - if videoPlayer == nil { - startVideoPlayer() - } } override func viewWillLayoutSubviews() { @@ -39,6 +63,85 @@ class ViewController: UIViewController { } } + @IBAction func startPresenter(_ sender: Any) { + TwilioVideo.audioDevice = audioDevice + isPresenter = true + connect(name: "presenter") + } + + @IBAction func startViewer(_ sender: Any) { + TwilioVideo.audioDevice = TVIDefaultAudioDevice() + isPresenter = false + connect(name: "viewer") + } + + func logMessage(messageText: String) { + } + + func connect(name: String) { + // Configure access token either from server or manually. + // If the default wasn't changed, try fetching from server. + if (accessToken == "TWILIO_ACCESS_TOKEN") { + let urlStringWithRole = tokenUrl + "?identity=" + name + do { + accessToken = try String(contentsOf:URL(string: urlStringWithRole)!) + } catch { + let message = "Failed to fetch access token" + print(message) + return + } + } + + // Prepare local media which we will share with Room Participants. + self.prepareLocalMedia() + // Preparing the connect options with the access token that we fetched (or hardcoded). + let connectOptions = TVIConnectOptions.init(token: accessToken) { (builder) in + + // Use the local media that we prepared earlier. + builder.videoTracks = self.localVideoTrack != nil ? [self.localVideoTrack!] : [TVILocalVideoTrack]() + if (!self.isPresenter!) { + builder.audioTracks = self.localAudioTrack != nil ? [self.localAudioTrack!] : [TVILocalAudioTrack]() + } + + + // The name of the Room where the Client will attempt to connect to. Please note that if you pass an empty + // Room `name`, the Client will create one for you. You can get the name or sid from any connected Room. + builder.roomName = "twilio" + } + + // Connect to the Room using the options we provided. + room = TwilioVideo.connect(with: connectOptions, delegate: self) + print("Attempting to connect to room") + + self.showRoomUI(inRoom: true) + } + + func prepareLocalMedia() { + + // We will share local audio and video when we connect to the Room. + + // Create an audio track. + if (!self.isPresenter! && localAudioTrack == nil) { + localAudioTrack = TVILocalAudioTrack.init() + + if (localAudioTrack == nil) { + print("Failed to create audio track") + } + } + + if (localVideoTrack == nil) { + // Preview our local camera track in the local video preview view. + camera = TVICameraCapturer(source: .frontCamera, delegate: nil) + localVideoTrack = TVILocalVideoTrack.init(capturer: camera!) + localVideoTrack.addRenderer(self.localView) + } + } + + func showRoomUI(inRoom: Bool) { + self.presenterButton.isHidden = true + self.viewerButton.isHidden = true + } + func startVideoPlayer() { if let player = self.videoPlayer { player.play() @@ -82,13 +185,6 @@ class ViewController: UIViewController { audioMix.inputParameters = [inputParameters] playerItem.audioMix = audioMix - - if ViewController.useAudioDevice { - // Fake start the device...? - let format = audioDevice.renderFormat() - print("Starting rendering with format:", format as Any) - audioDevice.startRendering(UnsafeMutableRawPointer(bitPattern: 1)!) - } } else { // Abort, retry, fail? } @@ -103,3 +199,184 @@ class ViewController: UIViewController { videoPlayerView = nil } } + +// MARK: TVIRoomDelegate +extension ViewController : TVIRoomDelegate { + func didConnect(to room: TVIRoom) { + + // Listen to events from existing `TVIRemoteParticipant`s + for remoteParticipant in room.remoteParticipants { + remoteParticipant.delegate = self + } + + if (room.remoteParticipants.count > 0 && self.isPresenter!) { + stopVideoPlayer() + startVideoPlayer() + } + + let connectMessage = "Connected to room \(room.name) as \(room.localParticipant?.identity ?? "")." + logMessage(messageText: connectMessage) + } + + func room(_ room: TVIRoom, didDisconnectWithError error: Error?) { + if let disconnectError = error { + logMessage(messageText: "Disconnected from \(room.name).\ncode = \((disconnectError as NSError).code) error = \(disconnectError.localizedDescription)") + } else { + logMessage(messageText: "Disconnected from \(room.name)") + } + + self.room = nil + + self.showRoomUI(inRoom: false) + } + + func room(_ room: TVIRoom, didFailToConnectWithError error: Error) { + logMessage(messageText: "Failed to connect to Room:\n\(error.localizedDescription)") + + self.room = nil + + self.showRoomUI(inRoom: false) + } + + func room(_ room: TVIRoom, participantDidConnect participant: TVIRemoteParticipant) { + participant.delegate = self + + logMessage(messageText: "Participant \(participant.identity) connected with \(participant.remoteAudioTracks.count) audio and \(participant.remoteVideoTracks.count) video tracks") + + if (room.remoteParticipants.count == 1 && self.isPresenter!) { + stopVideoPlayer() + startVideoPlayer() + } + } + + func room(_ room: TVIRoom, participantDidDisconnect participant: TVIRemoteParticipant) { + logMessage(messageText: "Room \(room.name), Participant \(participant.identity) disconnected") + } +} + +// MARK: TVIRemoteParticipantDelegate +extension ViewController : TVIRemoteParticipantDelegate { + + func remoteParticipant(_ participant: TVIRemoteParticipant, + publishedVideoTrack publication: TVIRemoteVideoTrackPublication) { + + // Remote Participant has offered to share the video Track. + + logMessage(messageText: "Participant \(participant.identity) published \(publication.trackName) video track") + } + + func remoteParticipant(_ participant: TVIRemoteParticipant, + unpublishedVideoTrack publication: TVIRemoteVideoTrackPublication) { + + // Remote Participant has stopped sharing the video Track. + + logMessage(messageText: "Participant \(participant.identity) unpublished \(publication.trackName) video track") + } + + func remoteParticipant(_ participant: TVIRemoteParticipant, + publishedAudioTrack publication: TVIRemoteAudioTrackPublication) { + + // Remote Participant has offered to share the audio Track. + + logMessage(messageText: "Participant \(participant.identity) published \(publication.trackName) audio track") + } + + func remoteParticipant(_ participant: TVIRemoteParticipant, + unpublishedAudioTrack publication: TVIRemoteAudioTrackPublication) { + + // Remote Participant has stopped sharing the audio Track. + + logMessage(messageText: "Participant \(participant.identity) unpublished \(publication.trackName) audio track") + } + + func subscribed(to videoTrack: TVIRemoteVideoTrack, + publication: TVIRemoteVideoTrackPublication, + for participant: TVIRemoteParticipant) { + + // We are subscribed to the remote Participant's video Track. We will start receiving the + // remote Participant's video frames now. + + logMessage(messageText: "Subscribed to \(publication.trackName) video track for Participant \(participant.identity)") + + // Start remote rendering, and add a touch handler. + videoTrack.addRenderer(self.remoteView) + } + + func unsubscribed(from videoTrack: TVIRemoteVideoTrack, + publication: TVIRemoteVideoTrackPublication, + for participant: TVIRemoteParticipant) { + + // We are unsubscribed from the remote Participant's video Track. We will no longer receive the + // remote Participant's video. + + logMessage(messageText: "Unsubscribed from \(publication.trackName) video track for Participant \(participant.identity)") + + // Stop remote rendering. + + } + + func subscribed(to audioTrack: TVIRemoteAudioTrack, + publication: TVIRemoteAudioTrackPublication, + for participant: TVIRemoteParticipant) { + + // We are subscribed to the remote Participant's audio Track. We will start receiving the + // remote Participant's audio now. + + logMessage(messageText: "Subscribed to \(publication.trackName) audio track for Participant \(participant.identity)") + } + + func unsubscribed(from audioTrack: TVIRemoteAudioTrack, + publication: TVIRemoteAudioTrackPublication, + for participant: TVIRemoteParticipant) { + + // We are unsubscribed from the remote Participant's audio Track. We will no longer receive the + // remote Participant's audio. + + logMessage(messageText: "Unsubscribed from \(publication.trackName) audio track for Participant \(participant.identity)") + } + + func remoteParticipant(_ participant: TVIRemoteParticipant, + enabledVideoTrack publication: TVIRemoteVideoTrackPublication) { + logMessage(messageText: "Participant \(participant.identity) enabled \(publication.trackName) video track") + } + + func remoteParticipant(_ participant: TVIRemoteParticipant, + disabledVideoTrack publication: TVIRemoteVideoTrackPublication) { + logMessage(messageText: "Participant \(participant.identity) disabled \(publication.trackName) video track") + } + + func remoteParticipant(_ participant: TVIRemoteParticipant, + enabledAudioTrack publication: TVIRemoteAudioTrackPublication) { + logMessage(messageText: "Participant \(participant.identity) enabled \(publication.trackName) audio track") + } + + func remoteParticipant(_ participant: TVIRemoteParticipant, + disabledAudioTrack publication: TVIRemoteAudioTrackPublication) { + // We will continue to record silence and/or recognize audio while a Track is disabled. + logMessage(messageText: "Participant \(participant.identity) disabled \(publication.trackName) audio track") + } + + func failedToSubscribe(toAudioTrack publication: TVIRemoteAudioTrackPublication, + error: Error, + for participant: TVIRemoteParticipant) { + logMessage(messageText: "FailedToSubscribe \(publication.trackName) audio track, error = \(String(describing: error))") + } + + func failedToSubscribe(toVideoTrack publication: TVIRemoteVideoTrackPublication, + error: Error, + for participant: TVIRemoteParticipant) { + logMessage(messageText: "FailedToSubscribe \(publication.trackName) video track, error = \(String(describing: error))") + } +} + +extension ViewController : TVICameraCapturerDelegate { + func cameraCapturer(_ capturer: TVICameraCapturer, didStartWith source: TVICameraCaptureSource) { + // Layout the camera preview with dimensions appropriate for our orientation. + self.view.setNeedsLayout() + } + + func cameraCapturer(_ capturer: TVICameraCapturer, didFailWithError error: Error) { + logMessage(messageText: "Capture failed with error.\ncode = \((error as NSError).code) error = \(error.localizedDescription)") + capturer.previewView.removeFromSuperview() + } +} From 12bdc2c88aad997a2578bf8e442a740841e755f9 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 16:42:05 -0700 Subject: [PATCH 22/94] Minor. --- CoViewingExample/ExampleAVPlayerSource.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index ac369808..3cb37027 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -56,7 +56,7 @@ class ExampleAVPlayerSource: NSObject { let pixelBuffer = output.copyPixelBuffer(forItemTime: targetItemTime, itemTimeForDisplay: &presentationTime) ExampleAVPlayerSource.frameCounter += 1 - if ExampleAVPlayerSource.frameCounter % 30 == 0 { + if ExampleAVPlayerSource.frameCounter % 500 == 0 { print("Copied new pixel buffer: ", pixelBuffer as Any) } } else { From 14f507cdaf8f9cd4dfbcc81032efaf2fd0d02677 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 16:47:35 -0700 Subject: [PATCH 23/94] WIP - Adding capturing capabilities. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 271 ++++++++++++++---- 1 file changed, 208 insertions(+), 63 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index c8eda7c3..e5f444e3 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -17,14 +17,36 @@ static size_t const kAudioSampleSize = 2; static uint32_t const kPreferredSampleRate = 48000; -typedef struct ExampleAVPlayerContext { +typedef struct ExampleAVPlayerAudioTapContext { + TPCircularBuffer *capturingBuffer; + dispatch_semaphore_t capturingInitSemaphore; + + TPCircularBuffer *renderingBuffer; + dispatch_semaphore_t renderingInitSemaphore; +} ExampleAVPlayerAudioTapContext; + +typedef struct ExampleAVPlayerRendererContext { TVIAudioDeviceContext deviceContext; size_t expectedFramesPerBuffer; size_t maxFramesPerBuffer; + + // The buffer of AVPlayer content that we will consume. TPCircularBuffer *playoutBuffer; -} ExampleAVPlayerContext; +} ExampleAVPlayerRendererContext; + +typedef struct ExampleAVPlayerCapturerContext { + TVIAudioDeviceContext deviceContext; + size_t expectedFramesPerBuffer; + size_t maxFramesPerBuffer; + + // Core Audio's VoiceProcessingIO audio unit. + AudioUnit audioUnit; + + // The buffer of AVPlayer content that we will consume. + TPCircularBuffer *recordingBuffer; +} ExampleAVPlayerCapturerContext; -// The RemoteIO audio unit uses bus 0 for ouptut, and bus 1 for input. +// The IO audio units use bus 0 for ouptut, and bus 1 for input. static int kOutputBus = 0; static int kInputBus = 1; // This is the maximum slice size for RemoteIO (as observed in the field). We will double check at initialization time. @@ -35,9 +57,13 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; @property (nonatomic, assign) AudioUnit audioUnit; -@property (nonatomic, assign, nullable) TPCircularBuffer *audioTapBuffer; +@property (nonatomic, assign, nullable) TPCircularBuffer *audioTapCapturingBuffer; +@property (nonatomic, assign, nullable) TPCircularBuffer *audioTapRenderingBuffer; + +@property (nonatomic, strong, nullable) TVIAudioFormat *capturingFormat; +@property (nonatomic, assign, nullable) ExampleAVPlayerCapturerContext *capturingContext; +@property (atomic, assign, nullable) ExampleAVPlayerRendererContext *renderingContext; @property (nonatomic, strong, nullable) TVIAudioFormat *renderingFormat; -@property (atomic, assign) ExampleAVPlayerContext *renderingContext; @end @@ -151,14 +177,15 @@ void process(MTAudioProcessingTapRef tap, @implementation ExampleAVPlayerAudioDevice -@synthesize audioTapBuffer = _audioTapBuffer; +@synthesize audioTapCapturingBuffer = _audioTapCapturingBuffer; #pragma mark - Init & Dealloc - (id)init { self = [super init]; if (self) { - _audioTapBuffer = malloc(sizeof(TPCircularBuffer)); + _audioTapCapturingBuffer = malloc(sizeof(TPCircularBuffer)); + _audioTapRenderingBuffer = malloc(sizeof(TPCircularBuffer)); } return self; } @@ -166,11 +193,12 @@ - (id)init { - (void)dealloc { [self unregisterAVAudioSessionObservers]; - free(_audioTapBuffer); + free(_audioTapCapturingBuffer); + free(_audioTapRenderingBuffer); } + (NSString *)description { - return @"ExampleCoreAudioDevice (stereo playback)"; + return @"ExampleAVPlayerAudioDevice"; } /* @@ -210,7 +238,8 @@ - (MTAudioProcessingTapRef)createProcessingTap { MTAudioProcessingTapCallbacks callbacks; callbacks.version = kMTAudioProcessingTapCallbacksVersion_0; - callbacks.clientInfo = (void *)(_audioTapBuffer); + // TODO: Context + callbacks.clientInfo = (void *)(_audioTapRenderingBuffer); callbacks.init = init; callbacks.prepare = prepare; callbacks.process = process; @@ -235,7 +264,7 @@ - (nullable TVIAudioFormat *)renderFormat { // Setup the AVAudioSession early. You could also defer to `startRendering:` and `stopRendering:`. [self setupAVAudioSession]; - _renderingFormat = [[self class] activeRenderingFormat]; + _renderingFormat = [[self class] activeFormat]; } return _renderingFormat; @@ -252,14 +281,15 @@ - (BOOL)initializeRenderer { - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { @synchronized(self) { NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); + // TODO: No longer true. NSAssert(self.audioUnit == NULL, @"The audio unit should not be created yet."); - self.renderingContext = malloc(sizeof(ExampleAVPlayerContext)); + self.renderingContext = malloc(sizeof(ExampleAVPlayerRendererContext)); self.renderingContext->deviceContext = context; self.renderingContext->maxFramesPerBuffer = _renderingFormat.framesPerBuffer; // TODO: Do we need to synchronize with the tap being started at this point? - self.renderingContext->playoutBuffer = _audioTapBuffer; + self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; [NSThread sleepForTimeInterval:0.2]; const NSTimeInterval sessionBufferDuration = [AVAudioSession sharedInstance].IOBufferDuration; @@ -267,7 +297,8 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { const size_t sessionFramesPerBuffer = (size_t)(sessionSampleRate * sessionBufferDuration + .5); self.renderingContext->expectedFramesPerBuffer = sessionFramesPerBuffer; - if (![self setupAudioUnit:self.renderingContext]) { + if (![self setupAudioUnitRendererContext:self.renderingContext + capturerContext:self.capturingContext]) { free(self.renderingContext); self.renderingContext = NULL; return NO; @@ -285,7 +316,7 @@ - (BOOL)stopRendering { [self stopAudioUnit]; @synchronized(self) { - NSAssert(self.renderingContext != NULL, @"Should have a rendering context."); + NSAssert(self.renderingContext != NULL, @"We should have a rendering context when stopping."); TVIAudioSessionDeactivated(self.renderingContext->deviceContext); [self teardownAudioUnit]; @@ -300,40 +331,86 @@ - (BOOL)stopRendering { #pragma mark - TVIAudioDeviceCapturer - (nullable TVIAudioFormat *)captureFormat { - /* - * We don't support capturing and return a nil format to indicate this. The other TVIAudioDeviceCapturer methods - * are simply stubs. - */ return nil; + if (!_capturingFormat) { + + /* + * Assume that the AVAudioSession has already been configured and started and that the values + * for sampleRate and IOBufferDuration are final. + */ + _capturingFormat = [[self class] activeFormat]; + } + + return _capturingFormat; } - (BOOL)initializeCapturer { return NO; +// return YES; } - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { - return NO; + @synchronized(self) { + NSAssert(self.capturingContext == NULL, @"We should not have a capturing context when starting."); + + // Restart the already setup graph. + if (_audioUnit) { + [self stopAudioUnit]; + [self teardownAudioUnit]; + } + + self.capturingContext = malloc(sizeof(ExampleAVPlayerCapturerContext)); + self.capturingContext->deviceContext = context; + self.capturingContext->maxFramesPerBuffer = _capturingFormat.framesPerBuffer; + + // TODO: Do we need to synchronize with the tap being started at this point? + self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; + [NSThread sleepForTimeInterval:0.2]; + + const NSTimeInterval sessionBufferDuration = [AVAudioSession sharedInstance].IOBufferDuration; + const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; + const size_t sessionFramesPerBuffer = (size_t)(sessionSampleRate * sessionBufferDuration + .5); + self.capturingContext->expectedFramesPerBuffer = sessionFramesPerBuffer; + + if (![self setupAudioUnitRendererContext:self.renderingContext + capturerContext:self.capturingContext]) { + free(self.capturingContext); + self.capturingContext = NULL; + return NO; + } + } + return YES; } - (BOOL)stopCapturing { - return NO; -} + @synchronized (self) { + NSAssert(self.capturingContext != NULL, @"We should have a capturing context when stopping."); -#pragma mark - Private (MTAudioProcessingTap) + if (!self.renderingContext) { + [self stopAudioUnit]; + TVIAudioSessionDeactivated(self.capturingContext->deviceContext); + [self teardownAudioUnit]; + + free(self.capturingContext); + self.capturingContext = NULL; + } + } + return YES; +} #pragma mark - Private (AudioUnit callbacks) -static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, - AudioUnitRenderActionFlags *actionFlags, - const AudioTimeStamp *timestamp, - UInt32 busNumber, - UInt32 numFrames, - AudioBufferList *bufferList) { +static OSStatus ExampleAVPlayerAudioDevicePlayoutCallback(void *refCon, + AudioUnitRenderActionFlags *actionFlags, + const AudioTimeStamp *timestamp, + UInt32 busNumber, + UInt32 numFrames, + AudioBufferList *bufferList) { assert(bufferList->mNumberBuffers == 1); assert(bufferList->mBuffers[0].mNumberChannels <= 2); assert(bufferList->mBuffers[0].mNumberChannels > 0); - ExampleAVPlayerContext *context = (ExampleAVPlayerContext *)refCon; + ExampleAVPlayerRendererContext *context = (ExampleAVPlayerRendererContext *)refCon; TPCircularBuffer *buffer = context->playoutBuffer; int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; UInt32 audioBufferSizeInBytes = bufferList->mBuffers[0].mDataByteSize; @@ -375,9 +452,50 @@ static OSStatus ExampleCoreAudioDevicePlayoutCallback(void *refCon, return noErr; } +static OSStatus ExampleAVPlayerAudioDeviceRecordingCallback(void *refCon, + AudioUnitRenderActionFlags *actionFlags, + const AudioTimeStamp *timestamp, + UInt32 busNumber, + UInt32 numFrames, + AudioBufferList *bufferList) { + + if (numFrames > kMaximumFramesPerBuffer) { + NSLog(@"Expected %u frames but got %u.", (unsigned int)kMaximumFramesPerBuffer, (unsigned int)numFrames); + return noErr; + } + + ExampleAVPlayerCapturerContext *context = (ExampleAVPlayerCapturerContext *)refCon; + + if (context->deviceContext == NULL) { + return noErr; + } + + // Render into the buffer provided by the AudioUnit. + OSStatus status = AudioUnitRender(context->audioUnit, + actionFlags, + timestamp, + 1, + numFrames, + bufferList); + + if (status != noErr) { + return status; + } + + // Copy the recorded samples. + int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; + UInt32 audioBufferSize = bufferList->mBuffers[0].mDataByteSize; + + if (context->deviceContext && audioBuffer) { + TVIAudioDeviceWriteCaptureData(context->deviceContext, audioBuffer, audioBufferSize); + } + + return noErr; +} + #pragma mark - Private (AVAudioSession and CoreAudio) -+ (nullable TVIAudioFormat *)activeRenderingFormat { ++ (nullable TVIAudioFormat *)activeFormat { /* * Use the pre-determined maximum frame size. AudioUnit callbacks are variable, and in most sitations will be close * to the `AVAudioSession.preferredIOBufferDuration` that we've requested. @@ -396,7 +514,7 @@ + (nullable TVIAudioFormat *)activeRenderingFormat { + (AudioComponentDescription)audioUnitDescription { AudioComponentDescription audioUnitDescription; audioUnitDescription.componentType = kAudioUnitType_Output; - audioUnitDescription.componentSubType = kAudioUnitSubType_RemoteIO; + audioUnitDescription.componentSubType = kAudioUnitSubType_VoiceProcessingIO; audioUnitDescription.componentManufacturer = kAudioUnitManufacturer_Apple; audioUnitDescription.componentFlags = 0; audioUnitDescription.componentFlagsMask = 0; @@ -417,7 +535,7 @@ - (void)setupAVAudioSession { } /* - * We want to be as close as possible to the 10 millisecond buffer size that the media engine needs. If there is + * We want to be as close as possible to the buffer size that the media engine needs. If there is * a mismatch then TwilioVideo will ensure that appropriately sized audio buffers are delivered. */ if (![session setPreferredIOBufferDuration:kPreferredIOBufferDuration error:&error]) { @@ -435,8 +553,8 @@ - (void)setupAVAudioSession { } } -- (BOOL)setupAudioUnit:(ExampleAVPlayerContext *)context { - // Find and instantiate the RemoteIO audio unit. +- (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)rendererContext + capturerContext:(ExampleAVPlayerCapturerContext *)capturerContext { AudioComponentDescription audioUnitDescription = [[self class] audioUnitDescription]; AudioComponent audioComponent = AudioComponentFindNext(NULL, &audioUnitDescription); @@ -447,60 +565,87 @@ - (BOOL)setupAudioUnit:(ExampleAVPlayerContext *)context { } /* - * Configure the RemoteIO audio unit. Our rendering format attempts to match what AVAudioSession requires to + * Configure the VoiceProcessingIO audio unit. Our rendering format attempts to match what AVAudioSession requires to * prevent any additional format conversions after the media engine has mixed our playout audio. */ AudioStreamBasicDescription streamDescription = self.renderingFormat.streamDescription; - UInt32 enableOutput = 1; + UInt32 enableOutput = rendererContext ? 1 : 0; status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, kOutputBus, &enableOutput, sizeof(enableOutput)); if (status != 0) { - NSLog(@"Could not enable output bus!"); + NSLog(@"Could not enable/disable output bus!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; return NO; } - status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, - kAudioUnitScope_Input, kOutputBus, - &streamDescription, sizeof(streamDescription)); - if (status != 0) { - NSLog(@"Could not enable output bus!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; + if (enableOutput) { + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, kOutputBus, + &streamDescription, sizeof(streamDescription)); + if (status != 0) { + NSLog(@"Could not set stream format on the output bus!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Setup the rendering callback. + AURenderCallbackStruct renderCallback; + renderCallback.inputProc = ExampleAVPlayerAudioDevicePlayoutCallback; + renderCallback.inputProcRefCon = (void *)(rendererContext); + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Output, kOutputBus, &renderCallback, + sizeof(renderCallback)); + if (status != 0) { + NSLog(@"Could not set rendering callback!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } } - // Disable input, we don't want it. - UInt32 enableInput = 0; + UInt32 enableInput = capturerContext ? 1 : 0; status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, kInputBus, &enableInput, sizeof(enableInput)); if (status != 0) { - NSLog(@"Could not disable input bus!"); + NSLog(@"Could not enable/disable input bus!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; return NO; } - // Setup the rendering callback. - AURenderCallbackStruct renderCallback; - renderCallback.inputProc = ExampleCoreAudioDevicePlayoutCallback; - renderCallback.inputProcRefCon = (void *)(context); - status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_SetRenderCallback, - kAudioUnitScope_Output, kOutputBus, &renderCallback, - sizeof(renderCallback)); - if (status != 0) { - NSLog(@"Could not set rendering callback!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; + if (enableInput) { + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, kInputBus, + &streamDescription, sizeof(streamDescription)); + if (status != 0) { + NSLog(@"Could not set stream format on the input bus!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Setup the capturing callback. + AURenderCallbackStruct capturerCallback; + capturerCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingCallback; + capturerCallback.inputProcRefCon = (void *)(capturerContext); + status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_SetInputCallback, + kAudioUnitScope_Input, kInputBus, &capturerCallback, + sizeof(capturerCallback)); + if (status != 0) { + NSLog(@"Could not set rendering callback!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } } - // Finally, initialize and start the RemoteIO audio unit. + // Finally, initialize and start the IO audio unit. status = AudioUnitInitialize(_audioUnit); if (status != 0) { NSLog(@"Could not initialize the audio unit!"); @@ -641,7 +786,7 @@ - (void)handleValidRouteChange { NSLog(@"A route change ocurred while the AudioUnit was started. Checking the active audio format."); // Determine if the format actually changed. We only care about sample rate and number of channels. - TVIAudioFormat *activeFormat = [[self class] activeRenderingFormat]; + TVIAudioFormat *activeFormat = [[self class] activeFormat]; if (![activeFormat isEqual:_renderingFormat]) { NSLog(@"The rendering format changed. Restarting with %@", activeFormat); From 231411953af23b295484c59372819108e72b5024 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 19:15:18 -0700 Subject: [PATCH 24/94] UI tweaks - always share audio, mirror camera. --- CoViewingExample/ExampleAVPlayerSource.swift | 2 +- CoViewingExample/ViewController.swift | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 3cb37027..cacc1ce6 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -57,7 +57,7 @@ class ExampleAVPlayerSource: NSObject { ExampleAVPlayerSource.frameCounter += 1 if ExampleAVPlayerSource.frameCounter % 500 == 0 { - print("Copied new pixel buffer: ", pixelBuffer as Any) +// print("Copied new pixel buffer: ", pixelBuffer as Any) } } else { // TODO: Consider suspending the timer and requesting a notification when media becomes available. diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 6cfc4772..e15dfd91 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -45,6 +45,8 @@ class ViewController: UIViewController { override func viewDidLoad() { super.viewDidLoad() + // We use the front facing camera for Co-Viewing. + localView.shouldMirror = true presenterButton.backgroundColor = UIColor.red presenterButton.titleLabel?.textColor = UIColor.white viewerButton.backgroundColor = UIColor.red @@ -99,10 +101,7 @@ class ViewController: UIViewController { // Use the local media that we prepared earlier. builder.videoTracks = self.localVideoTrack != nil ? [self.localVideoTrack!] : [TVILocalVideoTrack]() - if (!self.isPresenter!) { - builder.audioTracks = self.localAudioTrack != nil ? [self.localAudioTrack!] : [TVILocalAudioTrack]() - } - + builder.audioTracks = self.localAudioTrack != nil ? [self.localAudioTrack!] : [TVILocalAudioTrack]() // The name of the Room where the Client will attempt to connect to. Please note that if you pass an empty // Room `name`, the Client will create one for you. You can get the name or sid from any connected Room. @@ -111,17 +110,15 @@ class ViewController: UIViewController { // Connect to the Room using the options we provided. room = TwilioVideo.connect(with: connectOptions, delegate: self) - print("Attempting to connect to room") + print("Attempting to connect to:", connectOptions.roomName as Any) self.showRoomUI(inRoom: true) } func prepareLocalMedia() { - - // We will share local audio and video when we connect to the Room. - + // All Participants share local audio and video when they connect to the Room. // Create an audio track. - if (!self.isPresenter! && localAudioTrack == nil) { + if (localAudioTrack == nil) { localAudioTrack = TVILocalAudioTrack.init() if (localAudioTrack == nil) { @@ -129,12 +126,15 @@ class ViewController: UIViewController { } } + // Create a camera video Track. if (localVideoTrack == nil) { // Preview our local camera track in the local video preview view. camera = TVICameraCapturer(source: .frontCamera, delegate: nil) localVideoTrack = TVILocalVideoTrack.init(capturer: camera!) localVideoTrack.addRenderer(self.localView) } + + // TODO: Create a player video Track. } func showRoomUI(inRoom: Bool) { From 2542a602f77fa1b8101c21ea43bab39d7e525c5a Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 19:15:56 -0700 Subject: [PATCH 25/94] Recording is almost working (distorted). --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 143 ++++++++++++------ 1 file changed, 99 insertions(+), 44 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index e5f444e3..b7745171 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -42,6 +42,9 @@ // Core Audio's VoiceProcessingIO audio unit. AudioUnit audioUnit; + // Buffer used to render into. + AudioBufferList *bufferList; + // The buffer of AVPlayer content that we will consume. TPCircularBuffer *recordingBuffer; } ExampleAVPlayerCapturerContext; @@ -57,9 +60,11 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; @property (nonatomic, assign) AudioUnit audioUnit; +@property (nonatomic, assign, nullable) ExampleAVPlayerAudioTapContext *audioTapContext; @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapCapturingBuffer; @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapRenderingBuffer; +@property (nonatomic, assign) AudioBufferList captureBufferList; @property (nonatomic, strong, nullable) TVIAudioFormat *capturingFormat; @property (nonatomic, assign, nullable) ExampleAVPlayerCapturerContext *capturingContext; @property (atomic, assign, nullable) ExampleAVPlayerRendererContext *renderingContext; @@ -79,7 +84,8 @@ void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { } void finalize(MTAudioProcessingTapRef tap) { - TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = context->renderingBuffer; TPCircularBufferCleanup(buffer); } @@ -91,7 +97,8 @@ void prepare(MTAudioProcessingTapRef tap, assert(processingFormat->mFormatID == kAudioFormatLinearPCM); // Defer init of the ring buffer memory until we understand the processing format. - TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = context->renderingBuffer; size_t bufferSize = processingFormat->mBytesPerFrame * maxFrames; // We need to add some overhead for the AudioBufferList data structures. @@ -121,7 +128,8 @@ void prepare(MTAudioProcessingTapRef tap, void unprepare(MTAudioProcessingTapRef tap) { // Prevent any more frames from being consumed. Note that this might end audio playback early. - TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = context->renderingBuffer; TPCircularBufferClear(buffer); free(audioFormat); @@ -139,7 +147,8 @@ void process(MTAudioProcessingTapRef tap, AudioBufferList *bufferListInOut, CMItemCount *numberFramesOut, MTAudioProcessingTapFlags *flagsOut) { - TPCircularBuffer *buffer = (TPCircularBuffer *)MTAudioProcessingTapGetStorage(tap); + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *buffer = context->renderingBuffer; CMTimeRange sourceRange; OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, @@ -195,6 +204,7 @@ - (void)dealloc { free(_audioTapCapturingBuffer); free(_audioTapRenderingBuffer); + free(_audioTapContext); } + (NSString *)description { @@ -202,7 +212,7 @@ + (NSString *)description { } /* - * Determine at runtime the maximum slice size used by RemoteIO. Setting the stream format and sample rate doesn't + * Determine at runtime the maximum slice size used by our audio unit. Setting the stream format and sample rate doesn't * appear to impact the maximum size so we prefer to read this value once at initialization time. */ + (void)initialize { @@ -234,18 +244,22 @@ + (void)initialize { #pragma mark - Public - (MTAudioProcessingTapRef)createProcessingTap { - MTAudioProcessingTapRef processingTap; + NSAssert(_audioTapContext == NULL, @"We should not already have an audio tap context when creating a tap."); + _audioTapContext = malloc(sizeof(ExampleAVPlayerAudioTapContext)); + _audioTapContext->capturingBuffer = _audioTapCapturingBuffer; + _audioTapContext->renderingBuffer = _audioTapRenderingBuffer; + MTAudioProcessingTapRef processingTap; MTAudioProcessingTapCallbacks callbacks; callbacks.version = kMTAudioProcessingTapCallbacksVersion_0; - // TODO: Context - callbacks.clientInfo = (void *)(_audioTapRenderingBuffer); callbacks.init = init; callbacks.prepare = prepare; callbacks.process = process; callbacks.unprepare = unprepare; callbacks.finalize = finalize; + callbacks.clientInfo = (void *)(_audioTapContext); + OSStatus status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, @@ -253,6 +267,8 @@ - (MTAudioProcessingTapRef)createProcessingTap { if (status == kCVReturnSuccess) { return processingTap; } else { + free(_audioTapContext); + _audioTapContext = NULL; return NULL; } } @@ -279,10 +295,16 @@ - (BOOL)initializeRenderer { } - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { + NSLog(@"%s", __PRETTY_FUNCTION__); + @synchronized(self) { NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); - // TODO: No longer true. - NSAssert(self.audioUnit == NULL, @"The audio unit should not be created yet."); + + // Restart the already setup graph. + if (_audioUnit) { + [self stopAudioUnit]; + [self teardownAudioUnit]; + } self.renderingContext = malloc(sizeof(ExampleAVPlayerRendererContext)); self.renderingContext->deviceContext = context; @@ -307,22 +329,25 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { BOOL success = [self startAudioUnit]; if (success) { -// TVIAudioSessionActivated(context); + TVIAudioSessionActivated(context); } return success; } - (BOOL)stopRendering { - [self stopAudioUnit]; + NSLog(@"%s", __PRETTY_FUNCTION__); @synchronized(self) { NSAssert(self.renderingContext != NULL, @"We should have a rendering context when stopping."); - TVIAudioSessionDeactivated(self.renderingContext->deviceContext); - [self teardownAudioUnit]; + if (!self.capturingContext) { + [self stopAudioUnit]; + TVIAudioSessionDeactivated(self.renderingContext->deviceContext); + [self teardownAudioUnit]; - free(self.renderingContext); - self.renderingContext = NULL; + free(self.renderingContext); + self.renderingContext = NULL; + } } return YES; @@ -331,7 +356,6 @@ - (BOOL)stopRendering { #pragma mark - TVIAudioDeviceCapturer - (nullable TVIAudioFormat *)captureFormat { - return nil; if (!_capturingFormat) { /* @@ -345,11 +369,23 @@ - (nullable TVIAudioFormat *)captureFormat { } - (BOOL)initializeCapturer { - return NO; -// return YES; + if (_captureBufferList.mNumberBuffers == 0) { + _captureBufferList.mNumberBuffers = 1; + + AudioBuffer *audioBuffer = &_captureBufferList.mBuffers[0]; + audioBuffer->mNumberChannels = kPreferredNumberOfChannels; + + size_t byteSize = kMaximumFramesPerBuffer * kPreferredNumberOfChannels * 2; + audioBuffer->mDataByteSize = (UInt32)byteSize; + audioBuffer->mData = malloc(byteSize); + } + + return YES; } - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { + NSLog(@"%s", __PRETTY_FUNCTION__); + @synchronized(self) { NSAssert(self.capturingContext == NULL, @"We should not have a capturing context when starting."); @@ -362,6 +398,7 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { self.capturingContext = malloc(sizeof(ExampleAVPlayerCapturerContext)); self.capturingContext->deviceContext = context; self.capturingContext->maxFramesPerBuffer = _capturingFormat.framesPerBuffer; + self.capturingContext->bufferList = &_captureBufferList; // TODO: Do we need to synchronize with the tap being started at this point? self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; @@ -377,12 +414,20 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { free(self.capturingContext); self.capturingContext = NULL; return NO; + } else { + self.capturingContext->audioUnit = _audioUnit; } } - return YES; + BOOL success = [self startAudioUnit]; + if (success) { + TVIAudioSessionActivated(context); + } + return success; } - (BOOL)stopCapturing { + NSLog(@"%s", __PRETTY_FUNCTION__); + @synchronized (self) { NSAssert(self.capturingContext != NULL, @"We should have a capturing context when stopping."); @@ -470,22 +515,26 @@ static OSStatus ExampleAVPlayerAudioDeviceRecordingCallback(void *refCon, return noErr; } - // Render into the buffer provided by the AudioUnit. + // Render into our recording buffer. + AudioBufferList *renderingBufferList = context->bufferList; + AudioBuffer *renderingBuffer = renderingBufferList->mBuffers; + UInt32 audioBufferSize = renderingBuffer->mDataByteSize; + + assert(numFrames <= context->expectedFramesPerBuffer); + + int8_t *audioBuffer = (int8_t *)renderingBuffer->mData; OSStatus status = AudioUnitRender(context->audioUnit, actionFlags, timestamp, 1, numFrames, - bufferList); + renderingBufferList); if (status != noErr) { return status; } // Copy the recorded samples. - int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; - UInt32 audioBufferSize = bufferList->mBuffers[0].mDataByteSize; - if (context->deviceContext && audioBuffer) { TVIAudioDeviceWriteCaptureData(context->deviceContext, audioBuffer, audioBufferSize); } @@ -542,7 +591,11 @@ - (void)setupAVAudioSession { NSLog(@"Error setting IOBuffer duration: %@", error); } - if (![session setCategory:AVAudioSessionCategoryPlayback error:&error]) { + if (![session setCategory:AVAudioSessionCategoryPlayAndRecord error:&error]) { + NSLog(@"Error setting session category: %@", error); + } + + if (![session setMode:AVAudioSessionModeVideoChat error:&error]) { NSLog(@"Error setting session category: %@", error); } @@ -559,8 +612,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer AudioComponent audioComponent = AudioComponentFindNext(NULL, &audioUnitDescription); OSStatus status = AudioComponentInstanceNew(audioComponent, &_audioUnit); - if (status != 0) { - NSLog(@"Could not find RemoteIO AudioComponent instance!"); + if (status != noErr) { + NSLog(@"Could not find the AudioComponent instance!"); return NO; } @@ -568,13 +621,11 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer * Configure the VoiceProcessingIO audio unit. Our rendering format attempts to match what AVAudioSession requires to * prevent any additional format conversions after the media engine has mixed our playout audio. */ - AudioStreamBasicDescription streamDescription = self.renderingFormat.streamDescription; - UInt32 enableOutput = rendererContext ? 1 : 0; status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, kOutputBus, &enableOutput, sizeof(enableOutput)); - if (status != 0) { + if (status != noErr) { NSLog(@"Could not enable/disable output bus!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; @@ -582,10 +633,12 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer } if (enableOutput) { + AudioStreamBasicDescription renderingFormatDescription = self.renderingFormat.streamDescription; + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, kOutputBus, - &streamDescription, sizeof(streamDescription)); - if (status != 0) { + &renderingFormatDescription, sizeof(renderingFormatDescription)); + if (status != noErr) { NSLog(@"Could not set stream format on the output bus!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; @@ -612,7 +665,7 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer kAudioUnitScope_Input, kInputBus, &enableInput, sizeof(enableInput)); - if (status != 0) { + if (status != noErr) { NSLog(@"Could not enable/disable input bus!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; @@ -620,10 +673,12 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer } if (enableInput) { + AudioStreamBasicDescription capturingFormatDescription = self.capturingFormat.streamDescription; + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, - kAudioUnitScope_Input, kInputBus, - &streamDescription, sizeof(streamDescription)); - if (status != 0) { + kAudioUnitScope_Output, kInputBus, + &capturingFormatDescription, sizeof(capturingFormatDescription)); + if (status != noErr) { NSLog(@"Could not set stream format on the input bus!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; @@ -637,8 +692,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Input, kInputBus, &capturerCallback, sizeof(capturerCallback)); - if (status != 0) { - NSLog(@"Could not set rendering callback!"); + if (status != noErr) { + NSLog(@"Could not set capturing callback!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; return NO; @@ -647,7 +702,7 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer // Finally, initialize and start the IO audio unit. status = AudioUnitInitialize(_audioUnit); - if (status != 0) { + if (status != noErr) { NSLog(@"Could not initialize the audio unit!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; @@ -659,8 +714,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer - (BOOL)startAudioUnit { OSStatus status = AudioOutputUnitStart(_audioUnit); - if (status != 0) { - NSLog(@"Could not start the audio unit!"); + if (status != noErr) { + NSLog(@"Could not start the audio unit. code: %d", status); return NO; } return YES; @@ -668,8 +723,8 @@ - (BOOL)startAudioUnit { - (BOOL)stopAudioUnit { OSStatus status = AudioOutputUnitStop(_audioUnit); - if (status != 0) { - NSLog(@"Could not stop the audio unit!"); + if (status != noErr) { + NSLog(@"Could not stop the audio unit. code: %d", status); return NO; } return YES; From 06fc4fbb9701d61e3022d946d1cb1101aa8ad816 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 19:50:47 -0700 Subject: [PATCH 26/94] Fix recording distortion. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index b7745171..85826ecc 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -42,8 +42,8 @@ // Core Audio's VoiceProcessingIO audio unit. AudioUnit audioUnit; - // Buffer used to render into. - AudioBufferList *bufferList; + // Buffer used to render audio samples into. + int16_t *audioBuffer; // The buffer of AVPlayer content that we will consume. TPCircularBuffer *recordingBuffer; @@ -64,7 +64,7 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapCapturingBuffer; @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapRenderingBuffer; -@property (nonatomic, assign) AudioBufferList captureBufferList; +@property (nonatomic, assign) int16_t *captureBuffer; @property (nonatomic, strong, nullable) TVIAudioFormat *capturingFormat; @property (nonatomic, assign, nullable) ExampleAVPlayerCapturerContext *capturingContext; @property (atomic, assign, nullable) ExampleAVPlayerRendererContext *renderingContext; @@ -295,7 +295,7 @@ - (BOOL)initializeRenderer { } - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { - NSLog(@"%s", __PRETTY_FUNCTION__); + NSLog(@"%s %@", __PRETTY_FUNCTION__, self.renderingFormat); @synchronized(self) { NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); @@ -324,6 +324,8 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { free(self.renderingContext); self.renderingContext = NULL; return NO; + } else if (self.capturingContext) { + self.capturingContext->audioUnit = _audioUnit; } } @@ -369,22 +371,16 @@ - (nullable TVIAudioFormat *)captureFormat { } - (BOOL)initializeCapturer { - if (_captureBufferList.mNumberBuffers == 0) { - _captureBufferList.mNumberBuffers = 1; - - AudioBuffer *audioBuffer = &_captureBufferList.mBuffers[0]; - audioBuffer->mNumberChannels = kPreferredNumberOfChannels; - + if (_captureBuffer == NULL) { size_t byteSize = kMaximumFramesPerBuffer * kPreferredNumberOfChannels * 2; - audioBuffer->mDataByteSize = (UInt32)byteSize; - audioBuffer->mData = malloc(byteSize); + _captureBuffer = malloc(byteSize); } return YES; } - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { - NSLog(@"%s", __PRETTY_FUNCTION__); + NSLog(@"%s %@", __PRETTY_FUNCTION__, self.capturingFormat); @synchronized(self) { NSAssert(self.capturingContext == NULL, @"We should not have a capturing context when starting."); @@ -396,9 +392,10 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { } self.capturingContext = malloc(sizeof(ExampleAVPlayerCapturerContext)); + memset(self.capturingContext, 0, sizeof(ExampleAVPlayerCapturerContext)); self.capturingContext->deviceContext = context; self.capturingContext->maxFramesPerBuffer = _capturingFormat.framesPerBuffer; - self.capturingContext->bufferList = &_captureBufferList; + self.capturingContext->audioBuffer = _captureBuffer; // TODO: Do we need to synchronize with the tap being started at this point? self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; @@ -438,6 +435,9 @@ - (BOOL)stopCapturing { free(self.capturingContext); self.capturingContext = NULL; + + free(self.captureBuffer); + self.captureBuffer = NULL; } } return YES; @@ -516,27 +516,33 @@ static OSStatus ExampleAVPlayerAudioDeviceRecordingCallback(void *refCon, } // Render into our recording buffer. - AudioBufferList *renderingBufferList = context->bufferList; - AudioBuffer *renderingBuffer = renderingBufferList->mBuffers; - UInt32 audioBufferSize = renderingBuffer->mDataByteSize; - assert(numFrames <= context->expectedFramesPerBuffer); + AudioBufferList renderingBufferList; + renderingBufferList.mNumberBuffers = 1; + + AudioBuffer *audioBuffer = &renderingBufferList.mBuffers[0]; + audioBuffer->mNumberChannels = kPreferredNumberOfChannels; + audioBuffer->mDataByteSize = (UInt32)context->maxFramesPerBuffer * kPreferredNumberOfChannels * 2; + audioBuffer->mData = context->audioBuffer; - int8_t *audioBuffer = (int8_t *)renderingBuffer->mData; OSStatus status = AudioUnitRender(context->audioUnit, actionFlags, timestamp, - 1, + busNumber, numFrames, - renderingBufferList); + &renderingBufferList); if (status != noErr) { + NSLog(@"Render failed with code: %d", status); return status; } // Copy the recorded samples. + int8_t *audioData = (int8_t *)audioBuffer->mData; + UInt32 audioDataByteSize = audioBuffer->mDataByteSize; + if (context->deviceContext && audioBuffer) { - TVIAudioDeviceWriteCaptureData(context->deviceContext, audioBuffer, audioBufferSize); + TVIAudioDeviceWriteCaptureData(context->deviceContext, audioData, audioDataByteSize); } return noErr; From 0aace234f939fc208847737b1fca879c67a7b09a Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Thu, 1 Nov 2018 21:33:23 -0700 Subject: [PATCH 27/94] Use a multichannel mixer for playback. * Add a rendering method for WebRTC audio. * Hook up both audio tap and rendering inputs to the mixer. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 169 ++++++++++++++++-- 1 file changed, 152 insertions(+), 17 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 85826ecc..d08a20bf 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -59,6 +59,7 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; @property (nonatomic, assign) AudioUnit audioUnit; +@property (nonatomic, assign) AudioUnit playbackMixer; @property (nonatomic, assign, nullable) ExampleAVPlayerAudioTapContext *audioTapContext; @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapCapturingBuffer; @@ -445,12 +446,12 @@ - (BOOL)stopCapturing { #pragma mark - Private (AudioUnit callbacks) -static OSStatus ExampleAVPlayerAudioDevicePlayoutCallback(void *refCon, - AudioUnitRenderActionFlags *actionFlags, - const AudioTimeStamp *timestamp, - UInt32 busNumber, - UInt32 numFrames, - AudioBufferList *bufferList) { +static OSStatus ExampleAVPlayerAudioDeviceAudioTapPlaybackCallback(void *refCon, + AudioUnitRenderActionFlags *actionFlags, + const AudioTimeStamp *timestamp, + UInt32 busNumber, + UInt32 numFrames, + AudioBufferList *bufferList) { assert(bufferList->mNumberBuffers == 1); assert(bufferList->mBuffers[0].mNumberChannels <= 2); assert(bufferList->mBuffers[0].mNumberChannels > 0); @@ -492,8 +493,35 @@ static OSStatus ExampleAVPlayerAudioDevicePlayoutCallback(void *refCon, memset(audioBuffer, 0, bytesRemaining); } - // TODO: Pull decoded, mixed audio data from the media engine into the AudioUnit's AudioBufferList. -// TVIAudioDeviceReadRenderData(context->deviceContext, audioBuffer, audioBufferSizeInBytes); + return noErr; +} + +static OSStatus ExampleAVPlayerAudioDeviceAudioRendererPlaybackCallback(void *refCon, + AudioUnitRenderActionFlags *actionFlags, + const AudioTimeStamp *timestamp, + UInt32 busNumber, + UInt32 numFrames, + AudioBufferList *bufferList) { + assert(bufferList->mNumberBuffers == 1); + assert(bufferList->mBuffers[0].mNumberChannels <= 2); + assert(bufferList->mBuffers[0].mNumberChannels > 0); + + ExampleAVPlayerCapturerContext *context = (ExampleAVPlayerCapturerContext *)refCon; + int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; + UInt32 audioBufferSizeInBytes = bufferList->mBuffers[0].mDataByteSize; + + // Render silence if there are temporary mismatches between CoreAudio and our rendering format. + if (numFrames > context->maxFramesPerBuffer) { + NSLog(@"Can handle a max of %u frames but got %u.", (unsigned int)context->maxFramesPerBuffer, (unsigned int)numFrames); + *actionFlags |= kAudioUnitRenderAction_OutputIsSilence; + memset(audioBuffer, 0, audioBufferSizeInBytes); + return noErr; + } + + // Pull decoded, mixed audio data from the media engine into the AudioUnit's AudioBufferList. + assert(numFrames <= context->maxFramesPerBuffer); + assert(audioBufferSizeInBytes == (bufferList->mBuffers[0].mNumberChannels * kAudioSampleSize * numFrames)); + TVIAudioDeviceReadRenderData(context->deviceContext, audioBuffer, audioBufferSizeInBytes); return noErr; } @@ -576,6 +604,16 @@ + (AudioComponentDescription)audioUnitDescription { return audioUnitDescription; } ++ (AudioComponentDescription)mixerAudioCompontentDescription { + AudioComponentDescription audioUnitDescription; + audioUnitDescription.componentType = kAudioUnitType_Mixer; + audioUnitDescription.componentSubType = kAudioUnitSubType_MultiChannelMixer; + audioUnitDescription.componentManufacturer = kAudioUnitManufacturer_Apple; + audioUnitDescription.componentFlags = 0; + audioUnitDescription.componentFlagsMask = 0; + return audioUnitDescription; +} + - (void)setupAVAudioSession { AVAudioSession *session = [AVAudioSession sharedInstance]; NSError *error = nil; @@ -641,6 +679,63 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer if (enableOutput) { AudioStreamBasicDescription renderingFormatDescription = self.renderingFormat.streamDescription; + // Setup playback mixer. + AudioComponentDescription mixerComponentDescription = [[self class] mixerAudioCompontentDescription]; + AudioComponent mixerComponent = AudioComponentFindNext(NULL, &mixerComponentDescription); + + OSStatus status = AudioComponentInstanceNew(mixerComponent, &_playbackMixer); + if (status != noErr) { + NSLog(@"Could not find the mixer AudioComponent instance!"); + return NO; + } + + // Configure I/O format. + status = AudioUnitSetProperty(_playbackMixer, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, kOutputBus, + &renderingFormatDescription, sizeof(renderingFormatDescription)); + if (status != noErr) { + NSLog(@"Could not set stream format on the mixer output bus!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + status = AudioUnitSetProperty(_playbackMixer, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, 0, + &renderingFormatDescription, sizeof(renderingFormatDescription)); + if (status != noErr) { + NSLog(@"Could not set stream format on the mixer input bus 0!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + status = AudioUnitSetProperty(_playbackMixer, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, 1, + &renderingFormatDescription, sizeof(renderingFormatDescription)); + if (status != noErr) { + NSLog(@"Could not set stream format on the mixer input bus 1!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Connection: Mixer Output 0 -> VoiceProcessingIO Input Scope, Output Bus + AudioUnitConnection mixerOutputConnection; + mixerOutputConnection.sourceAudioUnit = _playbackMixer; + mixerOutputConnection.sourceOutputNumber = kOutputBus; + mixerOutputConnection.destInputNumber = kOutputBus; + + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_MakeConnection, + kAudioUnitScope_Input, kOutputBus, + &mixerOutputConnection, sizeof(mixerOutputConnection)); + if (status != noErr) { + NSLog(@"Could not connect the mixer output to voice processing input!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, kOutputBus, &renderingFormatDescription, sizeof(renderingFormatDescription)); @@ -651,15 +746,39 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer return NO; } - // Setup the rendering callback. - AURenderCallbackStruct renderCallback; - renderCallback.inputProc = ExampleAVPlayerAudioDevicePlayoutCallback; - renderCallback.inputProcRefCon = (void *)(rendererContext); - status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_SetRenderCallback, - kAudioUnitScope_Output, kOutputBus, &renderCallback, - sizeof(renderCallback)); + // Setup the rendering callbacks. + UInt32 elementCount = 2; + status = AudioUnitSetProperty(_playbackMixer, kAudioUnitProperty_ElementCount, + kAudioUnitScope_Input, 0, &elementCount, + sizeof(elementCount)); + if (status != 0) { + NSLog(@"Could not set input element count!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + AURenderCallbackStruct audioTapRenderCallback; + audioTapRenderCallback.inputProc = ExampleAVPlayerAudioDeviceAudioTapPlaybackCallback; + audioTapRenderCallback.inputProcRefCon = (void *)(rendererContext); + status = AudioUnitSetProperty(_playbackMixer, kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Input, 0, &audioTapRenderCallback, + sizeof(audioTapRenderCallback)); + if (status != 0) { + NSLog(@"Could not set audio tap rendering callback!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + AURenderCallbackStruct audioRendererRenderCallback; + audioRendererRenderCallback.inputProc = ExampleAVPlayerAudioDeviceAudioRendererPlaybackCallback; + audioRendererRenderCallback.inputProcRefCon = (void *)(rendererContext); + status = AudioUnitSetProperty(_playbackMixer, kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Input, 1, &audioRendererRenderCallback, + sizeof(audioRendererRenderCallback)); if (status != 0) { - NSLog(@"Could not set rendering callback!"); + NSLog(@"Could not set audio renderer rendering callback!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; return NO; @@ -706,7 +825,7 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer } } - // Finally, initialize and start the IO audio unit. + // Finally, initialize the IO audio unit and mixer (if present). status = AudioUnitInitialize(_audioUnit); if (status != noErr) { NSLog(@"Could not initialize the audio unit!"); @@ -715,6 +834,16 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer return NO; } + if (_playbackMixer) { + status = AudioUnitInitialize(_playbackMixer); + if (status != noErr) { + NSLog(@"Could not initialize the mixer audio unit!"); + AudioComponentInstanceDispose(_playbackMixer); + _playbackMixer = NULL; + return NO; + } + } + return YES; } @@ -742,6 +871,12 @@ - (void)teardownAudioUnit { AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; } + + if (_playbackMixer) { + AudioUnitUninitialize(_playbackMixer); + AudioComponentInstanceDispose(_playbackMixer); + _playbackMixer = NULL; + } } #pragma mark - NSNotification Observers From 15d44dab3ae3738522308e5cd016612e52a7d8a1 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 2 Nov 2018 08:48:25 -0700 Subject: [PATCH 28/94] Print UI messages to the console for now. --- CoViewingExample/ViewController.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index e15dfd91..1efa522d 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -78,6 +78,7 @@ class ViewController: UIViewController { } func logMessage(messageText: String) { + print(messageText) } func connect(name: String) { From a54a4401dd2fabe51c64738c87c617336c33e2d1 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 2 Nov 2018 10:20:09 -0700 Subject: [PATCH 29/94] ExampleAVPlayerSource is a TVIVideoCapturer. --- CoViewingExample/ExampleAVPlayerSource.swift | 44 +++++++++++++++++--- CoViewingExample/ViewController.swift | 8 ++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index cacc1ce6..605e2702 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -6,14 +6,16 @@ // import AVFoundation +import TwilioVideo -class ExampleAVPlayerSource: NSObject { +class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { private let sampleQueue: DispatchQueue private var outputTimer: CADisplayLink? = nil private var videoOutput: AVPlayerItemVideoOutput? = nil + private var captureConsumer: TVIVideoCaptureConsumer? = nil - static private var frameCounter = UInt32(0) + private var frameCounter = UInt32(0) init(item: AVPlayerItem) { sampleQueue = DispatchQueue(label: "", qos: DispatchQoS.userInteractive, @@ -55,9 +57,16 @@ class ExampleAVPlayerSource: NSObject { var presentationTime = CMTime.zero let pixelBuffer = output.copyPixelBuffer(forItemTime: targetItemTime, itemTimeForDisplay: &presentationTime) - ExampleAVPlayerSource.frameCounter += 1 - if ExampleAVPlayerSource.frameCounter % 500 == 0 { -// print("Copied new pixel buffer: ", pixelBuffer as Any) + if let consumer = self.captureConsumer, + let buffer = pixelBuffer { + guard let frame = TVIVideoFrame(timestamp: targetItemTime, + buffer: buffer, + orientation: TVIVideoOrientation.up) else { + assertionFailure("We couldn't create a TVIVideoFrame with a valid CVPixelBuffer.") + return + } + + consumer.consumeCapturedFrame(frame) } } else { // TODO: Consider suspending the timer and requesting a notification when media becomes available. @@ -67,6 +76,31 @@ class ExampleAVPlayerSource: NSObject { @objc func stopTimer() { outputTimer?.invalidate() } + + public var isScreencast: Bool { + get { + return false + } + } + + public var supportedFormats: [TVIVideoFormat] { + get { + return [TVIVideoFormat()] + } + } + + func startCapture(_ format: TVIVideoFormat, consumer: TVIVideoCaptureConsumer) { + DispatchQueue.main.async { + self.captureConsumer = consumer; + consumer.captureDidStart(true) + } + } + + func stopCapture() { + DispatchQueue.main.async { + self.captureConsumer = nil + } + } } extension ExampleAVPlayerSource: AVPlayerItemOutputPullDelegate { diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 1efa522d..b74c4976 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -23,6 +23,7 @@ class ViewController: UIViewController { var room: TVIRoom? var camera: TVICameraCapturer? var localVideoTrack: TVILocalVideoTrack! + var playerVideoTrack: TVILocalVideoTrack? var localAudioTrack: TVILocalAudioTrack! var audioDevice: ExampleAVPlayerAudioDevice = ExampleAVPlayerAudioDevice() @@ -186,6 +187,13 @@ class ViewController: UIViewController { audioMix.inputParameters = [inputParameters] playerItem.audioMix = audioMix + + // Create and publish video track. + if let track = TVILocalVideoTrack(capturer: videoPlayerSource!) { + playerVideoTrack = track + self.room!.localParticipant!.publishVideoTrack(track) + } + } else { // Abort, retry, fail? } From 26f8c456359928f79bb957df9f5415d1af6c547c Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 2 Nov 2018 10:07:08 -0700 Subject: [PATCH 30/94] WIP - Add a recording mixer. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 208 +++++++++++++----- 1 file changed, 157 insertions(+), 51 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index d08a20bf..aa2d641c 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -60,6 +60,7 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; @property (nonatomic, assign) AudioUnit audioUnit; @property (nonatomic, assign) AudioUnit playbackMixer; +@property (nonatomic, assign) AudioUnit recordingMixer; @property (nonatomic, assign, nullable) ExampleAVPlayerAudioTapContext *audioTapContext; @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapCapturingBuffer; @@ -86,8 +87,10 @@ void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { void finalize(MTAudioProcessingTapRef tap) { ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = context->renderingBuffer; - TPCircularBufferCleanup(buffer); + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + TPCircularBuffer *renderingBuffer = context->renderingBuffer; + TPCircularBufferCleanup(capturingBuffer); + TPCircularBufferCleanup(renderingBuffer); } void prepare(MTAudioProcessingTapRef tap, @@ -99,7 +102,8 @@ void prepare(MTAudioProcessingTapRef tap, // Defer init of the ring buffer memory until we understand the processing format. ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = context->renderingBuffer; + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + TPCircularBuffer *renderingBuffer = context->renderingBuffer; size_t bufferSize = processingFormat->mBytesPerFrame * maxFrames; // We need to add some overhead for the AudioBufferList data structures. @@ -108,7 +112,8 @@ void prepare(MTAudioProcessingTapRef tap, bufferSize *= 12; // TODO: If we are re-allocating then check the size? - TPCircularBufferInit(buffer, bufferSize); + TPCircularBufferInit(capturingBuffer, bufferSize); + TPCircularBufferInit(renderingBuffer, bufferSize); audioFormat = malloc(sizeof(AudioStreamBasicDescription)); memcpy(audioFormat, processingFormat, sizeof(AudioStreamBasicDescription)); @@ -130,9 +135,11 @@ void prepare(MTAudioProcessingTapRef tap, void unprepare(MTAudioProcessingTapRef tap) { // Prevent any more frames from being consumed. Note that this might end audio playback early. ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = context->renderingBuffer; + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + TPCircularBuffer *renderingBuffer = context->renderingBuffer; - TPCircularBufferClear(buffer); + TPCircularBufferClear(capturingBuffer); + TPCircularBufferClear(renderingBuffer); free(audioFormat); audioFormat = NULL; @@ -149,8 +156,8 @@ void process(MTAudioProcessingTapRef tap, CMItemCount *numberFramesOut, MTAudioProcessingTapFlags *flagsOut) { ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *buffer = context->renderingBuffer; - + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + TPCircularBuffer *renderingBuffer = context->renderingBuffer; CMTimeRange sourceRange; OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, @@ -167,22 +174,23 @@ void process(MTAudioProcessingTapRef tap, UInt32 framesToCopy = (UInt32)*numberFramesOut; // TODO: Assumptions about our producer's format. UInt32 bytesToCopy = framesToCopy * 4; - AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesToCopy, NULL); - if (producerBufferList == NULL) { - // TODO: - return; - } - status = AudioConverterConvertComplexBuffer(formatConverter, framesToCopy, bufferListInOut, producerBufferList); - if (status != kCVReturnSuccess) { - // TODO: Do we still produce the buffer list? - return; - } + for (int i=0; i < 2; i++) { + TPCircularBuffer *buffer = i == 0 ? capturingBuffer : renderingBuffer; + AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesToCopy, NULL); + if (producerBufferList == NULL) { + // TODO: + return; + } - TPCircularBufferProduceAudioBufferList(buffer, NULL); + status = AudioConverterConvertComplexBuffer(formatConverter, framesToCopy, bufferListInOut, producerBufferList); + if (status != kCVReturnSuccess) { + // TODO: Do we still produce the buffer list? + return; + } - // TODO: Silence the audio returned to AVPlayer just in case? -// memset(NULL, 0, numberFramesOut * ) + TPCircularBufferProduceAudioBufferList(buffer, NULL); + } } @implementation ExampleAVPlayerAudioDevice @@ -525,47 +533,51 @@ static OSStatus ExampleAVPlayerAudioDeviceAudioRendererPlaybackCallback(void *re return noErr; } -static OSStatus ExampleAVPlayerAudioDeviceRecordingCallback(void *refCon, - AudioUnitRenderActionFlags *actionFlags, - const AudioTimeStamp *timestamp, - UInt32 busNumber, - UInt32 numFrames, - AudioBufferList *bufferList) { - - if (numFrames > kMaximumFramesPerBuffer) { - NSLog(@"Expected %u frames but got %u.", (unsigned int)kMaximumFramesPerBuffer, (unsigned int)numFrames); - return noErr; - } - +static OSStatus ExampleAVPlayerAudioDeviceRecordingInputCallback(void *refCon, + AudioUnitRenderActionFlags *actionFlags, + const AudioTimeStamp *timestamp, + UInt32 busNumber, + UInt32 numFrames, + AudioBufferList *bufferList) { ExampleAVPlayerCapturerContext *context = (ExampleAVPlayerCapturerContext *)refCon; - if (context->deviceContext == NULL) { return noErr; } - // Render into our recording buffer. - - AudioBufferList renderingBufferList; - renderingBufferList.mNumberBuffers = 1; - - AudioBuffer *audioBuffer = &renderingBufferList.mBuffers[0]; - audioBuffer->mNumberChannels = kPreferredNumberOfChannels; - audioBuffer->mDataByteSize = (UInt32)context->maxFramesPerBuffer * kPreferredNumberOfChannels * 2; - audioBuffer->mData = context->audioBuffer; + if (numFrames > context->maxFramesPerBuffer) { + NSLog(@"Expected %u frames but got %u.", (unsigned int)context->maxFramesPerBuffer, (unsigned int)numFrames); + return noErr; + } OSStatus status = AudioUnitRender(context->audioUnit, actionFlags, timestamp, busNumber, numFrames, - &renderingBufferList); + bufferList); + return status; +} - if (status != noErr) { - NSLog(@"Render failed with code: %d", status); - return status; + +static OSStatus ExampleAVPlayerAudioDeviceRecordingOutputCallback(void *refCon, + AudioUnitRenderActionFlags *actionFlags, + const AudioTimeStamp *timestamp, + UInt32 busNumber, + UInt32 numFrames, + AudioBufferList *bufferList) { + + if (numFrames > kMaximumFramesPerBuffer) { + NSLog(@"Expected %u frames but got %u.", (unsigned int)kMaximumFramesPerBuffer, (unsigned int)numFrames); + return noErr; } - // Copy the recorded samples. + ExampleAVPlayerCapturerContext *context = (ExampleAVPlayerCapturerContext *)refCon; + if (context->deviceContext == NULL) { + return noErr; + } + + // Deliver the samples (via copying) to WebRTC. + AudioBuffer *audioBuffer = &bufferList->mBuffers[0]; int8_t *audioData = (int8_t *)audioBuffer->mData; UInt32 audioDataByteSize = audioBuffer->mDataByteSize; @@ -799,6 +811,65 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer if (enableInput) { AudioStreamBasicDescription capturingFormatDescription = self.capturingFormat.streamDescription; + Float64 sampleRate = capturingFormatDescription.mSampleRate; + + // Setup recording mixer. + AudioComponentDescription mixerComponentDescription = [[self class] mixerAudioCompontentDescription]; + AudioComponent mixerComponent = AudioComponentFindNext(NULL, &mixerComponentDescription); + + OSStatus status = AudioComponentInstanceNew(mixerComponent, &_recordingMixer); + if (status != noErr) { + NSLog(@"Could not find the mixer AudioComponent instance!"); + return NO; + } + + // Configure I/O format. + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SampleRate, + kAudioUnitScope_Output, kOutputBus, + &sampleRate, sizeof(sampleRate)); + if (status != noErr) { + NSLog(@"Could not set sample rate on the mixer output bus!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Set the sample rate for the inputs. We are not supposed to set a format? +// status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SampleRate, +// kAudioUnitScope_Input, 0, +// &sampleRate, sizeof(sampleRate)); +// if (status != noErr) { +// NSLog(@"Could not set sample rate on the mixer input bus 0!"); +// AudioComponentInstanceDispose(_audioUnit); +// _audioUnit = NULL; +// return NO; +// } + +// status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_StreamFormat, +// kAudioUnitScope_Input, 1, +// &capturingFormatDescription, sizeof(capturingFormatDescription)); +// if (status != noErr) { +// NSLog(@"Could not set stream format on the mixer input bus 1!"); +// AudioComponentInstanceDispose(_audioUnit); +// _audioUnit = NULL; +// return NO; +// } + + // Connection: VoiceProcessingIO Output Scope, Input Bus -> Mixer Input Scope, Bus 0 + AudioUnitConnection mixerInputConnection; + mixerInputConnection.sourceAudioUnit = _audioUnit; + mixerInputConnection.sourceOutputNumber = kInputBus; + mixerInputConnection.destInputNumber = 0; + + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_MakeConnection, + kAudioUnitScope_Input, 0, + &mixerInputConnection, sizeof(mixerInputConnection)); + if (status != noErr) { + NSLog(@"Could not connect voice processing output scope, input bus, to the mixer input!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, kInputBus, @@ -810,9 +881,34 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer return NO; } - // Setup the capturing callback. + // Setup the rendering callbacks for the mixer. + UInt32 elementCount = 1; + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_ElementCount, + kAudioUnitScope_Input, 0, &elementCount, + sizeof(elementCount)); + if (status != 0) { + NSLog(@"Could not set input element count!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + AURenderCallbackStruct mixerOutputRenderCallback; + mixerOutputRenderCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingOutputCallback; + mixerOutputRenderCallback.inputProcRefCon = (void *)(capturerContext); + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Output, 0, &mixerOutputRenderCallback, + sizeof(mixerOutputRenderCallback)); + if (status != 0) { + NSLog(@"Could not set mixer output rendering callback!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + + // Setup the I/O input callback. AURenderCallbackStruct capturerCallback; - capturerCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingCallback; + capturerCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingInputCallback; capturerCallback.inputProcRefCon = (void *)(capturerContext); status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Input, kInputBus, &capturerCallback, @@ -837,13 +933,23 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer if (_playbackMixer) { status = AudioUnitInitialize(_playbackMixer); if (status != noErr) { - NSLog(@"Could not initialize the mixer audio unit!"); + NSLog(@"Could not initialize the playback mixer audio unit!"); AudioComponentInstanceDispose(_playbackMixer); _playbackMixer = NULL; return NO; } } + if (_recordingMixer) { + status = AudioUnitInitialize(_recordingMixer); + if (status != noErr) { + NSLog(@"Could not initialize the recording mixer audio unit!"); + AudioComponentInstanceDispose(_recordingMixer); + _recordingMixer = NULL; + return NO; + } + } + return YES; } From d08d26cb6cf34becfe105d838c6dd214bd245f19 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 2 Nov 2018 10:34:06 -0700 Subject: [PATCH 31/94] Init is at least working. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index aa2d641c..4bb88aab 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -845,27 +845,11 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer // return NO; // } -// status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_StreamFormat, -// kAudioUnitScope_Input, 1, -// &capturingFormatDescription, sizeof(capturingFormatDescription)); -// if (status != noErr) { -// NSLog(@"Could not set stream format on the mixer input bus 1!"); -// AudioComponentInstanceDispose(_audioUnit); -// _audioUnit = NULL; -// return NO; -// } - - // Connection: VoiceProcessingIO Output Scope, Input Bus -> Mixer Input Scope, Bus 0 - AudioUnitConnection mixerInputConnection; - mixerInputConnection.sourceAudioUnit = _audioUnit; - mixerInputConnection.sourceOutputNumber = kInputBus; - mixerInputConnection.destInputNumber = 0; - - status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_MakeConnection, + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, - &mixerInputConnection, sizeof(mixerInputConnection)); + &capturingFormatDescription, sizeof(capturingFormatDescription)); if (status != noErr) { - NSLog(@"Could not connect voice processing output scope, input bus, to the mixer input!"); + NSLog(@"Could not set stream format on the mixer input bus 1!"); AudioComponentInstanceDispose(_audioUnit); _audioUnit = NULL; return NO; @@ -881,6 +865,22 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer return NO; } + // Connection: VoiceProcessingIO Output Scope, Input Bus -> Mixer Input Scope, Bus 0 + AudioUnitConnection mixerInputConnection; + mixerInputConnection.sourceAudioUnit = _audioUnit; + mixerInputConnection.sourceOutputNumber = kInputBus; + mixerInputConnection.destInputNumber = 0; + + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_MakeConnection, + kAudioUnitScope_Input, 0, + &mixerInputConnection, sizeof(mixerInputConnection)); + if (status != noErr) { + NSLog(@"Could not connect voice processing output scope, input bus, to the mixer input!"); + AudioComponentInstanceDispose(_audioUnit); + _audioUnit = NULL; + return NO; + } + // Setup the rendering callbacks for the mixer. UInt32 elementCount = 1; status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_ElementCount, From a265fabeaa8d08b0ba4a8e89166fdace82403e90 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 2 Nov 2018 10:39:31 -0700 Subject: [PATCH 32/94] Render into our buffer list. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 4bb88aab..472b2614 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -549,12 +549,22 @@ static OSStatus ExampleAVPlayerAudioDeviceRecordingInputCallback(void *refCon, return noErr; } + // Render into our recording buffer. + + AudioBufferList renderingBufferList; + renderingBufferList.mNumberBuffers = 1; + + AudioBuffer *audioBuffer = &renderingBufferList.mBuffers[0]; + audioBuffer->mNumberChannels = kPreferredNumberOfChannels; + audioBuffer->mDataByteSize = (UInt32)context->maxFramesPerBuffer * kPreferredNumberOfChannels * 2; + audioBuffer->mData = context->audioBuffer; + OSStatus status = AudioUnitRender(context->audioUnit, actionFlags, timestamp, busNumber, numFrames, - bufferList); + &renderingBufferList); return status; } From 4552a1c2f87cc1b9c21ca638154dbf94dc82c096 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 2 Nov 2018 10:56:26 -0700 Subject: [PATCH 33/94] Internal refactor to prepare for generic output. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 337 +++++++++--------- 1 file changed, 177 insertions(+), 160 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 472b2614..54e626bb 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -58,9 +58,10 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; -@property (nonatomic, assign) AudioUnit audioUnit; +@property (nonatomic, assign) AudioUnit voiceProcessingIO; @property (nonatomic, assign) AudioUnit playbackMixer; @property (nonatomic, assign) AudioUnit recordingMixer; +@property (nonatomic, assign) AudioUnit recordingOutput; @property (nonatomic, assign, nullable) ExampleAVPlayerAudioTapContext *audioTapContext; @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapCapturingBuffer; @@ -310,7 +311,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); // Restart the already setup graph. - if (_audioUnit) { + if (_voiceProcessingIO) { [self stopAudioUnit]; [self teardownAudioUnit]; } @@ -334,7 +335,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { self.renderingContext = NULL; return NO; } else if (self.capturingContext) { - self.capturingContext->audioUnit = _audioUnit; + self.capturingContext->audioUnit = _voiceProcessingIO; } } @@ -395,7 +396,7 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { NSAssert(self.capturingContext == NULL, @"We should not have a capturing context when starting."); // Restart the already setup graph. - if (_audioUnit) { + if (_voiceProcessingIO) { [self stopAudioUnit]; [self teardownAudioUnit]; } @@ -421,7 +422,7 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { self.capturingContext = NULL; return NO; } else { - self.capturingContext->audioUnit = _audioUnit; + self.capturingContext->audioUnit = _voiceProcessingIO; } } BOOL success = [self startAudioUnit]; @@ -636,6 +637,16 @@ + (AudioComponentDescription)mixerAudioCompontentDescription { return audioUnitDescription; } ++ (AudioComponentDescription)genericOutputAudioCompontentDescription { + AudioComponentDescription audioUnitDescription; + audioUnitDescription.componentType = kAudioUnitType_Output; + audioUnitDescription.componentSubType = kAudioUnitSubType_GenericOutput; + audioUnitDescription.componentManufacturer = kAudioUnitManufacturer_Apple; + audioUnitDescription.componentFlags = 0; + audioUnitDescription.componentFlagsMask = 0; + return audioUnitDescription; +} + - (void)setupAVAudioSession { AVAudioSession *session = [AVAudioSession sharedInstance]; NSError *error = nil; @@ -672,12 +683,140 @@ - (void)setupAVAudioSession { } } +- (OSStatus)setupCapturer:(ExampleAVPlayerCapturerContext *)capturerContext { + UInt32 enableInput = capturerContext ? 1 : 0; + OSStatus status = AudioUnitSetProperty(_voiceProcessingIO, kAudioOutputUnitProperty_EnableIO, + kAudioUnitScope_Input, kInputBus, &enableInput, + sizeof(enableInput)); + + if (status != noErr) { + NSLog(@"Could not enable/disable input bus!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + + if (enableInput) { + AudioStreamBasicDescription capturingFormatDescription = self.capturingFormat.streamDescription; + Float64 sampleRate = capturingFormatDescription.mSampleRate; + + // Setup recording mixer. + AudioComponentDescription mixerComponentDescription = [[self class] mixerAudioCompontentDescription]; + AudioComponent mixerComponent = AudioComponentFindNext(NULL, &mixerComponentDescription); + + OSStatus status = AudioComponentInstanceNew(mixerComponent, &_recordingMixer); + if (status != noErr) { + NSLog(@"Could not find the mixer AudioComponent instance!"); + return status; + } + + // Configure I/O format. + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SampleRate, + kAudioUnitScope_Output, kOutputBus, + &sampleRate, sizeof(sampleRate)); + if (status != noErr) { + NSLog(@"Could not set sample rate on the mixer output bus!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + + // Set the sample rate for the inputs. We are not supposed to set a format? + // status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SampleRate, + // kAudioUnitScope_Input, 0, + // &sampleRate, sizeof(sampleRate)); + // if (status != noErr) { + // NSLog(@"Could not set sample rate on the mixer input bus 0!"); + // AudioComponentInstanceDispose(_audioUnit); + // _audioUnit = NULL; + // return NO; + // } + + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, 0, + &capturingFormatDescription, sizeof(capturingFormatDescription)); + if (status != noErr) { + NSLog(@"Could not set stream format on the mixer input bus 1!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + + status = AudioUnitSetProperty(_voiceProcessingIO, kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, kInputBus, + &capturingFormatDescription, sizeof(capturingFormatDescription)); + if (status != noErr) { + NSLog(@"Could not set stream format on the input bus!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + + // Connection: VoiceProcessingIO Output Scope, Input Bus -> Mixer Input Scope, Bus 0 + AudioUnitConnection mixerInputConnection; + mixerInputConnection.sourceAudioUnit = _voiceProcessingIO; + mixerInputConnection.sourceOutputNumber = kInputBus; + mixerInputConnection.destInputNumber = 0; + + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_MakeConnection, + kAudioUnitScope_Input, 0, + &mixerInputConnection, sizeof(mixerInputConnection)); + if (status != noErr) { + NSLog(@"Could not connect voice processing output scope, input bus, to the mixer input!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + + // Setup the rendering callbacks for the mixer. + UInt32 elementCount = 1; + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_ElementCount, + kAudioUnitScope_Input, 0, &elementCount, + sizeof(elementCount)); + if (status != 0) { + NSLog(@"Could not set input element count!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + + AURenderCallbackStruct mixerOutputRenderCallback; + mixerOutputRenderCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingOutputCallback; + mixerOutputRenderCallback.inputProcRefCon = (void *)(capturerContext); + status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Output, 0, &mixerOutputRenderCallback, + sizeof(mixerOutputRenderCallback)); + if (status != 0) { + NSLog(@"Could not set mixer output rendering callback!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + + // Setup the I/O input callback. + AURenderCallbackStruct capturerCallback; + capturerCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingInputCallback; + capturerCallback.inputProcRefCon = (void *)(capturerContext); + status = AudioUnitSetProperty(_voiceProcessingIO, kAudioOutputUnitProperty_SetInputCallback, + kAudioUnitScope_Input, kInputBus, &capturerCallback, + sizeof(capturerCallback)); + if (status != noErr) { + NSLog(@"Could not set capturing callback!"); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; + return status; + } + } + + return status; +} + - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)rendererContext capturerContext:(ExampleAVPlayerCapturerContext *)capturerContext { AudioComponentDescription audioUnitDescription = [[self class] audioUnitDescription]; AudioComponent audioComponent = AudioComponentFindNext(NULL, &audioUnitDescription); - OSStatus status = AudioComponentInstanceNew(audioComponent, &_audioUnit); + OSStatus status = AudioComponentInstanceNew(audioComponent, &_voiceProcessingIO); if (status != noErr) { NSLog(@"Could not find the AudioComponent instance!"); return NO; @@ -688,13 +827,13 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer * prevent any additional format conversions after the media engine has mixed our playout audio. */ UInt32 enableOutput = rendererContext ? 1 : 0; - status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_EnableIO, + status = AudioUnitSetProperty(_voiceProcessingIO, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, kOutputBus, &enableOutput, sizeof(enableOutput)); if (status != noErr) { NSLog(@"Could not enable/disable output bus!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -717,8 +856,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer &renderingFormatDescription, sizeof(renderingFormatDescription)); if (status != noErr) { NSLog(@"Could not set stream format on the mixer output bus!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -727,8 +866,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer &renderingFormatDescription, sizeof(renderingFormatDescription)); if (status != noErr) { NSLog(@"Could not set stream format on the mixer input bus 0!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -737,8 +876,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer &renderingFormatDescription, sizeof(renderingFormatDescription)); if (status != noErr) { NSLog(@"Could not set stream format on the mixer input bus 1!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -748,23 +887,23 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer mixerOutputConnection.sourceOutputNumber = kOutputBus; mixerOutputConnection.destInputNumber = kOutputBus; - status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_MakeConnection, + status = AudioUnitSetProperty(_voiceProcessingIO, kAudioUnitProperty_MakeConnection, kAudioUnitScope_Input, kOutputBus, &mixerOutputConnection, sizeof(mixerOutputConnection)); if (status != noErr) { NSLog(@"Could not connect the mixer output to voice processing input!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } - status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, + status = AudioUnitSetProperty(_voiceProcessingIO, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, kOutputBus, &renderingFormatDescription, sizeof(renderingFormatDescription)); if (status != noErr) { NSLog(@"Could not set stream format on the output bus!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -775,8 +914,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer sizeof(elementCount)); if (status != 0) { NSLog(@"Could not set input element count!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -788,8 +927,8 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer sizeof(audioTapRenderCallback)); if (status != 0) { NSLog(@"Could not set audio tap rendering callback!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -801,142 +940,20 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer sizeof(audioRendererRenderCallback)); if (status != 0) { NSLog(@"Could not set audio renderer rendering callback!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } } - UInt32 enableInput = capturerContext ? 1 : 0; - status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_EnableIO, - kAudioUnitScope_Input, kInputBus, &enableInput, - sizeof(enableInput)); - - if (status != noErr) { - NSLog(@"Could not enable/disable input bus!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - - if (enableInput) { - AudioStreamBasicDescription capturingFormatDescription = self.capturingFormat.streamDescription; - Float64 sampleRate = capturingFormatDescription.mSampleRate; - - // Setup recording mixer. - AudioComponentDescription mixerComponentDescription = [[self class] mixerAudioCompontentDescription]; - AudioComponent mixerComponent = AudioComponentFindNext(NULL, &mixerComponentDescription); - - OSStatus status = AudioComponentInstanceNew(mixerComponent, &_recordingMixer); - if (status != noErr) { - NSLog(@"Could not find the mixer AudioComponent instance!"); - return NO; - } - - // Configure I/O format. - status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SampleRate, - kAudioUnitScope_Output, kOutputBus, - &sampleRate, sizeof(sampleRate)); - if (status != noErr) { - NSLog(@"Could not set sample rate on the mixer output bus!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - - // Set the sample rate for the inputs. We are not supposed to set a format? -// status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SampleRate, -// kAudioUnitScope_Input, 0, -// &sampleRate, sizeof(sampleRate)); -// if (status != noErr) { -// NSLog(@"Could not set sample rate on the mixer input bus 0!"); -// AudioComponentInstanceDispose(_audioUnit); -// _audioUnit = NULL; -// return NO; -// } - - status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_StreamFormat, - kAudioUnitScope_Input, 0, - &capturingFormatDescription, sizeof(capturingFormatDescription)); - if (status != noErr) { - NSLog(@"Could not set stream format on the mixer input bus 1!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - - status = AudioUnitSetProperty(_audioUnit, kAudioUnitProperty_StreamFormat, - kAudioUnitScope_Output, kInputBus, - &capturingFormatDescription, sizeof(capturingFormatDescription)); - if (status != noErr) { - NSLog(@"Could not set stream format on the input bus!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - - // Connection: VoiceProcessingIO Output Scope, Input Bus -> Mixer Input Scope, Bus 0 - AudioUnitConnection mixerInputConnection; - mixerInputConnection.sourceAudioUnit = _audioUnit; - mixerInputConnection.sourceOutputNumber = kInputBus; - mixerInputConnection.destInputNumber = 0; - - status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_MakeConnection, - kAudioUnitScope_Input, 0, - &mixerInputConnection, sizeof(mixerInputConnection)); - if (status != noErr) { - NSLog(@"Could not connect voice processing output scope, input bus, to the mixer input!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - - // Setup the rendering callbacks for the mixer. - UInt32 elementCount = 1; - status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_ElementCount, - kAudioUnitScope_Input, 0, &elementCount, - sizeof(elementCount)); - if (status != 0) { - NSLog(@"Could not set input element count!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - - AURenderCallbackStruct mixerOutputRenderCallback; - mixerOutputRenderCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingOutputCallback; - mixerOutputRenderCallback.inputProcRefCon = (void *)(capturerContext); - status = AudioUnitSetProperty(_recordingMixer, kAudioUnitProperty_SetRenderCallback, - kAudioUnitScope_Output, 0, &mixerOutputRenderCallback, - sizeof(mixerOutputRenderCallback)); - if (status != 0) { - NSLog(@"Could not set mixer output rendering callback!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - - // Setup the I/O input callback. - AURenderCallbackStruct capturerCallback; - capturerCallback.inputProc = ExampleAVPlayerAudioDeviceRecordingInputCallback; - capturerCallback.inputProcRefCon = (void *)(capturerContext); - status = AudioUnitSetProperty(_audioUnit, kAudioOutputUnitProperty_SetInputCallback, - kAudioUnitScope_Input, kInputBus, &capturerCallback, - sizeof(capturerCallback)); - if (status != noErr) { - NSLog(@"Could not set capturing callback!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; - return NO; - } - } + [self setupCapturer:self.capturingContext]; // Finally, initialize the IO audio unit and mixer (if present). - status = AudioUnitInitialize(_audioUnit); + status = AudioUnitInitialize(_voiceProcessingIO); if (status != noErr) { NSLog(@"Could not initialize the audio unit!"); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; return NO; } @@ -964,7 +981,7 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer } - (BOOL)startAudioUnit { - OSStatus status = AudioOutputUnitStart(_audioUnit); + OSStatus status = AudioOutputUnitStart(_voiceProcessingIO); if (status != noErr) { NSLog(@"Could not start the audio unit. code: %d", status); return NO; @@ -973,7 +990,7 @@ - (BOOL)startAudioUnit { } - (BOOL)stopAudioUnit { - OSStatus status = AudioOutputUnitStop(_audioUnit); + OSStatus status = AudioOutputUnitStop(_voiceProcessingIO); if (status != noErr) { NSLog(@"Could not stop the audio unit. code: %d", status); return NO; @@ -982,10 +999,10 @@ - (BOOL)stopAudioUnit { } - (void)teardownAudioUnit { - if (_audioUnit) { - AudioUnitUninitialize(_audioUnit); - AudioComponentInstanceDispose(_audioUnit); - _audioUnit = NULL; + if (_voiceProcessingIO) { + AudioUnitUninitialize(_voiceProcessingIO); + AudioComponentInstanceDispose(_voiceProcessingIO); + _voiceProcessingIO = NULL; } if (_playbackMixer) { @@ -1091,7 +1108,7 @@ - (void)handleValidRouteChange { // Nothing to process while we are interrupted. We will interrogate the AVAudioSession once the interruption ends. if (self.isInterrupted) { return; - } else if (_audioUnit == NULL) { + } else if (_voiceProcessingIO == NULL) { return; } From 325a448e054430adcf2f0708e71e6ac12f512bf7 Mon Sep 17 00:00:00 2001 From: Piyush Tank Date: Fri, 2 Nov 2018 11:48:50 -0700 Subject: [PATCH 34/94] Player track rendering at remote side (#3) --- CoViewingExample/Base.lproj/Main.storyboard | 11 +++++++++++ CoViewingExample/ViewController.swift | 20 +++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/CoViewingExample/Base.lproj/Main.storyboard b/CoViewingExample/Base.lproj/Main.storyboard index bea94353..8cb7cf0e 100644 --- a/CoViewingExample/Base.lproj/Main.storyboard +++ b/CoViewingExample/Base.lproj/Main.storyboard @@ -17,6 +17,10 @@ + + + + + - - @@ -84,22 +88,21 @@ + + + - - - - + + + - + - + - - - @@ -109,10 +112,10 @@ - + - + diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index c711424f..b4775acf 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -46,21 +46,17 @@ class ViewController: UIViewController { static var useAudioDevice = true static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! +// static let kRemoteContentURL = URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8")! override func viewDidLoad() { super.viewDidLoad() - // We use the front facing camera for Co-Viewing. let red = UIColor(red: 226.0/255.0, green: 29.0/255.0, blue: 37.0/255.0, alpha: 1.0) - localView.shouldMirror = true presenterButton.backgroundColor = red - presenterButton.titleLabel?.textColor = UIColor.white - viewerButton.backgroundColor = red - viewerButton.titleLabel?.textColor = UIColor.white self.remotePlayerView.contentMode = UIView.ContentMode.scaleAspectFit self.remotePlayerView.isHidden = true self.hangupButton.backgroundColor = red @@ -162,6 +158,8 @@ class ViewController: UIViewController { camera = TVICameraCapturer(source: .frontCamera, delegate: nil) localVideoTrack = TVILocalVideoTrack.init(capturer: camera!) localVideoTrack.addRenderer(self.localView) + // We use the front facing camera only. Set mirroring each time since the renderer might be reused. + localView.shouldMirror = true } #endif } @@ -349,6 +347,7 @@ extension ViewController : TVIRoomDelegate { self.localVideoTrack = nil; self.localAudioTrack = nil; self.playerVideoTrack = nil; + self.accessToken = "TWILIO_ACCESS_TOKEN" } func room(_ room: TVIRoom, didFailToConnectWithError error: Error) { From 0fe18fc89678cfc73462773eea16046b7c806eec Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sat, 3 Nov 2018 22:26:52 -0700 Subject: [PATCH 53/94] Cleanup usage of self.audioDevice. --- CoViewingExample/ViewController.swift | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index b4775acf..a3dd24be 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -28,7 +28,7 @@ class ViewController: UIViewController { let kPlayerTrackName = "player-track" - var audioDevice: ExampleAVPlayerAudioDevice = ExampleAVPlayerAudioDevice() + var audioDevice: ExampleAVPlayerAudioDevice? var videoPlayer: AVPlayer? = nil var videoPlayerAudioTap: ExampleAVPlayerAudioTap? = nil var videoPlayerSource: ExampleAVPlayerSource? = nil @@ -44,7 +44,6 @@ class ViewController: UIViewController { @IBOutlet weak var remotePlayerView: TVIVideoView! @IBOutlet weak var hangupButton: UIButton! - static var useAudioDevice = true static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! // static let kRemoteContentURL = URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8")! @@ -85,13 +84,17 @@ class ViewController: UIViewController { } @IBAction func startPresenter(_ sender: Any) { - self.audioDevice = ExampleAVPlayerAudioDevice() - TwilioVideo.audioDevice = self.audioDevice + if self.audioDevice == nil { + let device = ExampleAVPlayerAudioDevice() + TwilioVideo.audioDevice = device + self.audioDevice = device + } isPresenter = true connect(name: "presenter") } @IBAction func startViewer(_ sender: Any) { + self.audioDevice = nil TwilioVideo.audioDevice = TVIDefaultAudioDevice() isPresenter = false connect(name: "viewer") @@ -245,7 +248,7 @@ class ViewController: UIViewController { let inputParameters = AVMutableAudioMixInputParameters(track: assetAudioTrack) // TODO: Memory management of the MTAudioProcessingTap. - inputParameters.audioTapProcessor = audioDevice.createProcessingTap()?.takeUnretainedValue() + inputParameters.audioTapProcessor = audioDevice!.createProcessingTap()?.takeUnretainedValue() audioMix.inputParameters = [inputParameters] playerItem.audioMix = audioMix } else { From d3787fb7206c3df112ff25fd3894f381883f0892 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sat, 3 Nov 2018 22:28:05 -0700 Subject: [PATCH 54/94] Cleanup more state on disconnect/failure. --- CoViewingExample/ViewController.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index a3dd24be..a41bfa4e 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -350,6 +350,7 @@ extension ViewController : TVIRoomDelegate { self.localVideoTrack = nil; self.localAudioTrack = nil; self.playerVideoTrack = nil; + self.room = nil self.accessToken = "TWILIO_ACCESS_TOKEN" } @@ -357,8 +358,10 @@ extension ViewController : TVIRoomDelegate { logMessage(messageText: "Failed to connect to Room:\n\(error.localizedDescription)") self.room = nil - self.showRoomUI(inRoom: false) + self.localVideoTrack = nil; + self.localAudioTrack = nil; + self.accessToken = "TWILIO_ACCESS_TOKEN" } func room(_ room: TVIRoom, participantDidConnect participant: TVIRemoteParticipant) { From cab5eed0d9066f8c987d462130fe398e5bd2bbbb Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 13:32:01 -0800 Subject: [PATCH 55/94] Several improvements. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Size local and remote views appropriately in landscape. * Use the presentation timestamp when delivering frames. * Add more test content. For now this is just in code. * Allow arbitrary loads for now. * Reduce viewer bandwidth usage to 800 kbps. * Reduce camera capture from 640x480 to 480x360. * Name camera track with “camera”. * Auto-hide the home indicator during playback. * Use a black background once the player starts. * Refactor of how audio mix setup works. --- CoViewingExample/Base.lproj/Main.storyboard | 14 +- CoViewingExample/ExampleAVPlayerSource.swift | 6 +- CoViewingExample/Info.plist | 5 + CoViewingExample/ViewController.swift | 146 +++++++++++++++++-- 4 files changed, 144 insertions(+), 27 deletions(-) diff --git a/CoViewingExample/Base.lproj/Main.storyboard b/CoViewingExample/Base.lproj/Main.storyboard index 7acc7665..3c5e6f10 100644 --- a/CoViewingExample/Base.lproj/Main.storyboard +++ b/CoViewingExample/Base.lproj/Main.storyboard @@ -67,7 +67,7 @@ - + @@ -75,12 +75,10 @@ - + - - @@ -89,23 +87,23 @@ - + - + - + - + diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 6180ba44..64bee395 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -35,8 +35,8 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { let attributes = [ // Note: It appears requesting IOSurface backing causes a crash on iPhone X / iOS 12.0.1. // kCVPixelBufferIOSurfacePropertiesKey as String : [], - kCVPixelBufferWidthKey as String : 640, - kCVPixelBufferHeightKey as String : 360, +// kCVPixelBufferWidthKey as String : 640, +// kCVPixelBufferHeightKey as String : 360, kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ] as [String : Any] @@ -61,7 +61,7 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { if let consumer = self.captureConsumer, let buffer = pixelBuffer { - guard let frame = TVIVideoFrame(timestamp: targetItemTime, + guard let frame = TVIVideoFrame(timestamp: presentationTime, buffer: buffer, orientation: TVIVideoOrientation.up) else { assertionFailure("We couldn't create a TVIVideoFrame with a valid CVPixelBuffer.") diff --git a/CoViewingExample/Info.plist b/CoViewingExample/Info.plist index 8a0cc2cb..e3a40ae1 100644 --- a/CoViewingExample/Info.plist +++ b/CoViewingExample/Info.plist @@ -10,6 +10,11 @@ $(DEVELOPMENT_LANGUAGE) CFBundleExecutable $(EXECUTABLE_NAME) + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + CFBundleIdentifier $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index a41bfa4e..14dfe585 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -36,6 +36,11 @@ class ViewController: UIViewController { var isPresenter:Bool? + @IBOutlet weak var localHeightConstraint: NSLayoutConstraint? + @IBOutlet weak var localWidthConstraint: NSLayoutConstraint? + @IBOutlet weak var remoteHeightConstraint: NSLayoutConstraint? + @IBOutlet weak var remoteWidthConstraint: NSLayoutConstraint? + @IBOutlet weak var presenterButton: UIButton! @IBOutlet weak var viewerButton: UIButton! @@ -44,8 +49,30 @@ class ViewController: UIViewController { @IBOutlet weak var remotePlayerView: TVIVideoView! @IBOutlet weak var hangupButton: UIButton! - static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! -// static let kRemoteContentURL = URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8")! + // Avengers: Infinity War Trailer (720p24 (1280x544) / Stereo 44.1 kHz) +// static let kRemoteContentURL = URL(string: "https://trailers.apple.com/movies/marvel/avengers-infinity-war/avengers-infinity-war-trailer-2_h720p.mov")! +// static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! + // BitDash / BitMovin demo content. The first stream is HLS and runs into the AVPlayer / AVAudioMix issue. + // Straight mp4 stream without AVAudioMix issue. Audio sync is a bit off in the source, but more so in our player. +// static let kRemoteContentURL = URL(string: "https://bitmovin-a.akamaihd.net/content/MI201109210084_1/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4")! +// static let kRemoteContentURL = URL(string: "https://b028.wpc.azureedge.net/80B028/Samples/a38e6323-95e9-4f1f-9b38-75eba91704e4/5f2ce531-d508-49fb-8152-647eba422aec.ism/Manifest(format=m3u8-aapl-v3)")! +// static let kRemoteContentURL = URL(string: "https://mnmedias.api.telequebec.tv/m3u8/29880.m3u8")! + // Video only source, but at 720p30 which is the max frame rate that we can capture. +// static let kRemoteContentURL = URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_720p30_2M.mp4")! + + // http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h720p.mov + static let kRemoteContentUrls = [ + "Avengers: Infinity War Trailer 3 (720p24, 44.1 kHz)" : URL(string: "https://trailers.apple.com/movies/marvel/avengers-infinity-war/avengers-infinity-war-trailer-2_h720p.mov")!, + "BitDash - Parkour (HLS)" : URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8")!, + "BitDash - Parkour (1080p25, 48 kHz)" : URL(string: "https://bitmovin-a.akamaihd.net/content/MI201109210084_1/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4")!, + "Interstellar Trailer 3 (720p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h720p.mov")!, + "Interstellar Trailer 3 (1080p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h1080p.mov")!, + "Tele Quebec (HLS)" : URL(string: "https://mnmedias.api.telequebec.tv/m3u8/29880.m3u8")!, + "Telecom ParisTech, GPAC (720p30)" : URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_720p30_2M.mp4")!, + "Telecom ParisTech, GPAC (1080p30)" : URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_1080p30_6M.mp4")!, + "Twilio: What is Cloud Communications? (1080p24, 44.1 kHz)" : URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! + ] + static let kRemoteContentURL = kRemoteContentUrls["Twilio: What is Cloud Communications? (1080p24, 44.1 kHz)"]! override func viewDidLoad() { super.viewDidLoad() @@ -67,10 +94,15 @@ class ViewController: UIViewController { hangupButton.layer.cornerRadius = 2; self.localView.contentMode = UIView.ContentMode.scaleAspectFit + self.localView.delegate = self + self.localWidthConstraint = self.localView.constraints.first + self.localHeightConstraint = self.localView.constraints.last self.remoteView.contentMode = UIView.ContentMode.scaleAspectFit + self.remoteView.delegate = self + self.remoteHeightConstraint = self.remoteView.constraints.first + self.remoteWidthConstraint = self.remoteView.constraints.last } - override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) } @@ -83,6 +115,38 @@ class ViewController: UIViewController { } } + override func updateViewConstraints() { + super.updateViewConstraints() + + if self.localView.hasVideoData { + let localDimensions = self.localView.videoDimensions + if localDimensions.width > localDimensions.height { + self.localWidthConstraint?.constant = 128 + self.localHeightConstraint?.constant = 96 + } else { + self.localWidthConstraint?.constant = 96 + self.localHeightConstraint?.constant = 128 + } + } + + if self.remoteView.hasVideoData { + let remoteDimensions = self.remoteView.videoDimensions + if remoteDimensions.width > remoteDimensions.height { + self.remoteWidthConstraint?.constant = 128 + self.remoteHeightConstraint?.constant = 96 + } else { + self.remoteWidthConstraint?.constant = 96 + self.remoteHeightConstraint?.constant = 128 + } + } + } + + override var prefersHomeIndicatorAutoHidden: Bool { + get { + return self.room != nil + } + } + @IBAction func startPresenter(_ sender: Any) { if self.audioDevice == nil { let device = ExampleAVPlayerAudioDevice() @@ -134,6 +198,11 @@ class ViewController: UIViewController { // The name of the Room where the Client will attempt to connect to. Please note that if you pass an empty // Room `name`, the Client will create one for you. You can get the name or sid from any connected Room. builder.roomName = "twilio" + + // Restrict video bandwidth used by viewers to improve presenter video. + if name == "viewer" { + builder.encodingParameters = TVIEncodingParameters(audioBitrate: 0, videoBitrate: 1024 * 900) + } } // Connect to the Room using the options we provided. @@ -158,8 +227,17 @@ class ViewController: UIViewController { #if !targetEnvironment(simulator) if (localVideoTrack == nil) { // Preview our local camera track in the local video preview view. + camera = TVICameraCapturer(source: .frontCamera, delegate: nil) - localVideoTrack = TVILocalVideoTrack.init(capturer: camera!) + let constraints = TVIVideoConstraints.init { (builder) in + builder.maxSize = TVIVideoConstraintsSize480x360 + builder.maxFrameRate = TVIVideoConstraintsFrameRate24 + } + + localVideoTrack = TVILocalVideoTrack(capturer: camera!, + enabled: true, + constraints: constraints, + name: "camera") localVideoTrack.addRenderer(self.localView) // We use the front facing camera only. Set mirroring each time since the renderer might be reused. localView.shouldMirror = true @@ -176,6 +254,7 @@ class ViewController: UIViewController { self.remoteView.isHidden = !inRoom self.presenterButton.isHidden = inRoom self.viewerButton.isHidden = inRoom + self.setNeedsUpdateOfHomeIndicatorAutoHidden() } func startVideoPlayer() { @@ -218,6 +297,7 @@ class ViewController: UIViewController { // We will rely on frame based layout to size and position `self.videoPlayerView`. self.view.insertSubview(playerView, at: 0) self.view.setNeedsLayout() + self.view.backgroundColor = UIColor.black } func setupVideoSource(item: AVPlayerItem) { @@ -234,8 +314,19 @@ class ViewController: UIViewController { } func setupAudioMix(player: AVPlayer, playerItem: AVPlayerItem) { + let audioAssetTrack = firstAudioAssetTrack(playerItem: playerItem) + print("Setup audio mix with AssetTrack:", audioAssetTrack != nil ? audioAssetTrack as Any : " none") + let audioMix = AVMutableAudioMix() + let inputParameters = AVMutableAudioMixInputParameters(track: audioAssetTrack) + // TODO: Memory management of the MTAudioProcessingTap. + inputParameters.audioTapProcessor = audioDevice!.createProcessingTap()?.takeUnretainedValue() + audioMix.inputParameters = [inputParameters] + playerItem.audioMix = audioMix + } + + func firstAudioAssetTrack(playerItem: AVPlayerItem) -> AVAssetTrack? { var audioAssetTracks: [AVAssetTrack] = [] for playerItemTrack in playerItem.tracks { if let assetTrack = playerItemTrack.assetTrack, @@ -243,16 +334,21 @@ class ViewController: UIViewController { audioAssetTracks.append(assetTrack) } } - - if let assetAudioTrack = audioAssetTracks.first { - let inputParameters = AVMutableAudioMixInputParameters(track: assetAudioTrack) - - // TODO: Memory management of the MTAudioProcessingTap. - inputParameters.audioTapProcessor = audioDevice!.createProcessingTap()?.takeUnretainedValue() - audioMix.inputParameters = [inputParameters] - playerItem.audioMix = audioMix + return audioAssetTracks.first + } + + func updateAudioMixParameters(playerItem: AVPlayerItem) { + // Update the audio mix to point to the first AVAssetTrack that we find. + if let audioAssetTrack = firstAudioAssetTrack(playerItem: playerItem), + let inputParameters = playerItem.audioMix?.inputParameters.first { + let mutableInputParameters = inputParameters as! AVMutableAudioMixInputParameters + mutableInputParameters.trackID = audioAssetTrack.trackID + print("Update the mix input parameters to use Track Id:", audioAssetTrack.trackID as Any, "\n", + "Asset:", audioAssetTrack.asset as Any, "\n", + "Audio Fallbacks:", audioAssetTrack.associatedTracks(ofType: AVAssetTrack.AssociationType.audioFallback), "\n", + "isPlayable:", audioAssetTrack.isPlayable) } else { - // Abort, retry, fail? + // TODO } } @@ -310,8 +406,10 @@ class ViewController: UIViewController { // Configure our audio capturer to receive audio samples from the AVPlayerItem. if playerItem.audioMix == nil, - playerItem.tracks.count > 0 { + firstAudioAssetTrack(playerItem: playerItem) != nil { setupAudioMix(player: videoPlayer!, playerItem: playerItem) + } else { + // Possibly update the existing mix? } } } @@ -346,11 +444,11 @@ extension ViewController : TVIRoomDelegate { stopVideoPlayer() - self.showRoomUI(inRoom: false) self.localVideoTrack = nil; self.localAudioTrack = nil; self.playerVideoTrack = nil; self.room = nil + self.showRoomUI(inRoom: false) self.accessToken = "TWILIO_ACCESS_TOKEN" } @@ -358,9 +456,9 @@ extension ViewController : TVIRoomDelegate { logMessage(messageText: "Failed to connect to Room:\n\(error.localizedDescription)") self.room = nil - self.showRoomUI(inRoom: false) self.localVideoTrack = nil; self.localAudioTrack = nil; + self.showRoomUI(inRoom: false) self.accessToken = "TWILIO_ACCESS_TOKEN" } @@ -428,6 +526,9 @@ extension ViewController : TVIRemoteParticipantDelegate { if (videoTrack.name == self.kPlayerTrackName) { self.remotePlayerView.isHidden = false videoTrack.addRenderer(self.remotePlayerView) + UIView.animate(withDuration: 0.2) { + self.view.backgroundColor = UIColor.black + } } else { videoTrack.addRenderer(self.remoteView) } @@ -511,3 +612,16 @@ extension ViewController : TVICameraCapturerDelegate { capturer.previewView.removeFromSuperview() } } + +extension ViewController : TVIVideoViewDelegate { + func videoViewDidReceiveData(_ view: TVIVideoView) { + if view == self.localView || view == self.remoteView { + self.view.setNeedsUpdateConstraints() + } + } + func videoView(_ view: TVIVideoView, videoDimensionsDidChange dimensions: CMVideoDimensions) { + if view == self.localView || view == self.remoteView { + self.view.setNeedsUpdateConstraints() + } + } +} From 7ada723a343ff88d15e3480217912b12d14b8147 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 18:14:48 -0800 Subject: [PATCH 56/94] Cleanup content, initial animation, and tap to fit/fill AVP video. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 8 ++--- CoViewingExample/ExampleAVPlayerView.swift | 21 +++++++++++- CoViewingExample/ViewController.swift | 32 +++++++++++-------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index b94ff80a..a7700162 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -383,8 +383,8 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { // Ensure that we wait for the audio tap buffer to become ready. if (_audioTapCapturingBuffer) { self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; - dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 100 * 1000 * 1000); - dispatch_semaphore_wait(_audioTapRenderingSemaphore, timeout); +// dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 100 * 1000 * 1000); +// dispatch_semaphore_wait(_audioTapRenderingSemaphore, timeout); } else { self.renderingContext->playoutBuffer = NULL; } @@ -477,8 +477,8 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { // Ensure that we wait for the audio tap buffer to become ready. if (_audioTapCapturingBuffer) { self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; - dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 100 * 1000 * 1000); - dispatch_semaphore_wait(_audioTapCapturingSemaphore, timeout); +// dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 100 * 1000 * 1000); +// dispatch_semaphore_wait(_audioTapCapturingSemaphore, timeout); } else { self.capturingContext->recordingBuffer = NULL; } diff --git a/CoViewingExample/ExampleAVPlayerView.swift b/CoViewingExample/ExampleAVPlayerView.swift index 6e7b95fc..202e51e2 100644 --- a/CoViewingExample/ExampleAVPlayerView.swift +++ b/CoViewingExample/ExampleAVPlayerView.swift @@ -28,8 +28,27 @@ class ExampleAVPlayerView: UIView { } } + override var contentMode: UIView.ContentMode { + set { + switch newValue { + case .scaleAspectFill: + playerLayer.videoGravity = .resizeAspectFill + case .scaleAspectFit: + playerLayer.videoGravity = .resizeAspect + case .scaleToFill: + playerLayer.videoGravity = .resize + default: + playerLayer.videoGravity = .resizeAspect + } + super.contentMode = newValue + } + + get { + return super.contentMode + } + } + override class var layerClass : AnyClass { return AVPlayerLayer.self } - } diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 14dfe585..7a789093 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -49,30 +49,24 @@ class ViewController: UIViewController { @IBOutlet weak var remotePlayerView: TVIVideoView! @IBOutlet weak var hangupButton: UIButton! - // Avengers: Infinity War Trailer (720p24 (1280x544) / Stereo 44.1 kHz) -// static let kRemoteContentURL = URL(string: "https://trailers.apple.com/movies/marvel/avengers-infinity-war/avengers-infinity-war-trailer-2_h720p.mov")! -// static let kRemoteContentURL = URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! - // BitDash / BitMovin demo content. The first stream is HLS and runs into the AVPlayer / AVAudioMix issue. - // Straight mp4 stream without AVAudioMix issue. Audio sync is a bit off in the source, but more so in our player. -// static let kRemoteContentURL = URL(string: "https://bitmovin-a.akamaihd.net/content/MI201109210084_1/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4")! -// static let kRemoteContentURL = URL(string: "https://b028.wpc.azureedge.net/80B028/Samples/a38e6323-95e9-4f1f-9b38-75eba91704e4/5f2ce531-d508-49fb-8152-647eba422aec.ism/Manifest(format=m3u8-aapl-v3)")! -// static let kRemoteContentURL = URL(string: "https://mnmedias.api.telequebec.tv/m3u8/29880.m3u8")! - // Video only source, but at 720p30 which is the max frame rate that we can capture. -// static let kRemoteContentURL = URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_720p30_2M.mp4")! - // http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h720p.mov static let kRemoteContentUrls = [ "Avengers: Infinity War Trailer 3 (720p24, 44.1 kHz)" : URL(string: "https://trailers.apple.com/movies/marvel/avengers-infinity-war/avengers-infinity-war-trailer-2_h720p.mov")!, + // HLS stream which runs into the AVPlayer / AVAudioMix issue. "BitDash - Parkour (HLS)" : URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8")!, + // Progressive download mp4 version. Demonstrates that 48 kHz support is incorrect right now. "BitDash - Parkour (1080p25, 48 kHz)" : URL(string: "https://bitmovin-a.akamaihd.net/content/MI201109210084_1/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4")!, + // Encoding in 1080p takes significantly more CPU than 720p "Interstellar Trailer 3 (720p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h720p.mov")!, "Interstellar Trailer 3 (1080p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h1080p.mov")!, + // HLS stream which runs into the AVPlayer / AVAudioMix issue. "Tele Quebec (HLS)" : URL(string: "https://mnmedias.api.telequebec.tv/m3u8/29880.m3u8")!, + // Video only source, but at 30 fps which is the max frame rate that we can capture. "Telecom ParisTech, GPAC (720p30)" : URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_720p30_2M.mp4")!, "Telecom ParisTech, GPAC (1080p30)" : URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_1080p30_6M.mp4")!, "Twilio: What is Cloud Communications? (1080p24, 44.1 kHz)" : URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! ] - static let kRemoteContentURL = kRemoteContentUrls["Twilio: What is Cloud Communications? (1080p24, 44.1 kHz)"]! + static let kRemoteContentURL = kRemoteContentUrls["Interstellar Trailer 3 (720p24, 44.1 kHz)"]! override func viewDidLoad() { super.viewDidLoad() @@ -292,12 +286,24 @@ class ViewController: UIViewController { let playerView = ExampleAVPlayerView(frame: CGRect.zero, player: player) videoPlayerView = playerView + let tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(handlePlayerTap)) + tapRecognizer.numberOfTapsRequired = 2 + videoPlayerView?.addGestureRecognizer(tapRecognizer) + setupVideoSource(item: playerItem) // We will rely on frame based layout to size and position `self.videoPlayerView`. self.view.insertSubview(playerView, at: 0) self.view.setNeedsLayout() - self.view.backgroundColor = UIColor.black + UIView.animate(withDuration: 0.2) { + self.view.backgroundColor = UIColor.black + } + } + + @objc func handlePlayerTap(recognizer: UITapGestureRecognizer) { + if let view = self.videoPlayerView { + view.contentMode = view.contentMode == .scaleAspectFit ? .scaleAspectFill : .scaleAspectFit + } } func setupVideoSource(item: AVPlayerItem) { From 0718c4bd4eeebb20b248b7faa7ae813b667775ed Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 20:30:50 -0800 Subject: [PATCH 57/94] Significant rework of ExampleAVPlayerSource. * Downscale progressive streamed content to be no larger than 960 pixels. * Request HLS content no larger than 960 pixels. * Use either a dispatch source or display link timer. * Fix scheduling of display link for content which is mapped to the display cadence (24, 25 Hz). --- CoViewingExample/ExampleAVPlayerSource.swift | 203 ++++++++++++++----- CoViewingExample/ViewController.swift | 19 +- 2 files changed, 168 insertions(+), 54 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 64bee395..6a3bba0d 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -10,73 +10,164 @@ import TwilioVideo class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { + private var captureConsumer: TVIVideoCaptureConsumer? = nil private let sampleQueue: DispatchQueue - private var outputTimer: CADisplayLink? = nil + private var timerSource: DispatchSourceTimer? = nil private var videoOutput: AVPlayerItemVideoOutput? = nil - private var captureConsumer: TVIVideoCaptureConsumer? = nil - private var frameCounter = UInt32(0) + private var lastPresentationTimestamp: CMTime? + private var outputTimer: CADisplayLink? = nil + + // 60 Hz = 16667, 23.976 Hz = 41708 + static let kFrameOutputInterval = DispatchTimeInterval.microseconds(16667) + static let kFrameOutputLeeway = DispatchTimeInterval.milliseconds(0) + static let kFrameOutputSuspendTimeout = Double(1.0) + static let kFrameOutputMaxDimension = CGFloat(960.0) + static let kFrameOutputMaxRect = CGRect(x: 0, y: 0, width: kFrameOutputMaxDimension, height: kFrameOutputMaxDimension) + static private var useDisplayLinkTimer = true init(item: AVPlayerItem) { - sampleQueue = DispatchQueue(label: "", qos: DispatchQoS.userInteractive, + sampleQueue = DispatchQueue(label: "com.twilio.avplayersource", qos: DispatchQoS.userInteractive, attributes: DispatchQueue.Attributes(rawValue: 0), autoreleaseFrequency: DispatchQueue.AutoreleaseFrequency.workItem, target: nil) - super.init() - let timer = CADisplayLink(target: self, - selector: #selector(ExampleAVPlayerSource.displayLinkDidFire(displayLink:))) - timer.preferredFramesPerSecond = 30 - timer.isPaused = true - timer.add(to: RunLoop.current, forMode: RunLoop.Mode.common) - outputTimer = timer - - // We request NV12 buffers downscaled to 480p for streaming. - let attributes = [ - // Note: It appears requesting IOSurface backing causes a crash on iPhone X / iOS 12.0.1. - // kCVPixelBufferIOSurfacePropertiesKey as String : [], -// kCVPixelBufferWidthKey as String : 640, -// kCVPixelBufferHeightKey as String : 360, - kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange - ] as [String : Any] + let presentationSize = item.presentationSize + let presentationPixels = presentationSize.width * presentationSize.height + print("Prepare for player item with size:", presentationSize, " pixels:", presentationPixels); + + /* + * We might request buffers downscaled for streaming. The output will be NV12, and backed by an IOSurface + * even though we dont explicitly include kCVPixelBufferIOSurfacePropertiesKey. + */ + let attributes: [String : Any] + + if (presentationSize.width > ExampleAVPlayerSource.kFrameOutputMaxDimension || + presentationSize.height > ExampleAVPlayerSource.kFrameOutputMaxDimension) { + let streamingRect = AVMakeRect(aspectRatio: presentationSize, insideRect: ExampleAVPlayerSource.kFrameOutputMaxRect) + print("Requesting downscaling to:", streamingRect.size, "."); + + attributes = [ + kCVPixelBufferWidthKey as String : Int(streamingRect.width), + kCVPixelBufferHeightKey as String : Int(streamingRect.height), + kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + ] as [String : Any] + } else { + attributes = [ + kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + ] as [String : Any] + } videoOutput = AVPlayerItemVideoOutput(pixelBufferAttributes: attributes) videoOutput?.setDelegate(self, queue: sampleQueue) - videoOutput?.requestNotificationOfMediaDataChange(withAdvanceInterval: 0.1) + + if ExampleAVPlayerSource.useDisplayLinkTimer { + addDisplayTimer() + } + videoOutput?.requestNotificationOfMediaDataChange(withAdvanceInterval: 0.02) item.add(videoOutput!) } - @objc func displayLinkDidFire(displayLink: CADisplayLink) { + func outputFrame(itemTimestamp: CMTime) { guard let output = videoOutput else { return } + guard let consumer = captureConsumer else { + return + } + if !output.hasNewPixelBuffer(forItemTime: itemTimestamp) { + // TODO: Consider suspending the timer and requesting a notification when media becomes available. + print("No frame for host timestamp:", CACurrentMediaTime(), "\n", + "Last presentation timestamp was:", lastPresentationTimestamp != nil ? lastPresentationTimestamp! : CMTime.zero) + return + } - let targetHostTime = displayLink.targetTimestamp - let targetItemTime = output.itemTime(forHostTime: targetHostTime) + var presentationTimestamp = CMTime.zero + let pixelBuffer = output.copyPixelBuffer(forItemTime: itemTimestamp, + itemTimeForDisplay: &presentationTimestamp) + if let buffer = pixelBuffer { + if let lastTime = lastPresentationTimestamp { + // TODO: Use this info to target our DispatchSource timestamps? +// let delta = presentationTimestamp - lastTime +// print("Frame delta was:", delta) +// let movieTime = CVBufferGetAttachment(buffer, kCVBufferMovieTimeKey, nil) +// print("Movie time was:", movieTime as Any) + } + lastPresentationTimestamp = presentationTimestamp - if output.hasNewPixelBuffer(forItemTime: targetItemTime) { - var presentationTime = CMTime.zero - let pixelBuffer = output.copyPixelBuffer(forItemTime: targetItemTime, itemTimeForDisplay: &presentationTime) + guard let frame = TVIVideoFrame(timestamp: presentationTimestamp, + buffer: buffer, + orientation: TVIVideoOrientation.up) else { + assertionFailure("We couldn't create a TVIVideoFrame with a valid CVPixelBuffer.") + return + } + consumer.consumeCapturedFrame(frame) + } - if let consumer = self.captureConsumer, - let buffer = pixelBuffer { - guard let frame = TVIVideoFrame(timestamp: presentationTime, - buffer: buffer, - orientation: TVIVideoOrientation.up) else { - assertionFailure("We couldn't create a TVIVideoFrame with a valid CVPixelBuffer.") - return - } + if ExampleAVPlayerSource.useDisplayLinkTimer { + outputTimer?.isPaused = false + } else if timerSource == nil { + startTimerSource(hostTime: CACurrentMediaTime()) + } + } + + func startTimerSource(hostTime: CFTimeInterval) { + print(#function) - consumer.consumeCapturedFrame(frame) + let source = DispatchSource.makeTimerSource(flags: DispatchSource.TimerFlags.strict, + queue: sampleQueue) + timerSource = source + + source.setEventHandler(handler: { + if let output = self.videoOutput { + let currentHostTime = CACurrentMediaTime() + let currentItemTime = output.itemTime(forHostTime: currentHostTime) + self.outputFrame(itemTimestamp: currentItemTime) } - } else { - // TODO: Consider suspending the timer and requesting a notification when media becomes available. + }) + + // Thread safe cleanup of temporary storage, in case of cancellation. + source.setCancelHandler(handler: { + }) + + // Schedule a first time source for the full interval. + let deadline = DispatchTime.now() + ExampleAVPlayerSource.kFrameOutputInterval + source.schedule(deadline: deadline, + repeating: ExampleAVPlayerSource.kFrameOutputInterval, + leeway: ExampleAVPlayerSource.kFrameOutputLeeway) + source.resume() + } + + func addDisplayTimer() { + let timer = CADisplayLink(target: self, + selector: #selector(ExampleAVPlayerSource.displayLinkDidFire(displayLink:))) + // Fire at the native v-sync cadence of our display. This is what AVPlayer is targeting anyways. + timer.preferredFramesPerSecond = 0 + timer.isPaused = true + timer.add(to: RunLoop.current, forMode: RunLoop.Mode.common) + outputTimer = timer + } + + @objc func displayLinkDidFire(displayLink: CADisplayLink) { + if let output = self.videoOutput { + // We want the video content targeted for the next v-sync. + let targetHostTime = displayLink.targetTimestamp + let currentItemTime = output.itemTime(forHostTime: targetHostTime) + self.outputFrame(itemTimestamp: currentItemTime) } } - @objc func stopTimer() { + @objc func stopTimerSource() { + print(#function) + + timerSource?.cancel() + timerSource = nil + } + + func stopDisplayTimer() { outputTimer?.invalidate() + outputTimer = nil } public var isScreencast: Bool { @@ -96,27 +187,47 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { } func startCapture(_ format: TVIVideoFormat, consumer: TVIVideoCaptureConsumer) { - DispatchQueue.main.async { - self.captureConsumer = consumer; - consumer.captureDidStart(true) - } + print(#function) + + self.captureConsumer = consumer; + consumer.captureDidStart(true) } func stopCapture() { - DispatchQueue.main.async { - self.captureConsumer = nil + print(#function) + + if ExampleAVPlayerSource.useDisplayLinkTimer { + stopDisplayTimer() + } else { + stopTimerSource() } + self.captureConsumer = nil } } extension ExampleAVPlayerSource: AVPlayerItemOutputPullDelegate { func outputMediaDataWillChange(_ sender: AVPlayerItemOutput) { print(#function) + // Begin to receive video frames. - outputTimer?.isPaused = false + let videoOutput = sender as! AVPlayerItemVideoOutput + let currentHostTime = CACurrentMediaTime() + let currentItemTime = videoOutput.itemTime(forHostTime: currentHostTime) + + // We might have been called back so late that the output already has a frame ready. + let hasFrame = videoOutput.hasNewPixelBuffer(forItemTime: currentItemTime) + if hasFrame { + outputFrame(itemTimestamp: currentItemTime) + } else if ExampleAVPlayerSource.useDisplayLinkTimer { + outputTimer?.isPaused = false + } else { + startTimerSource(hostTime: currentHostTime); + } } func outputSequenceWasFlushed(_ output: AVPlayerItemOutput) { + print(#function) + // TODO: Flush and output a black frame while we wait. } } diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 7a789093..a6041e07 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -266,8 +266,8 @@ class ViewController: UIViewController { print("Created asset with tracks:", asset.tracks as Any) let playerItem = AVPlayerItem(asset: asset, automaticallyLoadedAssetKeys: assetKeysToPreload) - // Prevent excessive bandwidth usage when the content is HLS. - playerItem.preferredMaximumResolution = CGSize(width: 640, height: 480) + // Prevent excessive resource usage when the content is HLS. We will downscale large progressively streamed content. + playerItem.preferredMaximumResolution = ExampleAVPlayerSource.kFrameOutputMaxRect.size // Register as an observer of the player item's status property playerItem.addObserver(self, forKeyPath: #keyPath(AVPlayerItem.status), @@ -290,8 +290,6 @@ class ViewController: UIViewController { tapRecognizer.numberOfTapsRequired = 2 videoPlayerView?.addGestureRecognizer(tapRecognizer) - setupVideoSource(item: playerItem) - // We will rely on frame based layout to size and position `self.videoPlayerView`. self.view.insertSubview(playerView, at: 0) self.view.setNeedsLayout() @@ -393,12 +391,17 @@ class ViewController: UIViewController { // Switch over the status switch status { case .readyToPlay: - // Player item is ready to play. - print("Playing asset with asset tracks: ", videoPlayer?.currentItem?.asset.tracks as Any) + // Player item is ready to play. + print("Ready. Playing asset with tracks: ", videoPlayer?.currentItem?.asset.tracks as Any) + // Defer video source setup until we've loaded the asset so that we can determine downscaling for progressive streaming content. + if self.videoPlayerSource == nil { + setupVideoSource(item: object as! AVPlayerItem) + } videoPlayer?.play() break case .failed: - // Player item failed. See error. + // Player item failed. See error. + // TODO: Show in the UI. print("Playback failed with error:", videoPlayer?.currentItem?.error as Any) break case .unknown: @@ -415,7 +418,7 @@ class ViewController: UIViewController { firstAudioAssetTrack(playerItem: playerItem) != nil { setupAudioMix(player: videoPlayer!, playerItem: playerItem) } else { - // Possibly update the existing mix? + // TODO: Possibly update the existing mix? } } } From 13fd7aa1df4fe944db52ab27a0c8bb1fe3b6df63 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 20:36:34 -0800 Subject: [PATCH 58/94] Comment out noisy logs. --- CoViewingExample/ExampleAVPlayerSource.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 6a3bba0d..4547d089 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -78,8 +78,8 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { } if !output.hasNewPixelBuffer(forItemTime: itemTimestamp) { // TODO: Consider suspending the timer and requesting a notification when media becomes available. - print("No frame for host timestamp:", CACurrentMediaTime(), "\n", - "Last presentation timestamp was:", lastPresentationTimestamp != nil ? lastPresentationTimestamp! : CMTime.zero) +// print("No frame for host timestamp:", CACurrentMediaTime(), "\n", +// "Last presentation timestamp was:", lastPresentationTimestamp != nil ? lastPresentationTimestamp! : CMTime.zero) return } From 3748e84520a1cfbd505216e031103c9d987c42bb Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 20:45:49 -0800 Subject: [PATCH 59/94] Add another trailer. --- CoViewingExample/ViewController.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index a6041e07..85c436ec 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -59,6 +59,8 @@ class ViewController: UIViewController { // Encoding in 1080p takes significantly more CPU than 720p "Interstellar Trailer 3 (720p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h720p.mov")!, "Interstellar Trailer 3 (1080p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h1080p.mov")!, + // Most trailers have a lot of cuts... this one not as many + "Mississippi Grind (720p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/independent/mississippigrind/mississippigrind-tlr1_h1080p.mov")!, // HLS stream which runs into the AVPlayer / AVAudioMix issue. "Tele Quebec (HLS)" : URL(string: "https://mnmedias.api.telequebec.tv/m3u8/29880.m3u8")!, // Video only source, but at 30 fps which is the max frame rate that we can capture. From 66d3878eeb953e41950f562a3a4215aa15ea6fb2 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 21:24:07 -0800 Subject: [PATCH 60/94] Remove remote player view for presenters. --- CoViewingExample/ViewController.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 85c436ec..535529fd 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -150,6 +150,8 @@ class ViewController: UIViewController { self.audioDevice = device } isPresenter = true + self.remotePlayerView.removeFromSuperview() + self.remotePlayerView = nil connect(name: "presenter") } From 34e2e22f287ea11a99cd76e8ad82e1416cc758d4 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 22:00:23 -0800 Subject: [PATCH 61/94] Fix initial state of ExampleAVPlayerView.contentMode. --- CoViewingExample/ExampleAVPlayerView.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerView.swift b/CoViewingExample/ExampleAVPlayerView.swift index 202e51e2..12a9acdb 100644 --- a/CoViewingExample/ExampleAVPlayerView.swift +++ b/CoViewingExample/ExampleAVPlayerView.swift @@ -13,13 +13,13 @@ class ExampleAVPlayerView: UIView { init(frame: CGRect, player: AVPlayer) { super.init(frame: frame) self.playerLayer.player = player - self.playerLayer.videoGravity = AVLayerVideoGravity.resizeAspect + self.contentMode = .scaleAspectFit } required init?(coder aDecoder: NSCoder) { super.init(coder: aDecoder) // It won't be possible to hookup an AVPlayer yet. - self.playerLayer.videoGravity = AVLayerVideoGravity.resizeAspect + self.contentMode = .scaleAspectFit } var playerLayer : AVPlayerLayer { From b4c855bf50d5790fecfbff0dfccced912bbd3ce5 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 4 Nov 2018 22:57:20 -0800 Subject: [PATCH 62/94] Hide the status bar as well. --- CoViewingExample/ViewController.swift | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 535529fd..f4ca277f 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -51,6 +51,8 @@ class ViewController: UIViewController { // http://movietrailers.apple.com/movies/paramount/interstellar/interstellar-tlr4_h720p.mov static let kRemoteContentUrls = [ + // Nice stereo separation in the trailer music. We only record in mono at the moment. + "American Animals Trailer 2 (720p24, 44.1 kHz)" : URL(string: "http://movietrailers.apple.com/movies/independent/american-animals/american-animals-trailer-2_h720p.mov")!, "Avengers: Infinity War Trailer 3 (720p24, 44.1 kHz)" : URL(string: "https://trailers.apple.com/movies/marvel/avengers-infinity-war/avengers-infinity-war-trailer-2_h720p.mov")!, // HLS stream which runs into the AVPlayer / AVAudioMix issue. "BitDash - Parkour (HLS)" : URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8")!, @@ -143,6 +145,12 @@ class ViewController: UIViewController { } } + override var prefersStatusBarHidden: Bool { + get { + return self.room != nil + } + } + @IBAction func startPresenter(_ sender: Any) { if self.audioDevice == nil { let device = ExampleAVPlayerAudioDevice() @@ -253,6 +261,7 @@ class ViewController: UIViewController { self.presenterButton.isHidden = inRoom self.viewerButton.isHidden = inRoom self.setNeedsUpdateOfHomeIndicatorAutoHidden() + self.setNeedsStatusBarAppearanceUpdate() } func startVideoPlayer() { From 5ea855a866dffa33945f923d02a201244117d832 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Mon, 5 Nov 2018 08:57:47 -0800 Subject: [PATCH 63/94] Add a conditional check. --- CoViewingExample/ViewController.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index f4ca277f..54c38522 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -158,8 +158,10 @@ class ViewController: UIViewController { self.audioDevice = device } isPresenter = true - self.remotePlayerView.removeFromSuperview() - self.remotePlayerView = nil + if self.remotePlayerView != nil { + self.remotePlayerView.removeFromSuperview() + self.remotePlayerView = nil + } connect(name: "presenter") } From a38df601f6380142f0d03013f6d91da7c0ff43f1 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 6 Nov 2018 17:13:16 -0800 Subject: [PATCH 64/94] Add .mp4 and .mov document handling + background modes. --- CoViewingExample/AppDelegate.swift | 16 ++++++++- CoViewingExample/Info.plist | 52 ++++++++++++++++++++++----- CoViewingExample/ViewController.swift | 16 ++++++++- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/CoViewingExample/AppDelegate.swift b/CoViewingExample/AppDelegate.swift index 8e7ad10a..b448d8f3 100644 --- a/CoViewingExample/AppDelegate.swift +++ b/CoViewingExample/AppDelegate.swift @@ -14,7 +14,21 @@ class AppDelegate: UIResponder, UIApplicationDelegate { func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { - // Override point for customization after application launch. + print("didFinishLaunchingWithOptions:", launchOptions as Any) + if let options = launchOptions, + let videoUrl = options[UIApplication.LaunchOptionsKey.url] as? URL { + let rootVC = window?.rootViewController as! ViewController + rootVC.startPresenter(contentUrl: videoUrl) + } + return true + } + + func application(_ app: UIApplication, open url: URL, options: [UIApplication.OpenURLOptionsKey : Any] = [:]) -> Bool { + print("app:openURL:", url, " options:", options as Any) + + let rootVC = window?.rootViewController as! ViewController + rootVC.startPresenter(contentUrl: url) + return true } diff --git a/CoViewingExample/Info.plist b/CoViewingExample/Info.plist index e3a40ae1..77483ff0 100644 --- a/CoViewingExample/Info.plist +++ b/CoViewingExample/Info.plist @@ -2,19 +2,37 @@ - NSCameraUsageDescription - ${PRODUCT_NAME} uses your camera to capture video which is shared with other Room Participants. - NSMicrophoneUsageDescription - ${PRODUCT_NAME} uses your microphone to capture audio which is shared with other Room Participants. CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) + CFBundleDocumentTypes + + + CFBundleTypeIconFiles + + CFBundleTypeName + mpeg4 + LSHandlerRank + Default + LSItemContentTypes + + public.mpeg-4 + + + + CFBundleTypeIconFiles + + CFBundleTypeName + quicktime + LSHandlerRank + Default + LSItemContentTypes + + com.apple.quicktime-movie + + + CFBundleExecutable $(EXECUTABLE_NAME) - NSAppTransportSecurity - - NSAllowsArbitraryLoads - - CFBundleIdentifier $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion @@ -29,6 +47,20 @@ 1 LSRequiresIPhoneOS + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + + NSCameraUsageDescription + ${PRODUCT_NAME} uses your camera to capture video which is shared with other Viewers. + NSMicrophoneUsageDescription + ${PRODUCT_NAME} shares your microphone with other Viewers. Tap to mute your audio at any time. + UIBackgroundModes + + audio + voip + UILaunchStoryboardName LaunchScreen UIMainStoryboardFile @@ -37,6 +69,8 @@ armv7 + UIRequiresPersistentWiFi + UISupportedInterfaceOrientations UIInterfaceOrientationPortrait diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 54c38522..0812ae23 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -33,6 +33,7 @@ class ViewController: UIViewController { var videoPlayerAudioTap: ExampleAVPlayerAudioTap? = nil var videoPlayerSource: ExampleAVPlayerSource? = nil var videoPlayerView: ExampleAVPlayerView? = nil + var videoPlayerUrl: URL? = nil var isPresenter:Bool? @@ -99,6 +100,10 @@ class ViewController: UIViewController { self.remoteView.delegate = self self.remoteHeightConstraint = self.remoteView.constraints.first self.remoteWidthConstraint = self.remoteView.constraints.last + + if let videoUrl = videoPlayerUrl { + startPresenter(contentUrl: videoUrl) + } } override func viewWillAppear(_ animated: Bool) { @@ -152,6 +157,15 @@ class ViewController: UIViewController { } @IBAction func startPresenter(_ sender: Any) { + startPresenter(contentUrl: ViewController.kRemoteContentURL) + } + + public func startPresenter(contentUrl: URL) { + videoPlayerUrl = contentUrl + if self.isViewLoaded == false { + return + } + if self.audioDevice == nil { let device = ExampleAVPlayerAudioDevice() TwilioVideo.audioDevice = device @@ -272,7 +286,7 @@ class ViewController: UIViewController { return } - let asset = AVAsset(url: ViewController.kRemoteContentURL) + let asset = AVAsset(url: videoPlayerUrl!) let assetKeysToPreload = [ "hasProtectedContent", "playable", From 60fbf7048c9e8f9fa09ada362f3c8a63f0113b5c Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 6 Nov 2018 17:58:21 -0800 Subject: [PATCH 65/94] Support stopping/starting of the audio device. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 47 ++++++++++++++----- CoViewingExample/ViewController.swift | 6 +++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index a7700162..47c00d74 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -77,6 +77,9 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, nullable) ExampleAVPlayerCapturerContext *capturingContext; @property (atomic, assign, nullable) ExampleAVPlayerRendererContext *renderingContext; @property (nonatomic, strong, nullable) TVIAudioFormat *renderingFormat; +@property (nonatomic, assign, readonly) BOOL wantsAudio; +@property (nonatomic, assign) BOOL wantsCapturing; +@property (nonatomic, assign) BOOL wantsRendering; @end @@ -258,6 +261,8 @@ - (id)init { _audioTapRenderingBuffer = calloc(1, sizeof(TPCircularBuffer)); _audioTapCapturingSemaphore = dispatch_semaphore_create(0); _audioTapRenderingSemaphore = dispatch_semaphore_create(0); + _wantsCapturing = NO; + _wantsRendering = NO; } return self; } @@ -306,6 +311,10 @@ + (void)initialize { #pragma mark - Public +- (BOOL)wantsAudio { + return _wantsCapturing || _wantsRendering; +} + - (MTAudioProcessingTapRef)createProcessingTap { if (_audioTap) { return _audioTap; @@ -368,23 +377,23 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { NSLog(@"%s %@", __PRETTY_FUNCTION__, self.renderingFormat); @synchronized(self) { - NSAssert(self.renderingContext == NULL, @"Should not have any rendering context."); - // Restart the already setup graph. if (_voiceProcessingIO) { [self stopAudioUnit]; [self teardownAudioUnit]; } - self.renderingContext = malloc(sizeof(ExampleAVPlayerRendererContext)); + self.wantsRendering = YES; + if (!self.renderingContext) { + self.renderingContext = malloc(sizeof(ExampleAVPlayerRendererContext)); + memset(self.renderingContext, 0, sizeof(ExampleAVPlayerRendererContext)); + } self.renderingContext->deviceContext = context; self.renderingContext->maxFramesPerBuffer = _renderingFormat.framesPerBuffer; // Ensure that we wait for the audio tap buffer to become ready. if (_audioTapCapturingBuffer) { self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; -// dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 100 * 1000 * 1000); -// dispatch_semaphore_wait(_audioTapRenderingSemaphore, timeout); } else { self.renderingContext->playoutBuffer = NULL; } @@ -398,6 +407,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { capturerContext:self.capturingContext]) { free(self.renderingContext); self.renderingContext = NULL; + self.wantsRendering = NO; return NO; } else if (self.capturingContext) { self.capturingContext->audioUnit = _voiceProcessingIO; @@ -417,12 +427,19 @@ - (BOOL)stopRendering { @synchronized(self) { NSAssert(self.renderingContext != NULL, @"We should have a rendering context when stopping."); + self.wantsRendering = NO; - if (!self.capturingContext) { + if (!self.wantsAudio) { [self stopAudioUnit]; TVIAudioSessionDeactivated(self.renderingContext->deviceContext); [self teardownAudioUnit]; + free(self.capturingContext); + self.capturingContext = NULL; + + free(self.captureBuffer); + self.captureBuffer = NULL; + free(self.renderingContext); self.renderingContext = NULL; } @@ -460,16 +477,17 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { NSLog(@"%s %@", __PRETTY_FUNCTION__, self.capturingFormat); @synchronized(self) { - NSAssert(self.capturingContext == NULL, @"We should not have a capturing context when starting."); - // Restart the already setup graph. if (_voiceProcessingIO) { [self stopAudioUnit]; [self teardownAudioUnit]; } - self.capturingContext = malloc(sizeof(ExampleAVPlayerCapturerContext)); - memset(self.capturingContext, 0, sizeof(ExampleAVPlayerCapturerContext)); + self.wantsCapturing = YES; + if (!self.capturingContext) { + self.capturingContext = malloc(sizeof(ExampleAVPlayerCapturerContext)); + memset(self.capturingContext, 0, sizeof(ExampleAVPlayerCapturerContext)); + } self.capturingContext->deviceContext = context; self.capturingContext->maxFramesPerBuffer = _capturingFormat.framesPerBuffer; self.capturingContext->audioBuffer = _captureBuffer; @@ -477,8 +495,6 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { // Ensure that we wait for the audio tap buffer to become ready. if (_audioTapCapturingBuffer) { self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; -// dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 100 * 1000 * 1000); -// dispatch_semaphore_wait(_audioTapCapturingSemaphore, timeout); } else { self.capturingContext->recordingBuffer = NULL; } @@ -492,6 +508,7 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { capturerContext:self.capturingContext]) { free(self.capturingContext); self.capturingContext = NULL; + self.wantsCapturing = NO; return NO; } else { self.capturingContext->audioUnit = _voiceProcessingIO; @@ -510,8 +527,9 @@ - (BOOL)stopCapturing { @synchronized (self) { NSAssert(self.capturingContext != NULL, @"We should have a capturing context when stopping."); + self.wantsCapturing = NO; - if (!self.renderingContext) { + if (!self.wantsAudio) { [self stopAudioUnit]; TVIAudioSessionDeactivated(self.capturingContext->deviceContext); [self teardownAudioUnit]; @@ -521,6 +539,9 @@ - (BOOL)stopCapturing { free(self.captureBuffer); self.captureBuffer = NULL; + + free(self.renderingContext); + self.renderingContext = NULL; } } return YES; diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 0812ae23..93014402 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -278,6 +278,12 @@ class ViewController: UIViewController { self.viewerButton.isHidden = inRoom self.setNeedsUpdateOfHomeIndicatorAutoHidden() self.setNeedsStatusBarAppearanceUpdate() + + if inRoom == false { + UIView.animate(withDuration: 0.2) { + self.view.backgroundColor = .white + } + } } func startVideoPlayer() { From f0aa053746d4b44cdc5a00e522ae06bb36f996d8 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Tue, 6 Nov 2018 18:16:17 -0800 Subject: [PATCH 66/94] Add test content. --- CoViewingExample/ViewController.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 93014402..a44ca3a4 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -57,6 +57,8 @@ class ViewController: UIViewController { "Avengers: Infinity War Trailer 3 (720p24, 44.1 kHz)" : URL(string: "https://trailers.apple.com/movies/marvel/avengers-infinity-war/avengers-infinity-war-trailer-2_h720p.mov")!, // HLS stream which runs into the AVPlayer / AVAudioMix issue. "BitDash - Parkour (HLS)" : URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8")!, + // 540p variant taken directly from the master playlist above. Still shows the AVPlayer issue. + "BitDash - Parkour (HLS, 540p)" : URL(string: "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa_video_540_1200000.m3u8")!, // Progressive download mp4 version. Demonstrates that 48 kHz support is incorrect right now. "BitDash - Parkour (1080p25, 48 kHz)" : URL(string: "https://bitmovin-a.akamaihd.net/content/MI201109210084_1/MI201109210084_mpeg-4_hd_high_1080p25_10mbits.mp4")!, // Encoding in 1080p takes significantly more CPU than 720p From 9edc99894847f2ed74cf7b945dcee3fa3229af50 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 7 Nov 2018 16:54:54 -0800 Subject: [PATCH 67/94] Temporary fix for video vs full range content. --- CoViewingExample/ExampleAVPlayerSource.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 4547d089..93a4c3aa 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -42,6 +42,7 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { */ let attributes: [String : Any] + // TODO: We need to interrogate the content and choose our range (video/full) appropriately. if (presentationSize.width > ExampleAVPlayerSource.kFrameOutputMaxDimension || presentationSize.height > ExampleAVPlayerSource.kFrameOutputMaxDimension) { let streamingRect = AVMakeRect(aspectRatio: presentationSize, insideRect: ExampleAVPlayerSource.kFrameOutputMaxRect) @@ -50,11 +51,11 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { attributes = [ kCVPixelBufferWidthKey as String : Int(streamingRect.width), kCVPixelBufferHeightKey as String : Int(streamingRect.height), - kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ] as [String : Any] } else { attributes = [ - kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ] as [String : Any] } From f67df1f63c79a08ac6a38b176eabfdee25835705 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 7 Nov 2018 16:55:08 -0800 Subject: [PATCH 68/94] Disallow opening documents in place. --- CoViewingExample/Info.plist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CoViewingExample/Info.plist b/CoViewingExample/Info.plist index 77483ff0..639683c4 100644 --- a/CoViewingExample/Info.plist +++ b/CoViewingExample/Info.plist @@ -84,5 +84,7 @@ UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight + LSSupportsOpeningDocumentsInPlace + From 79b0b13b8dbdc9f08b009c6ebaa58d7f577757c4 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 7 Nov 2018 16:55:31 -0800 Subject: [PATCH 69/94] Improve some view controller teardown. --- CoViewingExample/ViewController.swift | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index a44ca3a4..e201dd91 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -490,9 +490,10 @@ extension ViewController : TVIRoomDelegate { stopVideoPlayer() - self.localVideoTrack = nil; - self.localAudioTrack = nil; - self.playerVideoTrack = nil; + self.localVideoTrack = nil + self.localAudioTrack = nil + self.playerVideoTrack = nil + self.videoPlayerSource = nil self.room = nil self.showRoomUI(inRoom: false) self.accessToken = "TWILIO_ACCESS_TOKEN" @@ -502,8 +503,8 @@ extension ViewController : TVIRoomDelegate { logMessage(messageText: "Failed to connect to Room:\n\(error.localizedDescription)") self.room = nil - self.localVideoTrack = nil; - self.localAudioTrack = nil; + self.localVideoTrack = nil + self.localAudioTrack = nil self.showRoomUI(inRoom: false) self.accessToken = "TWILIO_ACCESS_TOKEN" } From 0adf3895f4a2fd7d6d4646ce34c91da1cc2f736b Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 7 Nov 2018 20:42:12 -0800 Subject: [PATCH 70/94] Refactor of audio pipeline. * Recording format conversion is almost working. * Use the mixer for playback format conversion. * Introduce ExampleAVPlayerAudioConverterContext. * Expand ExampleAVPlayerAudioTapContext. * Pass explicit context to the audio graph when the tap is prepared. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 411 ++++++++++++++---- 1 file changed, 325 insertions(+), 86 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 47c00d74..c9bfbbc3 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -17,12 +17,31 @@ static size_t const kAudioSampleSize = 2; static uint32_t const kPreferredSampleRate = 48000; +typedef struct ExampleAVPlayerAudioConverterContext { + AudioBufferList *cacheBuffers; + UInt32 cachePackets; + AudioBufferList *sourceBuffers; + // Keep track if we are iterating through the source to provide data to a converter. + UInt32 sourcePackets; +} ExampleAVPlayerAudioConverterContext; + typedef struct ExampleAVPlayerAudioTapContext { + __weak ExampleAVPlayerAudioDevice *audioDevice; + BOOL audioTapPrepared; + TPCircularBuffer *capturingBuffer; + AudioConverterRef captureFormatConverter; dispatch_semaphore_t capturingInitSemaphore; + BOOL capturingSampleRateConversion; TPCircularBuffer *renderingBuffer; + AudioConverterRef renderFormatConverter; dispatch_semaphore_t renderingInitSemaphore; + + // Cached source audio, in case we need to perform a sample rate conversion and can't consume all the samples in one go. + AudioBufferList *sourceCache; + UInt32 sourceCacheFrames; + AudioStreamBasicDescription sourceFormat; } ExampleAVPlayerAudioTapContext; typedef struct ExampleAVPlayerRendererContext { @@ -60,6 +79,8 @@ @interface ExampleAVPlayerAudioDevice() +- (void)audioTapDidPrepare:(const AudioStreamBasicDescription *)audioFormat; + @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; @property (nonatomic, assign) AudioUnit playbackMixer; @property (nonatomic, assign) AudioUnit voiceProcessingIO; @@ -85,31 +106,197 @@ @interface ExampleAVPlayerAudioDevice() #pragma mark - MTAudioProcessingTap -// TODO: Bad robot. -static AudioStreamBasicDescription *audioFormat = NULL; -static AudioConverterRef captureFormatConverter = NULL; -static AudioConverterRef formatConverter = NULL; +AudioBufferList *AudioBufferListCreate(const AudioStreamBasicDescription *audioFormat, int frameCount) { + int numberOfBuffers = audioFormat->mFormatFlags & kAudioFormatFlagIsNonInterleaved ? audioFormat->mChannelsPerFrame : 1; + AudioBufferList *audio = malloc(sizeof(AudioBufferList) + (numberOfBuffers - 1) * sizeof(AudioBuffer)); + if (!audio) { + return NULL; + } + audio->mNumberBuffers = numberOfBuffers; + + int channelsPerBuffer = audioFormat->mFormatFlags & kAudioFormatFlagIsNonInterleaved ? 1 : audioFormat->mChannelsPerFrame; + int bytesPerBuffer = audioFormat->mBytesPerFrame * frameCount; + for (int i = 0; i < numberOfBuffers; i++) { + if (bytesPerBuffer > 0) { + audio->mBuffers[i].mData = calloc(bytesPerBuffer, 1); + if (!audio->mBuffers[i].mData) { + for (int j = 0; j < i; j++ ) { + free(audio->mBuffers[j].mData); + } + free(audio); + return NULL; + } + } else { + audio->mBuffers[i].mData = NULL; + } + audio->mBuffers[i].mDataByteSize = bytesPerBuffer; + audio->mBuffers[i].mNumberChannels = channelsPerBuffer; + } + return audio; +} + +void AudioBufferListFree(AudioBufferList *bufferList ) { + for (int i=0; imNumberBuffers; i++) { + if (bufferList->mBuffers[i].mData != NULL) { + free(bufferList->mBuffers[i].mData); + } + } + free(bufferList); +} + +OSStatus ExampleAVPlayerAudioDeviceAudioConverterInputDataProc(AudioConverterRef inAudioConverter, + UInt32 *ioNumberDataPackets, + AudioBufferList *ioData, + AudioStreamPacketDescription * _Nullable *outDataPacketDescription, + void *inUserData) { + // Give the converter what they asked for. They might not consume all of our source in one callback. + UInt32 minimumPackets = *ioNumberDataPackets; + ExampleAVPlayerAudioConverterContext *context = inUserData; + AudioBufferList *sourceBufferList = (AudioBufferList *)context->sourceBuffers; + AudioBufferList *cacheBufferList = (AudioBufferList *)context->cacheBuffers; + assert(sourceBufferList->mNumberBuffers == ioData->mNumberBuffers); + UInt32 bytesPerChannel = 4; + printf("Convert at least %d input packets.\n", minimumPackets); + + for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { + // TODO: What if the cached packets are more than what is requested? + if (context->cachePackets > 0) { + // Copy the minimum packets from the source to the back of our cache, and return the continuous samples to the converter. + AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; + AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; + + UInt32 sourceFramesToCopy = minimumPackets - context->cachePackets; + UInt32 sourceBytesToCopy = sourceFramesToCopy * bytesPerChannel; + UInt32 cachedBytes = context->cachePackets * bytesPerChannel; + assert(sourceBytesToCopy <= cacheBuffer->mDataByteSize - cachedBytes); + void *cacheData = cacheBuffer->mData + cachedBytes; + memcpy(cacheData, sourceBuffer->mData, sourceBytesToCopy); + ioData->mBuffers[i] = *cacheBuffer; + } else { + ioData->mBuffers[i] = sourceBufferList->mBuffers[i]; + } + } + + if (minimumPackets < context->sourcePackets) { + // Copy the remainder of the source which was not used into the front of our cache. + + UInt32 packetsToCopy = context->sourcePackets - minimumPackets; + for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { + AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; + AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; + assert(cacheBuffer->mDataByteSize >= sourceBuffer->mDataByteSize); + UInt32 bytesToCopy = packetsToCopy * bytesPerChannel; + void *sourceData = sourceBuffer->mData + (minimumPackets * bytesPerChannel); + memcpy(cacheBuffer->mData, sourceData, bytesToCopy); + } + context->cachePackets = packetsToCopy; + } + +// *ioNumberDataPackets = inputBufferList->mBuffers[0].mDataByteSize / (UInt32)(4); + return noErr; +} + +static inline void AVPlayerAudioDeviceProduceFilledFrames(TPCircularBuffer *buffer, + AudioConverterRef converter, + AudioBufferList *bufferListIn, + AudioBufferList *sourceCache, + UInt32 *cachedSourceFrames, + UInt32 framesIn, + UInt32 bytesPerFrameOut) { + // Start with input buffer size as our argument. + // TODO: Does non-interleaving count towards the size (*2)? + UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; + printf("Input is %d bytes (%d frames).\n", desiredIoBufferSize, framesIn); + UInt32 propertySizeIo = sizeof(desiredIoBufferSize); + AudioConverterGetProperty(converter, + kAudioConverterPropertyCalculateOutputBufferSize, + &propertySizeIo, &desiredIoBufferSize); + + UInt32 framesOut = desiredIoBufferSize / bytesPerFrameOut; + UInt32 bytesOut = framesOut * bytesPerFrameOut; + printf("Converter wants an output of %d bytes (%d frames, %d bytes per frames).\n", + desiredIoBufferSize, framesOut, bytesPerFrameOut); + + AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); + if (producerBufferList == NULL) { + return; + } + producerBufferList->mBuffers[0].mNumberChannels = bytesPerFrameOut / 2; + + OSStatus status; + UInt32 ioPacketSize = framesOut; + printf("Ready to fill output buffer of frames: %d, bytes: %d with input buffer of frames: %d, bytes: %d.\n", + framesOut, bytesOut, framesIn, framesIn * 4 * bufferListIn->mNumberBuffers); + ExampleAVPlayerAudioConverterContext context; + context.sourceBuffers = bufferListIn; + context.cacheBuffers = sourceCache; + context.sourcePackets = framesIn; + // TODO: Update this each time! + context.cachePackets = *cachedSourceFrames; + status = AudioConverterFillComplexBuffer(converter, + ExampleAVPlayerAudioDeviceAudioConverterInputDataProc, + &context, + &ioPacketSize, + producerBufferList, + NULL); + // Adjust for what the format converter actually produced, in case it was different than what we asked for. + producerBufferList->mBuffers[0].mDataByteSize = ioPacketSize * bytesPerFrameOut; + printf("Output was: %d packets / %d bytes. Consumed input packets: %d. Cached input packets: %d.\n", + ioPacketSize, ioPacketSize * bytesPerFrameOut, context.sourcePackets, context.cachePackets); + + // TODO: Do we still produce the buffer list after a failure? + if (status == kCVReturnSuccess) { + *cachedSourceFrames = context.cachePackets; + TPCircularBufferProduceAudioBufferList(buffer, NULL); + } else { + printf("Error converting buffers: %d\n", status); + } +} + +static inline void AVPlayerAudioDeviceProduceConvertedFrames(TPCircularBuffer *buffer, + AudioConverterRef converter, + AudioBufferList *bufferListIn, + UInt32 framesIn, + UInt32 channelsOut) { + UInt32 bytesOut = framesIn * channelsOut * 2; + AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); + if (producerBufferList == NULL) { + return; + } + producerBufferList->mBuffers[0].mNumberChannels = channelsOut; + + OSStatus status = AudioConverterConvertComplexBuffer(converter, + framesIn, + bufferListIn, + producerBufferList); + + // TODO: Do we still produce the buffer list after a failure? + if (status == kCVReturnSuccess) { + TPCircularBufferProduceAudioBufferList(buffer, NULL); + } else { + printf("Error converting buffers: %d\n", status); + } +} -void init(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { +void AVPlayerProcessingTapInit(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { NSLog(@"Init audio tap."); // Provide access to our device in the Callbacks. *tapStorageOut = clientInfo; } -void finalize(MTAudioProcessingTapRef tap) { +void AVPlayerProcessingTapFinalize(MTAudioProcessingTapRef tap) { NSLog(@"Finalize audio tap."); ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + context->audioTapPrepared = NO; TPCircularBuffer *capturingBuffer = context->capturingBuffer; TPCircularBuffer *renderingBuffer = context->renderingBuffer; - dispatch_semaphore_signal(context->capturingInitSemaphore); - dispatch_semaphore_signal(context->renderingInitSemaphore); TPCircularBufferCleanup(capturingBuffer); TPCircularBufferCleanup(renderingBuffer); } -void prepare(MTAudioProcessingTapRef tap, +void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, CMItemCount maxFrames, const AudioStreamBasicDescription *processingFormat) { NSLog(@"Preparing with frames: %d, channels: %d, bits/channel: %d, sample rate: %0.1f", @@ -125,7 +312,7 @@ void prepare(MTAudioProcessingTapRef tap, // We need to add some overhead for the AudioBufferList data structures. bufferSize += 2048; // TODO: Size the buffer appropriately, as we may need to accumulate more than maxFrames due to bursty processing. - bufferSize *= 12; + bufferSize *= 20; // TODO: If we are re-allocating then check the size? TPCircularBufferInit(capturingBuffer, bufferSize); @@ -133,17 +320,19 @@ void prepare(MTAudioProcessingTapRef tap, dispatch_semaphore_signal(context->capturingInitSemaphore); dispatch_semaphore_signal(context->renderingInitSemaphore); - audioFormat = malloc(sizeof(AudioStreamBasicDescription)); - memcpy(audioFormat, processingFormat, sizeof(AudioStreamBasicDescription)); + AudioBufferList *cacheBufferList = AudioBufferListCreate(processingFormat, (int)maxFrames); + context->sourceCache = cacheBufferList; + context->sourceCacheFrames = 0; + context->sourceFormat = *processingFormat; - TVIAudioFormat *playbackFormat = [[TVIAudioFormat alloc] initWithChannels:processingFormat->mChannelsPerFrame + TVIAudioFormat *playbackFormat = [[TVIAudioFormat alloc] initWithChannels:kPreferredNumberOfChannels sampleRate:processingFormat->mSampleRate framesPerBuffer:maxFrames]; AudioStreamBasicDescription preferredPlaybackDescription = [playbackFormat streamDescription]; BOOL requiresFormatConversion = preferredPlaybackDescription.mFormatFlags != processingFormat->mFormatFlags; if (requiresFormatConversion) { - OSStatus status = AudioConverterNew(processingFormat, &preferredPlaybackDescription, &formatConverter); + OSStatus status = AudioConverterNew(processingFormat, &preferredPlaybackDescription, &context->renderFormatConverter); if (status != 0) { NSLog(@"Failed to create AudioConverter: %d", (int)status); return; @@ -151,20 +340,28 @@ void prepare(MTAudioProcessingTapRef tap, } TVIAudioFormat *recordingFormat = [[TVIAudioFormat alloc] initWithChannels:1 - sampleRate:processingFormat->mSampleRate + sampleRate:(Float64)kPreferredSampleRate framesPerBuffer:maxFrames]; AudioStreamBasicDescription preferredRecordingDescription = [recordingFormat streamDescription]; + BOOL requiresSampleRateConversion = processingFormat->mSampleRate != preferredRecordingDescription.mSampleRate; + context->capturingSampleRateConversion = requiresSampleRateConversion; - if (requiresFormatConversion) { - OSStatus status = AudioConverterNew(processingFormat, &preferredRecordingDescription, &captureFormatConverter); + if (requiresFormatConversion || requiresSampleRateConversion) { + OSStatus status = AudioConverterNew(processingFormat, &preferredRecordingDescription, &context->captureFormatConverter); if (status != 0) { NSLog(@"Failed to create AudioConverter: %d", (int)status); return; } + UInt32 primingMethod = kConverterPrimeMethod_None; + status = AudioConverterSetProperty(context->captureFormatConverter, kAudioConverterPrimeMethod, + sizeof(UInt32), &primingMethod); } + + context->audioTapPrepared = YES; + [context->audioDevice audioTapDidPrepare:processingFormat]; } -void unprepare(MTAudioProcessingTapRef tap) { +void AVPlayerProcessingTapUnprepare(MTAudioProcessingTapRef tap) { NSLog(@"Unpreparing audio tap."); // Prevent any more frames from being consumed. Note that this might end audio playback early. @@ -174,29 +371,30 @@ void unprepare(MTAudioProcessingTapRef tap) { TPCircularBufferClear(capturingBuffer); TPCircularBufferClear(renderingBuffer); - free(audioFormat); - audioFormat = NULL; + if (context->sourceCache) { + AudioBufferListFree(context->sourceCache); + context->sourceCache = NULL; + context->sourceCacheFrames = 0; + } - if (formatConverter != NULL) { - AudioConverterDispose(formatConverter); - formatConverter = NULL; + if (context->renderFormatConverter != NULL) { + AudioConverterDispose(context->renderFormatConverter); + context->renderFormatConverter = NULL; } - if (captureFormatConverter != NULL) { - AudioConverterDispose(captureFormatConverter); - captureFormatConverter = NULL; + if (context->captureFormatConverter != NULL) { + AudioConverterDispose(context->captureFormatConverter); + context->captureFormatConverter = NULL; } } -void process(MTAudioProcessingTapRef tap, - CMItemCount numberFrames, - MTAudioProcessingTapFlags flags, - AudioBufferList *bufferListInOut, - CMItemCount *numberFramesOut, - MTAudioProcessingTapFlags *flagsOut) { +void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, + CMItemCount numberFrames, + MTAudioProcessingTapFlags flags, + AudioBufferList *bufferListInOut, + CMItemCount *numberFramesOut, + MTAudioProcessingTapFlags *flagsOut) { ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *capturingBuffer = context->capturingBuffer; - TPCircularBuffer *renderingBuffer = context->renderingBuffer; CMTimeRange sourceRange; OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, @@ -212,39 +410,23 @@ void process(MTAudioProcessingTapRef tap, UInt32 framesToCopy = (UInt32)*numberFramesOut; - // TODO: Assumptions about our producer's format. - // Produce renderer buffers. - UInt32 bytesToCopy = framesToCopy * 4; - AudioBufferList *rendererProducerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(renderingBuffer, 1, bytesToCopy, NULL); - if (rendererProducerBufferList != NULL) { - status = AudioConverterConvertComplexBuffer(formatConverter, - framesToCopy, bufferListInOut, rendererProducerBufferList); - if (status != kCVReturnSuccess) { - // TODO: Do we still produce the buffer list? - return; - } - - TPCircularBufferProduceAudioBufferList(renderingBuffer, NULL); - } - - // Produce capturer buffers. - bytesToCopy = framesToCopy * 2; - AudioBufferList *capturerProducerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(capturingBuffer, 1, bytesToCopy, NULL); - if (capturerProducerBufferList != NULL) { - status = AudioConverterConvertComplexBuffer(captureFormatConverter, - framesToCopy, bufferListInOut, capturerProducerBufferList); - if (status != kCVReturnSuccess) { - // TODO: Do we still produce the buffer list? - return; - } + // Produce renderer buffers. These are interleaved, signed integer frames in the source's sample rate. + TPCircularBuffer *renderingBuffer = context->renderingBuffer; + AVPlayerAudioDeviceProduceConvertedFrames(renderingBuffer, context->renderFormatConverter, bufferListInOut, framesToCopy, 2); - TPCircularBufferProduceAudioBufferList(capturingBuffer, NULL); + // Produce capturer buffers. We will perform a sample rate conversion if needed. + UInt32 bytesPerFrameOut = 2; + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + if (context->capturingSampleRateConversion) { + AVPlayerAudioDeviceProduceFilledFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, bytesPerFrameOut); + } else { + AVPlayerAudioDeviceProduceConvertedFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, framesToCopy, 1); } - // Flush converter buffers on discontinuity. + // Flush converters on a discontinuity. This is especially important for priming a sample rate converter. if (*flagsOut & kMTAudioProcessingTapFlag_EndOfStream) { - AudioConverterReset(formatConverter); - AudioConverterReset(captureFormatConverter); + AudioConverterReset(context->renderFormatConverter); + AudioConverterReset(context->captureFormatConverter); } } @@ -263,6 +445,14 @@ - (id)init { _audioTapRenderingSemaphore = dispatch_semaphore_create(0); _wantsCapturing = NO; _wantsRendering = NO; + + _audioTapContext = calloc(1, sizeof(ExampleAVPlayerAudioTapContext)); + _audioTapContext->capturingBuffer = _audioTapCapturingBuffer; + _audioTapContext->capturingInitSemaphore = _audioTapCapturingSemaphore; + _audioTapContext->renderingBuffer = _audioTapRenderingBuffer; + _audioTapContext->renderingInitSemaphore = _audioTapRenderingSemaphore; + _audioTapContext->audioDevice = self; + _audioTapContext->audioTapPrepared = NO; } return self; } @@ -315,27 +505,61 @@ - (BOOL)wantsAudio { return _wantsCapturing || _wantsRendering; } +- (void)audioTapDidPrepare:(const AudioStreamBasicDescription *)processingDescription { + NSLog(@"%s", __PRETTY_FUNCTION__); + + // TODO: Multiple contexts. + @synchronized (self) { + TVIAudioDeviceContext *context = _capturingContext ? _capturingContext->deviceContext : _renderingContext ? _renderingContext->deviceContext : NULL; + if (context) { + TVIAudioDeviceExecuteWorkerBlock(context, ^{ + [self restartAudioUnit]; + }); + } + } +} + +- (void)restartAudioUnit { + BOOL restart = NO; + @synchronized (self) { + if (self.wantsAudio) { + restart = YES; + [self stopAudioUnit]; + [self teardownAudioUnit]; + if (self.renderingContext) { + self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; + } + if (self.capturingContext) { + self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; + } + if ([self setupAudioUnitRendererContext:self.renderingContext + capturerContext:self.capturingContext]) { + if (self.capturingContext) { + self.capturingContext->audioUnit = _voiceProcessingIO; + self.capturingContext->audioConverter = _captureConverter; + } + } else { + return; + } + } + } + + [self startAudioUnit]; +} + - (MTAudioProcessingTapRef)createProcessingTap { if (_audioTap) { return _audioTap; } - if (!_audioTapContext) { - _audioTapContext = malloc(sizeof(ExampleAVPlayerAudioTapContext)); - _audioTapContext->capturingBuffer = _audioTapCapturingBuffer; - _audioTapContext->capturingInitSemaphore = _audioTapCapturingSemaphore; - _audioTapContext->renderingBuffer = _audioTapRenderingBuffer; - _audioTapContext->renderingInitSemaphore = _audioTapRenderingSemaphore; - } - MTAudioProcessingTapRef processingTap; MTAudioProcessingTapCallbacks callbacks; callbacks.version = kMTAudioProcessingTapCallbacksVersion_0; - callbacks.init = init; - callbacks.prepare = prepare; - callbacks.process = process; - callbacks.unprepare = unprepare; - callbacks.finalize = finalize; + callbacks.init = AVPlayerProcessingTapInit; + callbacks.prepare = AVPlayerProcessingTapPrepare; + callbacks.process = AVPlayerProcessingTapProcess; + callbacks.unprepare = AVPlayerProcessingTapUnprepare; + callbacks.finalize = AVPlayerProcessingTapFinalize; callbacks.clientInfo = (void *)(_audioTapContext); OSStatus status = MTAudioProcessingTapCreate(kCFAllocatorDefault, @@ -346,8 +570,6 @@ - (MTAudioProcessingTapRef)createProcessingTap { _audioTap = processingTap; return processingTap; } else { - free(_audioTapContext); - _audioTapContext = NULL; return NULL; } } @@ -392,7 +614,7 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { self.renderingContext->maxFramesPerBuffer = _renderingFormat.framesPerBuffer; // Ensure that we wait for the audio tap buffer to become ready. - if (_audioTapCapturingBuffer) { + if (self.audioTapContext->audioTapPrepared) { self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; } else { self.renderingContext->playoutBuffer = NULL; @@ -493,7 +715,7 @@ - (BOOL)startCapturing:(nonnull TVIAudioDeviceContext)context { self.capturingContext->audioBuffer = _captureBuffer; // Ensure that we wait for the audio tap buffer to become ready. - if (_audioTapCapturingBuffer) { + if (self.audioTapContext->audioTapPrepared) { self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; } else { self.capturingContext->recordingBuffer = NULL; @@ -561,7 +783,7 @@ static void ExampleAVPlayerAudioDeviceDequeueFrames(TPCircularBuffer *buffer, format.mBytesPerFrame = format.mChannelsPerFrame * format.mBitsPerChannel / 8; format.mFormatID = kAudioFormatLinearPCM; format.mFormatFlags = kAudioFormatFlagIsPacked | kAudioFormatFlagIsSignedInteger; - format.mSampleRate = 44100; + format.mSampleRate = kPreferredSampleRate; UInt32 framesInOut = numFrames; if (buffer->buffer != NULL) { @@ -591,6 +813,7 @@ static OSStatus ExampleAVPlayerAudioDeviceAudioTapPlaybackCallback(void *refCon, assert(bufferList->mBuffers[0].mNumberChannels > 0); ExampleAVPlayerRendererContext *context = (ExampleAVPlayerRendererContext *)refCon; + TPCircularBuffer *buffer = context->playoutBuffer; UInt32 audioBufferSizeInBytes = bufferList->mBuffers[0].mDataByteSize; // Render silence if there are temporary mismatches between CoreAudio and our rendering format. @@ -600,11 +823,13 @@ static OSStatus ExampleAVPlayerAudioDeviceAudioTapPlaybackCallback(void *refCon, int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; memset(audioBuffer, 0, audioBufferSizeInBytes); return noErr; + } else if (buffer == nil) { + *actionFlags |= kAudioUnitRenderAction_OutputIsSilence; + memset(bufferList->mBuffers[0].mData, 0, audioBufferSizeInBytes); + return noErr; } - TPCircularBuffer *buffer = context->playoutBuffer; ExampleAVPlayerAudioDeviceDequeueFrames(buffer, numFrames, bufferList); - return noErr; } @@ -674,6 +899,16 @@ static OSStatus ExampleAVPlayerAudioDeviceRecordingInputCallback(void *refCon, return status; } + // Early return with microphone only recording. + if (context->recordingBuffer == NULL) { + if (context->deviceContext) { + TVIAudioDeviceWriteCaptureData(context->deviceContext, + microphoneAudioBuffer->mData, + microphoneAudioBuffer->mDataByteSize); + } + return noErr; + } + // Dequeue the AVPlayer audio. AudioBufferList playerBufferList; playerBufferList.mNumberBuffers = 1; @@ -735,8 +970,7 @@ + (nullable TVIAudioFormat *)activeFormat { * to the `AVAudioSession.preferredIOBufferDuration` that we've requested. */ const size_t sessionFramesPerBuffer = kMaximumFramesPerBuffer; -// const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; - const double sessionSampleRate = 44100.; + const double sessionSampleRate = [AVAudioSession sharedInstance].sampleRate; const NSInteger sessionOutputChannels = [AVAudioSession sharedInstance].outputNumberOfChannels; size_t rendererChannels = sessionOutputChannels >= TVIAudioChannelsStereo ? TVIAudioChannelsStereo : TVIAudioChannelsMono; @@ -917,6 +1151,10 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer if (enableOutput) { AudioStreamBasicDescription renderingFormatDescription = self.renderingFormat.streamDescription; + AudioStreamBasicDescription playerFormatDescription = renderingFormatDescription; + if (self.renderingContext->playoutBuffer) { + playerFormatDescription.mSampleRate = self.audioTapContext->sourceFormat.mSampleRate; + } // Setup playback mixer. AudioComponentDescription mixerComponentDescription = [[self class] mixerAudioCompontentDescription]; @@ -941,7 +1179,7 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer status = AudioUnitSetProperty(_playbackMixer, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 0, - &renderingFormatDescription, sizeof(renderingFormatDescription)); + &playerFormatDescription, sizeof(playerFormatDescription)); if (status != noErr) { NSLog(@"Could not set stream format on the mixer input bus 0!"); AudioComponentInstanceDispose(_voiceProcessingIO); @@ -1112,6 +1350,7 @@ - (void)handleAudioInterruption:(NSNotification *)notification { AVAudioSessionInterruptionType type = [notification.userInfo[AVAudioSessionInterruptionTypeKey] unsignedIntegerValue]; @synchronized(self) { + // TODO: Multiple contexts. // If the worker block is executed, then context is guaranteed to be valid. TVIAudioDeviceContext context = self.renderingContext ? self.renderingContext->deviceContext : NULL; if (context) { From b96a6bd931a6038fc3a2e9c929d06852491cd6ed Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 7 Nov 2018 21:30:56 -0800 Subject: [PATCH 71/94] Revert video level changes. --- CoViewingExample/ExampleAVPlayerSource.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 93a4c3aa..53e60b66 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -51,11 +51,11 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { attributes = [ kCVPixelBufferWidthKey as String : Int(streamingRect.width), kCVPixelBufferHeightKey as String : Int(streamingRect.height), - kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ] as [String : Any] } else { attributes = [ - kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ] as [String : Any] } From 4bc0459952e4d0e9d4e7e965a1152df1760bd076 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 7 Nov 2018 21:38:50 -0800 Subject: [PATCH 72/94] Separate MTAudioProcessingTap code into a separate file. --- CoViewingExample.xcodeproj/project.pbxproj | 6 + .../AudioDevices/ExampleAVPlayerAudioDevice.h | 2 + .../AudioDevices/ExampleAVPlayerAudioDevice.m | 360 +----------------- .../ExampleAVPlayerProcessingTap.h | 47 +++ .../ExampleAVPlayerProcessingTap.m | 347 +++++++++++++++++ 5 files changed, 405 insertions(+), 357 deletions(-) create mode 100644 CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h create mode 100644 CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m diff --git a/CoViewingExample.xcodeproj/project.pbxproj b/CoViewingExample.xcodeproj/project.pbxproj index f300ebeb..667607e5 100644 --- a/CoViewingExample.xcodeproj/project.pbxproj +++ b/CoViewingExample.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ 8A395E4D2187D2B300437980 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8A395E4B2187D2B300437980 /* LaunchScreen.storyboard */; }; 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */; }; 8A395E572187F04C00437980 /* ExampleAVPlayerSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8A395E562187F04C00437980 /* ExampleAVPlayerSource.swift */; }; + 8AF48A832193FC5B007B1A84 /* ExampleAVPlayerProcessingTap.m in Sources */ = {isa = PBXBuildFile; fileRef = 8AF48A822193FC5B007B1A84 /* ExampleAVPlayerProcessingTap.m */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -32,6 +33,8 @@ 8A395E4E2187D2B300437980 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 8A395E542187D52400437980 /* ExampleAVPlayerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerView.swift; sourceTree = ""; }; 8A395E562187F04C00437980 /* ExampleAVPlayerSource.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleAVPlayerSource.swift; sourceTree = ""; }; + 8AF48A812193FC5B007B1A84 /* ExampleAVPlayerProcessingTap.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ExampleAVPlayerProcessingTap.h; sourceTree = ""; }; + 8AF48A822193FC5B007B1A84 /* ExampleAVPlayerProcessingTap.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ExampleAVPlayerProcessingTap.m; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -51,6 +54,8 @@ 8A34C1D82189496A00F22BE9 /* AudioDevices-Bridging-Header.h */, 8A34C1D72189496A00F22BE9 /* ExampleAVPlayerAudioDevice.h */, 8A34C1D92189496A00F22BE9 /* ExampleAVPlayerAudioDevice.m */, + 8AF48A812193FC5B007B1A84 /* ExampleAVPlayerProcessingTap.h */, + 8AF48A822193FC5B007B1A84 /* ExampleAVPlayerProcessingTap.m */, ); path = AudioDevices; sourceTree = ""; @@ -163,6 +168,7 @@ 8A395E432187D2B200437980 /* AppDelegate.swift in Sources */, 8A395E552187D52400437980 /* ExampleAVPlayerView.swift in Sources */, 8A34C1D52189333400F22BE9 /* ExampleAVPlayerAudioTap.swift in Sources */, + 8AF48A832193FC5B007B1A84 /* ExampleAVPlayerProcessingTap.m in Sources */, 8A34C1DA2189496A00F22BE9 /* ExampleAVPlayerAudioDevice.m in Sources */, 8A395E572187F04C00437980 /* ExampleAVPlayerSource.swift in Sources */, ); diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h index 3a11cc41..e2fc4d51 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h @@ -14,6 +14,8 @@ */ @interface ExampleAVPlayerAudioDevice : NSObject +- (void)audioTapDidPrepare; + /* * Creates a processing tap bound to the device instance. * diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index c9bfbbc3..584ed02b 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -7,6 +7,7 @@ #import "ExampleAVPlayerAudioDevice.h" +#import "ExampleAVPlayerProcessingTap.h" #import "TPCircularBuffer+AudioBufferList.h" // We want to get as close to 20 msec buffers as possible, to match the behavior of TVIDefaultAudioDevice. @@ -14,36 +15,9 @@ // We will use stereo playback where available. Some audio routes may be restricted to mono only. static size_t const kPreferredNumberOfChannels = 2; // An audio sample is a signed 16-bit integer. -static size_t const kAudioSampleSize = 2; +static size_t const kAudioSampleSize = sizeof(SInt16); static uint32_t const kPreferredSampleRate = 48000; -typedef struct ExampleAVPlayerAudioConverterContext { - AudioBufferList *cacheBuffers; - UInt32 cachePackets; - AudioBufferList *sourceBuffers; - // Keep track if we are iterating through the source to provide data to a converter. - UInt32 sourcePackets; -} ExampleAVPlayerAudioConverterContext; - -typedef struct ExampleAVPlayerAudioTapContext { - __weak ExampleAVPlayerAudioDevice *audioDevice; - BOOL audioTapPrepared; - - TPCircularBuffer *capturingBuffer; - AudioConverterRef captureFormatConverter; - dispatch_semaphore_t capturingInitSemaphore; - BOOL capturingSampleRateConversion; - - TPCircularBuffer *renderingBuffer; - AudioConverterRef renderFormatConverter; - dispatch_semaphore_t renderingInitSemaphore; - - // Cached source audio, in case we need to perform a sample rate conversion and can't consume all the samples in one go. - AudioBufferList *sourceCache; - UInt32 sourceCacheFrames; - AudioStreamBasicDescription sourceFormat; -} ExampleAVPlayerAudioTapContext; - typedef struct ExampleAVPlayerRendererContext { // Used to pull audio from the media engine. TVIAudioDeviceContext deviceContext; @@ -79,8 +53,6 @@ @interface ExampleAVPlayerAudioDevice() -- (void)audioTapDidPrepare:(const AudioStreamBasicDescription *)audioFormat; - @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; @property (nonatomic, assign) AudioUnit playbackMixer; @property (nonatomic, assign) AudioUnit voiceProcessingIO; @@ -104,332 +76,6 @@ - (void)audioTapDidPrepare:(const AudioStreamBasicDescription *)audioFormat; @end -#pragma mark - MTAudioProcessingTap - -AudioBufferList *AudioBufferListCreate(const AudioStreamBasicDescription *audioFormat, int frameCount) { - int numberOfBuffers = audioFormat->mFormatFlags & kAudioFormatFlagIsNonInterleaved ? audioFormat->mChannelsPerFrame : 1; - AudioBufferList *audio = malloc(sizeof(AudioBufferList) + (numberOfBuffers - 1) * sizeof(AudioBuffer)); - if (!audio) { - return NULL; - } - audio->mNumberBuffers = numberOfBuffers; - - int channelsPerBuffer = audioFormat->mFormatFlags & kAudioFormatFlagIsNonInterleaved ? 1 : audioFormat->mChannelsPerFrame; - int bytesPerBuffer = audioFormat->mBytesPerFrame * frameCount; - for (int i = 0; i < numberOfBuffers; i++) { - if (bytesPerBuffer > 0) { - audio->mBuffers[i].mData = calloc(bytesPerBuffer, 1); - if (!audio->mBuffers[i].mData) { - for (int j = 0; j < i; j++ ) { - free(audio->mBuffers[j].mData); - } - free(audio); - return NULL; - } - } else { - audio->mBuffers[i].mData = NULL; - } - audio->mBuffers[i].mDataByteSize = bytesPerBuffer; - audio->mBuffers[i].mNumberChannels = channelsPerBuffer; - } - return audio; -} - -void AudioBufferListFree(AudioBufferList *bufferList ) { - for (int i=0; imNumberBuffers; i++) { - if (bufferList->mBuffers[i].mData != NULL) { - free(bufferList->mBuffers[i].mData); - } - } - free(bufferList); -} - -OSStatus ExampleAVPlayerAudioDeviceAudioConverterInputDataProc(AudioConverterRef inAudioConverter, - UInt32 *ioNumberDataPackets, - AudioBufferList *ioData, - AudioStreamPacketDescription * _Nullable *outDataPacketDescription, - void *inUserData) { - // Give the converter what they asked for. They might not consume all of our source in one callback. - UInt32 minimumPackets = *ioNumberDataPackets; - ExampleAVPlayerAudioConverterContext *context = inUserData; - AudioBufferList *sourceBufferList = (AudioBufferList *)context->sourceBuffers; - AudioBufferList *cacheBufferList = (AudioBufferList *)context->cacheBuffers; - assert(sourceBufferList->mNumberBuffers == ioData->mNumberBuffers); - UInt32 bytesPerChannel = 4; - printf("Convert at least %d input packets.\n", minimumPackets); - - for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { - // TODO: What if the cached packets are more than what is requested? - if (context->cachePackets > 0) { - // Copy the minimum packets from the source to the back of our cache, and return the continuous samples to the converter. - AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; - AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; - - UInt32 sourceFramesToCopy = minimumPackets - context->cachePackets; - UInt32 sourceBytesToCopy = sourceFramesToCopy * bytesPerChannel; - UInt32 cachedBytes = context->cachePackets * bytesPerChannel; - assert(sourceBytesToCopy <= cacheBuffer->mDataByteSize - cachedBytes); - void *cacheData = cacheBuffer->mData + cachedBytes; - memcpy(cacheData, sourceBuffer->mData, sourceBytesToCopy); - ioData->mBuffers[i] = *cacheBuffer; - } else { - ioData->mBuffers[i] = sourceBufferList->mBuffers[i]; - } - } - - if (minimumPackets < context->sourcePackets) { - // Copy the remainder of the source which was not used into the front of our cache. - - UInt32 packetsToCopy = context->sourcePackets - minimumPackets; - for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { - AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; - AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; - assert(cacheBuffer->mDataByteSize >= sourceBuffer->mDataByteSize); - UInt32 bytesToCopy = packetsToCopy * bytesPerChannel; - void *sourceData = sourceBuffer->mData + (minimumPackets * bytesPerChannel); - memcpy(cacheBuffer->mData, sourceData, bytesToCopy); - } - context->cachePackets = packetsToCopy; - } - -// *ioNumberDataPackets = inputBufferList->mBuffers[0].mDataByteSize / (UInt32)(4); - return noErr; -} - -static inline void AVPlayerAudioDeviceProduceFilledFrames(TPCircularBuffer *buffer, - AudioConverterRef converter, - AudioBufferList *bufferListIn, - AudioBufferList *sourceCache, - UInt32 *cachedSourceFrames, - UInt32 framesIn, - UInt32 bytesPerFrameOut) { - // Start with input buffer size as our argument. - // TODO: Does non-interleaving count towards the size (*2)? - UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; - printf("Input is %d bytes (%d frames).\n", desiredIoBufferSize, framesIn); - UInt32 propertySizeIo = sizeof(desiredIoBufferSize); - AudioConverterGetProperty(converter, - kAudioConverterPropertyCalculateOutputBufferSize, - &propertySizeIo, &desiredIoBufferSize); - - UInt32 framesOut = desiredIoBufferSize / bytesPerFrameOut; - UInt32 bytesOut = framesOut * bytesPerFrameOut; - printf("Converter wants an output of %d bytes (%d frames, %d bytes per frames).\n", - desiredIoBufferSize, framesOut, bytesPerFrameOut); - - AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); - if (producerBufferList == NULL) { - return; - } - producerBufferList->mBuffers[0].mNumberChannels = bytesPerFrameOut / 2; - - OSStatus status; - UInt32 ioPacketSize = framesOut; - printf("Ready to fill output buffer of frames: %d, bytes: %d with input buffer of frames: %d, bytes: %d.\n", - framesOut, bytesOut, framesIn, framesIn * 4 * bufferListIn->mNumberBuffers); - ExampleAVPlayerAudioConverterContext context; - context.sourceBuffers = bufferListIn; - context.cacheBuffers = sourceCache; - context.sourcePackets = framesIn; - // TODO: Update this each time! - context.cachePackets = *cachedSourceFrames; - status = AudioConverterFillComplexBuffer(converter, - ExampleAVPlayerAudioDeviceAudioConverterInputDataProc, - &context, - &ioPacketSize, - producerBufferList, - NULL); - // Adjust for what the format converter actually produced, in case it was different than what we asked for. - producerBufferList->mBuffers[0].mDataByteSize = ioPacketSize * bytesPerFrameOut; - printf("Output was: %d packets / %d bytes. Consumed input packets: %d. Cached input packets: %d.\n", - ioPacketSize, ioPacketSize * bytesPerFrameOut, context.sourcePackets, context.cachePackets); - - // TODO: Do we still produce the buffer list after a failure? - if (status == kCVReturnSuccess) { - *cachedSourceFrames = context.cachePackets; - TPCircularBufferProduceAudioBufferList(buffer, NULL); - } else { - printf("Error converting buffers: %d\n", status); - } -} - -static inline void AVPlayerAudioDeviceProduceConvertedFrames(TPCircularBuffer *buffer, - AudioConverterRef converter, - AudioBufferList *bufferListIn, - UInt32 framesIn, - UInt32 channelsOut) { - UInt32 bytesOut = framesIn * channelsOut * 2; - AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); - if (producerBufferList == NULL) { - return; - } - producerBufferList->mBuffers[0].mNumberChannels = channelsOut; - - OSStatus status = AudioConverterConvertComplexBuffer(converter, - framesIn, - bufferListIn, - producerBufferList); - - // TODO: Do we still produce the buffer list after a failure? - if (status == kCVReturnSuccess) { - TPCircularBufferProduceAudioBufferList(buffer, NULL); - } else { - printf("Error converting buffers: %d\n", status); - } -} - -void AVPlayerProcessingTapInit(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { - NSLog(@"Init audio tap."); - - // Provide access to our device in the Callbacks. - *tapStorageOut = clientInfo; -} - -void AVPlayerProcessingTapFinalize(MTAudioProcessingTapRef tap) { - NSLog(@"Finalize audio tap."); - - ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - context->audioTapPrepared = NO; - TPCircularBuffer *capturingBuffer = context->capturingBuffer; - TPCircularBuffer *renderingBuffer = context->renderingBuffer; - TPCircularBufferCleanup(capturingBuffer); - TPCircularBufferCleanup(renderingBuffer); -} - -void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, - CMItemCount maxFrames, - const AudioStreamBasicDescription *processingFormat) { - NSLog(@"Preparing with frames: %d, channels: %d, bits/channel: %d, sample rate: %0.1f", - (int)maxFrames, processingFormat->mChannelsPerFrame, processingFormat->mBitsPerChannel, processingFormat->mSampleRate); - assert(processingFormat->mFormatID == kAudioFormatLinearPCM); - - // Defer init of the ring buffer memory until we understand the processing format. - ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *capturingBuffer = context->capturingBuffer; - TPCircularBuffer *renderingBuffer = context->renderingBuffer; - - size_t bufferSize = processingFormat->mBytesPerFrame * maxFrames; - // We need to add some overhead for the AudioBufferList data structures. - bufferSize += 2048; - // TODO: Size the buffer appropriately, as we may need to accumulate more than maxFrames due to bursty processing. - bufferSize *= 20; - - // TODO: If we are re-allocating then check the size? - TPCircularBufferInit(capturingBuffer, bufferSize); - TPCircularBufferInit(renderingBuffer, bufferSize); - dispatch_semaphore_signal(context->capturingInitSemaphore); - dispatch_semaphore_signal(context->renderingInitSemaphore); - - AudioBufferList *cacheBufferList = AudioBufferListCreate(processingFormat, (int)maxFrames); - context->sourceCache = cacheBufferList; - context->sourceCacheFrames = 0; - context->sourceFormat = *processingFormat; - - TVIAudioFormat *playbackFormat = [[TVIAudioFormat alloc] initWithChannels:kPreferredNumberOfChannels - sampleRate:processingFormat->mSampleRate - framesPerBuffer:maxFrames]; - AudioStreamBasicDescription preferredPlaybackDescription = [playbackFormat streamDescription]; - BOOL requiresFormatConversion = preferredPlaybackDescription.mFormatFlags != processingFormat->mFormatFlags; - - if (requiresFormatConversion) { - OSStatus status = AudioConverterNew(processingFormat, &preferredPlaybackDescription, &context->renderFormatConverter); - if (status != 0) { - NSLog(@"Failed to create AudioConverter: %d", (int)status); - return; - } - } - - TVIAudioFormat *recordingFormat = [[TVIAudioFormat alloc] initWithChannels:1 - sampleRate:(Float64)kPreferredSampleRate - framesPerBuffer:maxFrames]; - AudioStreamBasicDescription preferredRecordingDescription = [recordingFormat streamDescription]; - BOOL requiresSampleRateConversion = processingFormat->mSampleRate != preferredRecordingDescription.mSampleRate; - context->capturingSampleRateConversion = requiresSampleRateConversion; - - if (requiresFormatConversion || requiresSampleRateConversion) { - OSStatus status = AudioConverterNew(processingFormat, &preferredRecordingDescription, &context->captureFormatConverter); - if (status != 0) { - NSLog(@"Failed to create AudioConverter: %d", (int)status); - return; - } - UInt32 primingMethod = kConverterPrimeMethod_None; - status = AudioConverterSetProperty(context->captureFormatConverter, kAudioConverterPrimeMethod, - sizeof(UInt32), &primingMethod); - } - - context->audioTapPrepared = YES; - [context->audioDevice audioTapDidPrepare:processingFormat]; -} - -void AVPlayerProcessingTapUnprepare(MTAudioProcessingTapRef tap) { - NSLog(@"Unpreparing audio tap."); - - // Prevent any more frames from being consumed. Note that this might end audio playback early. - ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - TPCircularBuffer *capturingBuffer = context->capturingBuffer; - TPCircularBuffer *renderingBuffer = context->renderingBuffer; - - TPCircularBufferClear(capturingBuffer); - TPCircularBufferClear(renderingBuffer); - if (context->sourceCache) { - AudioBufferListFree(context->sourceCache); - context->sourceCache = NULL; - context->sourceCacheFrames = 0; - } - - if (context->renderFormatConverter != NULL) { - AudioConverterDispose(context->renderFormatConverter); - context->renderFormatConverter = NULL; - } - - if (context->captureFormatConverter != NULL) { - AudioConverterDispose(context->captureFormatConverter); - context->captureFormatConverter = NULL; - } -} - -void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, - CMItemCount numberFrames, - MTAudioProcessingTapFlags flags, - AudioBufferList *bufferListInOut, - CMItemCount *numberFramesOut, - MTAudioProcessingTapFlags *flagsOut) { - ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); - CMTimeRange sourceRange; - OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, - numberFrames, - bufferListInOut, - flagsOut, - &sourceRange, - numberFramesOut); - - if (status != kCVReturnSuccess) { - // TODO - return; - } - - UInt32 framesToCopy = (UInt32)*numberFramesOut; - - // Produce renderer buffers. These are interleaved, signed integer frames in the source's sample rate. - TPCircularBuffer *renderingBuffer = context->renderingBuffer; - AVPlayerAudioDeviceProduceConvertedFrames(renderingBuffer, context->renderFormatConverter, bufferListInOut, framesToCopy, 2); - - // Produce capturer buffers. We will perform a sample rate conversion if needed. - UInt32 bytesPerFrameOut = 2; - TPCircularBuffer *capturingBuffer = context->capturingBuffer; - if (context->capturingSampleRateConversion) { - AVPlayerAudioDeviceProduceFilledFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, bytesPerFrameOut); - } else { - AVPlayerAudioDeviceProduceConvertedFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, framesToCopy, 1); - } - - // Flush converters on a discontinuity. This is especially important for priming a sample rate converter. - if (*flagsOut & kMTAudioProcessingTapFlag_EndOfStream) { - AudioConverterReset(context->renderFormatConverter); - AudioConverterReset(context->captureFormatConverter); - } -} - @implementation ExampleAVPlayerAudioDevice @synthesize audioTapCapturingBuffer = _audioTapCapturingBuffer; @@ -505,7 +151,7 @@ - (BOOL)wantsAudio { return _wantsCapturing || _wantsRendering; } -- (void)audioTapDidPrepare:(const AudioStreamBasicDescription *)processingDescription { +- (void)audioTapDidPrepare { NSLog(@"%s", __PRETTY_FUNCTION__); // TODO: Multiple contexts. diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h new file mode 100644 index 00000000..4d60fe6b --- /dev/null +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h @@ -0,0 +1,47 @@ +// +// ExampleAVPlayerProcessingTap.h +// CoViewingExample +// +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +#import +#import + +@class ExampleAVPlayerAudioDevice; + +typedef struct ExampleAVPlayerAudioTapContext { + __weak ExampleAVPlayerAudioDevice *audioDevice; + BOOL audioTapPrepared; + + TPCircularBuffer *capturingBuffer; + AudioConverterRef captureFormatConverter; + dispatch_semaphore_t capturingInitSemaphore; + BOOL capturingSampleRateConversion; + + TPCircularBuffer *renderingBuffer; + AudioConverterRef renderFormatConverter; + dispatch_semaphore_t renderingInitSemaphore; + + // Cached source audio, in case we need to perform a sample rate conversion and can't consume all the samples in one go. + AudioBufferList *sourceCache; + UInt32 sourceCacheFrames; + AudioStreamBasicDescription sourceFormat; +} ExampleAVPlayerAudioTapContext; + +void AVPlayerProcessingTapInit(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut); + +void AVPlayerProcessingTapFinalize(MTAudioProcessingTapRef tap); + +void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, + CMItemCount maxFrames, + const AudioStreamBasicDescription *processingFormat); + +void AVPlayerProcessingTapUnprepare(MTAudioProcessingTapRef tap); + +void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, + CMItemCount numberFrames, + MTAudioProcessingTapFlags flags, + AudioBufferList *bufferListInOut, + CMItemCount *numberFramesOut, + MTAudioProcessingTapFlags *flagsOut); diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m new file mode 100644 index 00000000..8dc14098 --- /dev/null +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -0,0 +1,347 @@ +// +// ExampleAVPlayerProcessingTap.m +// CoViewingExample +// +// Copyright © 2018 Twilio Inc. All rights reserved. +// + +#import "ExampleAVPlayerProcessingTap.h" + +#import "ExampleAVPlayerAudioDevice.h" +#import "TPCircularBuffer+AudioBufferList.h" + +static size_t const kPreferredNumberOfChannels = 2; +static uint32_t const kPreferredSampleRate = 48000; + +typedef struct ExampleAVPlayerAudioConverterContext { + AudioBufferList *cacheBuffers; + UInt32 cachePackets; + AudioBufferList *sourceBuffers; + // Keep track if we are iterating through the source to provide data to a converter. + UInt32 sourcePackets; +} ExampleAVPlayerAudioConverterContext; + +AudioBufferList *AudioBufferListCreate(const AudioStreamBasicDescription *audioFormat, int frameCount) { + int numberOfBuffers = audioFormat->mFormatFlags & kAudioFormatFlagIsNonInterleaved ? audioFormat->mChannelsPerFrame : 1; + AudioBufferList *audio = malloc(sizeof(AudioBufferList) + (numberOfBuffers - 1) * sizeof(AudioBuffer)); + if (!audio) { + return NULL; + } + audio->mNumberBuffers = numberOfBuffers; + + int channelsPerBuffer = audioFormat->mFormatFlags & kAudioFormatFlagIsNonInterleaved ? 1 : audioFormat->mChannelsPerFrame; + int bytesPerBuffer = audioFormat->mBytesPerFrame * frameCount; + for (int i = 0; i < numberOfBuffers; i++) { + if (bytesPerBuffer > 0) { + audio->mBuffers[i].mData = calloc(bytesPerBuffer, 1); + if (!audio->mBuffers[i].mData) { + for (int j = 0; j < i; j++ ) { + free(audio->mBuffers[j].mData); + } + free(audio); + return NULL; + } + } else { + audio->mBuffers[i].mData = NULL; + } + audio->mBuffers[i].mDataByteSize = bytesPerBuffer; + audio->mBuffers[i].mNumberChannels = channelsPerBuffer; + } + return audio; +} + +void AudioBufferListFree(AudioBufferList *bufferList ) { + for (int i=0; imNumberBuffers; i++) { + if (bufferList->mBuffers[i].mData != NULL) { + free(bufferList->mBuffers[i].mData); + } + } + free(bufferList); +} + +OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConverter, + UInt32 *ioNumberDataPackets, + AudioBufferList *ioData, + AudioStreamPacketDescription * _Nullable *outDataPacketDescription, + void *inUserData) { + // Give the converter what they asked for. They might not consume all of our source in one callback. + UInt32 minimumPackets = *ioNumberDataPackets; + ExampleAVPlayerAudioConverterContext *context = inUserData; + AudioBufferList *sourceBufferList = (AudioBufferList *)context->sourceBuffers; + AudioBufferList *cacheBufferList = (AudioBufferList *)context->cacheBuffers; + assert(sourceBufferList->mNumberBuffers == ioData->mNumberBuffers); + UInt32 bytesPerChannel = 4; + printf("Convert at least %d input packets.\n", minimumPackets); + + for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { + // TODO: What if the cached packets are more than what is requested? + if (context->cachePackets > 0) { + // Copy the minimum packets from the source to the back of our cache, and return the continuous samples to the converter. + AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; + AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; + + UInt32 sourceFramesToCopy = minimumPackets - context->cachePackets; + UInt32 sourceBytesToCopy = sourceFramesToCopy * bytesPerChannel; + UInt32 cachedBytes = context->cachePackets * bytesPerChannel; + assert(sourceBytesToCopy <= cacheBuffer->mDataByteSize - cachedBytes); + void *cacheData = cacheBuffer->mData + cachedBytes; + memcpy(cacheData, sourceBuffer->mData, sourceBytesToCopy); + ioData->mBuffers[i] = *cacheBuffer; + } else { + ioData->mBuffers[i] = sourceBufferList->mBuffers[i]; + } + } + + if (minimumPackets < context->sourcePackets) { + // Copy the remainder of the source which was not used into the front of our cache. + + UInt32 packetsToCopy = context->sourcePackets - minimumPackets; + for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { + AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; + AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; + assert(cacheBuffer->mDataByteSize >= sourceBuffer->mDataByteSize); + UInt32 bytesToCopy = packetsToCopy * bytesPerChannel; + void *sourceData = sourceBuffer->mData + (minimumPackets * bytesPerChannel); + memcpy(cacheBuffer->mData, sourceData, bytesToCopy); + } + context->cachePackets = packetsToCopy; + } + + // *ioNumberDataPackets = inputBufferList->mBuffers[0].mDataByteSize / (UInt32)(4); + return noErr; +} + +static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, + AudioConverterRef converter, + AudioBufferList *bufferListIn, + AudioBufferList *sourceCache, + UInt32 *cachedSourceFrames, + UInt32 framesIn, + UInt32 bytesPerFrameOut) { + // Start with input buffer size as our argument. + // TODO: Does non-interleaving count towards the size (*2)? + UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; + printf("Input is %d bytes (%d frames).\n", desiredIoBufferSize, framesIn); + UInt32 propertySizeIo = sizeof(desiredIoBufferSize); + AudioConverterGetProperty(converter, + kAudioConverterPropertyCalculateOutputBufferSize, + &propertySizeIo, &desiredIoBufferSize); + + UInt32 framesOut = desiredIoBufferSize / bytesPerFrameOut; + UInt32 bytesOut = framesOut * bytesPerFrameOut; + printf("Converter wants an output of %d bytes (%d frames, %d bytes per frames).\n", + desiredIoBufferSize, framesOut, bytesPerFrameOut); + + AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); + if (producerBufferList == NULL) { + return; + } + producerBufferList->mBuffers[0].mNumberChannels = bytesPerFrameOut / 2; + + OSStatus status; + UInt32 ioPacketSize = framesOut; + printf("Ready to fill output buffer of frames: %d, bytes: %d with input buffer of frames: %d, bytes: %d.\n", + framesOut, bytesOut, framesIn, framesIn * 4 * bufferListIn->mNumberBuffers); + ExampleAVPlayerAudioConverterContext context; + context.sourceBuffers = bufferListIn; + context.cacheBuffers = sourceCache; + context.sourcePackets = framesIn; + context.cachePackets = *cachedSourceFrames; + status = AudioConverterFillComplexBuffer(converter, + AVPlayerAudioTapConverterInputDataProc, + &context, + &ioPacketSize, + producerBufferList, + NULL); + // Adjust for what the format converter actually produced, in case it was different than what we asked for. + producerBufferList->mBuffers[0].mDataByteSize = ioPacketSize * bytesPerFrameOut; + printf("Output was: %d packets / %d bytes. Consumed input packets: %d. Cached input packets: %d.\n", + ioPacketSize, ioPacketSize * bytesPerFrameOut, context.sourcePackets, context.cachePackets); + + // TODO: Do we still produce the buffer list after a failure? + if (status == kCVReturnSuccess) { + *cachedSourceFrames = context.cachePackets; + TPCircularBufferProduceAudioBufferList(buffer, NULL); + } else { + printf("Error converting buffers: %d\n", status); + } +} + +static inline void AVPlayerAudioTapProduceConvertedFrames(TPCircularBuffer *buffer, + AudioConverterRef converter, + AudioBufferList *bufferListIn, + UInt32 framesIn, + UInt32 channelsOut) { + UInt32 bytesOut = framesIn * channelsOut * 2; + AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); + if (producerBufferList == NULL) { + return; + } + producerBufferList->mBuffers[0].mNumberChannels = channelsOut; + + OSStatus status = AudioConverterConvertComplexBuffer(converter, + framesIn, + bufferListIn, + producerBufferList); + + // TODO: Do we still produce the buffer list after a failure? + if (status == kCVReturnSuccess) { + TPCircularBufferProduceAudioBufferList(buffer, NULL); + } else { + printf("Error converting buffers: %d\n", status); + } +} + +#pragma mark - MTAudioProcessingTap + +void AVPlayerProcessingTapInit(MTAudioProcessingTapRef tap, void *clientInfo, void **tapStorageOut) { + NSLog(@"Init audio tap."); + + // Provide access to our device in the Callbacks. + *tapStorageOut = clientInfo; +} + +void AVPlayerProcessingTapFinalize(MTAudioProcessingTapRef tap) { + NSLog(@"Finalize audio tap."); + + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + context->audioTapPrepared = NO; + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + TPCircularBuffer *renderingBuffer = context->renderingBuffer; + TPCircularBufferCleanup(capturingBuffer); + TPCircularBufferCleanup(renderingBuffer); +} + +void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, + CMItemCount maxFrames, + const AudioStreamBasicDescription *processingFormat) { + NSLog(@"Preparing with frames: %d, channels: %d, bits/channel: %d, sample rate: %0.1f", + (int)maxFrames, processingFormat->mChannelsPerFrame, processingFormat->mBitsPerChannel, processingFormat->mSampleRate); + assert(processingFormat->mFormatID == kAudioFormatLinearPCM); + + // Defer init of the ring buffer memory until we understand the processing format. + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + TPCircularBuffer *renderingBuffer = context->renderingBuffer; + + size_t bufferSize = processingFormat->mBytesPerFrame * maxFrames; + // We need to add some overhead for the AudioBufferList data structures. + bufferSize += 2048; + // TODO: Size the buffer appropriately, as we may need to accumulate more than maxFrames due to bursty processing. + bufferSize *= 20; + + // TODO: If we are re-allocating then check the size? + TPCircularBufferInit(capturingBuffer, bufferSize); + TPCircularBufferInit(renderingBuffer, bufferSize); + dispatch_semaphore_signal(context->capturingInitSemaphore); + dispatch_semaphore_signal(context->renderingInitSemaphore); + + AudioBufferList *cacheBufferList = AudioBufferListCreate(processingFormat, (int)maxFrames); + context->sourceCache = cacheBufferList; + context->sourceCacheFrames = 0; + context->sourceFormat = *processingFormat; + + TVIAudioFormat *playbackFormat = [[TVIAudioFormat alloc] initWithChannels:kPreferredNumberOfChannels + sampleRate:processingFormat->mSampleRate + framesPerBuffer:maxFrames]; + AudioStreamBasicDescription preferredPlaybackDescription = [playbackFormat streamDescription]; + BOOL requiresFormatConversion = preferredPlaybackDescription.mFormatFlags != processingFormat->mFormatFlags; + + if (requiresFormatConversion) { + OSStatus status = AudioConverterNew(processingFormat, &preferredPlaybackDescription, &context->renderFormatConverter); + if (status != 0) { + NSLog(@"Failed to create AudioConverter: %d", (int)status); + return; + } + } + + TVIAudioFormat *recordingFormat = [[TVIAudioFormat alloc] initWithChannels:1 + sampleRate:(Float64)kPreferredSampleRate + framesPerBuffer:maxFrames]; + AudioStreamBasicDescription preferredRecordingDescription = [recordingFormat streamDescription]; + BOOL requiresSampleRateConversion = processingFormat->mSampleRate != preferredRecordingDescription.mSampleRate; + context->capturingSampleRateConversion = requiresSampleRateConversion; + + if (requiresFormatConversion || requiresSampleRateConversion) { + OSStatus status = AudioConverterNew(processingFormat, &preferredRecordingDescription, &context->captureFormatConverter); + if (status != 0) { + NSLog(@"Failed to create AudioConverter: %d", (int)status); + return; + } + UInt32 primingMethod = kConverterPrimeMethod_None; + status = AudioConverterSetProperty(context->captureFormatConverter, kAudioConverterPrimeMethod, + sizeof(UInt32), &primingMethod); + } + + context->audioTapPrepared = YES; + [context->audioDevice audioTapDidPrepare]; +} + +void AVPlayerProcessingTapUnprepare(MTAudioProcessingTapRef tap) { + NSLog(@"Unpreparing audio tap."); + + // Prevent any more frames from being consumed. Note that this might end audio playback early. + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + TPCircularBuffer *renderingBuffer = context->renderingBuffer; + + TPCircularBufferClear(capturingBuffer); + TPCircularBufferClear(renderingBuffer); + if (context->sourceCache) { + AudioBufferListFree(context->sourceCache); + context->sourceCache = NULL; + context->sourceCacheFrames = 0; + } + + if (context->renderFormatConverter != NULL) { + AudioConverterDispose(context->renderFormatConverter); + context->renderFormatConverter = NULL; + } + + if (context->captureFormatConverter != NULL) { + AudioConverterDispose(context->captureFormatConverter); + context->captureFormatConverter = NULL; + } +} + +void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, + CMItemCount numberFrames, + MTAudioProcessingTapFlags flags, + AudioBufferList *bufferListInOut, + CMItemCount *numberFramesOut, + MTAudioProcessingTapFlags *flagsOut) { + ExampleAVPlayerAudioTapContext *context = (ExampleAVPlayerAudioTapContext *)MTAudioProcessingTapGetStorage(tap); + CMTimeRange sourceRange; + OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, + numberFrames, + bufferListInOut, + flagsOut, + &sourceRange, + numberFramesOut); + + if (status != kCVReturnSuccess) { + // TODO + return; + } + + UInt32 framesToCopy = (UInt32)*numberFramesOut; + + // Produce renderer buffers. These are interleaved, signed integer frames in the source's sample rate. + TPCircularBuffer *renderingBuffer = context->renderingBuffer; + AVPlayerAudioTapProduceConvertedFrames(renderingBuffer, context->renderFormatConverter, bufferListInOut, framesToCopy, 2); + + // Produce capturer buffers. We will perform a sample rate conversion if needed. + UInt32 bytesPerFrameOut = 2; + TPCircularBuffer *capturingBuffer = context->capturingBuffer; + if (context->capturingSampleRateConversion) { + AVPlayerAudioTapProduceFilledFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, bytesPerFrameOut); + } else { + AVPlayerAudioTapProduceConvertedFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, framesToCopy, 1); + } + + // Flush converters on a discontinuity. This is especially important for priming a sample rate converter. + if (*flagsOut & kMTAudioProcessingTapFlag_EndOfStream) { + AudioConverterReset(context->renderFormatConverter); + AudioConverterReset(context->captureFormatConverter); + } +} From e631fd40e800d4a76a749c18820501d3bad3e852 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Wed, 7 Nov 2018 23:27:21 -0800 Subject: [PATCH 73/94] WIP - SRC --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 12 ++++- .../ExampleAVPlayerProcessingTap.m | 52 ++++++++++++++----- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 584ed02b..461df4da 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -559,12 +559,20 @@ static OSStatus ExampleAVPlayerAudioDeviceRecordingInputCallback(void *refCon, AudioBufferList playerBufferList; playerBufferList.mNumberBuffers = 1; AudioBuffer *playerAudioBuffer = &playerBufferList.mBuffers[0]; - playerAudioBuffer->mNumberChannels = 1; - playerAudioBuffer->mDataByteSize = (UInt32)numFrames * 2; + playerAudioBuffer->mNumberChannels = kPreferredNumberOfChannels; + playerAudioBuffer->mDataByteSize = (UInt32)numFrames * playerAudioBuffer->mNumberChannels * kAudioSampleSize; playerAudioBuffer->mData = context->audioBuffer; ExampleAVPlayerAudioDeviceDequeueFrames(context->recordingBuffer, numFrames, &playerBufferList); + // Early return to test player audio. + // Deliver the samples (via copying) to WebRTC. + if (context->deviceContext) { + TVIAudioDeviceWriteCaptureData(context->deviceContext, playerAudioBuffer->mData, playerAudioBuffer->mDataByteSize); + return noErr; + } + + // Convert the mono AVPlayer and Microphone sources into a stereo stream. AudioConverterRef converter = context->audioConverter; diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m index 8dc14098..54132d6a 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -64,21 +64,31 @@ OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConvert AudioBufferList *ioData, AudioStreamPacketDescription * _Nullable *outDataPacketDescription, void *inUserData) { + if (*ioNumberDataPackets == 8) { + *ioNumberDataPackets = 0; + return noErr; + } // Give the converter what they asked for. They might not consume all of our source in one callback. UInt32 minimumPackets = *ioNumberDataPackets; ExampleAVPlayerAudioConverterContext *context = inUserData; + assert(context->sourcePackets >= *ioNumberDataPackets); + printf("Convert at least %d input packets. Providing %d packets.\n", *ioNumberDataPackets, context->sourcePackets); + minimumPackets = context->sourcePackets; + *ioNumberDataPackets = context->sourcePackets; AudioBufferList *sourceBufferList = (AudioBufferList *)context->sourceBuffers; AudioBufferList *cacheBufferList = (AudioBufferList *)context->cacheBuffers; assert(sourceBufferList->mNumberBuffers == ioData->mNumberBuffers); UInt32 bytesPerChannel = 4; - printf("Convert at least %d input packets.\n", minimumPackets); for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { - // TODO: What if the cached packets are more than what is requested? - if (context->cachePackets > 0) { + if (context->cachePackets > minimumPackets) { + // TODO: What if the cached packets are more than what is requested? + assert(false); + } else if (context->cachePackets > 0) { // Copy the minimum packets from the source to the back of our cache, and return the continuous samples to the converter. AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; + AudioBuffer *outputBuffer = &ioData->mBuffers[i]; UInt32 sourceFramesToCopy = minimumPackets - context->cachePackets; UInt32 sourceBytesToCopy = sourceFramesToCopy * bytesPerChannel; @@ -87,12 +97,23 @@ OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConvert void *cacheData = cacheBuffer->mData + cachedBytes; memcpy(cacheData, sourceBuffer->mData, sourceBytesToCopy); ioData->mBuffers[i] = *cacheBuffer; + outputBuffer->mNumberChannels = sourceBuffer->mNumberChannels; + outputBuffer->mDataByteSize = minimumPackets * bytesPerChannel; + outputBuffer->mData = cacheBuffer->mData; } else { - ioData->mBuffers[i] = sourceBufferList->mBuffers[i]; +// ioData->mBuffers[i] = sourceBufferList->mBuffers[i]; + UInt32 sourceFrames = minimumPackets; + UInt32 sourceBytes = sourceFrames * bytesPerChannel; + + AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; + AudioBuffer *outputBuffer = &ioData->mBuffers[i]; + outputBuffer->mNumberChannels = sourceBuffer->mNumberChannels; + outputBuffer->mDataByteSize = sourceBytes; + outputBuffer->mData = sourceBuffer->mData; } } - if (minimumPackets < context->sourcePackets) { + if (context->sourcePackets - minimumPackets > 0) { // Copy the remainder of the source which was not used into the front of our cache. UInt32 packetsToCopy = context->sourcePackets - minimumPackets; @@ -107,7 +128,6 @@ OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConvert context->cachePackets = packetsToCopy; } - // *ioNumberDataPackets = inputBufferList->mBuffers[0].mDataByteSize / (UInt32)(4); return noErr; } @@ -117,17 +137,21 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, AudioBufferList *sourceCache, UInt32 *cachedSourceFrames, UInt32 framesIn, - UInt32 bytesPerFrameOut) { + UInt32 channelsOut) { // Start with input buffer size as our argument. // TODO: Does non-interleaving count towards the size (*2)? UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; +// UInt32 desiredIoBufferSize = framesIn * 4; printf("Input is %d bytes (%d frames).\n", desiredIoBufferSize, framesIn); UInt32 propertySizeIo = sizeof(desiredIoBufferSize); AudioConverterGetProperty(converter, kAudioConverterPropertyCalculateOutputBufferSize, &propertySizeIo, &desiredIoBufferSize); - UInt32 framesOut = desiredIoBufferSize / bytesPerFrameOut; + UInt32 bytesPerFrameOut = channelsOut * sizeof(SInt16); + UInt32 framesOut = (desiredIoBufferSize) / bytesPerFrameOut; +// UInt32 framesOut = (desiredIoBufferSize + (bytesPerFrameOut - 1)) / bytesPerFrameOut; +// framesOut += framesOut % 2; UInt32 bytesOut = framesOut * bytesPerFrameOut; printf("Converter wants an output of %d bytes (%d frames, %d bytes per frames).\n", desiredIoBufferSize, framesOut, bytesPerFrameOut); @@ -136,7 +160,7 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, if (producerBufferList == NULL) { return; } - producerBufferList->mBuffers[0].mNumberChannels = bytesPerFrameOut / 2; + producerBufferList->mBuffers[0].mNumberChannels = channelsOut; OSStatus status; UInt32 ioPacketSize = framesOut; @@ -162,6 +186,7 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, if (status == kCVReturnSuccess) { *cachedSourceFrames = context.cachePackets; TPCircularBufferProduceAudioBufferList(buffer, NULL); +// printf("Produced filled buffers!\n"); } else { printf("Error converting buffers: %d\n", status); } @@ -255,7 +280,7 @@ void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, } } - TVIAudioFormat *recordingFormat = [[TVIAudioFormat alloc] initWithChannels:1 + TVIAudioFormat *recordingFormat = [[TVIAudioFormat alloc] initWithChannels:2 sampleRate:(Float64)kPreferredSampleRate framesPerBuffer:maxFrames]; AudioStreamBasicDescription preferredRecordingDescription = [recordingFormat streamDescription]; @@ -268,7 +293,7 @@ void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, NSLog(@"Failed to create AudioConverter: %d", (int)status); return; } - UInt32 primingMethod = kConverterPrimeMethod_None; + UInt32 primingMethod = kConverterPrimeMethod_Normal; status = AudioConverterSetProperty(context->captureFormatConverter, kAudioConverterPrimeMethod, sizeof(UInt32), &primingMethod); } @@ -331,12 +356,11 @@ void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, AVPlayerAudioTapProduceConvertedFrames(renderingBuffer, context->renderFormatConverter, bufferListInOut, framesToCopy, 2); // Produce capturer buffers. We will perform a sample rate conversion if needed. - UInt32 bytesPerFrameOut = 2; TPCircularBuffer *capturingBuffer = context->capturingBuffer; if (context->capturingSampleRateConversion) { - AVPlayerAudioTapProduceFilledFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, bytesPerFrameOut); + AVPlayerAudioTapProduceFilledFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, kPreferredNumberOfChannels); } else { - AVPlayerAudioTapProduceConvertedFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, framesToCopy, 1); + AVPlayerAudioTapProduceConvertedFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, framesToCopy, kPreferredNumberOfChannels); } // Flush converters on a discontinuity. This is especially important for priming a sample rate converter. From 65a42bb3e7b675214061c221c8fb276114ba3ed7 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 14:25:29 -0800 Subject: [PATCH 74/94] =?UTF-8?q?Use=2048khz=20sample=20content=20that=20d?= =?UTF-8?q?oesn=E2=80=99t=20need=20resampling.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CoViewingExample/ViewController.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index e201dd91..63582c4f 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -73,7 +73,7 @@ class ViewController: UIViewController { "Telecom ParisTech, GPAC (1080p30)" : URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_1080p30_6M.mp4")!, "Twilio: What is Cloud Communications? (1080p24, 44.1 kHz)" : URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! ] - static let kRemoteContentURL = kRemoteContentUrls["Interstellar Trailer 3 (720p24, 44.1 kHz)"]! + static let kRemoteContentURL = kRemoteContentUrls["BitDash - Parkour (1080p25, 48 kHz)"]! override func viewDidLoad() { super.viewDidLoad() From c7275b8b294a189cad93339de9ebb16647ed486f Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 15:12:55 -0800 Subject: [PATCH 75/94] Working sample rate conversion to 48 kHz. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 2 - .../ExampleAVPlayerProcessingTap.h | 3 +- .../ExampleAVPlayerProcessingTap.m | 93 ++++++++++--------- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 461df4da..ddf2b0d9 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -94,9 +94,7 @@ - (id)init { _audioTapContext = calloc(1, sizeof(ExampleAVPlayerAudioTapContext)); _audioTapContext->capturingBuffer = _audioTapCapturingBuffer; - _audioTapContext->capturingInitSemaphore = _audioTapCapturingSemaphore; _audioTapContext->renderingBuffer = _audioTapRenderingBuffer; - _audioTapContext->renderingInitSemaphore = _audioTapRenderingSemaphore; _audioTapContext->audioDevice = self; _audioTapContext->audioTapPrepared = NO; } diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h index 4d60fe6b..a997f450 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h @@ -16,12 +16,11 @@ typedef struct ExampleAVPlayerAudioTapContext { TPCircularBuffer *capturingBuffer; AudioConverterRef captureFormatConverter; - dispatch_semaphore_t capturingInitSemaphore; BOOL capturingSampleRateConversion; + BOOL captureFormatConvertIsPrimed; TPCircularBuffer *renderingBuffer; AudioConverterRef renderFormatConverter; - dispatch_semaphore_t renderingInitSemaphore; // Cached source audio, in case we need to perform a sample rate conversion and can't consume all the samples in one go. AudioBufferList *sourceCache; diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m index 54132d6a..5b11b666 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -19,6 +19,7 @@ AudioBufferList *sourceBuffers; // Keep track if we are iterating through the source to provide data to a converter. UInt32 sourcePackets; + UInt32 sourcePacketIndex; } ExampleAVPlayerAudioConverterContext; AudioBufferList *AudioBufferListCreate(const AudioStreamBasicDescription *audioFormat, int frameCount) { @@ -64,44 +65,30 @@ OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConvert AudioBufferList *ioData, AudioStreamPacketDescription * _Nullable *outDataPacketDescription, void *inUserData) { - if (*ioNumberDataPackets == 8) { - *ioNumberDataPackets = 0; - return noErr; - } + UInt32 bytesPerChannel = 4; + // Give the converter what they asked for. They might not consume all of our source in one callback. UInt32 minimumPackets = *ioNumberDataPackets; ExampleAVPlayerAudioConverterContext *context = inUserData; - assert(context->sourcePackets >= *ioNumberDataPackets); - printf("Convert at least %d input packets. Providing %d packets.\n", *ioNumberDataPackets, context->sourcePackets); - minimumPackets = context->sourcePackets; - *ioNumberDataPackets = context->sourcePackets; + + assert(context->sourcePackets + context->cachePackets >= *ioNumberDataPackets); + printf("Convert at least %d input packets. We have %d source packets, %d cached packets.\n", *ioNumberDataPackets, context->sourcePackets, context->cachePackets); +// minimumPackets = context->sourcePackets; AudioBufferList *sourceBufferList = (AudioBufferList *)context->sourceBuffers; AudioBufferList *cacheBufferList = (AudioBufferList *)context->cacheBuffers; assert(sourceBufferList->mNumberBuffers == ioData->mNumberBuffers); - UInt32 bytesPerChannel = 4; for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { - if (context->cachePackets > minimumPackets) { - // TODO: What if the cached packets are more than what is requested? - assert(false); - } else if (context->cachePackets > 0) { - // Copy the minimum packets from the source to the back of our cache, and return the continuous samples to the converter. + if (context->cachePackets > 0) { AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; - AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; AudioBuffer *outputBuffer = &ioData->mBuffers[i]; - - UInt32 sourceFramesToCopy = minimumPackets - context->cachePackets; - UInt32 sourceBytesToCopy = sourceFramesToCopy * bytesPerChannel; UInt32 cachedBytes = context->cachePackets * bytesPerChannel; - assert(sourceBytesToCopy <= cacheBuffer->mDataByteSize - cachedBytes); - void *cacheData = cacheBuffer->mData + cachedBytes; - memcpy(cacheData, sourceBuffer->mData, sourceBytesToCopy); - ioData->mBuffers[i] = *cacheBuffer; - outputBuffer->mNumberChannels = sourceBuffer->mNumberChannels; - outputBuffer->mDataByteSize = minimumPackets * bytesPerChannel; + UInt32 cachedFrames = context->cachePackets; + outputBuffer->mNumberChannels = cacheBuffer->mNumberChannels; + outputBuffer->mDataByteSize = cachedBytes; outputBuffer->mData = cacheBuffer->mData; + *ioNumberDataPackets = cachedFrames; } else { -// ioData->mBuffers[i] = sourceBufferList->mBuffers[i]; UInt32 sourceFrames = minimumPackets; UInt32 sourceBytes = sourceFrames * bytesPerChannel; @@ -109,30 +96,37 @@ OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConvert AudioBuffer *outputBuffer = &ioData->mBuffers[i]; outputBuffer->mNumberChannels = sourceBuffer->mNumberChannels; outputBuffer->mDataByteSize = sourceBytes; - outputBuffer->mData = sourceBuffer->mData; + outputBuffer->mData = sourceBuffer->mData + (context->sourcePacketIndex * bytesPerChannel * sourceBuffer->mNumberChannels); } } - if (context->sourcePackets - minimumPackets > 0) { - // Copy the remainder of the source which was not used into the front of our cache. - - UInt32 packetsToCopy = context->sourcePackets - minimumPackets; - for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { - AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; - AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; - assert(cacheBuffer->mDataByteSize >= sourceBuffer->mDataByteSize); - UInt32 bytesToCopy = packetsToCopy * bytesPerChannel; - void *sourceData = sourceBuffer->mData + (minimumPackets * bytesPerChannel); - memcpy(cacheBuffer->mData, sourceData, bytesToCopy); - } - context->cachePackets = packetsToCopy; + if (context->cachePackets > 0) { + context->cachePackets = 0; + } else { + context->sourcePacketIndex += *ioNumberDataPackets; } +// if (context->sourcePackets - minimumPackets > 0) { +// // Copy the remainder of the source which was not used into the front of our cache. +// +// UInt32 packetsToCopy = context->sourcePackets - minimumPackets; +// for (UInt32 i = 0; i < sourceBufferList->mNumberBuffers; i++) { +// AudioBuffer *cacheBuffer = &cacheBufferList->mBuffers[i]; +// AudioBuffer *sourceBuffer = &sourceBufferList->mBuffers[i]; +// assert(cacheBuffer->mDataByteSize >= sourceBuffer->mDataByteSize); +// UInt32 bytesToCopy = packetsToCopy * bytesPerChannel; +// void *sourceData = sourceBuffer->mData + (minimumPackets * bytesPerChannel); +// memcpy(cacheBuffer->mData, sourceData, bytesToCopy); +// } +// context->cachePackets = packetsToCopy; +// } + return noErr; } static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, AudioConverterRef converter, + BOOL isConverterPrimed, AudioBufferList *bufferListIn, AudioBufferList *sourceCache, UInt32 *cachedSourceFrames, @@ -140,9 +134,17 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, UInt32 channelsOut) { // Start with input buffer size as our argument. // TODO: Does non-interleaving count towards the size (*2)? + // Give us a little more priming than we need (~8 frames). + UInt32 primeFrames = 8; + UInt32 sourceFrames = framesIn; + if (!isConverterPrimed) { + framesIn -= primeFrames; + } else if (*cachedSourceFrames > 0) { + framesIn += *cachedSourceFrames; + } UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; // UInt32 desiredIoBufferSize = framesIn * 4; - printf("Input is %d bytes (%d frames).\n", desiredIoBufferSize, framesIn); + printf("Input is %d bytes (%d total frames, %d cached frames).\n", desiredIoBufferSize, framesIn, *cachedSourceFrames); UInt32 propertySizeIo = sizeof(desiredIoBufferSize); AudioConverterGetProperty(converter, kAudioConverterPropertyCalculateOutputBufferSize, @@ -169,7 +171,8 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, ExampleAVPlayerAudioConverterContext context; context.sourceBuffers = bufferListIn; context.cacheBuffers = sourceCache; - context.sourcePackets = framesIn; + context.sourcePackets = sourceFrames; + context.sourcePacketIndex = 0; context.cachePackets = *cachedSourceFrames; status = AudioConverterFillComplexBuffer(converter, AVPlayerAudioTapConverterInputDataProc, @@ -186,7 +189,6 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, if (status == kCVReturnSuccess) { *cachedSourceFrames = context.cachePackets; TPCircularBufferProduceAudioBufferList(buffer, NULL); -// printf("Produced filled buffers!\n"); } else { printf("Error converting buffers: %d\n", status); } @@ -258,8 +260,6 @@ void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, // TODO: If we are re-allocating then check the size? TPCircularBufferInit(capturingBuffer, bufferSize); TPCircularBufferInit(renderingBuffer, bufferSize); - dispatch_semaphore_signal(context->capturingInitSemaphore); - dispatch_semaphore_signal(context->renderingInitSemaphore); AudioBufferList *cacheBufferList = AudioBufferListCreate(processingFormat, (int)maxFrames); context->sourceCache = cacheBufferList; @@ -326,6 +326,7 @@ void AVPlayerProcessingTapUnprepare(MTAudioProcessingTapRef tap) { if (context->captureFormatConverter != NULL) { AudioConverterDispose(context->captureFormatConverter); context->captureFormatConverter = NULL; + context->captureFormatConvertIsPrimed = NO; } } @@ -358,7 +359,8 @@ void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, // Produce capturer buffers. We will perform a sample rate conversion if needed. TPCircularBuffer *capturingBuffer = context->capturingBuffer; if (context->capturingSampleRateConversion) { - AVPlayerAudioTapProduceFilledFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, kPreferredNumberOfChannels); + AVPlayerAudioTapProduceFilledFrames(capturingBuffer, context->captureFormatConverter, context->captureFormatConvertIsPrimed, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, kPreferredNumberOfChannels); + context->captureFormatConvertIsPrimed = YES; } else { AVPlayerAudioTapProduceConvertedFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, framesToCopy, kPreferredNumberOfChannels); } @@ -367,5 +369,6 @@ void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, if (*flagsOut & kMTAudioProcessingTapFlag_EndOfStream) { AudioConverterReset(context->renderFormatConverter); AudioConverterReset(context->captureFormatConverter); + context->captureFormatConvertIsPrimed = NO; } } From c5294b4b39f818d1ee7d9c40da6202a32aceb922 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 15:35:17 -0800 Subject: [PATCH 76/94] Use a 20 millisecond duration for all example TVIAudioDevices. --- .../AudioDevices/ExampleAVAudioEngineDevice.m | 6 +++--- AudioDeviceExample/AudioDevices/ExampleCoreAudioDevice.m | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/AudioDeviceExample/AudioDevices/ExampleAVAudioEngineDevice.m b/AudioDeviceExample/AudioDevices/ExampleAVAudioEngineDevice.m index 4c02ab1f..41efd06f 100644 --- a/AudioDeviceExample/AudioDevices/ExampleAVAudioEngineDevice.m +++ b/AudioDeviceExample/AudioDevices/ExampleAVAudioEngineDevice.m @@ -7,8 +7,8 @@ #import "ExampleAVAudioEngineDevice.h" -// We want to get as close to 10 msec buffers as possible because this is what the media engine prefers. -static double const kPreferredIOBufferDuration = 0.01; +// We want to get as close to 20 millisecond buffers as possible because this is what the media engine prefers. +static double const kPreferredIOBufferDuration = 0.02; // We will use mono playback and recording where available. static size_t const kPreferredNumberOfChannels = 1; @@ -558,7 +558,7 @@ - (void)setupAVAudioSession { } /* - * We want to be as close as possible to the 10 millisecond buffer size that the media engine needs. If there is + * We will operate our graph at roughly double the duration that the media engine natively operates in. If there is * a mismatch then TwilioVideo will ensure that appropriately sized audio buffers are delivered. */ if (![session setPreferredIOBufferDuration:kPreferredIOBufferDuration error:&error]) { diff --git a/AudioDeviceExample/AudioDevices/ExampleCoreAudioDevice.m b/AudioDeviceExample/AudioDevices/ExampleCoreAudioDevice.m index b96a83ed..fadb1413 100644 --- a/AudioDeviceExample/AudioDevices/ExampleCoreAudioDevice.m +++ b/AudioDeviceExample/AudioDevices/ExampleCoreAudioDevice.m @@ -7,8 +7,8 @@ #import "ExampleCoreAudioDevice.h" -// We want to get as close to 10 msec buffers as possible because this is what the media engine prefers. -static double const kPreferredIOBufferDuration = 0.01; +// We want to get as close to 20 msec buffers as possible because this is what the media engine prefers. +static double const kPreferredIOBufferDuration = 0.02; // We will use stereo playback where available. Some audio routes may be restricted to mono only. static size_t const kPreferredNumberOfChannels = 2; // An audio sample is a signed 16-bit integer. @@ -245,7 +245,7 @@ - (void)setupAVAudioSession { } /* - * We want to be as close as possible to the 10 millisecond buffer size that the media engine needs. If there is + * We will operate our graph at roughly double the duration that the media engine natively operates in. If there is * a mismatch then TwilioVideo will ensure that appropriately sized audio buffers are delivered. */ if (![session setPreferredIOBufferDuration:kPreferredIOBufferDuration error:&error]) { From 6659a3eb38927ace37ba2b4b2620f175a5b82c74 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 15:37:20 -0800 Subject: [PATCH 77/94] Use more bandwidth for presenter audio, restore 44.1 kHz content. --- CoViewingExample/ViewController.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 63582c4f..d9b848b0 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -73,7 +73,7 @@ class ViewController: UIViewController { "Telecom ParisTech, GPAC (1080p30)" : URL(string: "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_1080p30_6M.mp4")!, "Twilio: What is Cloud Communications? (1080p24, 44.1 kHz)" : URL(string: "https://s3-us-west-1.amazonaws.com/avplayervideo/What+Is+Cloud+Communications.mov")! ] - static let kRemoteContentURL = kRemoteContentUrls["BitDash - Parkour (1080p25, 48 kHz)"]! + static let kRemoteContentURL = kRemoteContentUrls["Mississippi Grind (720p24, 44.1 kHz)"]! override func viewDidLoad() { super.viewDidLoad() @@ -223,9 +223,11 @@ class ViewController: UIViewController { // Room `name`, the Client will create one for you. You can get the name or sid from any connected Room. builder.roomName = "twilio" - // Restrict video bandwidth used by viewers to improve presenter video. + // Restrict video bandwidth used by viewers to improve presenter video. Use more bandwidth for presenter audio. if name == "viewer" { builder.encodingParameters = TVIEncodingParameters(audioBitrate: 0, videoBitrate: 1024 * 900) + } else { + builder.encodingParameters = TVIEncodingParameters(audioBitrate: 1024 * 96, videoBitrate: 0) } } From 6b70a34e80dcc57399f2b55082bb815d001f1786 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 15:40:33 -0800 Subject: [PATCH 78/94] Comment out printf statements in realtime code. --- .../AudioDevices/ExampleAVPlayerProcessingTap.m | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m index 5b11b666..f64300e5 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -72,7 +72,7 @@ OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConvert ExampleAVPlayerAudioConverterContext *context = inUserData; assert(context->sourcePackets + context->cachePackets >= *ioNumberDataPackets); - printf("Convert at least %d input packets. We have %d source packets, %d cached packets.\n", *ioNumberDataPackets, context->sourcePackets, context->cachePackets); +// printf("Convert at least %d input packets. We have %d source packets, %d cached packets.\n", *ioNumberDataPackets, context->sourcePackets, context->cachePackets); // minimumPackets = context->sourcePackets; AudioBufferList *sourceBufferList = (AudioBufferList *)context->sourceBuffers; AudioBufferList *cacheBufferList = (AudioBufferList *)context->cacheBuffers; @@ -143,8 +143,7 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, framesIn += *cachedSourceFrames; } UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; -// UInt32 desiredIoBufferSize = framesIn * 4; - printf("Input is %d bytes (%d total frames, %d cached frames).\n", desiredIoBufferSize, framesIn, *cachedSourceFrames); +// printf("Input is %d bytes (%d total frames, %d cached frames).\n", desiredIoBufferSize, framesIn, *cachedSourceFrames); UInt32 propertySizeIo = sizeof(desiredIoBufferSize); AudioConverterGetProperty(converter, kAudioConverterPropertyCalculateOutputBufferSize, @@ -155,8 +154,8 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, // UInt32 framesOut = (desiredIoBufferSize + (bytesPerFrameOut - 1)) / bytesPerFrameOut; // framesOut += framesOut % 2; UInt32 bytesOut = framesOut * bytesPerFrameOut; - printf("Converter wants an output of %d bytes (%d frames, %d bytes per frames).\n", - desiredIoBufferSize, framesOut, bytesPerFrameOut); +// printf("Converter wants an output of %d bytes (%d frames, %d bytes per frames).\n", +// desiredIoBufferSize, framesOut, bytesPerFrameOut); AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); if (producerBufferList == NULL) { @@ -166,8 +165,8 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, OSStatus status; UInt32 ioPacketSize = framesOut; - printf("Ready to fill output buffer of frames: %d, bytes: %d with input buffer of frames: %d, bytes: %d.\n", - framesOut, bytesOut, framesIn, framesIn * 4 * bufferListIn->mNumberBuffers); +// printf("Ready to fill output buffer of frames: %d, bytes: %d with input buffer of frames: %d, bytes: %d.\n", +// framesOut, bytesOut, framesIn, framesIn * 4 * bufferListIn->mNumberBuffers); ExampleAVPlayerAudioConverterContext context; context.sourceBuffers = bufferListIn; context.cacheBuffers = sourceCache; @@ -182,8 +181,8 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, NULL); // Adjust for what the format converter actually produced, in case it was different than what we asked for. producerBufferList->mBuffers[0].mDataByteSize = ioPacketSize * bytesPerFrameOut; - printf("Output was: %d packets / %d bytes. Consumed input packets: %d. Cached input packets: %d.\n", - ioPacketSize, ioPacketSize * bytesPerFrameOut, context.sourcePackets, context.cachePackets); +// printf("Output was: %d packets / %d bytes. Consumed input packets: %d. Cached input packets: %d.\n", +// ioPacketSize, ioPacketSize * bytesPerFrameOut, context.sourcePackets, context.cachePackets); // TODO: Do we still produce the buffer list after a failure? if (status == kCVReturnSuccess) { From a51753158314fdde2609bb2550e08c6322d96d2e Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 20:25:27 -0800 Subject: [PATCH 79/94] Review feedback - document important audio device properties. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index ddf2b0d9..15f605c9 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -53,15 +53,56 @@ @interface ExampleAVPlayerAudioDevice() +/** + Indicates that our AVAudioSession and audio graph have been interrupted. When an interruption ends we will take steps + to restart our audio graph. + */ @property (nonatomic, assign, getter=isInterrupted) BOOL interrupted; + +/** + A multi-channel mixer which takes as input: + + 1. Decoded LPCM audio from Twilio. Remote audio is downmixed to `renderingFormat` by the media engine. + 2. Decoded, format converted LPCM audio from our MTAudioProcessingTap. + + The mixer's output is connected to the input of the VoiceProcessingIO's output bus. + */ @property (nonatomic, assign) AudioUnit playbackMixer; + +/** + A VoiceProcessingIO audio unit which performs several important functions. + + Input Graph + 1. Record from the microphone. + 2. Echo cancellation of the loudspeaker output from the microphone input. + 3. Deliver mixed, recorded samples from the microphone and AVPlayer to Twilio. + + Output Graph + 1. Pull audio from the output of `playbackMixer`. + + The mixer's output is connected to the input of the VoiceProcessingIO's output bus. + */ @property (nonatomic, assign) AudioUnit voiceProcessingIO; + +/** + The tap used to access audio samples from AVPlayer. This is where we produce audio for playback and recording. + */ @property (nonatomic, assign, nullable) MTAudioProcessingTapRef audioTap; + +/** + A context which contains the state needed for the processing tap's C functions. + */ @property (nonatomic, assign, nullable) ExampleAVPlayerAudioTapContext *audioTapContext; -@property (nonatomic, strong, nullable) dispatch_semaphore_t audioTapCapturingSemaphore; + +/** + A circular buffer used to feed the recording side of the audio graph with frames produced by our processing tap. + */ @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapCapturingBuffer; -@property (nonatomic, strong, nullable) dispatch_semaphore_t audioTapRenderingSemaphore; + +/** + A circular buffer used to feed the playback side of the audio graph with frames produced by our processing tap. + */ @property (nonatomic, assign, nullable) TPCircularBuffer *audioTapRenderingBuffer; @property (nonatomic, assign) AudioConverterRef captureConverter; @@ -70,8 +111,22 @@ @interface ExampleAVPlayerAudioDevice() @property (nonatomic, assign, nullable) ExampleAVPlayerCapturerContext *capturingContext; @property (atomic, assign, nullable) ExampleAVPlayerRendererContext *renderingContext; @property (nonatomic, strong, nullable) TVIAudioFormat *renderingFormat; + +/** + A convenience getter that indicates if either `wantsCapturing` or `wantsRendering` are true. + */ @property (nonatomic, assign, readonly) BOOL wantsAudio; + +/** + Indicates that our audio device has been requested to capture audio by Twilio. Capturing occurs when you publish + a TVILocalAudioTrack in a Group Room, or a Peer-to-Peer Room with 1 or more Participant. + */ @property (nonatomic, assign) BOOL wantsCapturing; + +/** + Indicates that our audio device has been requested to render audio by Twilio. Rendering occurs when one or more Remote + Participants publish a TVIRemoteAudioTrack in a Room. + */ @property (nonatomic, assign) BOOL wantsRendering; @end From b9a2c6cd5e91096bd99acc0904ab4a2627164356 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 20:27:46 -0800 Subject: [PATCH 80/94] Remove dispatch_semaphore, address ASBD naming. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 24 +++++++++---------- .../ExampleAVPlayerProcessingTap.m | 1 - 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 15f605c9..7fd0e0cc 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -142,8 +142,6 @@ - (id)init { if (self) { _audioTapCapturingBuffer = calloc(1, sizeof(TPCircularBuffer)); _audioTapRenderingBuffer = calloc(1, sizeof(TPCircularBuffer)); - _audioTapCapturingSemaphore = dispatch_semaphore_create(0); - _audioTapRenderingSemaphore = dispatch_semaphore_create(0); _wantsCapturing = NO; _wantsRendering = NO; @@ -755,20 +753,20 @@ - (void)setupAVAudioSession { } - (AudioStreamBasicDescription)microphoneInputStreamDescription { - AudioStreamBasicDescription capturingFormatDescription = self.capturingFormat.streamDescription; - capturingFormatDescription.mBytesPerFrame = 2; - capturingFormatDescription.mBytesPerPacket = 2; - capturingFormatDescription.mChannelsPerFrame = 1; - return capturingFormatDescription; + AudioStreamBasicDescription formatDescription = self.capturingFormat.streamDescription; + formatDescription.mBytesPerFrame = 2; + formatDescription.mBytesPerPacket = 2; + formatDescription.mChannelsPerFrame = 1; + return formatDescription; } - (AudioStreamBasicDescription)nonInterleavedStereoStreamDescription { - AudioStreamBasicDescription capturingFormatDescription = self.capturingFormat.streamDescription; - capturingFormatDescription.mBytesPerFrame = 2; - capturingFormatDescription.mBytesPerPacket = 2; - capturingFormatDescription.mChannelsPerFrame = 2; - capturingFormatDescription.mFormatFlags |= kAudioFormatFlagIsNonInterleaved; - return capturingFormatDescription; + AudioStreamBasicDescription formatDescription = self.capturingFormat.streamDescription; + formatDescription.mBytesPerFrame = 2; + formatDescription.mBytesPerPacket = 2; + formatDescription.mChannelsPerFrame = 2; + formatDescription.mFormatFlags |= kAudioFormatFlagIsNonInterleaved; + return formatDescription; } - (OSStatus)setupAudioCapturer:(ExampleAVPlayerCapturerContext *)capturerContext { diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m index f64300e5..9e258716 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -73,7 +73,6 @@ OSStatus AVPlayerAudioTapConverterInputDataProc(AudioConverterRef inAudioConvert assert(context->sourcePackets + context->cachePackets >= *ioNumberDataPackets); // printf("Convert at least %d input packets. We have %d source packets, %d cached packets.\n", *ioNumberDataPackets, context->sourcePackets, context->cachePackets); -// minimumPackets = context->sourcePackets; AudioBufferList *sourceBufferList = (AudioBufferList *)context->sourceBuffers; AudioBufferList *cacheBufferList = (AudioBufferList *)context->cacheBuffers; assert(sourceBufferList->mNumberBuffers == ioData->mNumberBuffers); From 1def76fb314e57f7df0b13aaace84aee5a4c2424 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 20:42:12 -0800 Subject: [PATCH 81/94] Use the regular token server URL. --- CoViewingExample/ExampleAVPlayerSource.swift | 4 ++-- CoViewingExample/ViewController.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index 53e60b66..b9a407f1 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -89,7 +89,7 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { itemTimeForDisplay: &presentationTimestamp) if let buffer = pixelBuffer { if let lastTime = lastPresentationTimestamp { - // TODO: Use this info to target our DispatchSource timestamps? + // TODO: Use this info for 3:2 pulldown to re-construct the proper timestamps without display cadence? // let delta = presentationTimestamp - lastTime // print("Frame delta was:", delta) // let movieTime = CVBufferGetAttachment(buffer, kCVBufferMovieTimeKey, nil) @@ -229,6 +229,6 @@ extension ExampleAVPlayerSource: AVPlayerItemOutputPullDelegate { func outputSequenceWasFlushed(_ output: AVPlayerItemOutput) { print(#function) - // TODO: Flush and output a black frame while we wait. + // TODO: Flush and output a black frame while we wait? } } diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index d9b848b0..c93925d4 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -17,7 +17,7 @@ class ViewController: UIViewController { var accessToken = "TWILIO_ACCESS_TOKEN" // Configure remote URL to fetch token from - var tokenUrl = "https://username:passowrd@simple-signaling.appspot.com/access-token" + var tokenUrl = "http://localhost:8000/token.php" // Video SDK components var room: TVIRoom? From 1a2628110d6ca5379f659824084edd3cfea3c296 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 20:44:41 -0800 Subject: [PATCH 82/94] Remove commented KVO code. --- CoViewingExample/ViewController.swift | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index c93925d4..1bc152b8 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -408,14 +408,6 @@ class ViewController: UIViewController { of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) { - // Only handle observations for the playerItemContext -// if object != videoPlayer?.currentItem { -// super.observeValue(forKeyPath: keyPath, -// of: object, -// change: change, -// context: context) -// return -// } if keyPath == #keyPath(AVPlayerItem.status) { let status: AVPlayerItem.Status From 9c51e003e48be5c7c8746c85d8681075482bd4e8 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 22:22:42 -0800 Subject: [PATCH 83/94] Explicitly request an IOSurface. --- CoViewingExample/ExampleAVPlayerSource.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index b9a407f1..df1e64b6 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -37,12 +37,10 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { print("Prepare for player item with size:", presentationSize, " pixels:", presentationPixels); /* - * We might request buffers downscaled for streaming. The output will be NV12, and backed by an IOSurface - * even though we dont explicitly include kCVPixelBufferIOSurfacePropertiesKey. + * We might request buffers downscaled for streaming. The output will be 8-bit 4:2:0 NV12. */ let attributes: [String : Any] - // TODO: We need to interrogate the content and choose our range (video/full) appropriately. if (presentationSize.width > ExampleAVPlayerSource.kFrameOutputMaxDimension || presentationSize.height > ExampleAVPlayerSource.kFrameOutputMaxDimension) { let streamingRect = AVMakeRect(aspectRatio: presentationSize, insideRect: ExampleAVPlayerSource.kFrameOutputMaxRect) @@ -51,10 +49,12 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { attributes = [ kCVPixelBufferWidthKey as String : Int(streamingRect.width), kCVPixelBufferHeightKey as String : Int(streamingRect.height), + kCVPixelBufferIOSurfacePropertiesKey as String : [ : ], kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ] as [String : Any] } else { attributes = [ + kCVPixelBufferIOSurfacePropertiesKey as String : [ : ], kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ] as [String : Any] } From 7156ed2e1809b85f57f5406fb26494bce0281b32 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Fri, 9 Nov 2018 22:24:58 -0800 Subject: [PATCH 84/94] Dynamically create and destroy remotePlayerView. --- CoViewingExample/Base.lproj/Main.storyboard | 20 ++---- CoViewingExample/ViewController.swift | 76 ++++++++++++++------- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/CoViewingExample/Base.lproj/Main.storyboard b/CoViewingExample/Base.lproj/Main.storyboard index 3c5e6f10..ae88ff9a 100644 --- a/CoViewingExample/Base.lproj/Main.storyboard +++ b/CoViewingExample/Base.lproj/Main.storyboard @@ -1,11 +1,11 @@ - + - + @@ -18,10 +18,6 @@ - - - - - + @@ -71,7 +71,7 @@ - + @@ -81,20 +81,20 @@ - + - + - + - + - - + + From 3555d2d97c55ecd398a5ed25f0a51a33ca17057a Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 11 Nov 2018 20:05:33 -0800 Subject: [PATCH 88/94] Comments and disable AVAudioMix update code. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 60 +++++++++---------- .../ExampleAVPlayerProcessingTap.m | 21 +++++-- CoViewingExample/ViewController.swift | 2 +- 3 files changed, 46 insertions(+), 37 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 7fd0e0cc..0f41905d 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -62,8 +62,8 @@ @interface ExampleAVPlayerAudioDevice() /** A multi-channel mixer which takes as input: - 1. Decoded LPCM audio from Twilio. Remote audio is downmixed to `renderingFormat` by the media engine. - 2. Decoded, format converted LPCM audio from our MTAudioProcessingTap. + 1. Decoded LPCM audio from Twilio. Remote audio is mixed and pulled from the media engine in `renderingFormat`. + 2. Decoded, format converted LPCM audio consumed from our MTAudioProcessingTap. The mixer's output is connected to the input of the VoiceProcessingIO's output bus. */ @@ -216,34 +216,6 @@ - (void)audioTapDidPrepare { } } -- (void)restartAudioUnit { - BOOL restart = NO; - @synchronized (self) { - if (self.wantsAudio) { - restart = YES; - [self stopAudioUnit]; - [self teardownAudioUnit]; - if (self.renderingContext) { - self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; - } - if (self.capturingContext) { - self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; - } - if ([self setupAudioUnitRendererContext:self.renderingContext - capturerContext:self.capturingContext]) { - if (self.capturingContext) { - self.capturingContext->audioUnit = _voiceProcessingIO; - self.capturingContext->audioConverter = _captureConverter; - } - } else { - return; - } - } - } - - [self startAudioUnit]; -} - - (MTAudioProcessingTapRef)createProcessingTap { if (_audioTap) { return _audioTap; @@ -1030,6 +1002,34 @@ - (void)teardownAudioUnit { } } +- (void)restartAudioUnit { + BOOL restart = NO; + @synchronized (self) { + if (self.wantsAudio) { + restart = YES; + [self stopAudioUnit]; + [self teardownAudioUnit]; + if (self.renderingContext) { + self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; + } + if (self.capturingContext) { + self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; + } + if ([self setupAudioUnitRendererContext:self.renderingContext + capturerContext:self.capturingContext]) { + if (self.capturingContext) { + self.capturingContext->audioUnit = _voiceProcessingIO; + self.capturingContext->audioConverter = _captureConverter; + } + } else { + return; + } + } + } + + [self startAudioUnit]; +} + #pragma mark - NSNotification Observers - (void)registerAVAudioSessionObservers { diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m index 9e258716..748a7439 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -144,9 +144,9 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; // printf("Input is %d bytes (%d total frames, %d cached frames).\n", desiredIoBufferSize, framesIn, *cachedSourceFrames); UInt32 propertySizeIo = sizeof(desiredIoBufferSize); - AudioConverterGetProperty(converter, - kAudioConverterPropertyCalculateOutputBufferSize, - &propertySizeIo, &desiredIoBufferSize); + status = AudioConverterGetProperty(converter, + kAudioConverterPropertyCalculateOutputBufferSize, + &propertySizeIo, &desiredIoBufferSize); UInt32 bytesPerFrameOut = channelsOut * sizeof(SInt16); UInt32 framesOut = (desiredIoBufferSize) / bytesPerFrameOut; @@ -162,7 +162,6 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, } producerBufferList->mBuffers[0].mNumberChannels = channelsOut; - OSStatus status; UInt32 ioPacketSize = framesOut; // printf("Ready to fill output buffer of frames: %d, bytes: %d with input buffer of frames: %d, bytes: %d.\n", // framesOut, bytesOut, framesIn, framesIn * 4 * bufferListIn->mNumberBuffers); @@ -357,10 +356,20 @@ void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, // Produce capturer buffers. We will perform a sample rate conversion if needed. TPCircularBuffer *capturingBuffer = context->capturingBuffer; if (context->capturingSampleRateConversion) { - AVPlayerAudioTapProduceFilledFrames(capturingBuffer, context->captureFormatConverter, context->captureFormatConvertIsPrimed, bufferListInOut, context->sourceCache, &context->sourceCacheFrames, framesToCopy, kPreferredNumberOfChannels); + AVPlayerAudioTapProduceFilledFrames(capturingBuffer, + context->captureFormatConverter, + context->captureFormatConvertIsPrimed, + bufferListInOut, context->sourceCache, + &context->sourceCacheFrames, + framesToCopy, + kPreferredNumberOfChannels); context->captureFormatConvertIsPrimed = YES; } else { - AVPlayerAudioTapProduceConvertedFrames(capturingBuffer, context->captureFormatConverter, bufferListInOut, framesToCopy, kPreferredNumberOfChannels); + AVPlayerAudioTapProduceConvertedFrames(capturingBuffer, + context->captureFormatConverter, + bufferListInOut, + framesToCopy, + kPreferredNumberOfChannels); } // Flush converters on a discontinuity. This is especially important for priming a sample rate converter. diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 00e39825..7b8ecd24 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -463,7 +463,7 @@ class ViewController: UIViewController { setupAudioMix(player: videoPlayer!, playerItem: playerItem) } else { // TODO: Possibly update the existing mix? - updateAudioMixParameters(playerItem: playerItem) +// updateAudioMixParameters(playerItem: playerItem) } } } From f924c8ebff440dbc6474d37064bc33fbf8b703a0 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 11 Nov 2018 20:14:13 -0800 Subject: [PATCH 89/94] Produce buffers with timestamps for playback. --- .../ExampleAVPlayerProcessingTap.m | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m index 748a7439..f0eb0c2c 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -144,9 +144,9 @@ static inline void AVPlayerAudioTapProduceFilledFrames(TPCircularBuffer *buffer, UInt32 desiredIoBufferSize = framesIn * 4 * bufferListIn->mNumberBuffers; // printf("Input is %d bytes (%d total frames, %d cached frames).\n", desiredIoBufferSize, framesIn, *cachedSourceFrames); UInt32 propertySizeIo = sizeof(desiredIoBufferSize); - status = AudioConverterGetProperty(converter, - kAudioConverterPropertyCalculateOutputBufferSize, - &propertySizeIo, &desiredIoBufferSize); + OSStatus status = AudioConverterGetProperty(converter, + kAudioConverterPropertyCalculateOutputBufferSize, + &propertySizeIo, &desiredIoBufferSize); UInt32 bytesPerFrameOut = channelsOut * sizeof(SInt16); UInt32 framesOut = (desiredIoBufferSize) / bytesPerFrameOut; @@ -195,6 +195,7 @@ static inline void AVPlayerAudioTapProduceConvertedFrames(TPCircularBuffer *buff AudioConverterRef converter, AudioBufferList *bufferListIn, UInt32 framesIn, + CMTimeRange *sourceRangeIn, UInt32 channelsOut) { UInt32 bytesOut = framesIn * channelsOut * 2; AudioBufferList *producerBufferList = TPCircularBufferPrepareEmptyAudioBufferList(buffer, 1, bytesOut, NULL); @@ -210,7 +211,10 @@ static inline void AVPlayerAudioTapProduceConvertedFrames(TPCircularBuffer *buff // TODO: Do we still produce the buffer list after a failure? if (status == kCVReturnSuccess) { - TPCircularBufferProduceAudioBufferList(buffer, NULL); + AudioTimeStamp timestamp = {0}; + timestamp.mFlags = kAudioTimeStampSampleTimeValid; + timestamp.mSampleTime = sourceRangeIn->start.value; + TPCircularBufferProduceAudioBufferList(buffer, ×tamp); } else { printf("Error converting buffers: %d\n", status); } @@ -341,9 +345,11 @@ void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, flagsOut, &sourceRange, numberFramesOut); - - if (status != kCVReturnSuccess) { - // TODO + if (status != noErr) { + // TODO: It might be useful to fill zeros here. + return; + } else if(CMTIMERANGE_IS_EMPTY(sourceRange) || + CMTIMERANGE_IS_INVALID(sourceRange)) { return; } @@ -351,7 +357,12 @@ void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, // Produce renderer buffers. These are interleaved, signed integer frames in the source's sample rate. TPCircularBuffer *renderingBuffer = context->renderingBuffer; - AVPlayerAudioTapProduceConvertedFrames(renderingBuffer, context->renderFormatConverter, bufferListInOut, framesToCopy, 2); + AVPlayerAudioTapProduceConvertedFrames(renderingBuffer, + context->renderFormatConverter, + bufferListInOut, + framesToCopy, + &sourceRange, + kPreferredNumberOfChannels); // Produce capturer buffers. We will perform a sample rate conversion if needed. TPCircularBuffer *capturingBuffer = context->capturingBuffer; @@ -369,6 +380,7 @@ void AVPlayerProcessingTapProcess(MTAudioProcessingTapRef tap, context->captureFormatConverter, bufferListInOut, framesToCopy, + &sourceRange, kPreferredNumberOfChannels); } From 6d97213ae0dad12ab2eab9944178c8ee9a9606e0 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 11 Nov 2018 20:19:00 -0800 Subject: [PATCH 90/94] Set the preferred number of input channels, cleanup logging. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 5 ++++- CoViewingExample/ViewController.swift | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 0f41905d..4602c1cc 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -14,6 +14,7 @@ static double const kPreferredIOBufferDuration = 0.02; // We will use stereo playback where available. Some audio routes may be restricted to mono only. static size_t const kPreferredNumberOfChannels = 2; +static size_t const kPreferredNumberOfInputChannels = 1; // An audio sample is a signed 16-bit integer. static size_t const kAudioSampleSize = sizeof(SInt16); static uint32_t const kPreferredSampleRate = 48000; @@ -721,7 +722,9 @@ - (void)setupAVAudioSession { NSLog(@"Error activating AVAudioSession: %@", error); } - // TODO: Set preferred input channels to 1? + if (![session setPreferredInputNumberOfChannels:kPreferredNumberOfInputChannels error:&error]) { + NSLog(@"Error setting preferred number of input channels: %@", error); + } } - (AudioStreamBasicDescription)microphoneInputStreamDescription { diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 7b8ecd24..4adf1fb1 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -368,8 +368,13 @@ class ViewController: UIViewController { } func setupAudioMix(player: AVPlayer, playerItem: AVPlayerItem) { - let audioAssetTrack = firstAudioAssetTrack(playerItem: playerItem) - print("Setup audio mix with AssetTrack:", audioAssetTrack != nil ? audioAssetTrack as Any : " none") + guard let audioAssetTrack = firstAudioAssetTrack(playerItem: playerItem) else { + return + } + print("Setup audio mix with AudioAssetTrack, Id:", audioAssetTrack.trackID as Any, "\n", + "Asset:", audioAssetTrack.asset as Any, "\n", + "Audio Fallbacks:", audioAssetTrack.associatedTracks(ofType: AVAssetTrack.AssociationType.audioFallback), "\n", + "isPlayable:", audioAssetTrack.isPlayable) let audioMix = AVMutableAudioMix() From d066e15f5c8b26d43f28c69984ff0e218980a70b Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 11 Nov 2018 22:51:48 -0800 Subject: [PATCH 91/94] Attempt to fix memory management of MTAudioProcessingTap. --- CoViewingExample/ExampleAVPlayerSource.swift | 2 +- CoViewingExample/ViewController.swift | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index c4e099bf..f75cbaed 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -28,7 +28,7 @@ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { private var outputTimer: CADisplayLink? = nil // Dispatch timer which fires at a pre-determined cadence `kFrameOutputInterval`. private var timerSource: DispatchSourceTimer? = nil - private var videoOutput: AVPlayerItemVideoOutput? = nil + var videoOutput: AVPlayerItemVideoOutput? = nil private let videoSampleQueue: DispatchQueue // Frame output/sampling interval for a DispatchSource. Note: 60 Hz = 16667, 23.976 Hz = 41708 diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 4adf1fb1..4818d4ec 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -379,7 +379,7 @@ class ViewController: UIViewController { let audioMix = AVMutableAudioMix() let inputParameters = AVMutableAudioMixInputParameters(track: audioAssetTrack) - // TODO: Memory management of the MTAudioProcessingTap. + // TODO: Is memory management of the MTAudioProcessingTap correct? inputParameters.audioTapProcessor = audioDevice!.createProcessingTap()?.takeUnretainedValue() audioMix.inputParameters = [inputParameters] playerItem.audioMix = audioMix @@ -412,10 +412,17 @@ class ViewController: UIViewController { } func stopVideoPlayer() { + print(#function) + videoPlayer?.pause() + videoPlayer?.currentItem?.removeObserver(self, forKeyPath: #keyPath(AVPlayerItem.status)) + videoPlayer?.currentItem?.removeObserver(self, forKeyPath: #keyPath(AVPlayerItem.tracks)) + videoPlayer?.currentItem?.remove((videoPlayerSource?.videoOutput)!) + videoPlayer?.currentItem?.audioMix = nil + videoPlayer?.replaceCurrentItem(with: nil) videoPlayer = nil - // Remove observers? + // TODO: Unpublish player video. // Remove player UI videoPlayerView?.removeFromSuperview() @@ -467,7 +474,8 @@ class ViewController: UIViewController { firstAudioAssetTrack(playerItem: playerItem) != nil { setupAudioMix(player: videoPlayer!, playerItem: playerItem) } else { - // TODO: Possibly update the existing mix? + // TODO: Possibly update the existing mix for HLS? + // This doesn't seem to fix the tap bug, nor does deferring mix creation. // updateAudioMixParameters(playerItem: playerItem) } } From 4980f30e4c26745e44343aa891391cca80fbf929 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 11 Nov 2018 23:07:31 -0800 Subject: [PATCH 92/94] Time synchronization for audio playback. * Pre-roll AVPlayer to coordinate start times between subsystems. * Use the audio master clock for AVPlayer. * Catch up when dequeuing old frames. --- .../AudioDevices/ExampleAVPlayerAudioDevice.h | 2 + .../AudioDevices/ExampleAVPlayerAudioDevice.m | 57 ++++++++++++++----- CoViewingExample/ViewController.swift | 50 +++++++++++++++- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h index e2fc4d51..d554fd38 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.h @@ -16,6 +16,8 @@ - (void)audioTapDidPrepare; +- (void)startAudioTapAtTime:(CMTime)startTime; + /* * Creates a processing tap bound to the device instance. * diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 4602c1cc..0529c7cb 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -27,6 +27,8 @@ // The buffer of AVPlayer content that we will consume. TPCircularBuffer *playoutBuffer; + AudioTimeStamp playoutStartTimestamp; + AudioTimeStamp playoutSampleTimestamp; } ExampleAVPlayerRendererContext; typedef struct ExampleAVPlayerCapturerContext { @@ -205,13 +207,14 @@ - (BOOL)wantsAudio { - (void)audioTapDidPrepare { NSLog(@"%s", __PRETTY_FUNCTION__); +} - // TODO: Multiple contexts. +- (void)startAudioTapAtTime:(CMTime)startTime { @synchronized (self) { TVIAudioDeviceContext *context = _capturingContext ? _capturingContext->deviceContext : _renderingContext ? _renderingContext->deviceContext : NULL; if (context) { TVIAudioDeviceExecuteWorkerBlock(context, ^{ - [self restartAudioUnit]; + [self restartAudioUnitAtTime:startTime]; }); } } @@ -287,6 +290,9 @@ - (BOOL)startRendering:(nonnull TVIAudioDeviceContext)context { if (self.audioTapContext->audioTapPrepared) { self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; } else { + AudioTimeStamp start = {0}; + start.mFlags = kAudioTimeStampNothingValid; + self.renderingContext->playoutStartTimestamp = start; self.renderingContext->playoutBuffer = NULL; } @@ -443,11 +449,12 @@ - (BOOL)stopCapturing { static void ExampleAVPlayerAudioDeviceDequeueFrames(TPCircularBuffer *buffer, UInt32 numFrames, + const AudioTimeStamp *timestamp, AudioBufferList *bufferList) { int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; // TODO: Include this format in the context? What if the formats are somehow not matched? - AudioStreamBasicDescription format; + AudioStreamBasicDescription format = {0}; format.mBitsPerChannel = 16; format.mChannelsPerFrame = bufferList->mBuffers[0].mNumberChannels; format.mBytesPerFrame = format.mChannelsPerFrame * format.mBitsPerChannel / 8; @@ -456,10 +463,13 @@ static void ExampleAVPlayerAudioDeviceDequeueFrames(TPCircularBuffer *buffer, format.mSampleRate = kPreferredSampleRate; UInt32 framesInOut = numFrames; - if (buffer->buffer != NULL) { - TPCircularBufferDequeueBufferListFrames(buffer, &framesInOut, bufferList, NULL, &format); + if (timestamp) { + AudioTimeStamp dequeuedTimestamp; + do { + TPCircularBufferDequeueBufferListFrames(buffer, &framesInOut, bufferList, &dequeuedTimestamp, &format); + } while (dequeuedTimestamp.mSampleTime < timestamp->mSampleTime); } else { - framesInOut = 0; + TPCircularBufferDequeueBufferListFrames(buffer, &framesInOut, bufferList, NULL, &format); } if (framesInOut != numFrames) { @@ -483,6 +493,8 @@ static OSStatus ExampleAVPlayerAudioDeviceAudioTapPlaybackCallback(void *refCon, assert(bufferList->mBuffers[0].mNumberChannels > 0); ExampleAVPlayerRendererContext *context = (ExampleAVPlayerRendererContext *)refCon; + AudioTimeStamp startTimestamp = context->playoutStartTimestamp; + BOOL readyToPlay = (startTimestamp.mFlags & kAudioTimeStampHostTimeValid) && (timestamp->mHostTime >= startTimestamp.mHostTime); TPCircularBuffer *buffer = context->playoutBuffer; UInt32 audioBufferSizeInBytes = bufferList->mBuffers[0].mDataByteSize; @@ -493,13 +505,20 @@ static OSStatus ExampleAVPlayerAudioDeviceAudioTapPlaybackCallback(void *refCon, int8_t *audioBuffer = (int8_t *)bufferList->mBuffers[0].mData; memset(audioBuffer, 0, audioBufferSizeInBytes); return noErr; - } else if (buffer == nil) { + } else if (buffer == nil || + !readyToPlay) { *actionFlags |= kAudioUnitRenderAction_OutputIsSilence; memset(bufferList->mBuffers[0].mData, 0, audioBufferSizeInBytes); return noErr; } - ExampleAVPlayerAudioDeviceDequeueFrames(buffer, numFrames, bufferList); + if (readyToPlay && context->playoutStartTimestamp.mSampleTime == 0) { + ExampleAVPlayerAudioDeviceDequeueFrames(buffer, numFrames, &context->playoutStartTimestamp, bufferList); + context->playoutStartTimestamp.mSampleTime += 1; + } else { + ExampleAVPlayerAudioDeviceDequeueFrames(buffer, numFrames, NULL, bufferList); + } + return noErr; } @@ -587,7 +606,7 @@ static OSStatus ExampleAVPlayerAudioDeviceRecordingInputCallback(void *refCon, playerAudioBuffer->mDataByteSize = (UInt32)numFrames * playerAudioBuffer->mNumberChannels * kAudioSampleSize; playerAudioBuffer->mData = context->audioBuffer; - ExampleAVPlayerAudioDeviceDequeueFrames(context->recordingBuffer, numFrames, &playerBufferList); + ExampleAVPlayerAudioDeviceDequeueFrames(context->recordingBuffer, numFrames, NULL, &playerBufferList); // Early return to test player audio. // Deliver the samples (via copying) to WebRTC. @@ -695,9 +714,9 @@ - (void)setupAVAudioSession { NSLog(@"Error setting sample rate: %@", error); } - NSInteger preferredOutputChannels = session.outputNumberOfChannels >= kPreferredNumberOfChannels ? kPreferredNumberOfChannels : session.outputNumberOfChannels; + size_t preferredOutputChannels = session.outputNumberOfChannels >= kPreferredNumberOfChannels ? kPreferredNumberOfChannels : session.outputNumberOfChannels; if (![session setPreferredOutputNumberOfChannels:preferredOutputChannels error:&error]) { - NSLog(@"Error setting number of output channels: %@", error); + NSLog(@"Error setting number of output channels to %zu: %@", preferredOutputChannels, error); } /* @@ -723,7 +742,7 @@ - (void)setupAVAudioSession { } if (![session setPreferredInputNumberOfChannels:kPreferredNumberOfInputChannels error:&error]) { - NSLog(@"Error setting preferred number of input channels: %@", error); + NSLog(@"Error setting preferred number of input channels to %zu: %@", kPreferredNumberOfChannels, error); } } @@ -1005,8 +1024,19 @@ - (void)teardownAudioUnit { } } -- (void)restartAudioUnit { +- (void)restartAudioUnitAtTime:(CMTime)startTime { BOOL restart = NO; + + AudioTimeStamp startTimestamp = {0}; + startTimestamp.mFlags = kAudioTimeStampHostTimeValid; + startTimestamp.mHostTime = CMClockConvertHostTimeToSystemUnits(startTime); + self.renderingContext->playoutStartTimestamp = startTimestamp; + + // TODO: Assumption, pass as an arg using the asset's current time and audio timescale? + AudioTimeStamp sampleTimestamp = {0}; + sampleTimestamp.mFlags = kAudioTimeStampSampleTimeValid; + sampleTimestamp.mSampleTime = 0; + @synchronized (self) { if (self.wantsAudio) { restart = YES; @@ -1014,6 +1044,7 @@ - (void)restartAudioUnit { [self teardownAudioUnit]; if (self.renderingContext) { self.renderingContext->playoutBuffer = _audioTapRenderingBuffer; + self.renderingContext->playoutSampleTimestamp = sampleTimestamp; } if (self.capturingContext) { self.capturingContext->recordingBuffer = _audioTapCapturingBuffer; diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 4818d4ec..3a8b5410 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -26,6 +26,8 @@ class ViewController: UIViewController { var playerVideoTrack: TVILocalVideoTrack? var localAudioTrack: TVILocalAudioTrack! + // How long we will spend in pre-roll, attempting to synchronize our AVPlayer and AudioUnit graph. + static let kPrerollDuration = Double(1.0) static let kPlayerTrackName = "player-track" // AVPlayer Audio/Video. @@ -35,6 +37,7 @@ class ViewController: UIViewController { var videoPlayerSource: ExampleAVPlayerSource? = nil var videoPlayerView: ExampleAVPlayerView? = nil var videoPlayerUrl: URL? = nil + var videoPlayerPreroll: Bool = false var isPresenter:Bool? @@ -317,6 +320,13 @@ class ViewController: UIViewController { let player = AVPlayer(playerItem: playerItem) player.volume = Float(0) + player.automaticallyWaitsToMinimizeStalling = false + + var audioClock: CMClock? = nil + let status = CMAudioClockCreate(allocator: nil, clockOut: &audioClock) + if (status == noErr) { + player.masterClock = audioClock; + } videoPlayer = player let playerView = ExampleAVPlayerView(frame: CGRect.zero, player: player) @@ -429,6 +439,36 @@ class ViewController: UIViewController { videoPlayerView = nil } + func prerollVideoPlayer() { + print("Preparing to play asset with Tracks:", videoPlayer?.currentItem?.asset.tracks as Any) + + videoPlayerPreroll = true + videoPlayer?.preroll(atRate: 1.0, completionHandler: { (success) in + if (success) { + // Start audio and video playback at a time synchronized with both parties. + // let now = CMClockGetTime(CMClockGetHostTimeClock()) + let now = CMClockGetTime((self.videoPlayer?.masterClock)!) + let start = now + CMTime(seconds: ViewController.kPrerollDuration, preferredTimescale: now.timescale) + + let audioAssetTrack = self.firstAudioAssetTrack(playerItem: (self.videoPlayer?.currentItem)!) + var range = CMTimeRange.invalid + if let assetTrack = audioAssetTrack { + range = assetTrack.timeRange + } + + print("Pre-roll success for item:", self.videoPlayer?.currentItem as Any, "\n", + "Current time:", self.videoPlayer?.currentItem?.currentTime() as Any, "\n", + "Audio asset range:", range as Any, "\n", + "\nStarting at:", start.seconds) + self.videoPlayer?.setRate(1.0, time: CMTime.invalid, atHostTime: start) + self.audioDevice?.startAudioTap(at: start) + } else { + print("Pre-roll failed, waiting to try again ...") + self.videoPlayerPreroll = false + } + }) + } + override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, @@ -448,12 +488,16 @@ class ViewController: UIViewController { switch status { case .readyToPlay: // Player item is ready to play. - print("Ready. Playing asset with tracks: ", videoPlayer?.currentItem?.asset.tracks as Any) + print("Ready to play asset.") // Defer video source setup until we've loaded the asset so that we can determine downscaling for progressive streaming content. if self.videoPlayerSource == nil { setupVideoSource(item: object as! AVPlayerItem) } - videoPlayer?.play() + + if videoPlayer?.rate == 0 && + videoPlayerPreroll == false { + self.prerollVideoPlayer() + } break case .failed: // Player item failed. See error. @@ -462,7 +506,7 @@ class ViewController: UIViewController { break case .unknown: // Player item is not yet ready. - print("Player status unknown.") + print("Player item status is unknown.") break } } else if keyPath == #keyPath(AVPlayerItem.tracks) { From 47be1d8895c20751213b353dbe9837d1cad7c2e5 Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sat, 1 Dec 2018 23:52:44 -0800 Subject: [PATCH 93/94] Arbitrary loads for media only. --- CoViewingExample/Info.plist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CoViewingExample/Info.plist b/CoViewingExample/Info.plist index 639683c4..9e96ba8f 100644 --- a/CoViewingExample/Info.plist +++ b/CoViewingExample/Info.plist @@ -49,7 +49,7 @@ NSAppTransportSecurity - NSAllowsArbitraryLoads + NSAllowsArbitraryLoadsForMedia NSCameraUsageDescription From d316d29919ca49fc80b4d6b98862a4894811449b Mon Sep 17 00:00:00 2001 From: Chris Eagleston Date: Sun, 2 Dec 2018 00:10:18 -0800 Subject: [PATCH 94/94] Support 1-channel output devices properly (like AirPods in HFP). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * AVPlayer playback was broken. * Don’t tie device recording format to AVAudioSession, we always want stereo. * AudioTap provides a rendering format, which is now used to configure the mixer input. --- .../AudioDevices/ExampleAVPlayerAudioDevice.m | 14 ++++++++++++-- .../AudioDevices/ExampleAVPlayerProcessingTap.h | 1 + .../AudioDevices/ExampleAVPlayerProcessingTap.m | 6 ++++-- CoViewingExample/ExampleAVPlayerSource.swift | 2 +- CoViewingExample/ViewController.swift | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m index 0529c7cb..49ffd72f 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerAudioDevice.m @@ -355,7 +355,7 @@ - (nullable TVIAudioFormat *)captureFormat { * Assume that the AVAudioSession has already been configured and started and that the values * for sampleRate and IOBufferDuration are final. */ - _capturingFormat = [[self class] activeFormat]; + _capturingFormat = [[self class] capturingFormat]; } return _capturingFormat; @@ -661,6 +661,16 @@ static OSStatus ExampleAVPlayerAudioDeviceRecordingInputCallback(void *refCon, #pragma mark - Private (AVAudioSession and CoreAudio) ++ (nonnull TVIAudioFormat *)capturingFormat { + /* + * Use the pre-determined maximum frame size. AudioUnit callbacks are variable, and in most sitations will be close + * to the `AVAudioSession.preferredIOBufferDuration` that we've requested. + */ + return [[TVIAudioFormat alloc] initWithChannels:kPreferredNumberOfChannels + sampleRate:kPreferredSampleRate + framesPerBuffer:kMaximumFramesPerBuffer]; +} + + (nullable TVIAudioFormat *)activeFormat { /* * Use the pre-determined maximum frame size. AudioUnit callbacks are variable, and in most sitations will be close @@ -852,7 +862,7 @@ - (BOOL)setupAudioUnitRendererContext:(ExampleAVPlayerRendererContext *)renderer AudioStreamBasicDescription renderingFormatDescription = self.renderingFormat.streamDescription; AudioStreamBasicDescription playerFormatDescription = renderingFormatDescription; if (self.renderingContext->playoutBuffer) { - playerFormatDescription.mSampleRate = self.audioTapContext->sourceFormat.mSampleRate; + playerFormatDescription = self.audioTapContext->renderingFormat; } // Setup playback mixer. diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h index a997f450..9ccf46d7 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.h @@ -21,6 +21,7 @@ typedef struct ExampleAVPlayerAudioTapContext { TPCircularBuffer *renderingBuffer; AudioConverterRef renderFormatConverter; + AudioStreamBasicDescription renderingFormat; // Cached source audio, in case we need to perform a sample rate conversion and can't consume all the samples in one go. AudioBufferList *sourceCache; diff --git a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m index f0eb0c2c..05574adf 100644 --- a/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m +++ b/CoViewingExample/AudioDevices/ExampleAVPlayerProcessingTap.m @@ -267,12 +267,14 @@ void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, context->sourceCacheFrames = 0; context->sourceFormat = *processingFormat; - TVIAudioFormat *playbackFormat = [[TVIAudioFormat alloc] initWithChannels:kPreferredNumberOfChannels + TVIAudioFormat *playbackFormat = [[TVIAudioFormat alloc] initWithChannels:processingFormat->mChannelsPerFrame sampleRate:processingFormat->mSampleRate framesPerBuffer:maxFrames]; AudioStreamBasicDescription preferredPlaybackDescription = [playbackFormat streamDescription]; BOOL requiresFormatConversion = preferredPlaybackDescription.mFormatFlags != processingFormat->mFormatFlags; + context->renderingFormat = preferredPlaybackDescription; + if (requiresFormatConversion) { OSStatus status = AudioConverterNew(processingFormat, &preferredPlaybackDescription, &context->renderFormatConverter); if (status != 0) { @@ -281,7 +283,7 @@ void AVPlayerProcessingTapPrepare(MTAudioProcessingTapRef tap, } } - TVIAudioFormat *recordingFormat = [[TVIAudioFormat alloc] initWithChannels:2 + TVIAudioFormat *recordingFormat = [[TVIAudioFormat alloc] initWithChannels:kPreferredNumberOfChannels sampleRate:(Float64)kPreferredSampleRate framesPerBuffer:maxFrames]; AudioStreamBasicDescription preferredRecordingDescription = [recordingFormat streamDescription]; diff --git a/CoViewingExample/ExampleAVPlayerSource.swift b/CoViewingExample/ExampleAVPlayerSource.swift index f75cbaed..cd5adfe6 100644 --- a/CoViewingExample/ExampleAVPlayerSource.swift +++ b/CoViewingExample/ExampleAVPlayerSource.swift @@ -17,7 +17,7 @@ import TwilioVideo * Please be aware that AVPlayer and its playback pipeline prepare content for presentation on your device, including * mapping frames to the display. For example, when playing 23.976 or 24 fps content a technique known as 3:2 pulldown * is used to time video samples for a 60 Hz iPhone display. Our capturer tags the frames with the best timing infromation - * that it has available - the presentation timestamps provided by AVPlayer's output. + * that it has available - the presentation timestamps provided by AVPlayerVideoItemOutput. */ class ExampleAVPlayerSource: NSObject, TVIVideoCapturer { diff --git a/CoViewingExample/ViewController.swift b/CoViewingExample/ViewController.swift index 3a8b5410..77d51631 100644 --- a/CoViewingExample/ViewController.swift +++ b/CoViewingExample/ViewController.swift @@ -39,7 +39,7 @@ class ViewController: UIViewController { var videoPlayerUrl: URL? = nil var videoPlayerPreroll: Bool = false - var isPresenter:Bool? + var isPresenter: Bool? @IBOutlet weak var localHeightConstraint: NSLayoutConstraint? @IBOutlet weak var localWidthConstraint: NSLayoutConstraint?