Add support for transparent photo and videos.

Author: v-grigoriev

Example/Example.xcodeproj/project.pbxproj

@@ -552,6 +552,7 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_STYLE = Automatic;
+				CURRENT_PROJECT_VERSION = 1;
 				DEVELOPMENT_TEAM = 5S8646274U;
 				INFOPLIST_FILE = Source/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
@@ -576,6 +577,7 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_STYLE = Automatic;
+				CURRENT_PROJECT_VERSION = 1;
 				DEVELOPMENT_TEAM = 5S8646274U;
 				INFOPLIST_FILE = Source/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 13.0;

Example/Pods/Pods.xcodeproj/project.pbxproj

@@ -18,6 +18,7 @@
 		D500F759257AB94100118C11 /* TestRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D500F758257AB94100118C11 /* TestRenderer.swift */; };
 		D500F774257ABE1200118C11 /* TestRecordableLayer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D500F773257ABE1200118C11 /* TestRecordableLayer.swift */; };
 		D55385CD254CB15900225EC0 /* PixelBufferOutput.swift in Sources */ = {isa = PBXBuildFile; fileRef = D55385CC254CB15900225EC0 /* PixelBufferOutput.swift */; };
+		D56915CF263F445B00591380 /* VideoSettings.CompressionProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = D56915CE263F445B00591380 /* VideoSettings.CompressionProperties.swift */; };
 		D56D805F25F16ECB00935848 /* AudioEngine.Player.swift in Sources */ = {isa = PBXBuildFile; fileRef = D56D805C25F16ECB00935848 /* AudioEngine.Player.swift */; };
 		D56D806025F16ECB00935848 /* AudioEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = D56D805D25F16ECB00935848 /* AudioEngine.swift */; };
 		D56D806125F16ECB00935848 /* AudioEngine.Player.State.swift in Sources */ = {isa = PBXBuildFile; fileRef = D56D805E25F16ECB00935848 /* AudioEngine.Player.State.swift */; };
@@ -137,6 +138,7 @@
 		D500F769257ABA0700118C11 /* SCNRecorderTests-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "SCNRecorderTests-Bridging-Header.h"; sourceTree = "<group>"; };
 		D500F773257ABE1200118C11 /* TestRecordableLayer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestRecordableLayer.swift; sourceTree = "<group>"; };
 		D55385CC254CB15900225EC0 /* PixelBufferOutput.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PixelBufferOutput.swift; sourceTree = "<group>"; };
+		D56915CE263F445B00591380 /* VideoSettings.CompressionProperties.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VideoSettings.CompressionProperties.swift; sourceTree = "<group>"; };
 		D56D805C25F16ECB00935848 /* AudioEngine.Player.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioEngine.Player.swift; sourceTree = "<group>"; };
 		D56D805D25F16ECB00935848 /* AudioEngine.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioEngine.swift; sourceTree = "<group>"; };
 		D56D805E25F16ECB00935848 /* AudioEngine.Player.State.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioEngine.Player.State.swift; sourceTree = "<group>"; };
@@ -286,6 +288,7 @@
 				D5B923AC2510CC9B00D35585 /* VideoRecording.State.swift */,
 				D5B923D02510CC9B00D35585 /* VideoSettings.swift */,
 				D5B923BE2510CC9B00D35585 /* VideoSettings.Codec.swift */,
+				D56915CE263F445B00591380 /* VideoSettings.CompressionProperties.swift */,
 				D5B9238B2510CC9A00D35585 /* VideoSettings.FileType.swift */,
 				D5B923C22510CC9B00D35585 /* VideoSettings.ScalingMode.swift */,
 				D56D805B25F16ECB00935848 /* AudioEngine */,
@@ -824,6 +827,7 @@
 				D5EE33F3257108C20056085A /* MetalPixelBufferProducer.swift in Sources */,
 				D5B923FB2510CC9C00D35585 /* Observable.swift in Sources */,
 				D5B923EB2510CC9C00D35585 /* VideoSettings.FileType.swift in Sources */,
+				D56915CF263F445B00591380 /* VideoSettings.CompressionProperties.swift in Sources */,
 				D5B923E12510CC9C00D35585 /* SCNView+MulticastDelegate.swift in Sources */,
 				D5B923F22510CC9C00D35585 /* UnfairAtomic.swift in Sources */,
 			);

SCNRecorder.xcodeproj/project.pbxproj

@@ -30,8 +30,9 @@ extension VideoSettings.Codec {
 
   var name: String {
     switch self {
-    case .h264: return "h264"
     case .hevc: return "hevc"
+    case .hevcWithAlpha: return "hevcWithAlpha"
+    case .h264: return "h264"
     case .jpeg: return "jpeg"
     }
   }

SCNRecorderTests/Extensions/VideoSetting.Codec+Helpers.swift

@@ -72,10 +72,17 @@ extension SceneRecorderTests {
     var codecs: [VideoSettings.Codec] = [.h264()]
 
     let supportHEVCEncoder = AVAssetExportSession.allExportPresets().contains(AVAssetExportPresetHEVC1920x1080)
-    if supportHEVCEncoder { codecs.append(.hevc) }
+    if supportHEVCEncoder {
+      codecs.append(.hevc())
+      if #available(iOS 13.0, *) {
+        if fileType == .mov {
+          codecs.append(.hevcWithAlpha())
+        }
+      }
+    }
 
     #if !targetEnvironment(simulator)
-    codecs.append(.jpeg)
+    codecs.append(.jpeg())
     #endif
 
     codecs.forEach { (codec) in

SCNRecorderTests/SceneRecorderTests.swift

@@ -25,16 +25,17 @@
 
 import Foundation
 import AVFoundation
+import MetalPerformanceShaders
 
 extension MTLPixelFormat {
 
-  // Undocumented format, something like bgr10_xr_srgb, was obtained on iPhone 7 iOS 12.1.4
-  static let rgb10a8_2p_xr10_srgb = MTLPixelFormat(rawValue: 551) ?? .bgr10_xr_srgb
+  // Undocumented format, something like bgra10_xr_srgb, was found on iPhone 7 iOS 12.1.4
+  static let rgb10a8_2p_xr10_srgb = MTLPixelFormat(rawValue: 551) ?? .bgra10_xr_srgb
 
   #if !targetEnvironment(simulator)
   static let supportedPixelFormats: Set<MTLPixelFormat> = Set([
     .bgra8Unorm, .bgra8Unorm_srgb,
-    .bgr10_xr, .bgr10_xr_srgb,
+    .bgr10_xr, .bgr10_xr_srgb
   ])
   #else
   static let supportedPixelFormats: Set<MTLPixelFormat> = Set([
@@ -44,8 +45,14 @@ extension MTLPixelFormat {
 
   var colorPrimaries: String {
     switch self {
-    case .bgr10_xr, .bgr10_xr_srgb: return AVVideoColorPrimaries_P3_D65
-    default: return AVVideoColorPrimaries_ITU_R_709_2
+    case .bgr10_xr,
+         .bgr10_xr_srgb,
+         .bgra10_xr,
+         .bgra10_xr_srgb,
+         .rgb10a8_2p_xr10_srgb:
+      return AVVideoColorPrimaries_P3_D65
+    default:
+      return AVVideoColorPrimaries_ITU_R_709_2
     }
   }
 
@@ -58,13 +65,28 @@ extension MTLPixelFormat {
   var supportedPixelFormat: MTLPixelFormat {
     if Self.supportedPixelFormats.contains(self) { return self }
 
+    // For some reason bgra8Unorm_srgb is not writable on simulators
+    #if targetEnvironment(simulator)
+    let bgra8Unorm_srgb = MTLPixelFormat.bgra8Unorm
+    #else
+    let bgra8Unorm_srgb = MTLPixelFormat.bgra8Unorm_srgb
+    #endif
+
+    // The recorder doesn't support bgra10_xr, bgra10_xr_srgb and rgb10a8_2p_xr10_srgb to be wrote directly.
+    // They should be converted to intermediate bgra8Unorm and bgra8Unorm_srgb formats to be recorded.
+    // Mostly becaouse, I can find a CoreVideo pixel format to be used for them.
+    // The drawback is loose of extended colors.
+    // In general any of https://developer.apple.com/documentation/metalperformanceshaders/image_filters#2793234
+    // formats can be used as intermediates, but for now, I don't see any reason to experiment.
+    // - Vlad
     switch self {
-    case .bgra10_xr: return .bgr10_xr
-    case .bgra10_xr_srgb, .rgb10a8_2p_xr10_srgb: return .bgr10_xr_srgb
-    default: return .bgra8Unorm_srgb
+    case .bgra10_xr: return .bgra8Unorm
+    case .bgra10_xr_srgb, .rgb10a8_2p_xr10_srgb: return bgra8Unorm_srgb
+    default: return bgra8Unorm_srgb
     }
   }
 
+  // A CoreVideo pixel format to be used as a format for storage for the content encoded as MTLPixelFormat
   var pixelFormatType: OSType {
     switch self {
 
@@ -78,4 +100,23 @@ extension MTLPixelFormat {
       return kCVPixelFormatType_32BGRA
     }
   }
+
+  // Assume that the alpha is premultiplied for formats with alpha channel.
+  // This is true for SceneKit, ARKit, but might be different for pure metal projects with custom shaders
+  var alphaType: MPSAlphaType {
+    switch self {
+
+    case .bgra8Unorm, .bgra8Unorm_srgb:
+      return .premultiplied
+
+    case .bgr10_xr, .bgr10_xr_srgb:
+      return .alphaIsOne
+
+    case .bgra10_xr, .bgra10_xr_srgb, .rgb10a8_2p_xr10_srgb:
+      return .premultiplied
+
+    default:
+      return .premultiplied
+    }
+  }
 }

Sources/Extensions/MTLPixelFormat+Helpers.swift

@@ -120,7 +120,11 @@ final class MetalPixelBufferProducer {
   ) throws {
     guard let commandBuffer = commandQueue.makeCommandBuffer() else { throw Error.noCommandBuffer }
 
-    let imageConversion = self.makeImageConversion()
+    let imageConversion = self.makeImageConversion(
+      sourceTexture: sourceTexture,
+      destinationTexture: destinationTexture.mtlTexture
+    )
+
     imageConversion.encode(
       commandBuffer: commandBuffer,
       sourceTexture: sourceTexture,
@@ -173,11 +177,14 @@ final class MetalPixelBufferProducer {
     return attachements
   }
 
-  func makeImageConversion() -> MPSImageConversion {
+  func makeImageConversion(
+    sourceTexture: MTLTexture,
+    destinationTexture: MTLTexture
+  ) -> MPSImageConversion {
     MPSImageConversion(
       device: device,
-      srcAlpha: .alphaIsOne,
-      destAlpha: .alphaIsOne,
+      srcAlpha: sourceTexture.pixelFormat.alphaType,
+      destAlpha: destinationTexture.pixelFormat.alphaType,
       backgroundColor: nil,
       conversionInfo: nil
     )