External ffmpeg support (#695)

* Do not use internal ffmpeg for client SDK build.

By default we should not build ffmpeg for client SDK. If you need to use ffmpeg
for d3d11va decoding, you need to specify owt_ffmpeg_header_root gn arg which
points to the source root of external ffmpeg, so the build process can find
ffmpeg headers there. Your application needs to explicitly link to ffmpeg libraries,
which includes but not limited to avutil.lib, avcodec.lib & avformat.lib.

* Update scripts for external FFmpeg.

* Roll WebRTC revision.

* Limit d3d11va_h264_decoder to Windows only.

---------

Co-authored-by: Johny Qiu <[email protected]>

Author: jianjunz

DEPS

@@ -98,7 +98,7 @@ deps = {
     Var('chromium_git') + '/chromium/src/third_party' + '@' + 'cd30703e732f3436f72f63c13f16ebb19803ddd6',
   # WebRTC-only dependencies (not present in Chromium).
   'src/third_party/webrtc':
-    Var('deps_owt_git') + '/owt-deps-webrtc' + '@' + '1f8b8174fdf33ea954ce2afcedd24f73ca0ef759',
+    Var('deps_owt_git') + '/owt-deps-webrtc' + '@' + '15bb85d7886c653bba004f2b2826bc471d07fd96',
   # Gradle 4.3-rc4. Used for testing Android Studio project generation for WebRTC.
   'src/third_party/webrtc/examples/androidtests/third_party/gradle': {
     'url': Var('chromium_git') + '/external/github.com/gradle/gradle.git' + '@' +

README.md

@@ -64,8 +64,9 @@ target_os = []
 Common build options shared by Windows and Linux:
   - By default `x86|Debug` library will be created. Specify `--arch x64` if you want to build x64 libraries; Specify `--scheme release` if release version of library is to be built.
   - The built binary will be under path specified by `--output_path`. If `--output_path` is not set, the built binary will be under `src/out` directory.
-  - The optional `--ssl_root` should be set to the root directory of lastest OpenSSL 3.0 series directory. If specified, build tool will search external OpenSSL headers in `ssl_root/include` and OpenSSL binaries in `ssl_root/lib`. But binaries are not included in OWT SDK, so applications still need to link libcrypto and libssl.
   - The argument `--sio_root` should be set to the root directory of Socket.IO cpp 3.x SDK, with headers in `include` sub-folder, libsioclient_tls.a in `lib` sub-folder.
+  - The optional `--ssl_root` should be set to the root directory of lastest OpenSSL 3.0 series directory. If specified, build tool will search external OpenSSL headers in `ssl_root/include` and OpenSSL binaries in `ssl_root/lib`. But binaries are not included in OWT SDK, so applications still need to link libcrypto and libssl.
+  - The optional `--ffmpeg_root` should be set to the root directory of FFmpeg, with headers in `include` sub-folder, and libs in `lib` sub-folder. Binary libraries are not necessary for building OWT SDK, but it's needed by your application or tests when this argument is specified. If this argument is not specified, FFmpeg will not be used. If neither `--ffmpeg_root` nor `--msdk_root` is specified, and external decoder doesn't support H.264, GN arg `rtc_use_h264` should be set to `false`, otherwise, a runtime error will occur.
   - Use `--gn_gen` to generate args.gn during the first build or when you change either `ssl_root`/`msdk_root` options.
   - The optional `--tests` will trigger unit tests after build.
   - Run `build*.py` with `--help` for argument description.
@@ -79,7 +80,6 @@ Update to latest macOS and Xcode. iOS SDK can only be built on macOS.
 #### Android
 This branch doesn't support Android build. owt-client-android depends on 5.0.x branch of this repository.
 
-
 ## How to contribute
 We warmly welcome community contributions to the owt-client-native repository. If you are willing to contribute your features and ideas to OWT, follow the process below:
 

scripts/build-win.py

@@ -4,8 +4,10 @@
 
 '''Script to build WebRTC libs on Windows.
 
-It builds OWT Windows SDK library which includes WebRTC lib, OWT base, p2p and conference
-lib.
+This script is a shortcut for building OWT Windows SDK library which includes
+WebRTC lib, OWT base, p2p and conference lib. It doesn't cover all
+configurations. Please update GN args and use ninja build manually if you have
+special configurations.
 
 Output lib is located in out/owt-debug(release).lib.
 '''
@@ -24,20 +26,15 @@
 NONPARALLEL_TEST_TARGET_LIST = ['webrtc_nonparallel_tests']
 
 GN_ARGS = [
-    'rtc_use_h264=true',
-    'ffmpeg_branding="Chrome"',
-    'rtc_use_h265=true',
     'is_component_build=false',
     'use_lld=false',
     'enable_libaom=true',
     'rtc_build_examples=false',
     'treat_warnings_as_errors=false',
-    # <stdatomic.h> is not yet supported when compiling as C.
-    'ffmpeg_use_unsafe_atomics=true',
     ]
 
 
-def gngen(arch, sio_root, ssl_root, msdk_root, quic_root, scheme, tests, runtime, cloud_gaming):
+def gngen(arch, sio_root, ffmpeg_root, ssl_root, msdk_root, quic_root, scheme, tests, runtime, cloud_gaming):
     gn_args = list(GN_ARGS)
     gn_args.append('target_cpu="%s"' % arch)
     using_llvm = False
@@ -89,6 +86,12 @@ def gngen(arch, sio_root, ssl_root, msdk_root, quic_root, scheme, tests, runtime
     if sio_root:
         # If sio_root is not specified, conference SDK is not able to build.
         gn_args.append('owt_sio_header_root="%s"' % (sio_root + r'\include'))
+    if ffmpeg_root:
+        gn_args.append('owt_ffmpeg_header_root="%s"'%(ffmpeg_root+r'\include'))
+    if ffmpeg_root or msdk_root or cloud_gaming:
+        gn_args.append('rtc_use_h264=true')
+    if msdk_root or cloud_gaming:
+        gn_args.append('rtc_use_h265=true')
     flattened_args = ' '.join(gn_args)
     ret = subprocess.call(['gn.bat', 'gen', getoutputpath(arch, scheme), '--args=%s' % flattened_args],
                           cwd=HOME_PATH, shell=False)
@@ -173,6 +176,7 @@ def main():
     parser.add_argument('--msdk_root', help='Path for MSDK.')
     parser.add_argument('--quic_root', help='Path to QUIC library. Not supported yet.')
     parser.add_argument('--sio_root', required=False, help='Path to Socket.IO cpp. Headers in include sub-folder, libsioclient_tls.a in lib sub-folder.')
+    parser.add_argument('--ffmpeg_root', required=False, help='Path to to root directory of FFmpeg, with headers in include sub-folder, and libs in lib sub-folder. Binary libraries are not necessary for building OWT SDK, but it is needed by your application or tests when this argument is specified.')
     parser.add_argument('--scheme', default='debug', choices=('debug', 'release'),
                         help='Schemes for building. Supported value: debug, release')
     parser.add_argument('--gn_gen', default=False, action='store_true',
@@ -202,7 +206,7 @@ def main():
         print('Invalid quic_root')
         return 1
     if opts.gn_gen:
-        if not gngen(opts.arch, opts.sio_root, opts.ssl_root, opts.msdk_root, opts.quic_root,
+        if not gngen(opts.arch, opts.sio_root, opts.ffmpeg_root, opts.ssl_root, opts.msdk_root, opts.quic_root,
                      opts.scheme, opts.tests, opts.runtime, opts.cloud_gaming):
             return 1
     if opts.sdk:

scripts/build_linux.py

@@ -4,8 +4,10 @@
 
 '''Script to build WebRTC libs on Linux.
 
-It builds OWT Linux SDK library which includes WebRTC lib, OWT base, p2p and conference
-lib.
+This script is a shortcut for building OWT Linux SDK library which includes
+WebRTC lib, OWT base, p2p and conference lib. It doesn't cover all
+configurations. Please update GN args and use ninja build manually if you have
+special configurations.
 
 Output lib is located in out/owt-debug(release).a.
 '''
@@ -24,16 +26,9 @@
 PARALLEL_TEST_TARGET_LIST = ['rtc_unittests', 'video_engine_tests']
 
 GN_ARGS = [
-    'rtc_use_h264=true',
-    'rtc_use_h265=true',
     'enable_libaom=true',
     'is_component_build=false',
     'rtc_build_examples=false',
-    # When is_clang is false, we're not using sysroot in tree.
-    'use_sysroot=false',
-    # For Linux build we expect application built with gcc/g++. Set SDK to use
-    # the same toolchain.
-    'is_clang=false',
     # Upstream only officially supports clang, so we need to suppress warnings
     # to avoid gcc/g++ specific build errors.
     'treat_warnings_as_errors=false',
@@ -45,7 +40,7 @@ def gen_lib_path(scheme):
     out_lib = OUT_LIB % {'scheme': scheme}
     return os.path.join(HOME_PATH + r'/out', out_lib)
 
-def gngen(arch, sio_root, ssl_root, msdk_root, quic_root, scheme, tests, use_gcc, fake_audio, shared, cloud_gaming):
+def gngen(arch, sio_root, ffmpeg_root, ssl_root, msdk_root, quic_root, scheme, tests, use_gcc, fake_audio, shared, cloud_gaming):
     gn_args = list(GN_ARGS)
     gn_args.append('target_cpu="%s"' % arch)
     if scheme == 'release':
@@ -68,20 +63,19 @@ def gngen(arch, sio_root, ssl_root, msdk_root, quic_root, scheme, tests, use_gcc
             msdk_lib = msdk_root + r'/lib64'
         else:
             return False
-        gn_args.append('owt_msdk_header_root="%s"' % (msdk_root + r'/include'))
+        gn_args.append('owt_msdk_header_root="%s"' % (msdk_root + '/include'))
         gn_args.append('owt_msdk_lib_root="%s"' % msdk_lib)
     else:
         print('msdk_root is not set.')
     if quic_root:
-        gn_args.append('owt_quic_header_root="%s"' % (quic_root + r'\include'))
+        gn_args.append('owt_quic_header_root="%s"' % (quic_root + '/include'))
         if scheme == 'release':
-            quic_lib = quic_root + r'\bin\release'
+            quic_lib = quic_root + '/bin/release'
         elif scheme == 'debug':
-            quic_lib = quic_root + r'\bin\debug'
+            quic_lib = quic_root + '/bin/debug'
         else:
             return False
         gn_args.append('owt_use_quic=true')
-
     if tests:
         gn_args.append('rtc_include_tests=true')
         gn_args.append('owt_include_tests=true')
@@ -96,11 +90,14 @@ def gngen(arch, sio_root, ssl_root, msdk_root, quic_root, scheme, tests, use_gcc
         gn_args.extend(['rtc_enable_protobuf=false', 'is_component_build=true'])
     else:
         gn_args.extend(['is_component_build=false'])
-    ffmpeg_branding_name = "Chrome"
     if cloud_gaming:
-        ffmpeg_branding_name = "OWT"
         gn_args.extend(['owt_cloud_gaming=true'])
-    gn_args.append('ffmpeg_branding="%s"' % ffmpeg_branding_name)
+    if ffmpeg_root:
+        gn_args.append('owt_ffmpeg_header_root="%s"'%(ffmpeg_root+'/include'))
+    if ffmpeg_root or msdk_root or cloud_gaming:
+        gn_args.append('rtc_use_h264=true')
+    if msdk_root or cloud_gaming:
+        gn_args.append('rtc_use_h265=true')
 
     flattened_args = ' '.join(gn_args)
     out = 'out/%s-%s' % (scheme, arch)
@@ -184,6 +181,7 @@ def main():
                         help='Target architecture. Supported value: x86, x64')
     parser.add_argument('--ssl_root', help='Path for OpenSSL.')
     parser.add_argument('--sio_root', required=False, help='Path to Socket.IO cpp. Headers in include sub-folder, libsioclient_tls.a in lib sub-folder.')
+    parser.add_argument('--ffmpeg_root', required=False, help='Path to to root directory of FFmpeg, with headers in include sub-folder, and libs in lib sub-folder. Binary libraries are not necessary for building OWT SDK, but it is needed by your application or tests when this argument is specified.')
     parser.add_argument('--msdk_root', help='Path for MSDK.')
     parser.add_argument('--quic_root', help='Path to QUIC library')
     parser.add_argument('--scheme', default='debug', choices=('debug', 'release'),
@@ -207,7 +205,7 @@ def main():
         print("sio_root is missing.")
         return 1
     if opts.gn_gen:
-        if not gngen(opts.arch, opts.sio_root, opts.ssl_root, opts.msdk_root, opts.quic_root, opts.scheme, opts.tests, opts.use_gcc, opts.fake_audio, opts.shared, opts.cloud_gaming):
+        if not gngen(opts.arch, opts.sio_root, opts.ffmpeg_root, opts.ssl_root, opts.msdk_root, opts.quic_root, opts.scheme, opts.tests, opts.use_gcc, opts.fake_audio, opts.shared, opts.cloud_gaming):
             return 1
     if opts.sdk:
          if not ninjabuild(opts.arch, opts.scheme, opts.shared):

talk/owt/BUILD.gn

@@ -12,6 +12,7 @@ declare_args() {
   owt_msdk_lib_root = ""
   owt_msdk_header_root = ""
   owt_sio_header_root = ""
+  owt_ffmpeg_header_root = ""
 }
 
 # Introduced for using libvpx config files. We only enable libvpx rate
@@ -234,6 +235,11 @@ static_library("owt_sdk_base") {
   if (rtc_include_internal_audio_device) {
     defines += [ "WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE" ]
   }
+
+  if (owt_ffmpeg_header_root != "") {
+    include_dirs += [ owt_ffmpeg_header_root ]
+  }
+
   if (rtc_use_x11) {
     defines += [ "WEBRTC_USE_X11" ]
   }
@@ -347,6 +353,17 @@ static_library("owt_sdk_base") {
       "sdk/base/linux/videorenderlinux.h",
     ]
   }
+
+  if (owt_ffmpeg_header_root != "") {
+    if (is_win) {
+      sources += [
+        "sdk/base/win/d3d11va_h264_decoder.cc",
+        "sdk/base/win/d3d11va_h264_decoder.h",
+      ]
+    }
+    defines += [ "OWT_USE_FFMPEG" ]
+  }
+
   if (!is_ios) {
     sources += [
       "sdk/base/customizedvideodecoderfactory.cc",

talk/owt/sdk/base/win/d3d11va_h264_decoder.cc

@@ -408,7 +408,8 @@ int32_t H264DXVADecoderImpl::Decode(const webrtc::EncodedImage& input_image,
         surface_handle_->cursor_data_size = cursor_data_size;
         surface_handle_->decode_start = decode_start_time;
         surface_handle_->decode_end = clock_->CurrentTime().ms_or(0);
-        surface_handle_->start_duration = input_image.bwe_stats_.start_duration_;
+        surface_handle_->start_duration =
+            input_image.bwe_stats_.start_duration_;
         surface_handle_->last_duration = input_image.bwe_stats_.last_duration_;
         surface_handle_->packet_loss = input_image.bwe_stats_.packets_lost_;
         surface_handle_->frame_size = input_image.size();

talk/owt/sdk/base/win/d3d11va_h264_decoder.h

@@ -82,7 +82,7 @@ class H264DXVADecoderImpl : public webrtc::H264Decoder {
   std::vector<uint8_t> current_cursor_data_;
   AVBufferRef* hw_device_ctx = nullptr;
   AVCodecContext* decoder_ctx = nullptr;
-  AVCodec* decoder = nullptr;
+  const AVCodec* decoder = nullptr;
   webrtc::Clock* clock_ = nullptr;
 };
 

talk/owt/sdk/base/win/msdkvideodecoderfactory.cc

@@ -66,7 +66,8 @@ MSDKVideoDecoderFactory::MSDKVideoDecoderFactory(ID3D11Device* d3d11_device_exte
 
 MSDKVideoDecoderFactory::~MSDKVideoDecoderFactory() {}
 
-std::unique_ptr<webrtc::VideoDecoder> MSDKVideoDecoderFactory::CreateVideoDecoder(
+std::unique_ptr<webrtc::VideoDecoder>
+MSDKVideoDecoderFactory::CreateVideoDecoder(
     const webrtc::SdpVideoFormat& format) {
   bool vp9_hw = false, vp8_hw = false, av1_hw = false, h264_hw = false;
 #ifdef WEBRTC_USE_H265
@@ -92,17 +93,28 @@ std::unique_ptr<webrtc::VideoDecoder> MSDKVideoDecoderFactory::CreateVideoDecode
     return webrtc::VP9Decoder::Create();
   } else if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName) &&
              !vp8_hw) {
-    RTC_LOG(LS_INFO) << "Not supporting HW VP8 decoder. Requesting SW decoding.";
+    RTC_LOG(LS_INFO)
+        << "Not supporting HW VP8 decoder. Requesting SW decoding.";
     return webrtc::VP8Decoder::Create();
-  } else if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName) && !h264_hw) {
+  }
+#ifdef OWT_USE_FFMPEG
+  else if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) {
+    return owt::base::H264DXVADecoderImpl::Create(cricket::VideoCodec(format));
+  }
+#else
+  else if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName) &&
+           !h264_hw) {
     return webrtc::H264Decoder::Create();
-  } else if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName) &&
-             !av1_hw) {
+  }
+#endif
+  else if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName) &&
+           !av1_hw) {
     return webrtc::CreateDav1dDecoder();
   }
 #ifdef WEBRTC_USE_H265
   // This should not happen. We do not return here but preceed with HW decoder.
-  else if (absl::EqualsIgnoreCase(format.name, cricket::kH265CodecName) && !h265_hw) {
+  else if (absl::EqualsIgnoreCase(format.name, cricket::kH265CodecName) &&
+           !h265_hw) {
     RTC_LOG(LS_ERROR) << "Returning null hevc encoder.";
   }
 #endif