Skip to content
This repository was archived by the owner on Oct 25, 2024. It is now read-only.

Commit b2c57b0

Browse files
jianjunzjianjunz
authored
D3D11 video decoder supports more codecs. (#714)
1 parent d8fe00d commit b2c57b0

File tree

9 files changed

+189
-68
lines changed

9 files changed

+189
-68
lines changed

‎scripts/build-win.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ def gngen(arch, sio_root, ffmpeg_root, ssl_root, msdk_root, quic_root, scheme, t
9797
gn_args.append('owt_ffmpeg_root="%s"'%(ffmpeg_root))
9898
if ffmpeg_root or msdk_root or cg_server:
9999
gn_args.append('rtc_use_h264=true')
100-
if msdk_root or cg_server:
100+
if ffmpeg_root or msdk_root or cg_server:
101101
gn_args.append('rtc_use_h265=true')
102102
if fake_audio:
103103
gn_args.append('rtc_include_internal_audio_device=false')

‎talk/owt/BUILD.gn

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -360,8 +360,8 @@ static_library("owt_sdk_base") {
360360
if (owt_ffmpeg_root != "") {
361361
if (is_win) {
362362
sources += [
363-
"sdk/base/win/d3d11va_h264_decoder.cc",
364-
"sdk/base/win/d3d11va_h264_decoder.h",
363+
"sdk/base/win/d3d11_video_decoder.cc",
364+
"sdk/base/win/d3d11_video_decoder.h",
365365
]
366366
}
367367
defines += [ "OWT_USE_FFMPEG" ]

‎talk/owt/sdk/base/mediautils.cc

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,22 @@ static const std::map<const std::string, const VideoCodec>
3838
{"h264", VideoCodec::kH264},
3939
{"h265", VideoCodec::kH265}};
4040

41+
#ifdef OWT_USE_FFMPEG
42+
static const std::map<const owt::base::VideoCodec, AVCodecID>
43+
video_codec_id_map = {{owt::base::VideoCodec::kVp8, AV_CODEC_ID_VP8},
44+
{owt::base::VideoCodec::kVp9, AV_CODEC_ID_VP9},
45+
{owt::base::VideoCodec::kH264, AV_CODEC_ID_H264},
46+
{owt::base::VideoCodec::kH265, AV_CODEC_ID_HEVC},
47+
{owt::base::VideoCodec::kAv1, AV_CODEC_ID_AV1}};
48+
static const std::map<const webrtc::VideoCodecType, AVCodecID>
49+
webrtc_video_codec_id_map = {
50+
{webrtc::VideoCodecType::kVideoCodecVP8, AV_CODEC_ID_VP8},
51+
{webrtc::VideoCodecType::kVideoCodecVP9, AV_CODEC_ID_VP9},
52+
{webrtc::VideoCodecType::kVideoCodecH264, AV_CODEC_ID_H264},
53+
{webrtc::VideoCodecType::kVideoCodecH265, AV_CODEC_ID_HEVC},
54+
{webrtc::VideoCodecType::kVideoCodecAV1, AV_CODEC_ID_AV1}};
55+
#endif
56+
4157
std::string MediaUtils::GetResolutionName(const Resolution& resolution) {
4258
for (auto it = resolution_name_map.begin(); it != resolution_name_map.end();
4359
++it) {
@@ -204,5 +220,25 @@ absl::optional<H265ProfileId> MediaUtils::ParseSdpForH265Profile(
204220
return StringToH265Profile(profile_str);
205221
}
206222

223+
#ifdef OWT_USE_FFMPEG
224+
absl::optional<AVCodecID> MediaUtils::GetFfmpegCodecId(
225+
const owt::base::VideoCodec& video_codec) {
226+
auto it = video_codec_id_map.find(video_codec);
227+
if (it == video_codec_id_map.end()) {
228+
return absl::nullopt;
229+
}
230+
return absl::make_optional(it->second);
231+
}
232+
233+
absl::optional<AVCodecID> MediaUtils::GetFfmpegCodecId(
234+
const webrtc::VideoCodecType& video_codec) {
235+
auto it = webrtc_video_codec_id_map.find(video_codec);
236+
if (it == webrtc_video_codec_id_map.end()) {
237+
return absl::nullopt;
238+
}
239+
return absl::make_optional(it->second);
240+
}
241+
#endif
242+
207243
} // namespace base
208244
} // namespace owt

‎talk/owt/sdk/base/mediautils.h

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,16 @@
55
#ifndef OWT_BASE_MEDIAUTILS_H_
66
#define OWT_BASE_MEDIAUTILS_H_
77

8+
#include <string>
89
#include "absl/types/optional.h"
10+
#include "api/video/video_codec_type.h"
911
#include "api/video_codecs/sdp_video_format.h"
10-
#include <string>
1112
#include "talk/owt/sdk/include/cpp/owt/base/commontypes.h"
13+
#ifdef OWT_USE_FFMPEG
14+
extern "C" {
15+
#include <libavcodec/avcodec.h>
16+
} // extern "C"
17+
#endif
1218

1319
namespace owt {
1420
namespace base {
@@ -179,6 +185,12 @@ class MediaUtils {
179185
const webrtc::SdpVideoFormat::Parameters& params);
180186
static absl::optional<H265ProfileId> ParseSdpForH265Profile(
181187
const webrtc::SdpVideoFormat::Parameters& params);
188+
#ifdef OWT_USE_FFMPEG
189+
static absl::optional<AVCodecID> GetFfmpegCodecId(
190+
const owt::base::VideoCodec& video_codec);
191+
static absl::optional<AVCodecID> GetFfmpegCodecId(
192+
const webrtc::VideoCodecType& video_codec);
193+
#endif
182194
};
183195
}
184196
}

‎talk/owt/sdk/base/win/d3d11va_h264_decoder.cc renamed to ‎talk/owt/sdk/base/win/d3d11_video_decoder.cc

Lines changed: 33 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@
22
//
33
// SPDX-License-Identifier: Apache-2.0
44

5-
#include "talk/owt/sdk/base/win/d3d11va_h264_decoder.h"
5+
#include "talk/owt/sdk/base/win/d3d11_video_decoder.h"
66
#include <algorithm>
77
#include <limits>
88

99
#include "system_wrappers/include/metrics.h"
10+
#include "talk/owt/sdk/base/mediautils.h"
1011
#include "talk/owt/sdk/base/nativehandlebuffer.h"
1112
#include "webrtc/api/video/color_space.h"
1213
#include "webrtc/api/video/i420_buffer.h"
@@ -29,10 +30,10 @@ const size_t kVPlaneIndex = 2;
2930
static const int kMaxSideDataListSize = 20;
3031

3132
// Used by histograms. Values of entries should not be changed.
32-
enum H264DecoderImplEvent {
33-
kH264DecoderEventInit = 0,
34-
kH264DecoderEventError = 1,
35-
kH264DecoderEventMax = 16,
33+
enum D3D11VideoDecoderEvent {
34+
kD3D11VideoDecoderEventInit = 0,
35+
kD3D11VideoDecoderEventError = 1,
36+
kD3D11VideoDecoderEventMax = 16,
3637
};
3738

3839
} // namespace
@@ -58,19 +59,19 @@ static enum AVPixelFormat get_hw_format(AVCodecContext* ctx,
5859
return AV_PIX_FMT_NONE;
5960
}
6061

61-
H264DXVADecoderImpl::H264DXVADecoderImpl(ID3D11Device* external_device)
62+
D3D11VideoDecoder::D3D11VideoDecoder(ID3D11Device* external_device)
6263
: decoded_image_callback_(nullptr),
6364
has_reported_init_(false),
6465
has_reported_error_(false),
6566
clock_(webrtc::Clock::GetRealTimeClock()) {
6667
surface_handle_.reset(new D3D11VAHandle());
6768
}
6869

69-
H264DXVADecoderImpl::~H264DXVADecoderImpl() {
70+
D3D11VideoDecoder::~D3D11VideoDecoder() {
7071
Release();
7172
}
7273

73-
int H264DXVADecoderImpl::InitHwContext(AVCodecContext* ctx,
74+
int D3D11VideoDecoder::InitHwContext(AVCodecContext* ctx,
7475
const enum AVHWDeviceType type) {
7576
int err = 0;
7677
AVBufferRef* device_ref = NULL;
@@ -148,7 +149,7 @@ int H264DXVADecoderImpl::InitHwContext(AVCodecContext* ctx,
148149
return err;
149150
}
150151

151-
int H264DXVADecoderImpl::PrepareHwDecoder(webrtc::VideoCodecType codec_type) {
152+
int D3D11VideoDecoder::PrepareHwDecoder(webrtc::VideoCodecType codec_type) {
152153
int ret = 0;
153154
enum AVHWDeviceType type;
154155

@@ -161,11 +162,13 @@ int H264DXVADecoderImpl::PrepareHwDecoder(webrtc::VideoCodecType codec_type) {
161162
}
162163
}
163164

164-
AVCodecID codec_id = codec_type == webrtc::VideoCodecType::kVideoCodecH264
165-
? AV_CODEC_ID_H264
166-
: AV_CODEC_ID_HEVC;
165+
auto codec_id = MediaUtils::GetFfmpegCodecId(codec_type);
166+
if (!codec_id.has_value()) {
167+
RTC_LOG(LS_ERROR) << "Unsupported codec type.";
168+
return -1;
169+
}
167170

168-
decoder = avcodec_find_decoder(codec_id);
171+
decoder = avcodec_find_decoder(codec_id.value());
169172
if (!decoder) {
170173
RTC_LOG(LS_ERROR) << "Decoder not found by avcodec_find_decoder.";
171174
return -1;
@@ -205,16 +208,8 @@ int H264DXVADecoderImpl::PrepareHwDecoder(webrtc::VideoCodecType codec_type) {
205208
return 0;
206209
}
207210

208-
bool H264DXVADecoderImpl::Configure(const Settings& settings) {
211+
bool D3D11VideoDecoder::Configure(const Settings& settings) {
209212
ReportInit();
210-
if (settings.codec_type() != webrtc::kVideoCodecH264
211-
#ifdef WEBRTC_USE_H265
212-
&& settings.codec_type() != webrtc::kVideoCodecH265
213-
#endif
214-
) {
215-
RTC_LOG(LS_ERROR) << "in H264DXVADecoderImpl: codec mismatch.";
216-
return false;
217-
}
218213

219214
// Release necessary in case of re-initializing.
220215
int32_t ret = Release();
@@ -231,7 +226,7 @@ bool H264DXVADecoderImpl::Configure(const Settings& settings) {
231226
return true;
232227
}
233228

234-
int32_t H264DXVADecoderImpl::Release() {
229+
int32_t D3D11VideoDecoder::Release() {
235230
// Do we need to flush the decoder? Currently we don't.
236231
if (decoder_ctx != nullptr) {
237232
avcodec_free_context(&decoder_ctx);
@@ -244,13 +239,13 @@ int32_t H264DXVADecoderImpl::Release() {
244239
return WEBRTC_VIDEO_CODEC_OK;
245240
}
246241

247-
int32_t H264DXVADecoderImpl::RegisterDecodeCompleteCallback(
242+
int32_t D3D11VideoDecoder::RegisterDecodeCompleteCallback(
248243
webrtc::DecodedImageCallback* callback) {
249244
decoded_image_callback_ = callback;
250245
return WEBRTC_VIDEO_CODEC_OK;
251246
}
252247

253-
int32_t H264DXVADecoderImpl::Decode(const webrtc::EncodedImage& input_image,
248+
int32_t D3D11VideoDecoder::Decode(const webrtc::EncodedImage& input_image,
254249
bool /*missing_frames*/,
255250
int64_t /*render_time_ms*/) {
256251
if (!IsInitialized()) {
@@ -394,7 +389,7 @@ int32_t H264DXVADecoderImpl::Decode(const webrtc::EncodedImage& input_image,
394389
}
395390

396391
// Helper function to extract the prefix-SEI.
397-
int64_t H264DXVADecoderImpl::GetSideData(const uint8_t* frame_data,
392+
int64_t D3D11VideoDecoder::GetSideData(const uint8_t* frame_data,
398393
size_t frame_size,
399394
std::vector<uint8_t>& side_data,
400395
std::vector<uint8_t>& cursor_data) {
@@ -460,35 +455,35 @@ int64_t H264DXVADecoderImpl::GetSideData(const uint8_t* frame_data,
460455
return payload_size;
461456
}
462457

463-
const char* H264DXVADecoderImpl::ImplementationName() const {
458+
const char* D3D11VideoDecoder::ImplementationName() const {
464459
return "OWTD3D11VA";
465460
}
466461

467-
bool H264DXVADecoderImpl::IsInitialized() const {
462+
bool D3D11VideoDecoder::IsInitialized() const {
468463
return decoder_ctx != nullptr;
469464
}
470465

471-
void H264DXVADecoderImpl::ReportInit() {
466+
void D3D11VideoDecoder::ReportInit() {
472467
if (has_reported_init_)
473468
return;
474-
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DXVADecoderImpl.Event",
475-
kH264DecoderEventInit,
476-
kH264DecoderEventMax);
469+
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.D3D11VideoDecoder.Event",
470+
kD3D11VideoDecoderEventInit,
471+
kD3D11VideoDecoderEventMax);
477472
has_reported_init_ = true;
478473
}
479474

480-
void H264DXVADecoderImpl::ReportError() {
475+
void D3D11VideoDecoder::ReportError() {
481476
if (has_reported_error_)
482477
return;
483-
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H264DXVADecoderImpl.Event",
484-
kH264DecoderEventError,
485-
kH264DecoderEventMax);
478+
RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.D3D11VideoDecoder.Event",
479+
kD3D11VideoDecoderEventError,
480+
kD3D11VideoDecoderEventMax);
486481
has_reported_error_ = true;
487482
}
488483

489-
std::unique_ptr<H264DXVADecoderImpl> H264DXVADecoderImpl::Create(
484+
std::unique_ptr<D3D11VideoDecoder> D3D11VideoDecoder::Create(
490485
cricket::VideoCodec format) {
491-
return absl::make_unique<H264DXVADecoderImpl>(nullptr);
486+
return absl::make_unique<D3D11VideoDecoder>(nullptr);
492487
}
493488

494489
} // namespace base

‎talk/owt/sdk/base/win/d3d11va_h264_decoder.h renamed to ‎talk/owt/sdk/base/win/d3d11_video_decoder.h

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
//
33
// SPDX-License-Identifier: Apache-2.0
44

5-
#ifndef OWT_BASE_WIN_D3D11VA_H264_DECODER_H_
6-
#define OWT_BASE_WIN_D3D11VA_H264_DECODER_H_
5+
#ifndef OWT_BASE_WIN_D3D11VA_DECODER_H_
6+
#define OWT_BASE_WIN_D3D11VA_DECODER_H_
77

88
#include <map>
99
#include <memory>
@@ -36,12 +36,12 @@ class Clock;
3636
namespace owt {
3737
namespace base {
3838

39-
class H264DXVADecoderImpl : public webrtc::H264Decoder {
39+
class D3D11VideoDecoder : public webrtc::VideoDecoder {
4040
public:
41-
static std::unique_ptr<H264DXVADecoderImpl> Create(
41+
static std::unique_ptr<D3D11VideoDecoder> Create(
4242
cricket::VideoCodec format);
43-
H264DXVADecoderImpl(ID3D11Device* external_device);
44-
~H264DXVADecoderImpl() override;
43+
D3D11VideoDecoder(ID3D11Device* external_device);
44+
~D3D11VideoDecoder() override;
4545

4646
bool Configure(const Settings& settings) override;
4747
int32_t Release() override;
@@ -91,4 +91,4 @@ class H264DXVADecoderImpl : public webrtc::H264Decoder {
9191
} // namespace base
9292
} // namespace owt
9393

94-
#endif // OWT_BASE_WIN_D3D11VA_H264_DECODER_H_
94+
#endif // OWT_BASE_WIN_D3D11VA_DECODER_H_

‎talk/owt/sdk/base/win/externalvideodecoderfactory.cc

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#include "modules/video_coding/codecs/vp9/include/vp9.h"
1313
#include "talk/owt/sdk/base/codecutils.h"
1414
#ifdef OWT_USE_FFMPEG
15-
#include "talk/owt/sdk/base/win/d3d11va_h264_decoder.h"
15+
#include "talk/owt/sdk/base/win/d3d11_video_decoder.h"
1616
#endif
1717
#ifdef OWT_USE_MSDK
1818
#include "talk/owt/sdk/base/win/msdkvideodecoder.h"
@@ -90,7 +90,6 @@ ExternalVideoDecoderFactory::CreateVideoDecoder(
9090
#ifdef WEBRTC_USE_H265
9191
else if (codec == webrtc::kVideoCodecH265) {
9292
h265_hw = true;
93-
RTC_LOG(LS_INFO) << "Enabling hardware hevc.";
9493
}
9594
#endif
9695
}
@@ -101,28 +100,21 @@ ExternalVideoDecoderFactory::CreateVideoDecoder(
101100
RTC_LOG(LS_INFO)
102101
<< "Not supporting HW VP8 decoder. Requesting SW decoding.";
103102
return webrtc::VP8Decoder::Create();
104-
}
105-
#ifdef OWT_USE_FFMPEG
106-
else if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) {
107-
return owt::base::H264DXVADecoderImpl::Create(cricket::VideoCodec(format));
108-
}
109-
#else
110-
else if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName) &&
103+
} else if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName) &&
111104
!h264_hw) {
112105
return webrtc::H264Decoder::Create();
113-
}
114-
#endif
115-
else if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName) &&
106+
} else if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName) &&
116107
!av1_hw) {
117108
return webrtc::CreateDav1dDecoder();
118109
}
119-
#ifdef WEBRTC_USE_H265
120-
// This should not happen. We do not return here but preceed with HW decoder.
121-
else if (absl::EqualsIgnoreCase(format.name, cricket::kH265CodecName) &&
122-
!h265_hw) {
123-
RTC_LOG(LS_ERROR) << "Returning null hevc encoder.";
124-
}
110+
if (vp8_hw || vp9_hw || h264_hw || h265_hw || av1_hw) {
111+
#if defined(OWT_USE_FFMPEG)
112+
return owt::base::D3D11VideoDecoder::Create(cricket::VideoCodec(format));
125113
#endif
114+
#if defined(OWT_USE_MSDK)
115+
return owt::base::MSDKVideoDecoder::Create(cricket::VideoCodec(format));
116+
#endif
117+
}
126118

127119
RTC_CHECK_NOTREACHED();
128120
}

‎talk/owt/sdk/base/win/externalvideodecoderfactory.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020

2121
namespace owt {
2222
namespace base {
23-
// Decoder factory for MSDK decoder and FFMPEG decoder.
23+
// Decoder factory for MSDK decoder and FFMPEG decoder. Only hardware decoders
24+
// will be created.
2425
class ExternalVideoDecoderFactory : public webrtc::VideoDecoderFactory {
2526
public:
2627
ExternalVideoDecoderFactory(ID3D11Device* d3d11_device_external);

0 commit comments

Comments
 (0)