Skip to content

Commit 47942dd

Browse files
halx99halx99
authored
Add render I420 video support, fix #2049 (#2050)
1 parent 6e09fdb commit 47942dd

File tree

10 files changed

+125
-28
lines changed

10 files changed

+125
-28
lines changed

‎core/media/AndroidMediaEngine.cpp

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,16 @@ JNIEXPORT void JNICALL Java_org_axmol_lib_AxmolMediaEngine_nativeHandleVideoSamp
4444
int outputY,
4545
int videoX,
4646
int videoY,
47-
int rotation)
47+
int rotation,
48+
int videoPF)
4849
{
4950
auto mediaEngine = (ax::AndroidMediaEngine*)((uintptr_t)pME);
5051
if (!mediaEngine)
5152
return;
5253

5354
auto sampleData = static_cast<uint8_t*>(env->GetDirectBufferAddress(sampleBuffer));
5455

55-
mediaEngine->handleVideoSample(sampleData, sampleLen, outputX, outputY, videoX, videoY, rotation);
56+
mediaEngine->handleVideoSample(sampleData, sampleLen, outputX, outputY, videoX, videoY, rotation, videoPF);
5657
}
5758

5859
JNIEXPORT void JNICALL Java_org_axmol_lib_AxmolMediaEngine_nativeSetDuration(JNIEnv* env,
@@ -161,7 +162,7 @@ bool AndroidMediaEngine::transferVideoFrame()
161162
auto& buffer = _frameBuffer2;
162163

163164
ax::MEVideoFrame frame{buffer.data(), buffer.data() + _outputDim.x * _outputDim.y, buffer.size(),
164-
ax::MEVideoPixelDesc{ax::MEVideoPixelFormat::NV12, _outputDim}, _videoDim};
165+
ax::MEVideoPixelDesc{static_cast<ax::MEVideoPixelFormat>(_videoPF), _outputDim}, _videoDim};
165166
frame._vpd._rotation = _videoRotation;
166167
assert(static_cast<int>(frame._dataLen) >= frame._vpd._dim.x * frame._vpd._dim.y * 3 / 2);
167168
_onVideoFrame(frame);
@@ -178,13 +179,15 @@ void AndroidMediaEngine::handleVideoSample(const uint8_t* buf,
178179
int outputY,
179180
int videoX,
180181
int videoY,
181-
int rotation)
182+
int rotation,
183+
int videoPF)
182184
{
183185
std::unique_lock<std::mutex> lck(_frameBuffer1Mtx);
184186
_frameBuffer1.assign(buf, buf + len);
185187
_outputDim.set(outputX, outputY);
186188
_videoDim.set(videoX, videoY);
187189
_videoRotation = rotation;
190+
_videoPF = videoPF;
188191
}
189192

190193
NS_AX_END

‎core/media/AndroidMediaEngine.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ class AndroidMediaEngine : public MediaEngine
5959
MEMediaState getState() const override;
6060
bool transferVideoFrame() override;
6161

62-
void handleVideoSample(const uint8_t* buf, size_t len, int outputX, int outputY, int videoX, int videoY, int rotation);
62+
void handleVideoSample(const uint8_t* buf, size_t len, int outputX, int outputY, int videoX, int videoY, int rotation, int videoPF);
6363
void updateCurrentTime(double currentTime) { _currentTime = currentTime; }
6464
void updateDuration(double duration) { _duration = duration; }
6565

@@ -71,6 +71,7 @@ class AndroidMediaEngine : public MediaEngine
7171
MEIntPoint _outputDim;
7272
MEIntPoint _videoDim;
7373
int _videoRotation{0};
74+
int _videoPF{-1};
7475

7576
yasio::byte_buffer _frameBuffer1; // for write
7677
yasio::byte_buffer _frameBuffer2; // for read

‎core/media/MediaEngine.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,12 @@ enum class MEMediaState
9898

9999
enum class MEVideoPixelFormat
100100
{
101-
INVALID,
102-
YUY2,
103-
NV12, // '420v' '420f'
101+
INVALID = -1,
104102
RGB32,
105103
BGR32,
104+
YUY2,
105+
NV12, // '420v' '420f'
106+
I420
106107
};
107108

108109
struct MEIntPoint

‎core/platform/android/java/src/org/axmol/lib/AxmolMediaEngine.java

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ of this software and associated documentation files (the "Software"), to deal
2626
import android.app.Activity;
2727
import android.content.Context;
2828
import android.graphics.Point;
29+
import android.media.MediaCodecInfo;
2930
import android.media.MediaFormat;
3031
import android.net.Uri;
3132
import android.os.Handler;
@@ -62,6 +63,10 @@ public class AxmolMediaEngine extends DefaultRenderersFactory implements Player.
6263
public static final int EVENT_STOPPED = 2;
6364
public static final int EVENT_ERROR = 3;
6465

66+
// The native video pixel formats, match with MEVideoPixelFormat
67+
public static final int VIDEO_PF_NV12 = 3;
68+
public static final int VIDEO_PF_I420 = 4;
69+
6570
/** Media has been closed and cannot be played again. */
6671
public static final int STATE_CLOSED = 0;
6772

@@ -100,11 +105,12 @@ public class AxmolMediaEngine extends DefaultRenderersFactory implements Player.
100105
private AtomicInteger mState = new AtomicInteger(STATE_CLOSED);
101106
Point mOutputDim = new Point(); // The output dim match with buffer
102107
Point mVideoDim = new Point(); // The video dim (validate image dim)
108+
private int mVideoPF = -1;
103109
private int mVideoRotation = 0;
104110

105111
/** ------ native methods ------- */
106112
public static native void nativeHandleEvent(long nativeObj, int arg1);
107-
public static native void nativeHandleVideoSample(long nativeObj, ByteBuffer sampleData, int sampleLen, int outputX, int outputY, int videoX, int videoY, int rotation);
113+
public static native void nativeHandleVideoSample(long nativeObj, ByteBuffer sampleData, int sampleLen, int outputX, int outputY, int videoX, int videoY, int rotation, int videoPF);
108114
public static native void nativeSetDuration(long nativeObj, double duration);
109115
public static native void nativeSetCurrentTime(long nativeObj, double currentTime);
110116

@@ -306,10 +312,23 @@ public void onVideoFrameAboutToBeRendered(
306312
/** update video informations */
307313
private void updateVideoMeta() {
308314
MediaFormat format = mOutputFormat;
309-
// String mimeType = format.getString(MediaFormat.KEY_MIME); // "video/raw" (NV12)
310-
// Integer colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT);
311-
// boolean NV12 = colorFormat == MediaCodecVideoRenderer.DESIRED_PIXEL_FORMAT;
312315
if(format != null) {
316+
// String mimeType = format.getString(MediaFormat.KEY_MIME); // "video/raw"
317+
// Note: some android 11 and older devices not response desired color format(NV12), instead will be YUV420P aka I420
318+
// refer: https://github.com/axmolengine/axmol/issues/2049
319+
Integer colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT);
320+
switch(colorFormat) {
321+
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
322+
mVideoPF = VIDEO_PF_NV12;
323+
break;
324+
case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
325+
mVideoPF = VIDEO_PF_I420;
326+
break;
327+
default:
328+
mVideoPF = VIDEO_PF_NV12;
329+
Log.w(TAG, String.format("Unsupported color format: %d, video render may incorrect!", colorFormat));
330+
}
331+
313332
mOutputDim.x = format.getInteger(MediaFormat.KEY_WIDTH);
314333
if (format.containsKey(MediaFormat.KEY_CROP_LEFT)
315334
&& format.containsKey(MediaFormat.KEY_CROP_RIGHT)) {
@@ -343,7 +362,7 @@ public void processVideoFrame(MediaCodecAdapter codec, int index, long presentat
343362
}
344363

345364
ByteBuffer tmpBuffer = codec.getOutputBuffer(index);
346-
nativeHandleVideoSample(mNativeObj, tmpBuffer, tmpBuffer.remaining(), mOutputDim.x, mOutputDim.y, mVideoDim.x, mVideoDim.y, mVideoRotation);
365+
nativeHandleVideoSample(mNativeObj, tmpBuffer, tmpBuffer.remaining(), mOutputDim.x, mOutputDim.y, mVideoDim.x, mVideoDim.y, mVideoRotation, mVideoPF);
347366

348367
AxmolEngine.getActivity().runOnUiThread(() -> {
349368
if (mPlayer != null) {

‎core/renderer/Shaders.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ AX_DLL const std::string_view hsv_frag = "hsv_fs"sv;
6262
AX_DLL const std::string_view dualSampler_hsv_frag = "dualSampler_hsv_fs"sv;
6363
AX_DLL const std::string_view videoTextureYUY2_frag = "videoTextureYUY2_fs"sv;
6464
AX_DLL const std::string_view videoTextureNV12_frag = "videoTextureNV12_fs"sv;
65+
AX_DLL const std::string_view videoTextureI420_frag = "videoTextureI420_fs"sv;
6566
AX_DLL const std::string_view lineColor_frag = "lineColor_fs"sv;
6667
AX_DLL const std::string_view lineColor_vert = "lineColor_vs"sv;
6768
AX_DLL const std::string_view color_frag = "color_fs"sv;

‎core/renderer/Shaders.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ extern AX_DLL const std::string_view dualSampler_hsv_frag;
7373

7474
extern AX_DLL const std::string_view videoTextureYUY2_frag;
7575
extern AX_DLL const std::string_view videoTextureNV12_frag;
76+
extern AX_DLL const std::string_view videoTextureI420_frag;
7677

7778
/* below is 3d shaders */
7879
extern AX_DLL const std::string_view lineColor_frag;

‎core/renderer/backend/Enums.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,7 @@ struct ProgramType
381381

382382
VIDEO_TEXTURE_YUY2,
383383
VIDEO_TEXTURE_NV12,
384+
VIDEO_TEXTURE_I420, // For some android 11 and older devices
384385
VIDEO_TEXTURE_BGR32,
385386

386387
BUILTIN_COUNT,

‎core/renderer/backend/ProgramManager.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ bool ProgramManager::init()
152152
VertexLayoutType::Sprite);
153153
registerProgram(ProgramType::VIDEO_TEXTURE_NV12, positionTextureColor_vert, videoTextureNV12_frag,
154154
VertexLayoutType::Sprite);
155+
registerProgram(ProgramType::VIDEO_TEXTURE_I420, positionTextureColor_vert, videoTextureI420_frag,
156+
VertexLayoutType::Sprite);
155157

156158
// The builtin dual sampler shader registry
157159
ProgramStateRegistry::getInstance()->registerProgram(ProgramType::POSITION_TEXTURE_COLOR,

‎core/renderer/shaders/videoTextureI420.frag

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#version 310 es
2+
precision highp float;
3+
precision highp int;
4+
5+
#include "base.glsl"
6+
#include "colorUtils.glsl"
7+
8+
layout(location = COLOR0) in vec4 v_color;
9+
layout(location = TEXCOORD0) in vec2 v_texCoord;
10+
11+
layout(binding = 0) uniform sampler2D u_tex0; // Y sample: LumaTexture
12+
layout(binding = 1) uniform sampler2D u_tex1; // U sample: ChromaTexture
13+
layout(binding = 2) uniform sampler2D u_tex2; // V sample: ChromaTexture
14+
15+
layout(std140) uniform fs_ub {
16+
mat4 colorTransform;
17+
};
18+
19+
layout(location = SV_Target0) out vec4 FragColor;
20+
21+
void main()
22+
{
23+
vec3 YUV;
24+
25+
YUV.x = texture(u_tex0, v_texCoord).x; // Y
26+
YUV.y = texture(u_tex1, v_texCoord).x; // U
27+
YUV.z = texture(u_tex2, v_texCoord).x; // V
28+
29+
/* Convert YUV to RGB */
30+
vec4 OutColor;
31+
OutColor.xyz = trasnformYUV(YUV, colorTransform);
32+
OutColor.w = 1.0;
33+
34+
FragColor = v_color * OutColor;
35+
}

‎core/ui/UIMediaPlayer.cpp

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,14 @@ USING_NS_AX;
5050
do \
5151
{ \
5252
decltype(value) __v = value; \
53-
auto __loc = (ps)->getUniformLocation(name); \
53+
auto __loc = (ps) -> getUniformLocation(name); \
5454
(ps)->setUniform(__loc, &__v, sizeof(__v)); \
5555
} while (false)
5656

5757
# define PS_SET_UNIFORM_R(ps, name, value) \
5858
do \
5959
{ \
60-
auto __loc = (ps)->getUniformLocation(name); \
60+
auto __loc = (ps) -> getUniformLocation(name); \
6161
(ps)->setUniform(__loc, &value, sizeof(value)); \
6262
} while (false)
6363

@@ -67,10 +67,11 @@ namespace
6767
{
6868
struct PrivateVideoDescriptor
6969
{
70-
MediaEngine* _engine = nullptr;
71-
Texture2D* _vtexture = nullptr;
72-
Texture2D* _vchromaTexture = nullptr;
73-
Sprite* _vrender = nullptr;
70+
MediaEngine* _engine = nullptr;
71+
Texture2D* _vtexture = nullptr;
72+
Texture2D* _vchromaTexture = nullptr;
73+
Texture2D* _vchroma2Texture = nullptr;
74+
Sprite* _vrender = nullptr;
7475

7576
MEVideoPixelDesc _vpixelDesc;
7677

@@ -518,11 +519,13 @@ void BasicMediaController::initRenderer()
518519
// loop. This is a work-around for a RenderTexture issue
519520
// when being created such places as a button click event handler
520521
// on Apple platforms/Metal renderer backend
521-
scheduleOnce([this](float){
522+
scheduleOnce(
523+
[this](float) {
522524
createControls();
523525
updateControlsForContentSize(_mediaPlayer->getContentSize());
524526
updateControllerState();
525-
}, 0.f, "__create_video_controls"sv);
527+
},
528+
0.f, "__create_video_controls"sv);
526529
}
527530

528531
void BasicMediaController::onPressStateChangedToPressed()
@@ -551,8 +554,10 @@ void BasicMediaController::onPressStateChangedToPressed()
551554
_controlPanel->runAction(Sequence::create(FadeOut::create(0.5f), nullptr));
552555
_mediaOverlay->runAction(Sequence::create(FadeOut::create(0.5f), nullptr));
553556
}
554-
}, 1.f, "__media_controller_fader"sv);
555-
}), nullptr));
557+
},
558+
1.f, "__media_controller_fader"sv);
559+
}),
560+
nullptr));
556561
}
557562

558563
void BasicMediaController::setContentSize(const Vec2& contentSize)
@@ -586,8 +591,7 @@ void BasicMediaController::updateControllerState()
586591
return;
587592

588593
auto state = _mediaPlayer->getState();
589-
if (state == MediaPlayer::MediaState::LOADING ||
590-
state == MediaPlayer::MediaState::CLOSED ||
594+
if (state == MediaPlayer::MediaState::LOADING || state == MediaPlayer::MediaState::CLOSED ||
591595
state == MediaPlayer::MediaState::ERROR)
592596
{
593597
_playButton->setVisible(false);
@@ -882,7 +886,8 @@ MediaPlayer::MediaPlayer()
882886
pvd->_vrender->setAutoUpdatePS(false);
883887
this->addProtectedChild(pvd->_vrender);
884888
/// setup media event callback
885-
pvd->_engine->setCallbacks([this, pvd](MEMediaEventType event) {
889+
pvd->_engine->setCallbacks(
890+
[this, pvd](MEMediaEventType event) {
886891
switch (event)
887892
{
888893
case MEMediaEventType::Playing:
@@ -895,7 +900,7 @@ MediaPlayer::MediaPlayer()
895900
break;
896901

897902
case MEMediaEventType::Stopped:
898-
onPlayEvent(pvd->_engine->isPlaybackEnded() ? (int) EventType::COMPLETED : (int) EventType::STOPPED);
903+
onPlayEvent(pvd->_engine->isPlaybackEnded() ? (int)EventType::COMPLETED : (int)EventType::STOPPED);
899904
break;
900905

901906
/* Raised by a media source when a presentation ends. This event signals that all streams in the
@@ -914,7 +919,8 @@ MediaPlayer::MediaPlayer()
914919
onPlayEvent((int)EventType::ERROR);
915920
break;
916921
}
917-
}, [this, pvd](const ax::MEVideoFrame& frame) {
922+
},
923+
[this, pvd](const ax::MEVideoFrame& frame) {
918924
auto pixelFormat = frame._vpd._PF;
919925
auto bPixelDescChnaged = !frame._vpd.equals(pvd->_vpixelDesc);
920926
if (bPixelDescChnaged)
@@ -925,10 +931,17 @@ MediaPlayer::MediaPlayer()
925931
pvd->_vtexture = new Texture2D(); // deault is Sampler Filter is: LINEAR
926932

927933
AX_SAFE_RELEASE_NULL(pvd->_vchromaTexture);
934+
AX_SAFE_RELEASE_NULL(pvd->_vchroma2Texture);
928935
if (pixelFormat >= MEVideoPixelFormat::YUY2)
929936
{ // use separated texture we can set differrent sample filter
930937
pvd->_vchromaTexture = new Texture2D(); // Sampler Filter: NEAREST
931938
pvd->_vchromaTexture->setAliasTexParameters();
939+
940+
if (pixelFormat == MEVideoPixelFormat::I420)
941+
{
942+
pvd->_vchroma2Texture = new Texture2D(); // Sampler Filter: NEAREST
943+
pvd->_vchroma2Texture->setAliasTexParameters();
944+
}
932945
}
933946

934947
auto programManager = ProgramManager::getInstance();
@@ -941,6 +954,9 @@ MediaPlayer::MediaPlayer()
941954
case MEVideoPixelFormat::NV12:
942955
pvd->_vrender->setProgramState(backend::ProgramType::VIDEO_TEXTURE_NV12);
943956
break;
957+
case MEVideoPixelFormat::I420:
958+
pvd->_vrender->setProgramState(backend::ProgramType::VIDEO_TEXTURE_I420);
959+
break;
944960
default:
945961
pvd->_vrender->setProgramState(backend::ProgramType::VIDEO_TEXTURE_RGB32);
946962
}
@@ -967,6 +983,18 @@ MediaPlayer::MediaPlayer()
967983
bufferDim.y >> 1, false, 0);
968984
break;
969985
}
986+
case MEVideoPixelFormat::I420:
987+
{
988+
pvd->_vtexture->updateWithData(frame._dataPointer, bufferDim.x * bufferDim.y, PixelFormat::R8,
989+
PixelFormat::R8, bufferDim.x, bufferDim.y, false, 0);
990+
const auto chromaTexDataSize = (bufferDim.x * bufferDim.y) >> 2;
991+
pvd->_vchromaTexture->updateWithData(frame._cbcrDataPointer, chromaTexDataSize, PixelFormat::R8,
992+
PixelFormat::R8, bufferDim.x >> 1, bufferDim.y >> 1, false, 0);
993+
pvd->_vchroma2Texture->updateWithData(frame._cbcrDataPointer + chromaTexDataSize, chromaTexDataSize,
994+
PixelFormat::R8, PixelFormat::R8, bufferDim.x >> 1,
995+
bufferDim.y >> 1, false, 0);
996+
break;
997+
}
970998
case MEVideoPixelFormat::RGB32:
971999
pvd->_vtexture->updateWithData(frame._dataPointer, frame._dataLen, PixelFormat::RGBA8,
9721000
PixelFormat::RGBA8, bufferDim.x, bufferDim.y, false, 0);
@@ -989,7 +1017,11 @@ MediaPlayer::MediaPlayer()
9891017
{
9901018
auto ps = pvd->_vrender->getProgramState();
9911019
PrivateVideoDescriptor::updateColorTransform(ps, frame._vpd._fullRange);
1020+
9921021
ps->setTexture(ps->getUniformLocation("u_tex1"), 1, pvd->_vchromaTexture->getBackendTexture());
1022+
1023+
if (pixelFormat == MEVideoPixelFormat::I420)
1024+
ps->setTexture(ps->getUniformLocation("u_tex2"), 2, pvd->_vchroma2Texture->getBackendTexture());
9931025
}
9941026

9951027
pvd->_scaleDirty = true;
@@ -1016,6 +1048,7 @@ MediaPlayer::~MediaPlayer()
10161048
AX_SAFE_RELEASE(pvd->_vrender);
10171049
AX_SAFE_RELEASE(pvd->_vtexture);
10181050
AX_SAFE_RELEASE(pvd->_vchromaTexture);
1051+
AX_SAFE_RELEASE(pvd->_vchroma2Texture);
10191052

10201053
if (g_mediaControlsTexture && g_mediaControlsTexture->getReferenceCount() == 1)
10211054
{

0 commit comments

Comments
 (0)