Skip to content

Commit a2cf1c7

Browse files
CoPrezCoPrez
authored
Update the native camera dimensions dynamically (#1203)
1 parent 75954f4 commit a2cf1c7

File tree

10 files changed

+142
-77
lines changed

10 files changed

+142
-77
lines changed

‎Plugins/NativeCamera/Source/Android/CameraDevice.cpp‎

Lines changed: 73 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,26 @@ namespace Babylon::Plugins
4646
return oesTexture;
4747
}
4848

49+
int GetCurrentSensorRotationDiff() {
50+
// Get the phone's current rotation so we can determine if the camera image needs to be rotated based on the sensor's natural orientation
51+
int32_t phoneRotation{GetAppContext().getSystemService<android::view::WindowManager>().getDefaultDisplay().getRotation() * 90};
52+
53+
// The sensor rotation dictates the orientation of the camera when the phone is in it's default orientation
54+
// Subtracting the phone's rotation from the camera's rotation will give us the current orientation
55+
// of the sensor. Then add 360 and modulus 360 to ensure we're always talking about positive degrees.
56+
int currentSensorRotationDiff{(sensorRotation - phoneRotation + 360) % 360};
57+
bool sensorIsPortrait{currentSensorRotationDiff == 90 || currentSensorRotationDiff == 270};
58+
if (facingUser && !sensorIsPortrait)
59+
{
60+
// Compensate for the front facing camera being naturally mirrored. In the portrait orientation
61+
// the mirrored behavior matches the browser, but in landscape it would result in the image rendering
62+
// upside down. Rotate the image by 180 to compensate.
63+
currentSensorRotationDiff = (currentSensorRotationDiff + 180) % 360;
64+
}
65+
66+
return currentSensorRotationDiff;
67+
}
68+
4969
Napi::Env env;
5070

5171
arcana::affinity threadAffinity{};
@@ -56,6 +76,8 @@ namespace Babylon::Plugins
5676
int32_t sensorRotation{};
5777
bool facingUser{};
5878
CameraDimensions cameraDimensions{};
79+
int32_t sensorRotationDiff{};
80+
bool updateTextureDimensions{true};
5981

6082
Graphics::DeviceContext* deviceContext{};
6183

@@ -73,7 +95,6 @@ namespace Babylon::Plugins
7395
GLuint cameraRGBATextureId{};
7496
GLuint cameraShaderProgramId{};
7597
GLuint frameBufferId{};
76-
const GLfloat* cameraUVs{};
7798

7899
EGLContext context{EGL_NO_CONTEXT};
79100
EGLDisplay display{};
@@ -186,25 +207,10 @@ namespace Babylon::Plugins
186207
return android::Permissions::CheckCameraPermissionAsync().then(arcana::inline_scheduler, arcana::cancellation::none(), [this, &track]()
187208
{
188209
// Get the phone's current rotation so we can determine if the camera image needs to be rotated based on the sensor's natural orientation
189-
int phoneRotation{GetAppContext().getSystemService<android::view::WindowManager>().getDefaultDisplay().getRotation() * 90};
190-
191-
// The sensor rotation dictates the orientation of the camera when the phone is in it's default orientation
192-
// Subtracting the phone's rotation from the camera's rotation will give us the current orientation
193-
// of the sensor. Then add 360 and modulus 360 to ensure we're always talking about positive degrees.
194-
int sensorRotationDiff{(m_impl->sensorRotation - phoneRotation + 360) % 360};
195-
bool sensorIsPortrait{sensorRotationDiff == 90 || sensorRotationDiff == 270};
196-
if (m_impl->facingUser && !sensorIsPortrait)
197-
{
198-
// Compensate for the front facing camera being naturally mirrored. In the portrait orientation
199-
// the mirrored behavior matches the browser, but in landscape it would result in the image rendering
200-
// upside down. Rotate the image by 180 to compensate.
201-
sensorRotationDiff = (sensorRotationDiff + 180) % 360;
202-
}
210+
m_impl->sensorRotationDiff = m_impl->GetCurrentSensorRotationDiff();
203211

204-
// To match the web implementation if the sensor is rotated into a portrait orientation then the width and height
205-
// of the video should be swapped
206-
m_impl->cameraDimensions.width = !sensorIsPortrait ? track.Width() : track.Height();
207-
m_impl->cameraDimensions.height = !sensorIsPortrait ? track.Height() : track.Width();
212+
m_impl->cameraDimensions.width = track.Width();
213+
m_impl->cameraDimensions.height = track.Height();
208214

209215
// Check if there is an already available context for this thread
210216
EGLContext currentContext = eglGetCurrentContext();
@@ -243,26 +249,6 @@ namespace Babylon::Plugins
243249
}
244250
}
245251

246-
glGenTextures(1, &m_impl->cameraRGBATextureId);
247-
glBindTexture(GL_TEXTURE_2D, m_impl->cameraRGBATextureId);
248-
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, m_impl->cameraDimensions.width, m_impl->cameraDimensions.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr);
249-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
250-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
251-
glGenerateMipmap(GL_TEXTURE_2D);
252-
253-
glBindTexture(GL_TEXTURE_2D, 0);
254-
255-
glGenFramebuffers(1, &m_impl->frameBufferId);
256-
glBindFramebuffer(GL_FRAMEBUFFER, m_impl->frameBufferId);
257-
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, m_impl->cameraRGBATextureId, 0);
258-
259-
glBindFramebuffer(GL_FRAMEBUFFER, 0);
260-
261-
m_impl->cameraUVs = sensorRotationDiff == 90 ? CAMERA_UVS_ROTATION_90 :
262-
sensorRotationDiff == 180 ? CAMERA_UVS_ROTATION_180 :
263-
sensorRotationDiff == 270 ? CAMERA_UVS_ROTATION_270 :
264-
CAMERA_UVS_ROTATION_0;
265-
266252
m_impl->cameraShaderProgramId = android::OpenGLHelpers::CreateShaderProgram(CAMERA_VERT_SHADER, CAMERA_FRAG_SHADER);
267253

268254
m_impl->cameraOESTextureId = m_impl->GenerateOESTexture();
@@ -315,7 +301,12 @@ namespace Babylon::Plugins
315301
throw std::runtime_error{"Unable to restore GL context for camera texture init."};
316302
}
317303

318-
return m_impl->cameraDimensions;
304+
// To match the web implementation if the sensor is rotated into a portrait orientation then the width and height
305+
// of the video should be swapped
306+
bool sensorIsPortrait{m_impl->sensorRotationDiff == 90 || m_impl->sensorRotationDiff == 270};
307+
return !sensorIsPortrait
308+
? CameraDimensions{m_impl->cameraDimensions.width, m_impl->cameraDimensions.height}
309+
: CameraDimensions{m_impl->cameraDimensions.height, m_impl->cameraDimensions.width};
319310
});
320311
}
321312

@@ -446,7 +437,7 @@ namespace Babylon::Plugins
446437
return cameraDevices;
447438
}
448439

449-
void CameraDevice::UpdateCameraTexture(bgfx::TextureHandle textureHandle)
440+
CameraDevice::CameraDimensions CameraDevice::UpdateCameraTexture(bgfx::TextureHandle textureHandle)
450441
{
451442
EGLContext currentContext = eglGetCurrentContext();
452443
if (m_impl->context != EGL_NO_CONTEXT)
@@ -458,17 +449,52 @@ namespace Babylon::Plugins
458449
}
459450
}
460451

452+
int currentSensorRotationDiff = m_impl->GetCurrentSensorRotationDiff();
453+
454+
// The UI Orientation has changed. Update our internal texture
455+
if (currentSensorRotationDiff != m_impl->sensorRotationDiff)
456+
{
457+
m_impl->sensorRotationDiff = currentSensorRotationDiff;
458+
m_impl->updateTextureDimensions = true;
459+
}
460+
461+
bool sensorIsPortrait{m_impl->sensorRotationDiff == 90 || m_impl->sensorRotationDiff == 270};
462+
463+
if (m_impl->updateTextureDimensions)
464+
{
465+
glGenTextures(1, &m_impl->cameraRGBATextureId);
466+
glBindTexture(GL_TEXTURE_2D, m_impl->cameraRGBATextureId);
467+
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, !sensorIsPortrait ? m_impl->cameraDimensions.width : m_impl->cameraDimensions.height, !sensorIsPortrait ? m_impl->cameraDimensions.height : m_impl->cameraDimensions.width, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr);
468+
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
469+
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
470+
glGenerateMipmap(GL_TEXTURE_2D);
471+
472+
glBindTexture(GL_TEXTURE_2D, 0);
473+
474+
glGenFramebuffers(1, &m_impl->frameBufferId);
475+
glBindFramebuffer(GL_FRAMEBUFFER, m_impl->frameBufferId);
476+
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, m_impl->cameraRGBATextureId, 0);
477+
478+
glBindFramebuffer(GL_FRAMEBUFFER, 0);
479+
480+
m_impl->updateTextureDimensions = false;
481+
}
482+
461483
m_impl->surfaceTexture.updateTexImage();
462484

463485
glBindFramebuffer(GL_FRAMEBUFFER, m_impl->frameBufferId);
464-
glViewport(0, 0, m_impl->cameraDimensions.width, m_impl->cameraDimensions.height);
486+
glViewport(0, 0, !sensorIsPortrait ? m_impl->cameraDimensions.width : m_impl->cameraDimensions.height, !sensorIsPortrait ? m_impl->cameraDimensions.height : m_impl->cameraDimensions.width);
465487
glUseProgram(m_impl->cameraShaderProgramId);
466488

467489
auto vertexPositionsUniformLocation{glGetUniformLocation(m_impl->cameraShaderProgramId, "positions")};
468490
glUniform2fv(vertexPositionsUniformLocation, CAMERA_VERTEX_COUNT, CAMERA_VERTEX_POSITIONS);
469491

470492
auto uvsUniformLocation{glGetUniformLocation(m_impl->cameraShaderProgramId, "uvs")};
471-
glUniform2fv(uvsUniformLocation, CAMERA_UVS_COUNT, m_impl->cameraUVs);
493+
glUniform2fv(uvsUniformLocation, CAMERA_UVS_COUNT,
494+
m_impl->sensorRotationDiff == 90 ? CAMERA_UVS_ROTATION_90 :
495+
m_impl->sensorRotationDiff == 180 ? CAMERA_UVS_ROTATION_180 :
496+
m_impl->sensorRotationDiff == 270 ? CAMERA_UVS_ROTATION_270 :
497+
CAMERA_UVS_ROTATION_0);
472498

473499
// Configure the camera texture
474500
auto cameraTextureUniformLocation{glGetUniformLocation(m_impl->cameraShaderProgramId, "cameraTexture")};
@@ -492,6 +518,10 @@ namespace Babylon::Plugins
492518
arcana::make_task(m_impl->deviceContext->BeforeRenderScheduler(), arcana::cancellation::none(), [rgbaTextureId = m_impl->cameraRGBATextureId, textureHandle] {
493519
bgfx::overrideInternal(textureHandle, rgbaTextureId);
494520
});
521+
522+
return !sensorIsPortrait
523+
? CameraDimensions{m_impl->cameraDimensions.width, m_impl->cameraDimensions.height}
524+
: CameraDimensions{m_impl->cameraDimensions.height, m_impl->cameraDimensions.width};
495525
}
496526

497527
void CameraDevice::Close()

‎Plugins/NativeCamera/Source/Apple/CameraDevice.mm‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
189189
id<MTLCommandBuffer> currentCommandBuffer{};
190190
bool isInitialized{false};
191191
bool refreshBgfxHandle{true};
192+
bgfx::TextureHandle textureHandle{};
192193

193194
arcana::background_dispatcher<32> cameraSessionDispatcher{};
194195
std::shared_ptr<arcana::cancellation_source> cancellationSource{std::make_shared<arcana::cancellation_source>()};
@@ -390,14 +391,7 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
390391
CMVideoFormatDescriptionRef videoFormatRef{static_cast<CMVideoFormatDescriptionRef>(resolution.m_impl->avDeviceFormat.formatDescription)};
391392
CMVideoDimensions dimensions{CMVideoFormatDescriptionGetDimensions(videoFormatRef)};
392393

393-
CameraDevice::CameraDimensions cameraDimensions{static_cast<uint32_t>(dimensions.width), static_cast<uint32_t>(dimensions.height)};
394-
395-
// For portrait orientations swap the height and width of the video format dimensions.
396-
if (m_impl->cameraTextureDelegate->VideoOrientation == AVCaptureVideoOrientationPortrait
397-
|| m_impl->cameraTextureDelegate->VideoOrientation == AVCaptureVideoOrientationPortraitUpsideDown)
398-
{
399-
std::swap(cameraDimensions.width, cameraDimensions.height);
400-
}
394+
m_impl->cameraDimensions = CameraDimensions{static_cast<uint32_t>(dimensions.width), static_cast<uint32_t>(dimensions.height)};
401395

402396
// Check for failed initialisation.
403397
if (!input)
@@ -406,7 +400,7 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
406400
}
407401

408402
// Kick off camera session on a background thread.
409-
return arcana::make_task(m_impl->cameraSessionDispatcher, arcana::cancellation::none(), [implObj = shared_from_this(), pixelFormat = resolution.m_impl->pixelFormat, input, cameraDimensions]() mutable {
403+
return arcana::make_task(m_impl->cameraSessionDispatcher, arcana::cancellation::none(), [implObj = shared_from_this(), pixelFormat = resolution.m_impl->pixelFormat, input]() mutable {
410404
if (implObj->m_impl->avCaptureSession == nil) {
411405
implObj->m_impl->avCaptureSession = [[AVCaptureSession alloc] init];
412406
} else {
@@ -436,11 +430,17 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
436430

437431
// Actually start the camera session.
438432
[implObj->m_impl->avCaptureSession startRunning];
439-
return cameraDimensions;
433+
434+
// To match the web implementation if the sensor is rotated into a portrait orientation then the width and height
435+
// of the video should be swapped
436+
return implObj->m_impl->cameraTextureDelegate->VideoOrientation == AVCaptureVideoOrientationLandscapeLeft ||
437+
implObj->m_impl->cameraTextureDelegate->VideoOrientation == AVCaptureVideoOrientationLandscapeRight ?
438+
CameraDimensions{implObj->m_impl->cameraDimensions.width, implObj->m_impl->cameraDimensions.height} :
439+
CameraDimensions{implObj->m_impl->cameraDimensions.width, implObj->m_impl->cameraDimensions.height};
440440
});
441441
}
442442

443-
void CameraDevice::UpdateCameraTexture(bgfx::TextureHandle textureHandle)
443+
CameraDevice::CameraDimensions CameraDevice::UpdateCameraTexture(bgfx::TextureHandle textureHandle)
444444
{
445445
// Hook into AfterRender to copy over the texture, ensuring that the textureHandle has already been initialized by bgfx.
446446
// Capture the cancellation token so that the shared pointer is kept alive when arcana checks internally for cancellation.
@@ -475,6 +475,13 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
475475
if (width == 0 || height == 0) {
476476
return;
477477
}
478+
479+
// Check if the we've been handed a new texture handle and if so refresh our override
480+
if (m_impl->textureHandle.idx != textureHandle.idx)
481+
{
482+
m_impl->refreshBgfxHandle = true;
483+
m_impl->textureHandle = textureHandle;
484+
}
478485

479486
// Recreate the output texture when the camera dimensions change.
480487
if (m_impl->textureRGBA == nil || m_impl->cameraDimensions.width != width || m_impl->cameraDimensions.height != height)
@@ -569,7 +576,13 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
569576
[m_impl->currentCommandBuffer commit];
570577
}
571578
});
572-
}
579+
580+
// To match the web implementation if the sensor is rotated into a portrait orientation then the width and height
581+
// of the video should be swapped
582+
return m_impl->cameraTextureDelegate->VideoOrientation == AVCaptureVideoOrientationLandscapeLeft ||
583+
m_impl->cameraTextureDelegate->VideoOrientation == AVCaptureVideoOrientationLandscapeRight ?
584+
CameraDimensions{m_impl->cameraDimensions.width, m_impl->cameraDimensions.height} :
585+
CameraDimensions{m_impl->cameraDimensions.width, m_impl->cameraDimensions.height}; }
573586

574587
void CameraDevice::Close()
575588
{

‎Plugins/NativeCamera/Source/CameraDevice.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ namespace Babylon::Plugins
5151

5252
arcana::task<CameraDimensions, std::exception_ptr> OpenAsync(const CameraTrack& track);
5353
void Close();
54-
void UpdateCameraTexture(bgfx::TextureHandle textureHandle);
54+
CameraDimensions UpdateCameraTexture(bgfx::TextureHandle textureHandle);
5555

5656
const std::vector<CameraTrack>& SupportedResolutions() const;
5757
const std::vector<std::unique_ptr<Capability>>& Capabilities() const;

‎Plugins/NativeCamera/Source/MediaStream.cpp‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -350,14 +350,25 @@ namespace Babylon::Plugins
350350
m_cameraDevice = nullptr;
351351
}
352352

353-
void MediaStream::UpdateTexture(bgfx::TextureHandle textureHandle)
353+
bool MediaStream::UpdateTexture(bgfx::TextureHandle textureHandle)
354354
{
355+
bool dimensionsChanged = false;
356+
355357
if (m_cameraDevice == nullptr)
356358
{
357359
// We don't have a cameraDevice selected yet.
358-
return;
360+
return dimensionsChanged;
361+
}
362+
363+
auto cameraDimensions{m_cameraDevice->UpdateCameraTexture(textureHandle)};
364+
365+
if (this->Width != cameraDimensions.width || this->Height != cameraDimensions.height)
366+
{
367+
dimensionsChanged = true;
368+
this->Width = cameraDimensions.width;
369+
this->Height = cameraDimensions.height;
359370
}
360371

361-
m_cameraDevice->UpdateCameraTexture(textureHandle);
372+
return dimensionsChanged;
362373
}
363374
}

‎Plugins/NativeCamera/Source/MediaStream.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,12 @@ namespace Babylon::Plugins
3131
Napi::Value GetSettings(const Napi::CallbackInfo& info);
3232
Napi::Value GetConstraints(const Napi::CallbackInfo& info);
3333
void Stop(const Napi::CallbackInfo& info);
34+
35+
// Update the camera texture and return true if the dimensions have changed, false otherwise
36+
bool UpdateTexture(bgfx::TextureHandle textureHandle);
3437

35-
void UpdateTexture(bgfx::TextureHandle textureHandle);
36-
37-
int Width{0};
38-
int Height{0};
38+
uint32_t Width{0};
39+
uint32_t Height{0};
3940

4041
private:
4142
arcana::task<void, std::exception_ptr> ApplyInitialConstraintsAsync(Napi::Env env, Napi::Object constraints);

‎Plugins/NativeCamera/Source/NativeVideo.cpp‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,22 @@ namespace Babylon::Plugins
3939

4040
Napi::Value NativeVideo::GetVideoWidth(const Napi::CallbackInfo& /*info*/)
4141
{
42-
return Napi::Value::From(Env(), m_width);
42+
if (!m_streamObject.Value().IsNull() && !m_streamObject.Value().IsUndefined())
43+
{
44+
return Napi::Value::From(Env(), MediaStream::Unwrap(m_streamObject.Value())->Width);
45+
}
46+
47+
return Napi::Value::From(Env(), 0);
4348
}
4449

4550
Napi::Value NativeVideo::GetVideoHeight(const Napi::CallbackInfo& /*info*/)
4651
{
47-
return Napi::Value::From(Env(), m_height);
52+
if (!m_streamObject.Value().IsNull() && !m_streamObject.Value().IsUndefined())
53+
{
54+
return Napi::Value::From(Env(), MediaStream::Unwrap(m_streamObject.Value())->Height);
55+
}
56+
57+
return Napi::Value::From(Env(), 0);
4858
}
4959

5060
void NativeVideo::SetAttribute(const Napi::CallbackInfo&)
@@ -72,7 +82,11 @@ namespace Babylon::Plugins
7282
// Only update the texture if we're playing and the srcObject is an instance of a MediaStream
7383
if (m_IsPlaying && !m_streamObject.Value().IsNull() && !m_streamObject.Value().IsUndefined())
7484
{
75-
MediaStream::Unwrap(m_streamObject.Value())->UpdateTexture(textureHandle);
85+
if (MediaStream::Unwrap(m_streamObject.Value())->UpdateTexture(textureHandle))
86+
{
87+
// The video dimensions have changed, raise the resize event to observers
88+
RaiseEvent("resize");
89+
}
7690
}
7791
}
7892

@@ -152,20 +166,18 @@ namespace Babylon::Plugins
152166
if (value.IsNull() || value.IsUndefined() || !value.As<Napi::Object>().InstanceOf(MediaStream::GetConstructor(env)))
153167
{
154168
m_streamObject = Napi::ObjectReference();
155-
this->m_width = 0;
156-
this->m_height = 0;
157169
this->m_isReady = false;
158170
this->m_IsPlaying = false;
159171
return;
160172
}
161173

162174
// We've received a MediaStream object
163175
m_streamObject = Napi::Persistent(value.As<Napi::Object>());
164-
auto mediaStream = MediaStream::Unwrap(m_streamObject.Value());
165-
166-
this->m_width = mediaStream->Width;
167-
this->m_height = mediaStream->Height;
176+
168177
this->m_isReady = true;
178+
179+
// Now that we have a src object the resize event should be fired to notify observers of the new video dimensions
180+
RaiseEvent("resize");
169181
}
170182

171183
Napi::Value NativeVideo::GetSrcObject(const Napi::CallbackInfo& info) {

0 commit comments

Comments
 (0)