Create dedicated XR view (#173)

Author: ryantrem

Modules/@babylonjs/react-native/android/src/main/cpp/BabylonNativeInterop.cpp

@@ -82,6 +82,21 @@ extern "C" JNIEXPORT void JNICALL Java_com_babylonreactnative_BabylonNativeInter
     Babylon::ResetView();
 }
 
+extern "C" JNIEXPORT void JNICALL Java_com_babylonreactnative_BabylonNativeInterop_00024BabylonNative_updateXRView(JNIEnv* env, jclass obj, jobject surface)
+{
+    ANativeWindow* windowPtr{};
+    if (surface)
+    {
+        windowPtr = ANativeWindow_fromSurface(env, surface);
+    }
+    Babylon::UpdateXRView(windowPtr);
+}
+
+extern "C" JNIEXPORT jboolean JNICALL Java_com_babylonreactnative_BabylonNativeInterop_00024BabylonNative_isXRActive(JNIEnv* env, jclass obj)
+{
+    return Babylon::IsXRActive();
+}
+
 extern "C" JNIEXPORT void JNICALL Java_com_babylonreactnative_BabylonNativeInterop_00024BabylonNative_setTouchButtonState(JNIEnv* env, jclass obj, jint pointerId, jboolean isDown, jint x, jint y)
 {
     Babylon::SetTouchButtonState(static_cast<uint32_t>(pointerId), isDown, static_cast<uint32_t>(x), static_cast<uint32_t>(y));

Modules/@babylonjs/react-native/android/src/main/java/com/babylonreactnative/BabylonNativeInterop.java

@@ -25,6 +25,8 @@ private static class BabylonNative {
         public static native void updateView(Surface surface);
         public static native void renderView();
         public static native void resetView();
+        public static native void updateXRView(Surface surface);
+        public static native boolean isXRActive();
         public static native void setTouchButtonState(int pointerId, boolean isDown, int x, int y);
         public static native void setTouchPosition(int pointerId, int x, int y);
     }
@@ -91,6 +93,14 @@ public static void resetView() {
         BabylonNative.resetView();
     }
 
+    public static void updateXRView(Surface surface) {
+        BabylonNative.updateXRView(surface);
+    }
+
+    public static boolean isXRActive() {
+        return BabylonNative.isXRActive();
+    }
+
     public static void reportMotionEvent(MotionEvent motionEvent) {
         int maskedAction = motionEvent.getActionMasked();
         boolean isPointerDown = maskedAction == MotionEvent.ACTION_DOWN || maskedAction == MotionEvent.ACTION_POINTER_DOWN;

Modules/@babylonjs/react-native/android/src/main/java/com/babylonreactnative/EngineView.java

@@ -12,23 +12,54 @@
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
+import android.widget.FrameLayout;
 
 import com.facebook.react.bridge.ReactContext;
 import com.facebook.react.uimanager.UIManagerModule;
 import com.facebook.react.uimanager.events.EventDispatcher;
 
 import java.io.ByteArrayOutputStream;
 
-public final class EngineView extends SurfaceView implements SurfaceHolder.Callback, View.OnTouchListener {
+public final class EngineView extends FrameLayout implements SurfaceHolder.Callback, View.OnTouchListener {
+    private static final FrameLayout.LayoutParams childViewLayoutParams = new FrameLayout.LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT);
+    private final SurfaceView primarySurfaceView;
+    private final SurfaceView xrSurfaceView;
     private final EventDispatcher reactEventDispatcher;
 
     public EngineView(ReactContext reactContext) {
         super(reactContext);
-        this.getHolder().addCallback(this);
+
+        this.primarySurfaceView = new SurfaceView(reactContext);
+        this.primarySurfaceView.setLayoutParams(EngineView.childViewLayoutParams);
+        this.primarySurfaceView.getHolder().addCallback(this);
+        this.addView(this.primarySurfaceView);
+
+        this.xrSurfaceView = new SurfaceView(reactContext);
+        this.xrSurfaceView.setLayoutParams(childViewLayoutParams);
+        this.xrSurfaceView.getHolder().addCallback(new SurfaceHolder.Callback() {
+            @Override
+            public void surfaceCreated(SurfaceHolder holder) {
+                // surfaceChanged is also called when the surface is created, so just do all the handling there
+            }
+
+            @Override
+            public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                BabylonNativeInterop.updateXRView(holder.getSurface());
+            }
+
+            @Override
+            public void surfaceDestroyed(SurfaceHolder holder) {
+                BabylonNativeInterop.updateXRView(null);
+            }
+        });
+        this.xrSurfaceView.setVisibility(View.INVISIBLE);
+        this.addView(this.xrSurfaceView);
+
         this.setOnTouchListener(this);
-        this.reactEventDispatcher = reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher();
 
-        setWillNotDraw(false);
+        this.setWillNotDraw(false);
+
+        this.reactEventDispatcher = reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher();
     }
 
     @Override
@@ -51,6 +82,18 @@ public boolean onTouch(View view, MotionEvent motionEvent) {
         return true;
     }
 
+    @Override
+    protected void onDraw(Canvas canvas) {
+        if (BabylonNativeInterop.isXRActive()) {
+            this.xrSurfaceView.setVisibility(View.VISIBLE);
+        } else {
+            this.xrSurfaceView.setVisibility(View.INVISIBLE);
+        }
+
+        BabylonNativeInterop.renderView();
+        invalidate();
+    }
+
     @TargetApi(24)
     public void takeSnapshot() {
         // Only supported on API level 24 and up, return a blank image.
@@ -70,8 +113,13 @@ public void takeSnapshot() {
         helperThread.start();
         final Handler helperThreadHandler = new Handler(helperThread.getLooper());
 
+        SurfaceView surfaceView = this.primarySurfaceView;
+        if (BabylonNativeInterop.isXRActive()) {
+            surfaceView = this.xrSurfaceView;
+        }
+
         // Request the pixel copy.
-        PixelCopy.request(this, bitmap, (copyResult) ->  {
+        PixelCopy.request(surfaceView, bitmap, (copyResult) ->  {
             // If the pixel copy was a success then convert the image to a base 64 encoded jpeg and fire the event.
             String encoded = "";
             if (copyResult == PixelCopy.SUCCESS) {
@@ -87,10 +135,4 @@ public void takeSnapshot() {
             helperThread.quitSafely();
         }, helperThreadHandler);
     }
-
-    @Override
-    protected void onDraw(Canvas canvas) {
-        BabylonNativeInterop.renderView();
-        invalidate();
-    }
 }

Modules/@babylonjs/react-native/ios/BabylonNativeInterop.h

@@ -8,5 +8,7 @@
 + (void)updateView:(MTKView*)mtkView;
 + (void)renderView;
 + (void)resetView;
++ (void)updateXRView:(MTKView*)mtkView;
++ (bool)isXRActive;
 + (void)reportTouchEvent:(MTKView*)mtkView touches:(NSSet<UITouch*>*)touches event:(UIEvent*)event;
 @end

Modules/@babylonjs/react-native/ios/BabylonNativeInterop.mm

@@ -70,6 +70,14 @@ + (void)resetView {
     Babylon::ResetView();
 }
 
++ (void)updateXRView:(MTKView*)mtkView {
+    Babylon::UpdateXRView((__bridge void*)mtkView);
+}
+
++ (bool)isXRActive {
+    return Babylon::IsXRActive();
+}
+
 + (void)reportTouchEvent:(MTKView*)mtkView touches:(NSSet<UITouch*>*)touches event:(UIEvent*)event {
     for (UITouch* touch in touches) {
         if (touch.view == mtkView) {

Modules/@babylonjs/react-native/ios/EngineViewManager.mm

@@ -15,7 +15,8 @@ @interface EngineView : MTKView
 @end
 
 @implementation EngineView {
-    RCTBridge* bridge;
+    const RCTBridge* bridge;
+    MTKView* xrView;
 }
 
 - (instancetype)init:(RCTBridge*)_bridge {
@@ -25,6 +26,13 @@ - (instancetype)init:(RCTBridge*)_bridge {
         super.translatesAutoresizingMaskIntoConstraints = false;
         super.colorPixelFormat = MTLPixelFormatBGRA8Unorm_sRGB;
         super.depthStencilPixelFormat = MTLPixelFormatDepth32Float;
+
+        xrView = [[MTKView alloc] initWithFrame:self.bounds device:self.device];
+        xrView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
+        xrView.userInteractionEnabled = false;
+        xrView.hidden = true;
+        [self addSubview:xrView];
+        [BabylonNativeInterop updateXRView:xrView];
     }
     return self;
 }
@@ -51,6 +59,12 @@ - (void)touchesCancelled:(NSSet<UITouch*>*)touches withEvent:(UIEvent*)event {
 }
 
 - (void)drawRect:(CGRect)rect {
+    if ([BabylonNativeInterop isXRActive]) {
+        xrView.hidden = false;
+    } else {
+        xrView.hidden = true;
+    }
+
     [BabylonNativeInterop renderView];
 }
 

Modules/@babylonjs/react-native/shared/BabylonNative.cpp

@@ -33,6 +33,7 @@ namespace Babylon
             : m_env{ Napi::Attach<facebook::jsi::Runtime&>(jsiRuntime) }
             , m_jsDispatcher{ std::move(jsDispatcher) }
             , m_isRunning{ std::make_shared<bool>(true) }
+            , m_isXRActive{ std::make_shared<bool>(false) }
         {
             // Initialize a JS promise that will be returned by whenInitialized, and completed when NativeEngine is initialized.
             CreateInitPromise();
@@ -41,7 +42,8 @@ namespace Babylon
             JsRuntime::CreateForJavaScript(m_env, CreateJsRuntimeDispatcher(m_env, jsiRuntime, m_jsDispatcher, m_isRunning));
 
             // Initialize Babylon Native plugins
-            Plugins::NativeXr::Initialize(m_env);
+            m_nativeXr.emplace(Plugins::NativeXr::Initialize(m_env));
+            m_nativeXr->SetSessionStateChangedCallback([isXRActive{ m_isXRActive }](bool isSessionActive) { *isXRActive = isSessionActive; });
             Plugins::NativeCapture::Initialize(m_env);
             m_nativeInput = &Plugins::NativeInput::CreateForJavaScript(m_env);
 
@@ -144,6 +146,16 @@ namespace Babylon
             m_nativeInput->TouchMove(pointerId, x, y);
         }
 
+        bool IsXRActive()
+        {
+            return *m_isXRActive;
+        }
+
+        void UpdateXRView(void* windowPtr)
+        {
+            m_nativeXr->UpdateWindow(windowPtr);
+        }
+
         jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& prop) override
         {
             const auto propName{ prop.utf8(runtime) };
@@ -183,6 +195,9 @@ namespace Babylon
         std::shared_ptr<bool> m_isRunning{};
         std::once_flag m_isGraphicsInitialized{};
         Plugins::NativeInput* m_nativeInput{};
+        std::optional<Plugins::NativeXr> m_nativeXr{};
+
+        std::shared_ptr<bool> m_isXRActive{};
     };
 
     namespace
@@ -270,4 +285,22 @@ namespace Babylon
             nativeModule->SetTouchPosition(pointerId, x, y);
         }
     }
+
+    bool IsXRActive()
+    {
+        if (auto nativeModule{ g_nativeModule.lock() })
+        {
+            return nativeModule->IsXRActive();
+        }
+
+        return false;
+    }
+
+    void UpdateXRView(void* windowPtr)
+    {
+        if (auto nativeModule{ g_nativeModule.lock() })
+        {
+            nativeModule->UpdateXRView(windowPtr);
+        }
+    }
 }

Modules/@babylonjs/react-native/shared/BabylonNative.h

@@ -15,6 +15,8 @@ namespace Babylon
     void SetMousePosition(uint32_t x, uint32_t y);
     void SetTouchButtonState(uint32_t pointerId, bool isDown, uint32_t x, uint32_t y);
     void SetTouchPosition(uint32_t pointerId, uint32_t x, uint32_t y);
+    bool IsXRActive();
+    void UpdateXRView(void* windowPtr);
 
     extern const uint32_t LEFT_MOUSE_BUTTON_ID;
     extern const uint32_t MIDDLE_MOUSE_BUTTON_ID;