diff --git a/code/mobile/android/PhoneVR/app/CMakeLists.txt b/code/mobile/android/PhoneVR/app/CMakeLists.txt index a5cf82bb..f4501bf4 100644 --- a/code/mobile/android/PhoneVR/app/CMakeLists.txt +++ b/code/mobile/android/PhoneVR/app/CMakeLists.txt @@ -21,6 +21,7 @@ file(GLOB_RECURSE GVR_SRC add_library(native-lib-alvr SHARED src/main/cpp/alvr_main.cpp + src/main/cpp/passthrough.cpp ${LIB_SRC} ) diff --git a/code/mobile/android/PhoneVR/app/build.gradle b/code/mobile/android/PhoneVR/app/build.gradle index da0e4d11..d6ec5e52 100644 --- a/code/mobile/android/PhoneVR/app/build.gradle +++ b/code/mobile/android/PhoneVR/app/build.gradle @@ -95,7 +95,7 @@ android { } project.gradle.taskGraph.whenReady { android.productFlavors.all { flavor -> - // Capitalize (as Gralde is case-sensitive). + // Capitalize (as Gradle is case-sensitive). def flavorName = flavor.name.substring(0, 1).toUpperCase() + flavor.name.substring(1) // At last, configure. @@ -132,6 +132,10 @@ android { } } +repositories { + maven { url 'https://jitpack.io' } +} + dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" @@ -140,6 +144,7 @@ dependencies { implementation 'com.google.android.material:material:1.2.1' implementation "androidx.tracing:tracing:1.1.0" + implementation 'androidx.preference:preference:1.2.1' testImplementation 'junit:junit:4.13.2' testImplementation 'junit:junit:4.12' androidTestImplementation "androidx.test.ext:junit-ktx:$extJUnitVersion" @@ -159,6 +164,7 @@ dependencies { implementation platform('com.google.firebase:firebase-bom:26.0.0') implementation 'com.google.firebase:firebase-analytics-ktx' + implementation 'com.github.PhilJay:MPAndroidChart:v3.1.0' } // The dependencies for NDK builds live inside the .aar files so they need to diff --git a/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml b/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml index 94ffd5b1..2749ca0e 100644 --- a/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml +++ b/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml @@ -46,6 +46,10 @@ android:theme="@style/AppTheme" tools:replace="android:extractNativeLibs" android:extractNativeLibs="true"> + #include "nlohmann/json.hpp" +#include "passthrough.h" #include "utils.h" using namespace nlohmann; @@ -38,6 +39,7 @@ struct NativeContext { bool running = false; bool streaming = false; + PassthroughInfo passthroughInfo = {}; std::thread inputThread; // Une one texture per eye, no need for swapchains. @@ -188,6 +190,10 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_initia Cardboard_initializeAndroid(CTX.javaVm, CTX.javaContext); CTX.headTracker = CardboardHeadTracker_create(); + + CTX.passthroughInfo.screenWidth = &(CTX.screenWidth); + CTX.passthroughInfo.screenHeight = &(CTX.screenHeight); + passthrough_createPlane(&(CTX.passthroughInfo)); } extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_destroyNative(JNIEnv *, @@ -233,11 +239,24 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_pauseN CardboardHeadTracker_pause(CTX.headTracker); } +extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_Passthrough_setPassthroughActiveNative( + JNIEnv *, jobject, jboolean activate) { + CTX.passthroughInfo.enabled = activate; + CTX.renderingParamsChanged = true; +} + extern "C" JNIEXPORT void JNICALL +Java_viritualisres_phonevr_Passthrough_setPassthroughSizeNative(JNIEnv *, jobject, jfloat size) { + CTX.passthroughInfo.passthroughSize = size; + passthrough_createPlane(&(CTX.passthroughInfo)); +} + +extern "C" JNIEXPORT jint JNICALL Java_viritualisres_phonevr_ALVRActivity_surfaceCreatedNative(JNIEnv *, jobject) { alvr_initialize_opengl(); - + GLuint camTex = passthrough_init(&(CTX.passthroughInfo)); CTX.glContextRecreated = true; + return camTex; } extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_setScreenResolutionNative( @@ -308,38 +327,41 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_render if (CTX.renderingParamsChanged && !CTX.glContextRecreated) { info("Pausing ALVR since glContext is not recreated, deleting textures"); alvr_pause_opengl(); - + passthrough_cleanup(&(CTX.passthroughInfo)); GL(glDeleteTextures(2, CTX.lobbyTextures)); } if (CTX.renderingParamsChanged || CTX.glContextRecreated) { - info("Rebuilding, binding textures, Resuming ALVR since glContextRecreated %b, " - "renderingParamsChanged %b", - CTX.renderingParamsChanged, - CTX.glContextRecreated); - GL(glGenTextures(2, CTX.lobbyTextures)); - - for (auto &lobbyTexture : CTX.lobbyTextures) { - GL(glBindTexture(GL_TEXTURE_2D, lobbyTexture)); - GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); - GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); - GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)); - GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)); - GL(glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RGB, - CTX.screenWidth / 2, - CTX.screenHeight, - 0, - GL_RGB, - GL_UNSIGNED_BYTE, - nullptr)); - } - - const uint32_t *targetViews[2] = {(uint32_t *) &CTX.lobbyTextures[0], - (uint32_t *) &CTX.lobbyTextures[1]}; - alvr_resume_opengl(CTX.screenWidth / 2, CTX.screenHeight, targetViews, 1, true); + if (CTX.passthroughInfo.enabled) { + passthrough_setup(&(CTX.passthroughInfo)); + } else { + info("Rebuilding, binding textures, Resuming ALVR since glContextRecreated %b, " + "renderingParamsChanged %b", + CTX.renderingParamsChanged, + CTX.glContextRecreated); + GL(glGenTextures(2, CTX.lobbyTextures)); + + for (auto &lobbyTexture : CTX.lobbyTextures) { + GL(glBindTexture(GL_TEXTURE_2D, lobbyTexture)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)); + GL(glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RGB, + CTX.screenWidth / 2, + CTX.screenHeight, + 0, + GL_RGB, + GL_UNSIGNED_BYTE, + nullptr)); + } + const uint32_t *targetViews[2] = {(uint32_t *) &CTX.lobbyTextures[0], + (uint32_t *) &CTX.lobbyTextures[1]}; + alvr_resume_opengl(CTX.screenWidth / 2, CTX.screenHeight, targetViews, 1, true); + } CTX.renderingParamsChanged = false; CTX.glContextRecreated = false; } @@ -481,7 +503,9 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_render viewsDesc.bottom_v = 0.0; } - if (CTX.streaming) { + if (CTX.passthroughInfo.enabled) { + passthrough_render(&(CTX.passthroughInfo), viewsDescs); + } else if (CTX.streaming) { void *streamHardwareBuffer = nullptr; AlvrViewParams dummyViewParams; diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp new file mode 100644 index 00000000..57c55f55 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp @@ -0,0 +1,198 @@ +#include "alvr_client_core.h" +// clang-format off +#include +#include +// clang-format on +#include + +#include "passthrough.h" +#include "utils.h" + +GLuint LoadGLShader(GLenum type, const char *shader_source) { + GLuint shader = GL(glCreateShader(type)); + + GL(glShaderSource(shader, 1, &shader_source, nullptr)); + GL(glCompileShader(shader)); + + // Get the compilation status. + GLint compile_status; + GL(glGetShaderiv(shader, GL_COMPILE_STATUS, &compile_status)); + + // If the compilation failed, delete the shader and show an error. + if (compile_status == 0) { + GLint info_len = 0; + GL(glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &info_len)); + if (info_len == 0) { + return 0; + } + + std::vector info_string(info_len); + GL(glGetShaderInfoLog(shader, info_string.size(), nullptr, info_string.data())); + // LOGE("Could not compile shader of type %d: %s", type, info_string.data()); + GL(glDeleteShader(shader)); + return 0; + } else { + return shader; + } +} + +namespace { + // Simple shaders to render camera Texture files without any lighting. + constexpr const char *camVertexShader = + R"glsl( + uniform mat4 u_MVP; + attribute vec4 a_Position; + attribute vec2 a_UV; + varying vec2 v_UV; + + void main() { + v_UV = a_UV; + gl_Position = a_Position; + })glsl"; + + constexpr const char *camFragmentShader = + R"glsl( + #extension GL_OES_EGL_image_external : require + precision mediump float; + varying vec2 v_UV; + uniform samplerExternalOES sTexture; + void main() { + gl_FragColor = texture2D(sTexture, v_UV); + })glsl"; + + static int passthroughProgram_ = 0; + static int texturePositionParam_ = 0; + static int textureUvParam_ = 0; + static int textureMvpParam_ = 0; + + float passthroughTexCoords[] = {0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0}; +} // namespace + +void passthrough_createPlane(PassthroughInfo *info) { + float size = info->passthroughSize; + float x0 = -size, y0 = size; // Top left + float x1 = size, y1 = size; // Top right + float x2 = size, y2 = -size; // Bottom right + float x3 = -size, y3 = -size; // Bottom left + + info->passthroughVertices[0] = x3; + info->passthroughVertices[1] = y3; + info->passthroughVertices[2] = x2; + info->passthroughVertices[3] = y2; + info->passthroughVertices[4] = x0; + info->passthroughVertices[5] = y0; + info->passthroughVertices[6] = x1; + info->passthroughVertices[7] = y1; +} + +GLuint passthrough_init(PassthroughInfo *info) { + const int obj_vertex_shader = LoadGLShader(GL_VERTEX_SHADER, camVertexShader); + const int obj_fragment_shader = LoadGLShader(GL_FRAGMENT_SHADER, camFragmentShader); + + passthroughProgram_ = GL(glCreateProgram()); + GL(glAttachShader(passthroughProgram_, obj_vertex_shader)); + GL(glAttachShader(passthroughProgram_, obj_fragment_shader)); + GL(glLinkProgram(passthroughProgram_)); + + GL(glUseProgram(passthroughProgram_)); + texturePositionParam_ = GL(glGetAttribLocation(passthroughProgram_, "a_Position")); + textureUvParam_ = GL(glGetAttribLocation(passthroughProgram_, "a_UV")); + textureMvpParam_ = GL(glGetUniformLocation(passthroughProgram_, "u_MVP")); + + GL(glGenTextures(1, &(info->cameraTexture))); + GL(glActiveTexture(GL_TEXTURE0)); + + GL(glBindTexture(GL_TEXTURE_EXTERNAL_OES, info->cameraTexture)); + GL(glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST)); + GL(glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR)); + GL(glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); + GL(glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); + + return info->cameraTexture; +} + +void passthrough_cleanup(PassthroughInfo *info) { + if (info->passthroughDepthRenderBuffer != 0) { + GL(glDeleteRenderbuffers(1, &(info->passthroughDepthRenderBuffer))); + info->passthroughDepthRenderBuffer = 0; + } + if (info->passthroughFramebuffer != 0) { + GL(glDeleteFramebuffers(1, &info->passthroughFramebuffer)); + info->passthroughFramebuffer = 0; + } + if (info->passthroughTexture != 0) { + GL(glDeleteTextures(1, &(info->passthroughTexture))); + info->passthroughTexture = 0; + } +} + +void passthrough_setup(PassthroughInfo *info) { + // Create render texture. + GL(glGenTextures(1, &(info->passthroughTexture))); + GL(glBindTexture(GL_TEXTURE_2D, info->passthroughTexture)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); + + GL(glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RGB, + *(info->screenWidth), + *(info->screenHeight), + 0, + GL_RGB, + GL_UNSIGNED_BYTE, + 0)); + + // Generate depth buffer to perform depth test. + GL(glGenRenderbuffers(1, &(info->passthroughDepthRenderBuffer))); + GL(glBindRenderbuffer(GL_RENDERBUFFER, info->passthroughDepthRenderBuffer)); + GL(glRenderbufferStorage( + GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, *(info->screenWidth), *(info->screenHeight))); + + // Create render target. + GL(glGenFramebuffers(1, &(info->passthroughFramebuffer))); + GL(glBindFramebuffer(GL_FRAMEBUFFER, info->passthroughFramebuffer)); + GL(glFramebufferTexture2D( + GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, info->passthroughTexture, 0)); + GL(glFramebufferRenderbuffer( + GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, info->passthroughDepthRenderBuffer)); +} + +void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription viewsDescs[]) { + GL(glBindFramebuffer(GL_FRAMEBUFFER, info->passthroughFramebuffer)); + + GL(glEnable(GL_DEPTH_TEST)); + GL(glEnable(GL_CULL_FACE)); + GL(glDisable(GL_SCISSOR_TEST)); + GL(glEnable(GL_BLEND)); + GL(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)); + GL(glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)); + + // Draw Passthrough video for each eye + for (int eye = 0; eye < 2; ++eye) { + GL(glViewport(eye == kLeft ? 0 : *(info->screenWidth) / 2, + 0, + *(info->screenWidth) / 2, + *(info->screenHeight))); + + GL(glUseProgram(passthroughProgram_)); + GL(glActiveTexture(GL_TEXTURE0)); + GL(glBindTexture(GL_TEXTURE_EXTERNAL_OES, info->cameraTexture)); + + // Draw Mesh + GL(glEnableVertexAttribArray(texturePositionParam_)); + GL(glVertexAttribPointer( + texturePositionParam_, 2, GL_FLOAT, false, 0, info->passthroughVertices)); + GL(glEnableVertexAttribArray(textureUvParam_)); + GL(glVertexAttribPointer(textureUvParam_, 2, GL_FLOAT, false, 0, passthroughTexCoords)); + + GL(glDrawArrays(GL_TRIANGLE_STRIP, 0, 4)); + + viewsDescs[eye].left_u = 0.5 * eye; // 0 for left, 0.5 for right + viewsDescs[eye].right_u = 0.5 + 0.5 * eye; // 0.5 for left, 1.0 for right + } + viewsDescs[0].texture = info->passthroughTexture; + viewsDescs[1].texture = info->passthroughTexture; +} \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h new file mode 100644 index 00000000..8b4164be --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h @@ -0,0 +1,28 @@ +#ifndef PHONEVR_PASSTHROUGH_H +#define PHONEVR_PASSTHROUGH_H + +#include "cardboard.h" + +struct PassthroughInfo { + bool enabled = false; + + GLuint cameraTexture = 0; + GLuint passthroughTexture = 0; + + GLuint passthroughDepthRenderBuffer = 0; + GLuint passthroughFramebuffer = 0; + + float passthroughVertices[8]; + float passthroughSize = 1.0; + + int *screenWidth = 0; + int *screenHeight = 0; +}; + +void passthrough_createPlane(PassthroughInfo *info); +GLuint passthrough_init(PassthroughInfo *info); +void passthrough_cleanup(PassthroughInfo *info); +void passthrough_setup(PassthroughInfo *info); +void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription viewDescs[]); + +#endif // PHONEVR_PASSTHROUGH_H diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h b/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h index bc31f22b..7362b876 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h @@ -12,12 +12,12 @@ const char LOG_TAG[] = "ALVR_PVR_NATIVE"; -void log(AlvrLogLevel level, - const char *FILE_NAME, - unsigned int LINE_NO, - const char *FUNC, - const char *format, - ...) { +static void log(AlvrLogLevel level, + const char *FILE_NAME, + unsigned int LINE_NO, + const char *FUNC, + const char *format, + ...) { va_list args; va_start(args, format); diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java index b5b09f56..3b7f7cb2 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java @@ -8,6 +8,7 @@ import android.content.IntentFilter; import android.content.SharedPreferences; import android.content.pm.PackageManager; +import android.hardware.SensorManager; import android.net.Uri; import android.opengl.GLSurfaceView; import android.os.BatteryManager; @@ -27,6 +28,7 @@ import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import androidx.core.app.ActivityCompat; +import androidx.preference.PreferenceManager; import java.util.Objects; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; @@ -45,6 +47,10 @@ public class ALVRActivity extends AppCompatActivity private GLSurfaceView glView; + private Passthrough passthrough = null; + + private SharedPreferences pref = null; + private final BatteryMonitor bMonitor = new BatteryMonitor(this); public static class BatteryMonitor extends BroadcastReceiver { @@ -134,6 +140,11 @@ public void onCreate(Bundle savedInstance) { // Prevents screen from dimming/locking. getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + + pref = PreferenceManager.getDefaultSharedPreferences(this); + passthrough = + new Passthrough( + pref, (SensorManager) getSystemService(SENSOR_SERVICE), width, height); } @Override @@ -149,6 +160,7 @@ protected void onPause() { pauseNative(); glView.onPause(); bMonitor.stopMonitoring(this); + passthrough.onPause(); } @Override @@ -163,6 +175,7 @@ protected void onResume() { return; } + passthrough.onResume(); glView.onResume(); resumeNative(); bMonitor.startMonitoring(this); @@ -173,6 +186,7 @@ protected void onDestroy() { super.onDestroy(); Log.d(TAG, "Destroying ALVR Activity"); destroyNative(); + passthrough.releaseCamera(); } @Override @@ -186,7 +200,8 @@ public void onWindowFocusChanged(boolean hasFocus) { private class Renderer implements GLSurfaceView.Renderer { @Override public void onSurfaceCreated(GL10 gl10, EGLConfig eglConfig) { - surfaceCreatedNative(); + int texID = surfaceCreatedNative(); + passthrough.createTexture(texID); } @Override @@ -196,6 +211,7 @@ public void onSurfaceChanged(GL10 gl10, int width, int height) { @Override public void onDrawFrame(GL10 gl10) { + passthrough.update(); renderNative(); } } @@ -218,6 +234,10 @@ public void showSettings(View view) { boolean isChecked = prefs.getBoolean("max_brightness", true); toggleBrightness.setChecked(isChecked); + MenuItem box = popup.getMenu().findItem(R.id.passthrough); + if (box != null) { + box.setChecked(pref.getBoolean("passthrough", false)); + } popup.setOnMenuItemClickListener(this); popup.show(); } @@ -244,6 +264,13 @@ public boolean onMenuItemClick(MenuItem item) { editor.putBoolean("max_brightness", item.isChecked()); editor.apply(); return true; + } else if (item.getItemId() == R.id.passthrough_settings) { + Intent settings = new Intent(this, PassthroughSettingsActivity.class); + startActivity(settings); + return true; + } else if (item.getItemId() == R.id.passthrough) { + passthrough.changePassthroughMode(); + return true; } return false; } @@ -293,7 +320,7 @@ private native void initializeNative( private native void pauseNative(); - private native void surfaceCreatedNative(); + private native int surfaceCreatedNative(); private native void setScreenResolutionNative(int width, int height); diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java new file mode 100644 index 00000000..0afd449e --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java @@ -0,0 +1,95 @@ +/* (C)2024 */ +package viritualisres.phonevr; + +import android.view.View; +import com.github.mikephil.charting.charts.LineChart; +import com.github.mikephil.charting.components.YAxis; +import com.github.mikephil.charting.data.Entry; +import com.github.mikephil.charting.data.LineData; +import com.github.mikephil.charting.data.LineDataSet; +import com.github.mikephil.charting.utils.ColorTemplate; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GraphAdapter { + + private final LineChart chart; + private List timestamps = new ArrayList<>(); + private Map> lines = new HashMap<>(); + private List keyOrder = new ArrayList<>(); + private List color = new ArrayList<>(); + private int len; + private float max = 10.f; + + public GraphAdapter(int len, View customView) { + this.len = len; + this.chart = customView.findViewById(R.id.chart); + } + + private int checkTimestamp(Long ts) { + if (!timestamps.contains(ts)) { + timestamps.add(ts); + if (timestamps.size() > this.len) { + timestamps.remove(0); + } + } + return timestamps.indexOf(ts) + 1; + } + + public void addValue(Long ts, String name, Float value) { + int position = checkTimestamp(ts); + if (!lines.containsKey(name)) { + lines.put(name, new ArrayList<>()); + keyOrder.add(name); + } + List list = lines.get(name); + list.add(value); + while (list.size() < position) { + list.add(0, 0.f); + } + if (list.size() > position) { + list.remove(0); + } + } + + public void addValue(Long ts, String name, boolean value) { + if (value) { + addValue(ts, name, max); + } else { + addValue(ts, name, 0.f); + } + } + + public void refresh() { + // List _lines = new ArrayList<>(); + LineData data = new LineData(); + for (int j = 0; j < keyOrder.size(); ++j) { + String name = keyOrder.get(j); + List elems = lines.get(name); + List values = new ArrayList<>(); + for (int i = 0; i < elems.size(); ++i) { + values.add(new Entry(i, elems.get(i))); + } + LineDataSet line = new LineDataSet(values, name); + if (j > 4) { + line.setColor(ColorTemplate.LIBERTY_COLORS[j - 5]); + } else { + line.setColor(ColorTemplate.JOYFUL_COLORS[j]); + } + line.setDrawCircles(false); + line.setLineWidth(2.f); + data.addDataSet(line); + } + chart.setData(data); + + YAxis yaxis = chart.getAxisLeft(); + chart.getAxisRight().setEnabled(false); + yaxis.setAxisMaximum(10); + yaxis.setAxisMinimum(0); + + chart.invalidate(); + // resetViewport(); + } +} diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java new file mode 100644 index 00000000..4075636a --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java @@ -0,0 +1,413 @@ +/* (C)2024 */ +package viritualisres.phonevr; + +import android.content.SharedPreferences; +import android.graphics.SurfaceTexture; +import android.hardware.Camera; +import android.hardware.Sensor; +import android.hardware.SensorEvent; +import android.hardware.SensorEventListener; +import android.hardware.SensorManager; +import android.os.Build; +import android.util.Log; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; + +public class Passthrough implements SensorEventListener { + + static { + System.loadLibrary("native-lib-alvr"); + } + + private static final String TAG = Passthrough.class.getSimpleName() + "-Java"; + + private Camera mCamera = null; + + private SurfaceTexture mTexture = null; + private Camera.Size mPreviewSize = null; + + private final SharedPreferences mPref; + + private final SensorManager mSensorManager; + + private GraphAdapter graphAdapter = null; + + private final int displayWidth; + + private final int displayHeight; + + private Sensor mAccelerometer = null; + + private long timestamp = 0; + + private long timestampAfterCandidate = 0; + + private long passthroughDelayInMs = 600; + + private float detectionLowerBound = 0.8f; + + private float detectionUpperBound = 4f; + + private final List[] accelStore = + new List[] {new ArrayList<>(), new ArrayList<>(), new ArrayList<>()}; + + private int halfSize = 0; + + private boolean triggerIsSet = false; + + private final List cumSums = new ArrayList<>(); + + public Passthrough( + SharedPreferences pref, + SensorManager sensorManager, + int displayWidth, + int displayHeight) { + this.displayWidth = displayWidth; + this.displayHeight = displayHeight; + mPref = pref; + mSensorManager = sensorManager; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + mAccelerometer = + mSensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER_UNCALIBRATED); + } + if (mAccelerometer == null) { + mAccelerometer = mSensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER); + } + } + + @Override + public void onSensorChanged(SensorEvent event) { + if (timestamp == 0) { + timestamp = event.timestamp; + } + + if (timestampAfterCandidate == 0) { + timestampAfterCandidate = event.timestamp - 1; + } + + boolean initialFull = event.timestamp - timestamp > passthroughDelayInMs * 1000000L; + boolean afterCandidateFull = + event.timestamp - timestampAfterCandidate > passthroughDelayInMs * 1000000L; + + // gather data for passthrough_delay_ms, we get a data point approx. each 60ms + String axesName[] = {"X", "Y", "Z"}; + for (int i = 0; i < accelStore.length; ++i) { + // derivative, absolute + accelStore[i].add(event.values[i]); + + if (initialFull) { + // remove the first element to only store the last X elements + accelStore[i].remove(0); + if (halfSize == 0) { + halfSize = accelStore[0].size() / 2; + } + // check if half size has at least 3 data points + if (halfSize < 4) { + accelStore[i].clear(); + timestamp = 0; + halfSize = 0; + } + } + } + + if (halfSize > 0) { + // Calculate derivative and absolute values + List[] accelAbs = + new List[] {new ArrayList<>(), new ArrayList<>(), new ArrayList<>()}; + for (int axis = 0; axis < accelStore.length; ++axis) { + for (int i = 1; i < accelStore[axis].size(); ++i) { + accelAbs[axis].add( + Math.abs(accelStore[axis].get(i) - accelStore[axis].get(i - 1))); + } + if (graphAdapter != null) { + graphAdapter.addValue( + event.timestamp, + axesName[axis], + accelAbs[axis].get(accelAbs[axis].size() - 1)); + } + } + + // get the mean for each axis and sum them up + float[] axisMean = new float[] {0.0f, 0.0f, 0.0f}; + for (int i = 0; i < accelAbs.length; ++i) { + for (int j = 0; j < accelAbs[i].size(); ++j) { + axisMean[i] += accelAbs[i].get(j); + } + axisMean[i] /= accelAbs[i].size(); + } + float sumMean = axisMean[0] + axisMean[1] + axisMean[2]; + cumSums.add(sumMean); + if (graphAdapter != null) { + graphAdapter.addValue(event.timestamp, "sumMean", sumMean); + } + + if (cumSums.size() < accelAbs[0].size()) { + return; + } else { + // only store the last X elements + cumSums.remove(0); + } + + if (!afterCandidateFull) { + // if a potential passthrough_change event is found, + // check if the device does not move much anymore + if (!triggerIsSet && sumMean < detectionLowerBound) { + triggerIsSet = true; + // if passthroughMode not already changed... change it + changePassthroughMode(); + } + } else { + triggerIsSet = false; + // here we check for potential passthrough_change events + // these may occur on any axis + for (List elem : accelAbs) { + // we look at two halves of the passthrough_delay + List t1 = elem.subList(0, halfSize + 1); + List t2 = elem.subList(halfSize, elem.size()); + + // and get the maxima + float max1 = Collections.max(t1); + float max2 = Collections.max(t2); + // and check whether the 2nd half goes belowe lower bound + int end2 = t2.indexOf(max2); + for (int i = end2; i < t2.size(); ++i) { + if (t2.get(i) < detectionLowerBound) { + end2 = i; + break; + } + } + + // We check + if (t1.get(halfSize - 1) > t2.get(0) // if halfSize is a local minimum + && t2.get(1) > t2.get(0) + && max1 - t1.get(0) >= detectionLowerBound // max - start > lower bound + && max1 - t1.get(t1.size() - 1) + >= detectionLowerBound // max - end > lower bound + && max2 - t2.get(0) >= detectionLowerBound + && max2 - t2.get(t2.size() - 1) >= detectionLowerBound + && max1 >= detectionLowerBound // max are in bounds + && max1 <= detectionUpperBound + && max2 >= detectionLowerBound + && max2 <= detectionUpperBound + && sumMean >= detectionLowerBound // sumMean is in bounds + && sumMean <= detectionUpperBound + && t2.get(end2) < max2 // the 2nd half goes below lower bound + && cumSums.get(0) + < Math.max( + detectionLowerBound - 0.3f, + 0.5f) // the first sumMean (of window) is below lower + // bounds + ) { + // If all is fulfilled we consider this a potential passthrough_change + // event. + // The sum must then decend under lower bound until passthrough_delay + timestampAfterCandidate = event.timestamp; + break; + } + } + } + } + if (graphAdapter != null) { + graphAdapter.addValue( + event.timestamp, "Candidate", timestampAfterCandidate == event.timestamp); + graphAdapter.addValue(event.timestamp, "Trigger", triggerIsSet); + graphAdapter.addValue(event.timestamp, "upperBound", detectionUpperBound); + graphAdapter.addValue(event.timestamp, "lowerBound", detectionLowerBound); + graphAdapter.refresh(); + this.updateSettings(); + } + + Log.v( + TAG + "-AccSensor", + String.format( + (Locale) null, + "%d: [%f, %f, %f]", + event.timestamp, + event.values[0], + event.values[1], + event.values[2])); + } + + @Override + public void onAccuracyChanged(Sensor sensor, int accuracy) {} + + protected void onResume(GraphAdapter graphAdapter) { + this.graphAdapter = graphAdapter; + this.onResume(); + } + + protected void updateSettings() { + try { + passthroughDelayInMs = Long.parseLong(mPref.getString("passthrough_delay", "600")); + } catch (Exception e) { + passthroughDelayInMs = 600; + } + try { + detectionLowerBound = Float.parseFloat(mPref.getString("passthrough_lower", "0.8")); + } catch (Exception e) { + detectionLowerBound = 0.8f; + } + try { + detectionUpperBound = Float.parseFloat(mPref.getString("passthrough_upper", "4")); + } catch (Exception e) { + detectionUpperBound = 4.f; + } + } + + protected void onResume() { + SharedPreferences.Editor edit = mPref.edit(); + edit.putBoolean("passthrough", false); + edit.apply(); + + timestamp = 0; + halfSize = 0; + if (mPref.getBoolean("passthrough_tap", true)) { + updateSettings(); + if (mAccelerometer != null) { + mSensorManager.registerListener( + this, mAccelerometer, SensorManager.SENSOR_DELAY_UI); + } + } else { + mSensorManager.unregisterListener(this); + } + } + + protected void setPassthroughMode(boolean passthrough) { + if (passthrough) { + int w = displayWidth / 2; + int h = displayHeight; + if (displayWidth < displayHeight) { + w = displayHeight / 2; + h = displayWidth; + } + openCamera(-1, w, h, mPref.getBoolean("passthrough_recording", true)); + + float size = 0.5f; + try { + size = Float.parseFloat(mPref.getString("passthrough_fraction", "0.5")); + } catch (RuntimeException e) { + } + + setPassthroughSizeNative(size); + } + setPassthroughActiveNative(passthrough); + + if (passthrough) { + startPreview(); + } else { + releaseCamera(); + } + } + + protected void changePassthroughMode() { + boolean pt = !mPref.getBoolean("passthrough", false); + SharedPreferences.Editor edit = mPref.edit(); + edit.putBoolean("passthrough", pt); + edit.apply(); + setPassthroughMode(pt); + } + + public void createTexture(int textureID) { + mTexture = new SurfaceTexture(textureID); + } + + public void openCamera(int camNr, int desiredWidth, int desiredHeight, boolean recordingHint) { + if (mCamera != null) { + // TODO maybe print only a warning + throw new RuntimeException("Camera already initialized"); + } + Camera.CameraInfo info = new Camera.CameraInfo(); + // TODO add possibility to choose camera + if (camNr < 0) { + int numCameras = Camera.getNumberOfCameras(); + for (int i = 0; i < numCameras; ++i) { + Camera.getCameraInfo(i, info); + if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { + mCamera = Camera.open(i); + break; + } + } + if (mCamera == null) { + // TODO log message "No back-facing camera found; opening default" + mCamera = Camera.open(); // open default + } + } else { + mCamera = Camera.open(camNr); + } + if (mCamera == null) { + throw new RuntimeException("Unable to open camera"); + } + // TODO the size stuff + Camera.Parameters params = mCamera.getParameters(); + List choices = params.getSupportedPreviewSizes(); + mPreviewSize = chooseOptimalSize(choices, desiredWidth, desiredHeight); + Log.d(TAG, "PreviewSize: " + mPreviewSize.width + "x" + mPreviewSize.height); + params.setPreviewSize(mPreviewSize.width, mPreviewSize.height); + params.setRecordingHint(recordingHint); // Can increase fps in passthrough mode + mCamera.setParameters(params); + } + + private Camera.Size chooseOptimalSize(List choices, int width, int height) { + Integer minSize = Integer.max(Integer.min(width, height), 320); + + // Collect the supported resolutions that are at least as big as the preview Surface + ArrayList bigEnough = new ArrayList<>(); + for (Camera.Size option : choices) { + if (option.width == width && option.height == height) { + return option; + } + if (option.height >= minSize && option.width >= minSize) { + bigEnough.add(option); + } + } + + Comparator comp = (a, b) -> (a.width * a.height) - (b.width * b.width); + // Pick the smallest of those, assuming we found any + if (bigEnough.size() > 0) { + return Collections.min(bigEnough, comp); + } else { + return choices.get(0); + } + } + + public void releaseCamera() { + if (mCamera != null) { + mCamera.setPreviewCallback(null); + mCamera.stopPreview(); + mCamera.release(); + mCamera = null; + } + } + + public void startPreview() { + if (mCamera != null) { + // Log.d(TAG, "starting camera preview") + try { + mCamera.setPreviewTexture(mTexture); + } catch (IOException e) { + throw new RuntimeException(e); + } + mCamera.startPreview(); + } + } + + public void onPause() { + mSensorManager.unregisterListener(this); + setPassthroughActiveNative(false); + releaseCamera(); + } + + public void update() { + if (mTexture != null) { + mTexture.updateTexImage(); + } + } + + private native void setPassthroughSizeNative(float size); + + private native void setPassthroughActiveNative(boolean activate); +} diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java new file mode 100644 index 00000000..d9e10118 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java @@ -0,0 +1,78 @@ +/* (C)2024 */ +package viritualisres.phonevr; + +import android.annotation.SuppressLint; +import android.content.SharedPreferences; +import android.hardware.SensorManager; +import android.os.Bundle; +import android.view.View; +import android.view.ViewGroup; +import androidx.annotation.NonNull; +import androidx.appcompat.app.ActionBar; +import androidx.appcompat.app.AppCompatActivity; +import androidx.preference.PreferenceFragmentCompat; +import androidx.preference.PreferenceGroupAdapter; +import androidx.preference.PreferenceManager; +import androidx.preference.PreferenceScreen; +import androidx.preference.PreferenceViewHolder; +import androidx.recyclerview.widget.RecyclerView; + +public class PassthroughSettingsActivity extends AppCompatActivity { + + private static SharedPreferences sharedPref; + private static SensorManager sensorManager; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + sharedPref = PreferenceManager.getDefaultSharedPreferences(this); + sensorManager = (SensorManager) getSystemService(SENSOR_SERVICE); + + setContentView(R.layout.activity_passthrough_settings); + if (savedInstanceState == null) { + getSupportFragmentManager() + .beginTransaction() + .replace(R.id.settings, new SettingsFragment()) + .commit(); + } + ActionBar actionBar = getSupportActionBar(); + if (actionBar != null) { + actionBar.setDisplayHomeAsUpEnabled(true); + } + } + + public static class SettingsFragment extends PreferenceFragmentCompat { + private Passthrough passthrough; + private GraphAdapter graphAdapter; + + public SettingsFragment() { + super(); + } + + @Override + public void onCreatePreferences(Bundle savedInstanceState, String rootKey) { + setPreferencesFromResource(R.xml.passthrough_settings, rootKey); + } + + @NonNull + @SuppressLint("RestrictedApi") + @Override + protected RecyclerView.Adapter onCreateAdapter( + PreferenceScreen pref) { + return new PreferenceGroupAdapter(pref) { + @NonNull + public PreferenceViewHolder onCreateViewHolder( + @NonNull ViewGroup parent, int viewType) { + PreferenceViewHolder holder = super.onCreateViewHolder(parent, viewType); + View customLayout = holder.itemView; + if (customLayout.getId() == R.id.chartLayout) { + graphAdapter = new GraphAdapter(100, customLayout); + passthrough = new Passthrough(sharedPref, sensorManager, 640, 480); + passthrough.onResume(graphAdapter); + } + return holder; + } + }; + } + } +} diff --git a/code/mobile/android/PhoneVR/app/src/main/res/layout/activity_passthrough_settings.xml b/code/mobile/android/PhoneVR/app/src/main/res/layout/activity_passthrough_settings.xml new file mode 100644 index 00000000..4a4f2472 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/layout/activity_passthrough_settings.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml b/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml new file mode 100644 index 00000000..6a180765 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml b/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml index 7deebc7b..7bd86d91 100644 --- a/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml +++ b/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml @@ -10,4 +10,11 @@ android:id="@+id/max_brightness_toggle" android:title="@string/max_brightness" android:checkable="true"/> + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/values/arrays.xml b/code/mobile/android/PhoneVR/app/src/main/res/values/arrays.xml new file mode 100644 index 00000000..6cf9ed48 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/values/arrays.xml @@ -0,0 +1,12 @@ + + + + Reply + Reply to all + + + + reply + reply_all + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml b/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml index c4657b69..28ad6797 100644 --- a/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml +++ b/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml @@ -41,4 +41,19 @@ Switch PhoneVR Server Remember choice Max Brightness + PassthroughSettingsActivity + Passthrough fraction + Enable passthrough + The fraction how much the passthrough should contain of the image (between 0.0 and 1.0). This depends on the FOV of the camera, for narrow FOV a lower value might be better, so that objects don\'t look to close... Play around with this value. + Set Recording Hint + If set to false the passthrough may have a greater FOV, but has less FPS which may cause motion sickness. + Experimental Double-Tap detection + If true passthrough can be switched through double-tap. + Double Tap Configuration + Delay in ms + Lower Bound + Upper Bound + Double Tap Test Graph + Passthrough Settings + Passthrough diff --git a/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml b/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml new file mode 100644 index 00000000..ed65e0bb --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file