From 78ca2c4ab715bb43a72f2803e38d3a52fc5cef87 Mon Sep 17 00:00:00 2001 From: Dominik Wetzel Date: Fri, 1 Mar 2024 16:01:36 +0100 Subject: [PATCH 1/8] An initial passthrough implementation with double tap detection. Combine 7 commits to one: * Initial Passthrough implementation with menu to select passthrough mode. * Made passthrough available via inflateMenu. Fixed crashes during change to passthrough mode. * Added recordingHint to Settings. Open Camera only on passthrough mode. * Added option to trigger passthrough by double tapping the headset on the side. * run make format * Moved passthrough code into separate file * split passthrough stuff into separate file, made variables camelCase --- .../mobile/android/PhoneVR/app/CMakeLists.txt | 1 + code/mobile/android/PhoneVR/app/build.gradle | 3 +- .../PhoneVR/app/src/main/AndroidManifest.xml | 4 + .../PhoneVR/app/src/main/cpp/alvr_main.cpp | 82 ++-- .../PhoneVR/app/src/main/cpp/passthrough.cpp | 196 +++++++++ .../PhoneVR/app/src/main/cpp/passthrough.h | 29 ++ .../viritualisres/phonevr/ALVRActivity.java | 34 +- .../viritualisres/phonevr/Passthrough.java | 389 ++++++++++++++++++ .../phonevr/PassthroughSettingsActivity.java | 33 ++ .../layout/activity_passthrough_settings.xml | 10 + .../app/src/main/res/menu/settings_menu.xml | 7 + .../app/src/main/res/values/arrays.xml | 12 + .../app/src/main/res/values/strings.xml | 14 + .../src/main/res/xml/passthrough_settings.xml | 45 ++ 14 files changed, 827 insertions(+), 32 deletions(-) create mode 100644 code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp create mode 100644 code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h create mode 100644 code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java create mode 100644 code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java create mode 100644 code/mobile/android/PhoneVR/app/src/main/res/layout/activity_passthrough_settings.xml create mode 100644 code/mobile/android/PhoneVR/app/src/main/res/values/arrays.xml create mode 100644 code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml diff --git a/code/mobile/android/PhoneVR/app/CMakeLists.txt b/code/mobile/android/PhoneVR/app/CMakeLists.txt index a5cf82bb..f4501bf4 100644 --- a/code/mobile/android/PhoneVR/app/CMakeLists.txt +++ b/code/mobile/android/PhoneVR/app/CMakeLists.txt @@ -21,6 +21,7 @@ file(GLOB_RECURSE GVR_SRC add_library(native-lib-alvr SHARED src/main/cpp/alvr_main.cpp + src/main/cpp/passthrough.cpp ${LIB_SRC} ) diff --git a/code/mobile/android/PhoneVR/app/build.gradle b/code/mobile/android/PhoneVR/app/build.gradle index da0e4d11..999ef85c 100644 --- a/code/mobile/android/PhoneVR/app/build.gradle +++ b/code/mobile/android/PhoneVR/app/build.gradle @@ -95,7 +95,7 @@ android { } project.gradle.taskGraph.whenReady { android.productFlavors.all { flavor -> - // Capitalize (as Gralde is case-sensitive). + // Capitalize (as Gradle is case-sensitive). def flavorName = flavor.name.substring(0, 1).toUpperCase() + flavor.name.substring(1) // At last, configure. @@ -140,6 +140,7 @@ dependencies { implementation 'com.google.android.material:material:1.2.1' implementation "androidx.tracing:tracing:1.1.0" + implementation 'androidx.preference:preference:1.2.1' testImplementation 'junit:junit:4.13.2' testImplementation 'junit:junit:4.12' androidTestImplementation "androidx.test.ext:junit-ktx:$extJUnitVersion" diff --git a/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml b/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml index 94ffd5b1..2749ca0e 100644 --- a/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml +++ b/code/mobile/android/PhoneVR/app/src/main/AndroidManifest.xml @@ -46,6 +46,10 @@ android:theme="@style/AppTheme" tools:replace="android:extractNativeLibs" android:extractNativeLibs="true"> + +#include +#include + +#include "passthrough.h" +#include "utils.h" + +GLuint LoadGLShader(GLenum type, const char *shader_source) { + GLuint shader = GL(glCreateShader(type)); + + GL(glShaderSource(shader, 1, &shader_source, nullptr)); + GL(glCompileShader(shader)); + + // Get the compilation status. + GLint compile_status; + GL(glGetShaderiv(shader, GL_COMPILE_STATUS, &compile_status)); + + // If the compilation failed, delete the shader and show an error. + if (compile_status == 0) { + GLint info_len = 0; + GL(glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &info_len)); + if (info_len == 0) { + return 0; + } + + std::vector info_string(info_len); + GL(glGetShaderInfoLog(shader, info_string.size(), nullptr, info_string.data())); + // LOGE("Could not compile shader of type %d: %s", type, info_string.data()); + GL(glDeleteShader(shader)); + return 0; + } else { + return shader; + } +} + +namespace { + // Simple shaders to render camera Texture files without any lighting. + constexpr const char *camVertexShader = + R"glsl( + uniform mat4 u_MVP; + attribute vec4 a_Position; + attribute vec2 a_UV; + varying vec2 v_UV; + + void main() { + v_UV = a_UV; + gl_Position = a_Position; + })glsl"; + + constexpr const char *camFragmentShader = + R"glsl( + #extension GL_OES_EGL_image_external : require + precision mediump float; + varying vec2 v_UV; + uniform samplerExternalOES sTexture; + void main() { + gl_FragColor = texture2D(sTexture, v_UV); + })glsl"; + + static int passthroughProgram_ = 0; + static int texturePositionParam_ = 0; + static int textureUvParam_ = 0; + static int textureMvpParam_ = 0; + + float passthroughTexCoords[] = {0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0}; +} // namespace + +void passthrough_createPlane(PassthroughInfo *info) { + float size = info->passthroughSize; + float x0 = -size, y0 = size; // Top left + float x1 = size, y1 = size; // Top right + float x2 = size, y2 = -size; // Bottom right + float x3 = -size, y3 = -size; // Bottom left + + info->passthroughVertices[0] = x3; + info->passthroughVertices[1] = y3; + info->passthroughVertices[2] = x2; + info->passthroughVertices[3] = y2; + info->passthroughVertices[4] = x0; + info->passthroughVertices[5] = y0; + info->passthroughVertices[6] = x1; + info->passthroughVertices[7] = y1; +} + +GLuint passthrough_init(PassthroughInfo *info) { + const int obj_vertex_shader = LoadGLShader(GL_VERTEX_SHADER, camVertexShader); + const int obj_fragment_shader = LoadGLShader(GL_FRAGMENT_SHADER, camFragmentShader); + + passthroughProgram_ = GL(glCreateProgram()); + GL(glAttachShader(passthroughProgram_, obj_vertex_shader)); + GL(glAttachShader(passthroughProgram_, obj_fragment_shader)); + GL(glLinkProgram(passthroughProgram_)); + + GL(glUseProgram(passthroughProgram_)); + texturePositionParam_ = GL(glGetAttribLocation(passthroughProgram_, "a_Position")); + textureUvParam_ = GL(glGetAttribLocation(passthroughProgram_, "a_UV")); + textureMvpParam_ = GL(glGetUniformLocation(passthroughProgram_, "u_MVP")); + + GL(glGenTextures(1, &(info->cameraTexture))); + GL(glActiveTexture(GL_TEXTURE0)); + + GL(glBindTexture(GL_TEXTURE_EXTERNAL_OES, info->cameraTexture)); + GL(glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST)); + GL(glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR)); + GL(glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); + GL(glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); + + return info->cameraTexture; +} + +void passthrough_cleanup(PassthroughInfo *info) { + if (info->passthroughDepthRenderBuffer != 0) { + GL(glDeleteRenderbuffers(1, &(info->passthroughDepthRenderBuffer))); + info->passthroughDepthRenderBuffer = 0; + } + if (info->passthroughFramebuffer != 0) { + GL(glDeleteFramebuffers(1, &info->passthroughFramebuffer)); + info->passthroughFramebuffer = 0; + } + if (info->passthroughTexture != 0) { + GL(glDeleteTextures(1, &(info->passthroughTexture))); + info->passthroughTexture = 0; + } +} + +void passthrough_setup(PassthroughInfo *info) { + // Create render texture. + GL(glGenTextures(1, &(info->passthroughTexture))); + GL(glBindTexture(GL_TEXTURE_2D, info->passthroughTexture)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); + GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); + + GL(glTexImage2D(GL_TEXTURE_2D, + 0, + GL_RGB, + *(info->screenWidth), + *(info->screenHeight), + 0, + GL_RGB, + GL_UNSIGNED_BYTE, + 0)); + + // Generate depth buffer to perform depth test. + GL(glGenRenderbuffers(1, &(info->passthroughDepthRenderBuffer))); + GL(glBindRenderbuffer(GL_RENDERBUFFER, info->passthroughDepthRenderBuffer)); + GL(glRenderbufferStorage( + GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, *(info->screenWidth), *(info->screenHeight))); + + // Create render target. + GL(glGenFramebuffers(1, &(info->passthroughFramebuffer))); + GL(glBindFramebuffer(GL_FRAMEBUFFER, info->passthroughFramebuffer)); + GL(glFramebufferTexture2D( + GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, info->passthroughTexture, 0)); + GL(glFramebufferRenderbuffer( + GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, info->passthroughDepthRenderBuffer)); +} + +void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription viewsDescs[]) { + GL(glBindFramebuffer(GL_FRAMEBUFFER, info->passthroughFramebuffer)); + + GL(glEnable(GL_DEPTH_TEST)); + GL(glEnable(GL_CULL_FACE)); + GL(glDisable(GL_SCISSOR_TEST)); + GL(glEnable(GL_BLEND)); + GL(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)); + GL(glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)); + + // Draw Passthrough video for each eye + for (int eye = 0; eye < 2; ++eye) { + GL(glViewport(eye == kLeft ? 0 : *(info->screenWidth) / 2, + 0, + *(info->screenWidth) / 2, + *(info->screenHeight))); + + GL(glUseProgram(passthroughProgram_)); + GL(glActiveTexture(GL_TEXTURE0)); + GL(glBindTexture(GL_TEXTURE_EXTERNAL_OES, info->cameraTexture)); + + // Draw Mesh + GL(glEnableVertexAttribArray(texturePositionParam_)); + GL(glVertexAttribPointer( + texturePositionParam_, 2, GL_FLOAT, false, 0, info->passthroughVertices)); + GL(glEnableVertexAttribArray(textureUvParam_)); + GL(glVertexAttribPointer(textureUvParam_, 2, GL_FLOAT, false, 0, passthroughTexCoords)); + + GL(glDrawArrays(GL_TRIANGLE_STRIP, 0, 4)); + + viewsDescs[eye].left_u = 0.5 * eye; // 0 for left, 0.5 for right + viewsDescs[eye].right_u = 0.5 + 0.5 * eye; // 0.5 for left, 1.0 for right + } + viewsDescs[0].texture = info->passthroughTexture; + viewsDescs[1].texture = info->passthroughTexture; +} \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h new file mode 100644 index 00000000..f7d8c5d4 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h @@ -0,0 +1,29 @@ +#ifndef PHONEVR_PASSTHROUGH_H + +#include "cardboard.h" + +struct PassthroughInfo { + bool enabled = false; + + GLuint cameraTexture = 0; + GLuint passthroughTexture = 0; + + GLuint passthroughDepthRenderBuffer = 0; + GLuint passthroughFramebuffer = 0; + + float passthroughVertices[8]; + float passthroughSize = 1.0; + + int *screenWidth = 0; + int *screenHeight = 0; +}; + +void passthrough_createPlane(PassthroughInfo *info); +GLuint passthrough_init(PassthroughInfo *info); +void passthrough_cleanup(PassthroughInfo *info); +void passthrough_setup(PassthroughInfo *info); +void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription viewDescs[]); + +#define PHONEVR_PASSTHROUGH_H + +#endif // PHONEVR_PASSTHROUGH_H diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java index b5b09f56..be457003 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java @@ -8,6 +8,7 @@ import android.content.IntentFilter; import android.content.SharedPreferences; import android.content.pm.PackageManager; +import android.hardware.SensorManager; import android.net.Uri; import android.opengl.GLSurfaceView; import android.os.BatteryManager; @@ -27,6 +28,7 @@ import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import androidx.core.app.ActivityCompat; +import androidx.preference.PreferenceManager; import java.util.Objects; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; @@ -45,6 +47,10 @@ public class ALVRActivity extends AppCompatActivity private GLSurfaceView glView; + private Passthrough passthrough = null; + + private SharedPreferences pref = null; + private final BatteryMonitor bMonitor = new BatteryMonitor(this); public static class BatteryMonitor extends BroadcastReceiver { @@ -134,6 +140,14 @@ public void onCreate(Bundle savedInstance) { // Prevents screen from dimming/locking. getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + + pref = PreferenceManager.getDefaultSharedPreferences(this); + passthrough = + new Passthrough( + pref, + (SensorManager) getSystemService(SENSOR_SERVICE), + width, + height); } @Override @@ -149,6 +163,7 @@ protected void onPause() { pauseNative(); glView.onPause(); bMonitor.stopMonitoring(this); + passthrough.onPause(); } @Override @@ -163,6 +178,7 @@ protected void onResume() { return; } + passthrough.onResume(); glView.onResume(); resumeNative(); bMonitor.startMonitoring(this); @@ -173,6 +189,7 @@ protected void onDestroy() { super.onDestroy(); Log.d(TAG, "Destroying ALVR Activity"); destroyNative(); + passthrough.releaseCamera(); } @Override @@ -186,7 +203,8 @@ public void onWindowFocusChanged(boolean hasFocus) { private class Renderer implements GLSurfaceView.Renderer { @Override public void onSurfaceCreated(GL10 gl10, EGLConfig eglConfig) { - surfaceCreatedNative(); + int texID = surfaceCreatedNative(); + passthrough.createTexture(texID); } @Override @@ -196,6 +214,7 @@ public void onSurfaceChanged(GL10 gl10, int width, int height) { @Override public void onDrawFrame(GL10 gl10) { + passthrough.update(); renderNative(); } } @@ -218,6 +237,10 @@ public void showSettings(View view) { boolean isChecked = prefs.getBoolean("max_brightness", true); toggleBrightness.setChecked(isChecked); + MenuItem box = popup.getMenu().findItem(R.id.passthrough); + if (box != null) { + box.setChecked(pref.getBoolean("passthrough", false)); + } popup.setOnMenuItemClickListener(this); popup.show(); } @@ -244,6 +267,13 @@ public boolean onMenuItemClick(MenuItem item) { editor.putBoolean("max_brightness", item.isChecked()); editor.apply(); return true; + } else if (item.getItemId() == R.id.passthrough_settings) { + Intent settings = new Intent(this, PassthroughSettingsActivity.class); + startActivity(settings); + return true; + } else if (item.getItemId() == R.id.passthrough) { + passthrough.changePassthroughMode(); + return true; } return false; } @@ -293,7 +323,7 @@ private native void initializeNative( private native void pauseNative(); - private native void surfaceCreatedNative(); + private native int surfaceCreatedNative(); private native void setScreenResolutionNative(int width, int height); diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java new file mode 100644 index 00000000..2892c429 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java @@ -0,0 +1,389 @@ +/* (C)2024 */ +package viritualisres.phonevr; + +import android.content.SharedPreferences; +import android.graphics.SurfaceTexture; +import android.hardware.Camera; +import android.hardware.Sensor; +import android.hardware.SensorEvent; +import android.hardware.SensorEventListener; +import android.hardware.SensorManager; +import android.os.Build; +import android.util.Log; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; + +public class Passthrough implements SensorEventListener { + + static { + System.loadLibrary("native-lib"); + } + + private static final String TAG = Passthrough.class.getSimpleName() + "-Java"; + + private Camera mCamera = null; + + private SurfaceTexture mTexture = null; + private Camera.Size mPreviewSize = null; + + private final SharedPreferences mPref; + + private final SensorManager mSensorManager; + + private final int displayWidth; + + private final int displayHeight; + + private Sensor mAccelerometer = null; + + private long timestamp = 0; + + private long timestampAfterCandidate = 0; + + private long passthroughDelayInMs = 600; + + private float detectionLowerBound = 0.8f; + + private float detectionUpperBound = 4f; + + private final List[] accelStore = + new List[] {new ArrayList<>(), new ArrayList<>(), new ArrayList<>()}; + + private int halfSize = 0; + + private boolean triggerIsSet = false; + + private final List cumSums = new ArrayList<>(); + + public Passthrough( + SharedPreferences pref, + SensorManager sensorManager, + int displayWidth, + int displayHeight) { + this.displayWidth = displayWidth; + this.displayHeight = displayHeight; + mPref = pref; + mSensorManager = sensorManager; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + mAccelerometer = + mSensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER_UNCALIBRATED); + } + if (mAccelerometer == null) { + mAccelerometer = mSensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER); + } + } + + @Override + public void onSensorChanged(SensorEvent event) { + if (timestamp == 0) { + timestamp = event.timestamp; + } + + if (timestampAfterCandidate == 0) { + timestampAfterCandidate = event.timestamp; + } + + boolean initialFull = event.timestamp - timestamp > passthroughDelayInMs * 1000000L; + boolean afterCandidateFull = + event.timestamp - timestampAfterCandidate > passthroughDelayInMs * 1000000L; + + // gather data for passthrough_delay_ms, we get a data point approx. each 60ms + for (int i = 0; i < accelStore.length; ++i) { + // derivative, absolute + accelStore[i].add(event.values[i]); + + if (initialFull) { + // remove the first element to only store the last X elements + accelStore[i].remove(0); + if (halfSize == 0) { + halfSize = accelStore[0].size() / 2; + } + // check if half size has at least 3 data points + if (halfSize < 4) { + accelStore[i].clear(); + timestamp = 0; + halfSize = 0; + } + } + } + + if (halfSize > 0) { + // Calculate derivative and absolute values + List[] accelAbs = + new List[] {new ArrayList<>(), new ArrayList<>(), new ArrayList<>()}; + for (int axis = 0; axis < accelStore.length; ++axis) { + for (int i = 1; i < accelStore[axis].size(); ++i) { + accelAbs[axis].add( + Math.abs(accelStore[axis].get(i) - accelStore[axis].get(i - 1))); + } + } + + // get the mean for each axis and sum them up + float[] axisMean = new float[] {0.0f, 0.0f, 0.0f}; + for (int i = 0; i < accelAbs.length; ++i) { + for (int j = 0; j < accelAbs[i].size(); ++j) { + axisMean[i] += accelAbs[i].get(j); + } + axisMean[i] /= accelAbs[i].size(); + } + float sumMean = axisMean[0] + axisMean[1] + axisMean[2]; + cumSums.add(sumMean); + + if (cumSums.size() < accelAbs[0].size()) { + return; + } else { + // only store the last X elements + cumSums.remove(0); + } + + if (!afterCandidateFull) { + // if a potential passthrough_change event is found, + // check if the device does not move much anymore + if (!triggerIsSet && sumMean < detectionLowerBound) { + triggerIsSet = true; + // if passthroughMode not already changed... change it + changePassthroughMode(); + } + } else { + triggerIsSet = false; + // here we check for potential passthrough_change events + // these may occur on any axis + for (List elem : accelAbs) { + // we look at two halves of the passthrough_delay + // we check: + // - if in the first and second half is a real peak + // - whether the peaks are in the bounds + // - if the summed mean is in the bounds + // - if the first summed mean is below lower_bound + // (to assure a resting stage before the triggering) + // - if the second maximum is decending under the lower bound until curr value + // If all is fulfilled we consider this a potential passthrough_change event. + // The sum must then decend under lower bound until passthrough_delay + int middle = halfSize - 1; + List t1 = elem.subList(0, halfSize); + List t2 = elem.subList(middle, elem.size()); + + float max1 = Collections.max(t1); + float max2 = Collections.max(t2); + + if (max1 - t1.get(0) < detectionLowerBound + || max1 - t1.get(t1.size() - 1) < detectionLowerBound + || max2 - t2.get(0) < detectionLowerBound + || max2 - t2.get(t2.size() - 1) < detectionLowerBound + || max1 < detectionLowerBound + || max1 > detectionUpperBound + || max2 < detectionLowerBound + || max2 > detectionUpperBound + || sumMean < detectionLowerBound + || sumMean > detectionUpperBound) { + continue; + } + + int end2 = 0; + for (int i = 0; i < t2.size(); ++i) { + if (t2.get(i) == max2) { + end2 = i; + } + if (end2 > 0) { + if (t2.get(i) < detectionLowerBound) { + end2 = i; + break; + } + } + } + + if (cumSums.get(0) < Math.max(detectionLowerBound - 0.3f, 0.5f) + && t2.get(end2) < max2) { + timestampAfterCandidate = event.timestamp; + break; + } + } + } + } + + Log.v( + TAG + "-AccSensor", + String.format( + (Locale) null, + "%d: [%f, %f, %f]", + event.timestamp, + event.values[0], + event.values[1], + event.values[2])); + } + + @Override + public void onAccuracyChanged(Sensor sensor, int accuracy) {} + + protected void onResume() { + SharedPreferences.Editor edit = mPref.edit(); + edit.putBoolean("passthrough", false); + edit.apply(); + + timestamp = 0; + halfSize = 0; + if (mPref.getBoolean("passthrough_tap", true)) { + try { + passthroughDelayInMs = Long.parseLong(mPref.getString("passthrough_delay", "600")); + } catch (Exception e) { + passthroughDelayInMs = 600; + } + try { + detectionLowerBound = Float.parseFloat(mPref.getString("passthrough_lower", "0.8")); + } catch (Exception e) { + detectionLowerBound = 0.8f; + } + try { + detectionUpperBound = Float.parseFloat(mPref.getString("passthrough_upper", "4")); + } catch (Exception e) { + detectionUpperBound = 4.f; + } + + if (mAccelerometer != null) { + mSensorManager.registerListener( + this, mAccelerometer, SensorManager.SENSOR_DELAY_UI); + } + } else { + mSensorManager.unregisterListener(this); + } + } + + protected void setPassthroughMode(boolean passthrough) { + if (passthrough) { + int w = displayWidth / 2; + int h = displayHeight; + if (displayWidth < displayHeight) { + w = displayHeight / 2; + h = displayWidth; + } + openCamera(-1, w, h, mPref.getBoolean("passthrough_recording", true)); + + float size = 0.5f; + try { + size = Float.parseFloat(mPref.getString("passthrough_fraction", "0.5")); + } catch (RuntimeException e) { + } + + setPassthroughSizeNative(size); + } + setPassthroughActiveNative(passthrough); + + if (passthrough) { + startPreview(); + } else { + releaseCamera(); + } + } + + protected void changePassthroughMode() { + boolean pt = !mPref.getBoolean("passthrough", false); + SharedPreferences.Editor edit = mPref.edit(); + edit.putBoolean("passthrough", pt); + edit.apply(); + setPassthroughMode(pt); + } + + public void createTexture(int textureID) { + mTexture = new SurfaceTexture(textureID); + } + + public void openCamera(int camNr, int desiredWidth, int desiredHeight, boolean recordingHint) { + if (mCamera != null) { + // TODO maybe print only a warning + throw new RuntimeException("Camera already initialized"); + } + Camera.CameraInfo info = new Camera.CameraInfo(); + // TODO add possibility to choose camera + if (camNr < 0) { + int numCameras = Camera.getNumberOfCameras(); + for (int i = 0; i < numCameras; ++i) { + Camera.getCameraInfo(i, info); + if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) { + mCamera = Camera.open(i); + break; + } + } + if (mCamera == null) { + // TODO log message "No back-facing camera found; opening default" + mCamera = Camera.open(); // open default + } + } else { + mCamera = Camera.open(camNr); + } + if (mCamera == null) { + throw new RuntimeException("Unable to open camera"); + } + // TODO the size stuff + Camera.Parameters params = mCamera.getParameters(); + List choices = params.getSupportedPreviewSizes(); + mPreviewSize = chooseOptimalSize(choices, desiredWidth, desiredHeight); + Log.d(TAG, "PreviewSize: " + mPreviewSize.width + "x" + mPreviewSize.height); + params.setPreviewSize(mPreviewSize.width, mPreviewSize.height); + params.setRecordingHint(recordingHint); // Can increase fps in passthrough mode + mCamera.setParameters(params); + } + + private Camera.Size chooseOptimalSize(List choices, int width, int height) { + Integer minSize = Integer.max(Integer.min(width, height), 320); + + // Collect the supported resolutions that are at least as big as the preview Surface + ArrayList bigEnough = new ArrayList<>(); + for (Camera.Size option : choices) { + if (option.width == width && option.height == height) { + return option; + } + if (option.height >= minSize && option.width >= minSize) { + bigEnough.add(option); + } + } + + Comparator comp = (a, b) -> (a.width * a.height) - (b.width * b.width); + // Pick the smallest of those, assuming we found any + if (bigEnough.size() > 0) { + return Collections.min(bigEnough, comp); + } else { + return choices.get(0); + } + } + + public void releaseCamera() { + if (mCamera != null) { + mCamera.setPreviewCallback(null); + mCamera.stopPreview(); + mCamera.release(); + mCamera = null; + } + } + + public void startPreview() { + if (mCamera != null) { + // Log.d(TAG, "starting camera preview") + try { + mCamera.setPreviewTexture(mTexture); + } catch (IOException e) { + throw new RuntimeException(e); + } + mCamera.startPreview(); + } + } + + public void onPause() { + mSensorManager.unregisterListener(this); + setPassthroughActiveNative(false); + releaseCamera(); + } + + public void update() { + if (mTexture != null) { + mTexture.updateTexImage(); + } + } + + private native void setPassthroughSizeNative(float size); + + private native void setPassthroughActiveNative(boolean activate); +} diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java new file mode 100644 index 00000000..ca61105a --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java @@ -0,0 +1,33 @@ +/* (C)2024 */ +package viritualisres.phonevr; + +import android.os.Bundle; +import androidx.appcompat.app.ActionBar; +import androidx.appcompat.app.AppCompatActivity; +import androidx.preference.PreferenceFragmentCompat; + +public class PassthroughSettingsActivity extends AppCompatActivity { + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_passthrough_settings); + if (savedInstanceState == null) { + getSupportFragmentManager() + .beginTransaction() + .replace(R.id.settings, new SettingsFragment()) + .commit(); + } + ActionBar actionBar = getSupportActionBar(); + if (actionBar != null) { + actionBar.setDisplayHomeAsUpEnabled(true); + } + } + + public static class SettingsFragment extends PreferenceFragmentCompat { + @Override + public void onCreatePreferences(Bundle savedInstanceState, String rootKey) { + setPreferencesFromResource(R.xml.passthrough_settings, rootKey); + } + } +} diff --git a/code/mobile/android/PhoneVR/app/src/main/res/layout/activity_passthrough_settings.xml b/code/mobile/android/PhoneVR/app/src/main/res/layout/activity_passthrough_settings.xml new file mode 100644 index 00000000..4a4f2472 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/layout/activity_passthrough_settings.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml b/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml index 7deebc7b..7bd86d91 100644 --- a/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml +++ b/code/mobile/android/PhoneVR/app/src/main/res/menu/settings_menu.xml @@ -10,4 +10,11 @@ android:id="@+id/max_brightness_toggle" android:title="@string/max_brightness" android:checkable="true"/> + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/values/arrays.xml b/code/mobile/android/PhoneVR/app/src/main/res/values/arrays.xml new file mode 100644 index 00000000..6cf9ed48 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/values/arrays.xml @@ -0,0 +1,12 @@ + + + + Reply + Reply to all + + + + reply + reply_all + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml b/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml index c4657b69..b923485d 100644 --- a/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml +++ b/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml @@ -41,4 +41,18 @@ Switch PhoneVR Server Remember choice Max Brightness + PassthroughSettingsActivity + Passthrough fraction + Enable passthrough + The fraction how much the passthrough should contain of the image (between 0.0 and 1.0). This depends on the FOV of the camera, for narrow FOV a lower value might be better, so that objects don\'t look to close... Play around with this value. + Set Recording Hint + If set to false the passthrough may have a greater FOV, but has less FPS which may cause motion sickness. + Experimental Double-Tap detection + If true passthrough can be switched through double-tap. + Double Tap Configuration + Delay in ms + Lower Bound + Upper Bound + Passthrough Settings + Passthrough diff --git a/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml b/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml new file mode 100644 index 00000000..90c53478 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + \ No newline at end of file From 4314ace8b8bc145cb4d878ff6c12b2b589edc3ac Mon Sep 17 00:00:00 2001 From: Dominik Wetzel Date: Tue, 7 May 2024 01:10:11 +0200 Subject: [PATCH 2/8] Minor passthrough algorithm refinement and made it buildable --- .../PhoneVR/app/src/main/cpp/passthrough.cpp | 2 +- .../PhoneVR/app/src/main/cpp/passthrough.h | 3 +- .../android/PhoneVR/app/src/main/cpp/utils.h | 2 +- .../viritualisres/phonevr/ALVRActivity.java | 5 +- .../viritualisres/phonevr/Passthrough.java | 71 +++++++++---------- 5 files changed, 37 insertions(+), 46 deletions(-) diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp index 927bbef0..db558953 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp @@ -3,8 +3,8 @@ #include #include -#include "passthrough.h" #include "utils.h" +#include "passthrough.h" GLuint LoadGLShader(GLenum type, const char *shader_source) { GLuint shader = GL(glCreateShader(type)); diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h index f7d8c5d4..8b4164be 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.h @@ -1,4 +1,5 @@ #ifndef PHONEVR_PASSTHROUGH_H +#define PHONEVR_PASSTHROUGH_H #include "cardboard.h" @@ -24,6 +25,4 @@ void passthrough_cleanup(PassthroughInfo *info); void passthrough_setup(PassthroughInfo *info); void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription viewDescs[]); -#define PHONEVR_PASSTHROUGH_H - #endif // PHONEVR_PASSTHROUGH_H diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h b/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h index bc31f22b..5bc99d00 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h @@ -12,7 +12,7 @@ const char LOG_TAG[] = "ALVR_PVR_NATIVE"; -void log(AlvrLogLevel level, +static void log(AlvrLogLevel level, const char *FILE_NAME, unsigned int LINE_NO, const char *FUNC, diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java index be457003..3b7f7cb2 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/ALVRActivity.java @@ -144,10 +144,7 @@ public void onCreate(Bundle savedInstance) { pref = PreferenceManager.getDefaultSharedPreferences(this); passthrough = new Passthrough( - pref, - (SensorManager) getSystemService(SENSOR_SERVICE), - width, - height); + pref, (SensorManager) getSystemService(SENSOR_SERVICE), width, height); } @Override diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java index 2892c429..bb9ab981 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java @@ -20,7 +20,7 @@ public class Passthrough implements SensorEventListener { static { - System.loadLibrary("native-lib"); + System.loadLibrary("native-lib-alvr"); } private static final String TAG = Passthrough.class.getSimpleName() + "-Java"; @@ -154,50 +154,45 @@ public void onSensorChanged(SensorEvent event) { // these may occur on any axis for (List elem : accelAbs) { // we look at two halves of the passthrough_delay - // we check: - // - if in the first and second half is a real peak - // - whether the peaks are in the bounds - // - if the summed mean is in the bounds - // - if the first summed mean is below lower_bound - // (to assure a resting stage before the triggering) - // - if the second maximum is decending under the lower bound until curr value - // If all is fulfilled we consider this a potential passthrough_change event. - // The sum must then decend under lower bound until passthrough_delay - int middle = halfSize - 1; - List t1 = elem.subList(0, halfSize); - List t2 = elem.subList(middle, elem.size()); + List t1 = elem.subList(0, halfSize + 1); + List t2 = elem.subList(halfSize, elem.size()); + // and get the maxima float max1 = Collections.max(t1); float max2 = Collections.max(t2); - - if (max1 - t1.get(0) < detectionLowerBound - || max1 - t1.get(t1.size() - 1) < detectionLowerBound - || max2 - t2.get(0) < detectionLowerBound - || max2 - t2.get(t2.size() - 1) < detectionLowerBound - || max1 < detectionLowerBound - || max1 > detectionUpperBound - || max2 < detectionLowerBound - || max2 > detectionUpperBound - || sumMean < detectionLowerBound - || sumMean > detectionUpperBound) { - continue; - } - - int end2 = 0; - for (int i = 0; i < t2.size(); ++i) { - if (t2.get(i) == max2) { + // and check whether the 2nd half goes belowe lower bound + int end2 = t2.indexOf(max2); + for (int i = end2; i < t2.size(); ++i) { + if (t2.get(i) < detectionLowerBound) { end2 = i; - } - if (end2 > 0) { - if (t2.get(i) < detectionLowerBound) { - end2 = i; - break; - } + break; } } - if (cumSums.get(0) < Math.max(detectionLowerBound - 0.3f, 0.5f) - && t2.get(end2) < max2) { + // We check + if (t1.get(halfSize - 1) > t2.get(0) // if halfSize is a local minimum + && t2.get(1) > t2.get(0) + && max1 - t1.get(0) >= detectionLowerBound // max - start > lower bound + && max1 - t1.get(t1.size() - 1) + >= detectionLowerBound // max - end > lower bound + && max2 - t2.get(0) >= detectionLowerBound + && max2 - t2.get(t2.size() - 1) >= detectionLowerBound + && max1 >= detectionLowerBound // max are in bounds + && max1 <= detectionUpperBound + && max2 >= detectionLowerBound + && max2 <= detectionUpperBound + && sumMean >= detectionLowerBound // sumMean is in bounds + && sumMean <= detectionUpperBound + && t2.get(end2) < max2 // the 2nd half goes below lower bound + && cumSums.get(0) + < Math.max( + detectionLowerBound - 0.3f, + 0.5f) // the first sumMean (of window) is below lower + // bounds + ) { + // If all is fulfilled we consider this a potential passthrough_change + // event. + // The sum must then decend under lower bound until passthrough_delay timestampAfterCandidate = event.timestamp; break; } From 68fe643ad03a92f375e04d221d026ca66b3f307b Mon Sep 17 00:00:00 2001 From: Dominik Wetzel Date: Wed, 24 Apr 2024 11:23:40 +0200 Subject: [PATCH 3/8] Added and tested hellocharts in PassthroughSettings --- code/mobile/android/PhoneVR/app/build.gradle | 1 + .../phonevr/PassthroughSettingsActivity.java | 102 ++++++++++++++++++ .../app/src/main/res/layout/chart_layout.xml | 12 +++ .../app/src/main/res/values/strings.xml | 1 + .../src/main/res/xml/passthrough_settings.xml | 10 ++ 5 files changed, 126 insertions(+) create mode 100644 code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml diff --git a/code/mobile/android/PhoneVR/app/build.gradle b/code/mobile/android/PhoneVR/app/build.gradle index 999ef85c..8161dded 100644 --- a/code/mobile/android/PhoneVR/app/build.gradle +++ b/code/mobile/android/PhoneVR/app/build.gradle @@ -134,6 +134,7 @@ android { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation fileTree(dir: "libraries", include: ['hellocharts-2.6.5.aar']) implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" implementation 'androidx.appcompat:appcompat:1.6.1' implementation 'androidx.constraintlayout:constraintlayout:2.0.4' diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java index ca61105a..5375d328 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java @@ -1,10 +1,27 @@ /* (C)2024 */ package viritualisres.phonevr; +import android.annotation.SuppressLint; import android.os.Bundle; +import android.view.View; +import android.view.ViewGroup; +import androidx.annotation.NonNull; import androidx.appcompat.app.ActionBar; import androidx.appcompat.app.AppCompatActivity; import androidx.preference.PreferenceFragmentCompat; +import androidx.preference.PreferenceGroupAdapter; +import androidx.preference.PreferenceScreen; +import androidx.preference.PreferenceViewHolder; +import androidx.recyclerview.widget.RecyclerView; +import ir.mahdiparastesh.hellocharts.model.Axis; +import ir.mahdiparastesh.hellocharts.model.Line; +import ir.mahdiparastesh.hellocharts.model.LineChartData; +import ir.mahdiparastesh.hellocharts.model.PointValue; +import ir.mahdiparastesh.hellocharts.model.Viewport; +import ir.mahdiparastesh.hellocharts.util.ChartUtils; +import ir.mahdiparastesh.hellocharts.view.LineChartView; +import java.util.ArrayList; +import java.util.List; public class PassthroughSettingsActivity extends AppCompatActivity { @@ -25,9 +42,94 @@ protected void onCreate(Bundle savedInstanceState) { } public static class SettingsFragment extends PreferenceFragmentCompat { + private final int numberOfLines = 4; + private final int numberOfPoints = 12; + private LineChartView chart = null; + + float[][] randomNumbersTab = new float[numberOfLines][numberOfPoints]; + @Override public void onCreatePreferences(Bundle savedInstanceState, String rootKey) { setPreferencesFromResource(R.xml.passthrough_settings, rootKey); } + + @NonNull + @SuppressLint("RestrictedApi") + @Override + protected RecyclerView.Adapter onCreateAdapter( + PreferenceScreen pref) { + return new PreferenceGroupAdapter(pref) { + @NonNull + public PreferenceViewHolder onCreateViewHolder( + @NonNull ViewGroup parent, int viewType) { + PreferenceViewHolder holder = super.onCreateViewHolder(parent, viewType); + View customLayout = holder.itemView; + if (customLayout.getId() == R.id.chartLayout) { + chart = customLayout.findViewById(R.id.chart); + generateValues(); + generateData(chart); + resetViewport(chart); + } + return holder; + } + }; + } + + private void generateValues() { + for (int i = 0; i < numberOfLines; ++i) { + for (int j = 0; j < numberOfPoints; ++j) { + randomNumbersTab[i][j] = (float) Math.random() * 100f; + } + } + } + + private void generateData(LineChartView chart) { + List lines = new ArrayList<>(); + for (int i = 0; i < numberOfLines; ++i) { + + List values = new ArrayList<>(); + for (int j = 0; j < numberOfPoints; ++j) { + values.add(new PointValue(j, randomNumbersTab[i][j])); + } + + Line line = new Line(values); + line.setColor(ChartUtils.COLORS[i]); + // line.setShape(shape); + // line.setCubic(isCubic); + // line.setFilled(isFilled); + // line.setHasLabels(hasLabels); + // line.setHasLabelsOnlyForSelected(hasLabelForSelected); + // line.setHasLines(hasLines); + // line.setHasPoints(hasPoints); + // line.setHasGradientToTransparent(hasGradientToTransparent); + // if (pointsHaveDifferentColor) + // line.setPointColor(ChartUtils.COLORS[(i + 1) % ChartUtils.COLORS.length]); + lines.add(line); + } + + LineChartData data = new LineChartData(lines); + + Axis axisX = new Axis(); + Axis axisY = new Axis().setHasLines(true); + axisX.setName("Axis X"); + axisY.setName("Axis Y"); + + data.setAxisXBottom(axisX); + data.setAxisYLeft(axisY); + + data.setBaseValue(Float.NEGATIVE_INFINITY); + chart.setLineChartData(data); + } + + private void resetViewport(LineChartView chart) { + // Reset viewport height range to (0,100) + final Viewport v = new Viewport(chart.getMaximumViewport()); + v.bottom = 0; + v.top = 100; + v.left = 0; + v.right = numberOfPoints - 1; + chart.setMaximumViewport(v); + chart.setCurrentViewport(v); + } } } diff --git a/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml b/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml new file mode 100644 index 00000000..fced2e18 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml b/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml index b923485d..28ad6797 100644 --- a/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml +++ b/code/mobile/android/PhoneVR/app/src/main/res/values/strings.xml @@ -53,6 +53,7 @@ Delay in ms Lower Bound Upper Bound + Double Tap Test Graph Passthrough Settings Passthrough diff --git a/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml b/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml index 90c53478..ed65e0bb 100644 --- a/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml +++ b/code/mobile/android/PhoneVR/app/src/main/res/xml/passthrough_settings.xml @@ -40,6 +40,16 @@ app:title="@string/upper_bound" app:useSimpleSummaryProvider="true" /> + + + + \ No newline at end of file From a77cee48ba448e67cfaac1d36eaf535120f2e14e Mon Sep 17 00:00:00 2001 From: Dominik Wetzel Date: Sat, 27 Apr 2024 23:36:21 +0200 Subject: [PATCH 4/8] Added a live graph for trigger detection --- .../viritualisres/phonevr/GraphAdapter.java | 104 ++++++++++++++++++ .../viritualisres/phonevr/Passthrough.java | 63 ++++++++--- .../phonevr/PassthroughSettingsActivity.java | 88 +++------------ 3 files changed, 167 insertions(+), 88 deletions(-) create mode 100644 code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java new file mode 100644 index 00000000..4e70cea8 --- /dev/null +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java @@ -0,0 +1,104 @@ +/* (C)2024 */ +package viritualisres.phonevr; + +import ir.mahdiparastesh.hellocharts.model.Axis; +import ir.mahdiparastesh.hellocharts.model.Line; +import ir.mahdiparastesh.hellocharts.model.LineChartData; +import ir.mahdiparastesh.hellocharts.model.PointValue; +import ir.mahdiparastesh.hellocharts.model.Viewport; +import ir.mahdiparastesh.hellocharts.util.ChartUtils; +import ir.mahdiparastesh.hellocharts.view.LineChartView; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GraphAdapter { + + private final LineChartView chart; + private List timestamps = new ArrayList<>(); + private Map> lines = new HashMap<>(); + private List keyOrder = new ArrayList<>(); + private List color = new ArrayList<>(); + private int len; + private float max = 10.f; + + public GraphAdapter(int len, LineChartView chart) { + this.len = len; + this.chart = chart; + } + + private int checkTimestamp(Long ts) { + if (!timestamps.contains(ts)) { + timestamps.add(ts); + if (timestamps.size() > this.len) { + timestamps.remove(0); + } + } + return timestamps.indexOf(ts) + 1; + } + + private void resetViewport() { + Viewport v = new Viewport(chart.getMaximumViewport()); + v.bottom = 0; + v.top = (int) max; + v.left = 0; + v.right = this.len - 1; + chart.setMaximumViewport(v); + chart.setCurrentViewport(v); + } + + public void addValue(Long ts, String name, Float value) { + int position = checkTimestamp(ts); + if (!lines.containsKey(name)) { + lines.put(name, new ArrayList<>()); + keyOrder.add(name); + color.add(ChartUtils.pickColor()); + } + List list = lines.get(name); + list.add(value); + while (list.size() < position) { + list.add(0, 0.f); + } + if (list.size() > position) { + list.remove(0); + } + } + + public void addValue(Long ts, String name, boolean value) { + if (value) { + addValue(ts, name, max); + } else { + addValue(ts, name, 0.f); + } + } + + public void refresh() { + List _lines = new ArrayList<>(); + for (int j = 0; j < keyOrder.size(); ++j) { + String name = keyOrder.get(j); + List elems = lines.get(name); + List values = new ArrayList<>(); + for (int i = 0; i < elems.size(); ++i) { + values.add(new PointValue(i, elems.get(i))); + } + Line line = new Line(values); + line.setColor(color.get(j)); + line.setHasPoints(false); + _lines.add(line); + } + LineChartData data = new LineChartData(_lines); + + Axis axisX = new Axis(); + Axis axisY = new Axis().setHasLines(true); + axisX.setName("Time"); + axisY.setName("Values"); + + data.setAxisXBottom(axisX); + data.setAxisYLeft(axisY); + + data.setBaseValue(Float.NEGATIVE_INFINITY); + chart.setLineChartData(data); + resetViewport(); + } +} diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java index bb9ab981..4075636a 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/Passthrough.java @@ -34,6 +34,8 @@ public class Passthrough implements SensorEventListener { private final SensorManager mSensorManager; + private GraphAdapter graphAdapter = null; + private final int displayWidth; private final int displayHeight; @@ -84,7 +86,7 @@ public void onSensorChanged(SensorEvent event) { } if (timestampAfterCandidate == 0) { - timestampAfterCandidate = event.timestamp; + timestampAfterCandidate = event.timestamp - 1; } boolean initialFull = event.timestamp - timestamp > passthroughDelayInMs * 1000000L; @@ -92,6 +94,7 @@ public void onSensorChanged(SensorEvent event) { event.timestamp - timestampAfterCandidate > passthroughDelayInMs * 1000000L; // gather data for passthrough_delay_ms, we get a data point approx. each 60ms + String axesName[] = {"X", "Y", "Z"}; for (int i = 0; i < accelStore.length; ++i) { // derivative, absolute accelStore[i].add(event.values[i]); @@ -120,6 +123,12 @@ public void onSensorChanged(SensorEvent event) { accelAbs[axis].add( Math.abs(accelStore[axis].get(i) - accelStore[axis].get(i - 1))); } + if (graphAdapter != null) { + graphAdapter.addValue( + event.timestamp, + axesName[axis], + accelAbs[axis].get(accelAbs[axis].size() - 1)); + } } // get the mean for each axis and sum them up @@ -132,6 +141,9 @@ public void onSensorChanged(SensorEvent event) { } float sumMean = axisMean[0] + axisMean[1] + axisMean[2]; cumSums.add(sumMean); + if (graphAdapter != null) { + graphAdapter.addValue(event.timestamp, "sumMean", sumMean); + } if (cumSums.size() < accelAbs[0].size()) { return; @@ -199,6 +211,15 @@ public void onSensorChanged(SensorEvent event) { } } } + if (graphAdapter != null) { + graphAdapter.addValue( + event.timestamp, "Candidate", timestampAfterCandidate == event.timestamp); + graphAdapter.addValue(event.timestamp, "Trigger", triggerIsSet); + graphAdapter.addValue(event.timestamp, "upperBound", detectionUpperBound); + graphAdapter.addValue(event.timestamp, "lowerBound", detectionLowerBound); + graphAdapter.refresh(); + this.updateSettings(); + } Log.v( TAG + "-AccSensor", @@ -214,6 +235,29 @@ public void onSensorChanged(SensorEvent event) { @Override public void onAccuracyChanged(Sensor sensor, int accuracy) {} + protected void onResume(GraphAdapter graphAdapter) { + this.graphAdapter = graphAdapter; + this.onResume(); + } + + protected void updateSettings() { + try { + passthroughDelayInMs = Long.parseLong(mPref.getString("passthrough_delay", "600")); + } catch (Exception e) { + passthroughDelayInMs = 600; + } + try { + detectionLowerBound = Float.parseFloat(mPref.getString("passthrough_lower", "0.8")); + } catch (Exception e) { + detectionLowerBound = 0.8f; + } + try { + detectionUpperBound = Float.parseFloat(mPref.getString("passthrough_upper", "4")); + } catch (Exception e) { + detectionUpperBound = 4.f; + } + } + protected void onResume() { SharedPreferences.Editor edit = mPref.edit(); edit.putBoolean("passthrough", false); @@ -222,22 +266,7 @@ protected void onResume() { timestamp = 0; halfSize = 0; if (mPref.getBoolean("passthrough_tap", true)) { - try { - passthroughDelayInMs = Long.parseLong(mPref.getString("passthrough_delay", "600")); - } catch (Exception e) { - passthroughDelayInMs = 600; - } - try { - detectionLowerBound = Float.parseFloat(mPref.getString("passthrough_lower", "0.8")); - } catch (Exception e) { - detectionLowerBound = 0.8f; - } - try { - detectionUpperBound = Float.parseFloat(mPref.getString("passthrough_upper", "4")); - } catch (Exception e) { - detectionUpperBound = 4.f; - } - + updateSettings(); if (mAccelerometer != null) { mSensorManager.registerListener( this, mAccelerometer, SensorManager.SENSOR_DELAY_UI); diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java index 5375d328..929bfdbe 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java @@ -2,6 +2,8 @@ package viritualisres.phonevr; import android.annotation.SuppressLint; +import android.content.SharedPreferences; +import android.hardware.SensorManager; import android.os.Bundle; import android.view.View; import android.view.ViewGroup; @@ -10,24 +12,23 @@ import androidx.appcompat.app.AppCompatActivity; import androidx.preference.PreferenceFragmentCompat; import androidx.preference.PreferenceGroupAdapter; +import androidx.preference.PreferenceManager; import androidx.preference.PreferenceScreen; import androidx.preference.PreferenceViewHolder; import androidx.recyclerview.widget.RecyclerView; -import ir.mahdiparastesh.hellocharts.model.Axis; -import ir.mahdiparastesh.hellocharts.model.Line; -import ir.mahdiparastesh.hellocharts.model.LineChartData; -import ir.mahdiparastesh.hellocharts.model.PointValue; -import ir.mahdiparastesh.hellocharts.model.Viewport; -import ir.mahdiparastesh.hellocharts.util.ChartUtils; import ir.mahdiparastesh.hellocharts.view.LineChartView; -import java.util.ArrayList; -import java.util.List; public class PassthroughSettingsActivity extends AppCompatActivity { + private static SharedPreferences sharedPref; + private static SensorManager sensorManager; + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); + sharedPref = PreferenceManager.getDefaultSharedPreferences(this); + sensorManager = (SensorManager) getSystemService(SENSOR_SERVICE); + setContentView(R.layout.activity_passthrough_settings); if (savedInstanceState == null) { getSupportFragmentManager() @@ -42,11 +43,13 @@ protected void onCreate(Bundle savedInstanceState) { } public static class SettingsFragment extends PreferenceFragmentCompat { - private final int numberOfLines = 4; - private final int numberOfPoints = 12; private LineChartView chart = null; + private Passthrough passthrough; + private GraphAdapter graphAdapter; - float[][] randomNumbersTab = new float[numberOfLines][numberOfPoints]; + public SettingsFragment() { + super(); + } @Override public void onCreatePreferences(Bundle savedInstanceState, String rootKey) { @@ -66,70 +69,13 @@ public PreferenceViewHolder onCreateViewHolder( View customLayout = holder.itemView; if (customLayout.getId() == R.id.chartLayout) { chart = customLayout.findViewById(R.id.chart); - generateValues(); - generateData(chart); - resetViewport(chart); + graphAdapter = new GraphAdapter(100, chart); + passthrough = new Passthrough(sharedPref, sensorManager, 640, 480); + passthrough.onResume(graphAdapter); } return holder; } }; } - - private void generateValues() { - for (int i = 0; i < numberOfLines; ++i) { - for (int j = 0; j < numberOfPoints; ++j) { - randomNumbersTab[i][j] = (float) Math.random() * 100f; - } - } - } - - private void generateData(LineChartView chart) { - List lines = new ArrayList<>(); - for (int i = 0; i < numberOfLines; ++i) { - - List values = new ArrayList<>(); - for (int j = 0; j < numberOfPoints; ++j) { - values.add(new PointValue(j, randomNumbersTab[i][j])); - } - - Line line = new Line(values); - line.setColor(ChartUtils.COLORS[i]); - // line.setShape(shape); - // line.setCubic(isCubic); - // line.setFilled(isFilled); - // line.setHasLabels(hasLabels); - // line.setHasLabelsOnlyForSelected(hasLabelForSelected); - // line.setHasLines(hasLines); - // line.setHasPoints(hasPoints); - // line.setHasGradientToTransparent(hasGradientToTransparent); - // if (pointsHaveDifferentColor) - // line.setPointColor(ChartUtils.COLORS[(i + 1) % ChartUtils.COLORS.length]); - lines.add(line); - } - - LineChartData data = new LineChartData(lines); - - Axis axisX = new Axis(); - Axis axisY = new Axis().setHasLines(true); - axisX.setName("Axis X"); - axisY.setName("Axis Y"); - - data.setAxisXBottom(axisX); - data.setAxisYLeft(axisY); - - data.setBaseValue(Float.NEGATIVE_INFINITY); - chart.setLineChartData(data); - } - - private void resetViewport(LineChartView chart) { - // Reset viewport height range to (0,100) - final Viewport v = new Viewport(chart.getMaximumViewport()); - v.bottom = 0; - v.top = 100; - v.left = 0; - v.right = numberOfPoints - 1; - chart.setMaximumViewport(v); - chart.setCurrentViewport(v); - } } } From 2f2495c5d658d220f53fdb717c4f3f0016d7118d Mon Sep 17 00:00:00 2001 From: Dominik Wetzel Date: Tue, 7 May 2024 13:14:45 +0200 Subject: [PATCH 5/8] Changed library to one in a repository (MPAndroicChart) --- code/mobile/android/PhoneVR/app/build.gradle | 6 +- .../viritualisres/phonevr/GraphAdapter.java | 69 ++++++++----------- .../phonevr/PassthroughSettingsActivity.java | 5 +- .../app/src/main/res/layout/chart_layout.xml | 4 +- 4 files changed, 38 insertions(+), 46 deletions(-) diff --git a/code/mobile/android/PhoneVR/app/build.gradle b/code/mobile/android/PhoneVR/app/build.gradle index 8161dded..d6ec5e52 100644 --- a/code/mobile/android/PhoneVR/app/build.gradle +++ b/code/mobile/android/PhoneVR/app/build.gradle @@ -132,9 +132,12 @@ android { } } +repositories { + maven { url 'https://jitpack.io' } +} + dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation fileTree(dir: "libraries", include: ['hellocharts-2.6.5.aar']) implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" implementation 'androidx.appcompat:appcompat:1.6.1' implementation 'androidx.constraintlayout:constraintlayout:2.0.4' @@ -161,6 +164,7 @@ dependencies { implementation platform('com.google.firebase:firebase-bom:26.0.0') implementation 'com.google.firebase:firebase-analytics-ktx' + implementation 'com.github.PhilJay:MPAndroidChart:v3.1.0' } // The dependencies for NDK builds live inside the .aar files so they need to diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java index 4e70cea8..0afd449e 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/GraphAdapter.java @@ -1,13 +1,13 @@ /* (C)2024 */ package viritualisres.phonevr; -import ir.mahdiparastesh.hellocharts.model.Axis; -import ir.mahdiparastesh.hellocharts.model.Line; -import ir.mahdiparastesh.hellocharts.model.LineChartData; -import ir.mahdiparastesh.hellocharts.model.PointValue; -import ir.mahdiparastesh.hellocharts.model.Viewport; -import ir.mahdiparastesh.hellocharts.util.ChartUtils; -import ir.mahdiparastesh.hellocharts.view.LineChartView; +import android.view.View; +import com.github.mikephil.charting.charts.LineChart; +import com.github.mikephil.charting.components.YAxis; +import com.github.mikephil.charting.data.Entry; +import com.github.mikephil.charting.data.LineData; +import com.github.mikephil.charting.data.LineDataSet; +import com.github.mikephil.charting.utils.ColorTemplate; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -15,7 +15,7 @@ public class GraphAdapter { - private final LineChartView chart; + private final LineChart chart; private List timestamps = new ArrayList<>(); private Map> lines = new HashMap<>(); private List keyOrder = new ArrayList<>(); @@ -23,9 +23,9 @@ public class GraphAdapter { private int len; private float max = 10.f; - public GraphAdapter(int len, LineChartView chart) { + public GraphAdapter(int len, View customView) { this.len = len; - this.chart = chart; + this.chart = customView.findViewById(R.id.chart); } private int checkTimestamp(Long ts) { @@ -38,22 +38,11 @@ private int checkTimestamp(Long ts) { return timestamps.indexOf(ts) + 1; } - private void resetViewport() { - Viewport v = new Viewport(chart.getMaximumViewport()); - v.bottom = 0; - v.top = (int) max; - v.left = 0; - v.right = this.len - 1; - chart.setMaximumViewport(v); - chart.setCurrentViewport(v); - } - public void addValue(Long ts, String name, Float value) { int position = checkTimestamp(ts); if (!lines.containsKey(name)) { lines.put(name, new ArrayList<>()); keyOrder.add(name); - color.add(ChartUtils.pickColor()); } List list = lines.get(name); list.add(value); @@ -74,31 +63,33 @@ public void addValue(Long ts, String name, boolean value) { } public void refresh() { - List _lines = new ArrayList<>(); + // List _lines = new ArrayList<>(); + LineData data = new LineData(); for (int j = 0; j < keyOrder.size(); ++j) { String name = keyOrder.get(j); List elems = lines.get(name); - List values = new ArrayList<>(); + List values = new ArrayList<>(); for (int i = 0; i < elems.size(); ++i) { - values.add(new PointValue(i, elems.get(i))); + values.add(new Entry(i, elems.get(i))); } - Line line = new Line(values); - line.setColor(color.get(j)); - line.setHasPoints(false); - _lines.add(line); + LineDataSet line = new LineDataSet(values, name); + if (j > 4) { + line.setColor(ColorTemplate.LIBERTY_COLORS[j - 5]); + } else { + line.setColor(ColorTemplate.JOYFUL_COLORS[j]); + } + line.setDrawCircles(false); + line.setLineWidth(2.f); + data.addDataSet(line); } - LineChartData data = new LineChartData(_lines); - - Axis axisX = new Axis(); - Axis axisY = new Axis().setHasLines(true); - axisX.setName("Time"); - axisY.setName("Values"); + chart.setData(data); - data.setAxisXBottom(axisX); - data.setAxisYLeft(axisY); + YAxis yaxis = chart.getAxisLeft(); + chart.getAxisRight().setEnabled(false); + yaxis.setAxisMaximum(10); + yaxis.setAxisMinimum(0); - data.setBaseValue(Float.NEGATIVE_INFINITY); - chart.setLineChartData(data); - resetViewport(); + chart.invalidate(); + // resetViewport(); } } diff --git a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java index 929bfdbe..d9e10118 100644 --- a/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java +++ b/code/mobile/android/PhoneVR/app/src/main/java/viritualisres/phonevr/PassthroughSettingsActivity.java @@ -16,7 +16,6 @@ import androidx.preference.PreferenceScreen; import androidx.preference.PreferenceViewHolder; import androidx.recyclerview.widget.RecyclerView; -import ir.mahdiparastesh.hellocharts.view.LineChartView; public class PassthroughSettingsActivity extends AppCompatActivity { @@ -43,7 +42,6 @@ protected void onCreate(Bundle savedInstanceState) { } public static class SettingsFragment extends PreferenceFragmentCompat { - private LineChartView chart = null; private Passthrough passthrough; private GraphAdapter graphAdapter; @@ -68,8 +66,7 @@ public PreferenceViewHolder onCreateViewHolder( PreferenceViewHolder holder = super.onCreateViewHolder(parent, viewType); View customLayout = holder.itemView; if (customLayout.getId() == R.id.chartLayout) { - chart = customLayout.findViewById(R.id.chart); - graphAdapter = new GraphAdapter(100, chart); + graphAdapter = new GraphAdapter(100, customLayout); passthrough = new Passthrough(sharedPref, sensorManager, 640, 480); passthrough.onResume(graphAdapter); } diff --git a/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml b/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml index fced2e18..6a180765 100644 --- a/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml +++ b/code/mobile/android/PhoneVR/app/src/main/res/layout/chart_layout.xml @@ -4,9 +4,9 @@ android:layout_width="match_parent" android:layout_height="match_parent"> - + android:layout_height="match_parent" /> \ No newline at end of file From 1b016775532f3155d9ae305a72fe69638917363a Mon Sep 17 00:00:00 2001 From: Dominik Wetzel Date: Wed, 8 May 2024 09:36:49 +0200 Subject: [PATCH 6/8] Make format --- .../PhoneVR/app/src/main/cpp/alvr_main.cpp | 4 +-- .../PhoneVR/app/src/main/cpp/passthrough.cpp | 26 +++++++++---------- .../android/PhoneVR/app/src/main/cpp/utils.h | 10 +++---- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/alvr_main.cpp b/code/mobile/android/PhoneVR/app/src/main/cpp/alvr_main.cpp index 548a8dd3..c57c6f95 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/alvr_main.cpp +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/alvr_main.cpp @@ -11,8 +11,8 @@ #include #include "nlohmann/json.hpp" -#include "utils.h" #include "passthrough.h" +#include "utils.h" using namespace nlohmann; @@ -341,7 +341,7 @@ extern "C" JNIEXPORT void JNICALL Java_viritualisres_phonevr_ALVRActivity_render CTX.glContextRecreated); GL(glGenTextures(2, CTX.lobbyTextures)); - for (auto &lobbyTexture: CTX.lobbyTextures) { + for (auto &lobbyTexture : CTX.lobbyTextures) { GL(glBindTexture(GL_TEXTURE_2D, lobbyTexture)); GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)); GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp index db558953..7a327700 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp @@ -3,8 +3,8 @@ #include #include -#include "utils.h" #include "passthrough.h" +#include "utils.h" GLuint LoadGLShader(GLenum type, const char *shader_source) { GLuint shader = GL(glCreateShader(type)); @@ -134,14 +134,14 @@ void passthrough_setup(PassthroughInfo *info) { GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)); GL(glTexImage2D(GL_TEXTURE_2D, - 0, - GL_RGB, - *(info->screenWidth), - *(info->screenHeight), - 0, - GL_RGB, - GL_UNSIGNED_BYTE, - 0)); + 0, + GL_RGB, + *(info->screenWidth), + *(info->screenHeight), + 0, + GL_RGB, + GL_UNSIGNED_BYTE, + 0)); // Generate depth buffer to perform depth test. GL(glGenRenderbuffers(1, &(info->passthroughDepthRenderBuffer))); @@ -171,9 +171,9 @@ void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription vi // Draw Passthrough video for each eye for (int eye = 0; eye < 2; ++eye) { GL(glViewport(eye == kLeft ? 0 : *(info->screenWidth) / 2, - 0, - *(info->screenWidth) / 2, - *(info->screenHeight))); + 0, + *(info->screenWidth) / 2, + *(info->screenHeight))); GL(glUseProgram(passthroughProgram_)); GL(glActiveTexture(GL_TEXTURE0)); @@ -182,7 +182,7 @@ void passthrough_render(PassthroughInfo *info, CardboardEyeTextureDescription vi // Draw Mesh GL(glEnableVertexAttribArray(texturePositionParam_)); GL(glVertexAttribPointer( - texturePositionParam_, 2, GL_FLOAT, false, 0, info->passthroughVertices)); + texturePositionParam_, 2, GL_FLOAT, false, 0, info->passthroughVertices)); GL(glEnableVertexAttribArray(textureUvParam_)); GL(glVertexAttribPointer(textureUvParam_, 2, GL_FLOAT, false, 0, passthroughTexCoords)); diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h b/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h index 5bc99d00..7362b876 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/utils.h @@ -13,11 +13,11 @@ const char LOG_TAG[] = "ALVR_PVR_NATIVE"; static void log(AlvrLogLevel level, - const char *FILE_NAME, - unsigned int LINE_NO, - const char *FUNC, - const char *format, - ...) { + const char *FILE_NAME, + unsigned int LINE_NO, + const char *FUNC, + const char *format, + ...) { va_list args; va_start(args, format); From 88ae3c32dcdd7631bc22e851add859df4f654559 Mon Sep 17 00:00:00 2001 From: Harsha Raghu Date: Wed, 8 May 2024 13:37:06 +0530 Subject: [PATCH 7/8] lint fix --- code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp index 7a327700..e1ab9478 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp @@ -1,6 +1,6 @@ #include "alvr_client_core.h" -#include #include +#include #include #include "passthrough.h" From 16e98a285b3d71af8e16bb8f525633ea42ea0239 Mon Sep 17 00:00:00 2001 From: Harsha Raghu Date: Wed, 8 May 2024 14:27:18 +0530 Subject: [PATCH 8/8] ignore lint for gl2ext This reverts commit 88ae3c32dcdd7631bc22e851add859df4f654559. --- code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp index e1ab9478..57c55f55 100644 --- a/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp +++ b/code/mobile/android/PhoneVR/app/src/main/cpp/passthrough.cpp @@ -1,6 +1,8 @@ #include "alvr_client_core.h" -#include +// clang-format off #include +#include +// clang-format on #include #include "passthrough.h"