Expo app

frontend/companion-app/App.tsx

@@ -1,6 +1,6 @@
 // Import necessary modules and components from the Expo and React Native libraries
 import { StatusBar } from "expo-status-bar";
-import { StyleSheet, Text, View, Image, SafeAreaView, TouchableOpacity, useWindowDimensions, ScrollView, TextStyle, Animated } from "react-native";
+import { StyleSheet, Text, View, Image, SafeAreaView, TouchableOpacity, useWindowDimensions, ScrollView, TextStyle, Animated, Platform } from "react-native";
 import React, { useEffect, useReducer, useRef, useState } from "react";
 import { startSession, synchronize } from "./components/Utils";
 import { Score_Select } from "./components/ScoreSelect";
@@ -17,6 +17,10 @@ import { SynthesizeButton } from "./components/SynthesizeButton";
 import Icon from 'react-native-vector-icons/Feather';
 import { ChromaMaker } from "./utils/features";
 import FontAwesome from 'react-native-vector-icons/FontAwesome';
+<<<<<<< HEAD
+import { ExpoMicProcessor } from './utils/ExpoMicProcessor';
+=======
+>>>>>>> main
 
 // Define the main application component
 export default function App() {
@@ -49,19 +53,103 @@ export default function App() {
     },
   );
 
-  // State used to store session token
-  const [sessionToken, setSessionToken] = useState<string>("")
+ const [chroma, setChroma] = useState<number[]>(new Array(12).fill(0)); // Initialize the chroma state as an array of 12 zeros (used to capture chroma vector at each chunk of audio).
+ const [started, setStarted] = useState(false); // state used to determine user selects live microphone option or not
+ const processor = useRef(new ExpoMicProcessor()).current; // Create a stable ExpoMicProcessor instance that persists across renders
+ const SAMPLE_RATE = 44100;  // Define sample rate for ChromaMaker
+ const N_FFT = 4096; // Define chunk size for ChromaMaker
+ const chromaMaker = useRef(new ChromaMaker(SAMPLE_RATE, N_FFT)).current; // Create a stable ChromaMaker instance that persists across renders
 
-  // Function used to generate session token using crypto API 
-  const generateSecureSessionToken = (): string => {
-    return window.crypto.randomUUID();
-  };
+  // Create an array of Animated.Value objects for a smooth height animation of each chroma bar.
+  // const animatedChroma = useRef(new Array(12).fill(0).map(() => new Animated.Value(0))).current;
+
+  // // Whenever the chroma state updates, animate each corresponding Animated.Value.
+  // useEffect(() => {
+  //   chroma.forEach((value, idx) => {
+  //     Animated.timing(animatedChroma[idx], {
+  //       toValue: value * 200, // scale factor to adjust maximum bar height
+  //       duration: 50,
+  //       useNativeDriver: false
+  //     }).start();
+  //   });
+  // }, [chroma]);
 
-  // On load, call generateSecureSessionToken function to generate and store new session token
-  useEffect(() => {
-    const newToken: string = generateSecureSessionToken();
-    setSessionToken(newToken)
-  }, []);
+ useEffect(() => {
+   let audioCtx: AudioContext; // Declare a reference to the AudioContext, which manages all audio processing
+   let micStream: MediaStream; // Declare a reference to the MediaStream from the user's microphone
+
+   // Web version of intializing miccrophone (Uses AudioWorklet Node)
+   const initWebAudio = async () => {
+     try {
+       micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); // Request access to user's microphone
+       audioCtx = new AudioContext(); // Create a new AudioContext for audio processing
+       await audioCtx.audioWorklet.addModule('./utils/mic-processor.js'); // Load the custom AudioWorkletProcessor
+       const source = audioCtx.createMediaStreamSource(micStream); // Create a source node from the microphone stream
+       const workletNode = new AudioWorkletNode(audioCtx, 'mic-processor');  // Create an AudioWorkletNode linked to our custom 'mic-processor'
+       source.connect(workletNode); // Connect the mic source to the worklet
+       workletNode.connect(audioCtx.destination); // connect worklet to output 
+
+      // Initialize the ChromaMaker for extracting chroma features
+       const n_fft = 4096;
+       const chromaMaker = new ChromaMaker(audioCtx.sampleRate, n_fft); 
+
+      // Handle incoming audio chunks from the worklet
+       workletNode.port.onmessage = (event) => {
+         const audioChunk = event.data as Float32Array;
+         try {
+            // Extract chroma features and update state
+           const chromaResult = chromaMaker.insert(audioChunk);
+           setChroma(chromaResult);
+         } catch (e) {
+           console.error('Chroma extraction error:', e);
+         }
+       };
+     } catch (err) {
+       console.error('Failed to initialize audio:', err);
+     }
+   };
+
+   // Mobile version of intializing miccrophone (Uses ExpoMicProcessor)
+   const initNativeAudio = async () => {
+    try {
+      await processor.init(); // ExpoMicProcessor intialization
+
+      processor.onmessage = ({ data }) => { // Once we get buffer of size 4096
+        const vec = chromaMaker.insert(data);  // Insert with ChromaMaker to get chroma vector
+        setChroma(vec); // Set chroma vector
+      };
+
+      await processor.start(); // Start recording 
+    } catch (err) {
+      console.error('Failed to initialize Native audio:', err);
+    }
+  };
+
+   // If "started" state is true, initialize audio processing based on platform
+   if (started) {
+    if (Platform.OS === 'web') {
+      initWebAudio();  // Use browser audio processor 
+    } else {
+      initNativeAudio(); // Use native Expo/React Native audio processor
+    }
+  }
+
+   // Cleanup: when the component unmounts or `started` becomes false, 
+   // stop the microphone stream and close the audio context to free up resources
+   return () => {
+
+    // Web version of microphone stop
+    if (Platform.OS === 'web') {
+      if (micStream) micStream.getTracks().forEach((track) => track.stop());
+      if (audioCtx) audioCtx.close();
+
+    // Mobile version of microphone stop
+    } else {
+      processor.stop();
+    }
+  };
+
+ }, [started]);
 
  // Initialize the chroma state as an array of 12 zeros (used to capture chroma vector at each chunk of audio).
  const [chroma, setChroma] = useState<number[]>(new Array(12).fill(0));
@@ -236,6 +324,7 @@ export default function App() {
   const backgroundColorAnim = useRef(new Animated.Value(0)).current; 
   const textColorAnim = useRef(new Animated.Value(0)).current; 
   const borderBottomAnim = useRef(new Animated.Value(0)).current;
+
   // const borderColorAnim = useRef(new Animated.Value(0)).current;
 
   // Interpolate background color based on light or dark mode
@@ -268,17 +357,14 @@ export default function App() {
     inputRange: [0, 1],
     outputRange: ["#2C3E50", "#FFFFFF"], // Light to dark
   });
-  // Interpolate header and footer container color based on light or dark mode
-  const menubarBackgroundColor =  backgroundColorAnim.interpolate({
-    inputRange: [0, 1],
-    outputRange: ["#2C3E50", "#1A252F"], // Light to dark
-  });
   // Interpolate border bottom color based on light or dark mode
   const borderBottomColor = borderBottomAnim.interpolate({
     inputRange: [0, 1],
     outputRange: ["#2C3E50", "#FFFFFF"], // Light to dark transition
   });
 
+
+
   // Toggles between light and dark mode by animating background, text, and border properties smoothly
   const toggleTheme = () => {
     const toValue = theme === "light" ? 1 : 0;
@@ -303,14 +389,45 @@ export default function App() {
     });
   };
   // Get device's width 
-  const { width } = useWindowDimensions()
+  const { width, height } = useWindowDimensions()
   // Boolean used for dynmaic display (row or column)
   const isSmallScreen = width < 960;
+<<<<<<< HEAD
+  // const noteLabels = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'];
+=======
+>>>>>>> main
 
   ////////////////////////////////////////////////////////////////////////////////
   // Render the component's UI
   ////////////////////////////////////////////////////////////////////////////////
   return (
+<<<<<<< HEAD
+
+    // BG Color for iphone padding - no padding if on landscape mode (top and bottom)
+    <SafeAreaView style={[styles.container, {backgroundColor: width < height? '#2C3E50': ""}]} >
+
+      {/* Account for top padding on Iphone */}
+      <SafeAreaView > 
+        {/* Header with image */}
+        <Animated.View style={[styles.menu_bar, {backgroundColor: '#2C3E50', height: isSmallScreen? 40: 80}, { position: 'relative', top: 0 }]}>
+          <Image source={require('./assets/companion.png')} style={[styles.logo, {height: isSmallScreen? 30: 100, width: isSmallScreen? 100: 200}]}/>
+          <View style={{ flexDirection: 'row', alignItems: 'center', gap: 10 }}>
+            <TouchableOpacity onPress={() => setStarted(!started)}>
+              <FontAwesome
+                name={started ? 'microphone' : 'microphone-slash'}
+                size={isSmallScreen ? 15 : 30}
+                color="white"
+              />
+            </TouchableOpacity>
+            <TouchableOpacity onPress={toggleTheme}>
+              <Icon name={theme === 'light' ? 'sun' : 'moon'} size={isSmallScreen? 15: 30} color="white" />
+            </TouchableOpacity>
+          </View>
+        </Animated.View>
+      </SafeAreaView>
+
+
+=======
     <SafeAreaView style={[styles.container]}>
       {/* Header with image */}
       <Animated.View style={[styles.menu_bar, {backgroundColor: menubarBackgroundColor, height: isSmallScreen? 40: 80}]}>
@@ -330,6 +447,7 @@ export default function App() {
         </View>
 
       </Animated.View>
+>>>>>>> main
 
         {/* Provides safe area insets for mobile devices */}
         <Animated.View style={[styles.container, { backgroundColor: containerBackgroundColor }]}>
@@ -344,7 +462,7 @@ export default function App() {
             {/* Sidebar for inputs and buttons (takes up little width) */}
             <Animated.View style={[styles.sidebar, { backgroundColor: sidebarBackgroundColor }, isSmallScreen ? styles.sidebarColumn : {}]}>
               { // List of scores, show when not in play mode
-              state.inPlayMode || <Score_Select state={state} dispatch={dispatch} textStyle={textColor} borderStyle={borderBottomColor}/> }
+              state.inPlayMode || <Score_Select state={state} dispatch={dispatch} textStyle={textColor} borderStyle={borderBottomColor} button_text_style={invertTextColor} button_format={[styles.button, {backgroundColor: buttonBackgroundColor}]}/> }
               <Return_Button
                 state={state}
                 dispatch={dispatch}
@@ -378,22 +496,47 @@ export default function App() {
 
             {/* Scroll View used for horizontal scolling */}
             <ScrollView
-              horizontal={true} 
+              horizontal={false} 
               showsHorizontalScrollIndicator={false} 
               contentContainerStyle={{ flexGrow: 1 }} // Ensure the content fills the container
             >
               {/* Actual content display (takes up remaining width after sidebar) */}
               <Animated.View style={[styles.mainContent, {backgroundColor: mainContentBackgroundColor}, isSmallScreen ? styles.mainContentColumn : {}]}>
-                <ScoreDisplay state={state} dispatch={dispatch} />
+                <ScoreDisplay state={state} dispatch={dispatch}/>
               </Animated.View>
             </ScrollView>
           </View>
+<<<<<<< HEAD
+
+          {/* Chroma Vector Display - commented out just in case we need to check if mic works */}
+          {/* <View style={chromaStyles.container}>
+            <Text style={chromaStyles.infoText}>Live Chroma Visualization</Text>
+            <View style={chromaStyles.chromaContainer}>
+              {chroma.map((value, idx) => (
+                <View key={idx} style={chromaStyles.chromaBarContainer}>
+                  <Animated.View style={[chromaStyles.chromaBar, { height: animatedChroma[idx] }]} />
+                  <Text style={chromaStyles.chromaLabel}>{noteLabels[idx]}</Text>
+                </View>
+              ))}
+            </View>
+
+          </View> */}
+=======
+>>>>>>> main
           {/* Footer display for status */}
           <StatusBar style="auto" />
           {/* Automatically adjusts the status bar style */}
         </ScrollView>
       </Animated.View>
+<<<<<<< HEAD
+
+      {/* Account for bottom padding on Iphone */}
+      {/* <SafeAreaView>
+        <AudioPlayer state={state}  menuStyle={{ backgroundColor: '#2C3E50' }}/>
+      </SafeAreaView> */}
+=======
       <AudioPlayer state={state}  menuStyle={{ backgroundColor: menubarBackgroundColor }}/>
+>>>>>>> main
     </SafeAreaView>
   );
 }
@@ -432,7 +575,7 @@ const styles = StyleSheet.create({
     gap: 10,
     flex: 1,
     padding: 20,
-    marginTop: 80 // account for fixed header
+    marginTop: 10 // account for fixed header
   },
   // Container displaying sidebar and main content (row form)
   contentWrapperRow: {
@@ -503,4 +646,67 @@ const styles = StyleSheet.create({
     color: "#FFFFFF",
     fontWeight: "bold",
   },
-});
+});
+
+// const chromaStyles = StyleSheet.create({
+//   container: {
+//     flex: 1,
+//     alignItems: 'center',
+//     justifyContent: 'center',
+//     backgroundColor: '#FAFAFA', // Slight off-white for a subtle card look
+//     marginVertical: 20,
+//     padding: 15,
+//     borderRadius: 10,
+//     shadowColor: "#000",
+//     shadowOffset: { width: 0, height: 2 },
+//     shadowOpacity: 0.1,
+//     shadowRadius: 3,
+//     elevation: 3,
+//   },
+//   startButton: {
+//     backgroundColor: '#2196F3',
+//     paddingVertical: 15,
+//     paddingHorizontal: 25,
+//     borderRadius: 10,
+//     marginBottom: 15, // Added margin for better separation
+//   },
+//   startButtonText: {
+//     color: '#fff',
+//     fontSize: 18,
+//     fontWeight: '600',
+//   },
+//   infoText: {
+//     marginTop: 10,
+//     fontSize: 16,
+//     fontWeight: '500',
+//     color: '#333', // Dark text for clear contrast
+//   },
+//   chromaContainer: {
+//     flexDirection: 'row',
+//     alignItems: 'flex-end',
+//     height: 220, // Slightly taller for clarity
+//     width: '95%',
+//     backgroundColor: '#F0F0F0', // Differentiated background color
+//     borderWidth: 1,
+//     borderColor: '#ccc',
+//     borderRadius: 10,
+//     marginTop: 20,
+//     padding: 10,
+//   },
+//   chromaBarContainer: {
+//     flex: 1,
+//     alignItems: 'center',
+//     marginHorizontal: 4,
+//   },
+//   chromaBar: {
+//     width: '100%',
+//     backgroundColor: '#2196F3',
+//     borderRadius: 4, // Soften edges of chroma bars
+//   },
+//   chromaLabel: {
+//     marginTop: 4,
+//     fontSize: 12,
+//     color: '#555',
+//     fontWeight: '500',
+//   },
+// });

frontend/companion-app/app.json

@@ -3,7 +3,7 @@
     "name": "companion-app",
     "slug": "companion-app",
     "version": "1.0.0",
-    "orientation": "portrait",
+    "orientation": "default",
     "icon": "./assets/icon.png",
     "userInterfaceStyle": "light",
     "splash": {