diff --git a/frontend/companion-app/App.tsx b/frontend/companion-app/App.tsx index 2e48094..983a409 100644 --- a/frontend/companion-app/App.tsx +++ b/frontend/companion-app/App.tsx @@ -1,14 +1,15 @@ // Import necessary modules and components from the Expo and React Native libraries import { StatusBar } from "expo-status-bar"; -import { StyleSheet, Text, View, SafeAreaView } from "react-native"; -import React, { useEffect, useReducer } from "react"; +import { StyleSheet, Text, View, Image, SafeAreaView } from "react-native"; +import React, { useEffect, useReducer, useRef, useState } from "react"; import { startSession, synchronize } from "./components/Utils"; import { Score_Select } from "./components/ScoreSelect"; +import { Return_Button } from "./components/ReturnButton"; import { Start_Stop_Button } from "./components/StartButton"; import { MeasureSetBox } from "./components/MeasureSetter"; import { TempoBox } from "./components/TempoBox"; import { Fraction } from "opensheetmusicdisplay"; -import AudioRecorder from "./components/AudioRecorder"; +import AudioRecorder from "./components/AudioRecorderStateVersion"; import { AudioPlayer } from "./components/AudioPlayer"; import reducer_function from "./Dispatch"; import ScoreDisplay from "./components/ScoreDisplay"; @@ -28,6 +29,7 @@ export default function App() { const [state, dispatch] = useReducer( reducer_function, // The reducer function is found in Dispatch.ts { + inPlayMode: false, // whether we are in play mode (and not score selection mode) playing: false, // whether the audio is playing resetMeasure: 1, // the measure to reset to playRate: 1.0, // the rate at which the audio is playing @@ -40,7 +42,7 @@ export default function App() { synth_tempo: 100, // the tempo of the synthesized audio tempo: 100, // the tempo in the tempo box (even if changed more recently) score_tempo: 100, // the tempo in the musical score - scores: [], // the list of scores to choose from + scores: [] // the list of scores to choose from }, ); @@ -61,41 +63,121 @@ export default function App() { fetchSessionToken(); }, []); - console.log(state); + //////////////////////////////////////////////////////////////////////////////// + // The lines below were modified, copied and pasted out of the audio recorder object + // (which never really needed a UI). + // *** THE ACT OF MOVING THEM FROM A COMPONENT TO APP.TSX MADE THE INTERFACE WORK *** + // *** Probably has to do with parent/child component state something idk *** + //////////////////////////////////////////////////////////////////////////////// + // Audio-related states and refs + // State for whether we have microphone permissions - is set to true on first trip to playmode + const [permission, setPermission] = useState(false); + // Assorted audio-related objects in need of reference + // Tend to be re-created upon starting a recording + const mediaRecorder = useRef( + new MediaRecorder(new MediaStream()), + ); + const [stream, setStream] = useState(new MediaStream()); + const [audioChunks, setAudioChunks] = useState([]); + + const audioContextRef = useRef(null); + const analyserRef = useRef(null); + const dataArrayRef = useRef(null); + const startTimeRef = useRef(null); + + // Audio-related functions ///////////////////////////////////////////////////////// - // The code below updates the timestamp but is not yet tied to the API - // This could be moved to any sub-component (e.g., ScoreDisplay, AudioPlayer) - // or made its own invisible component - OR, - // we will re-synchronize whenever the audiorecorder posts, in which case this should - // be handled there - const UPDATE_INTERVAL = 500; // milliseconds between updates to timestamp and rate + // This function sends a synchronization request and updates the state with the result + const UPDATE_INTERVAL = 100; - useEffect(() => { - const getAPIData = async () => { - const { - playback_rate: newPlayRate, - estimated_position: estimated_position, - } = await synchronize(state.sessionToken, [0], state.timestamp); - console.log("New play rate:", newPlayRate); - console.log("New timestamp:", estimated_position); + const getAPIData = async () => { + analyserRef.current?.getByteTimeDomainData(dataArrayRef.current); + const { + playback_rate: newPlayRate, + estimated_position: estimated_position, + } = await synchronize(state.sessionToken, Array.from(dataArrayRef.current), state.timestamp); - dispatch({ - type: "increment", - time: estimated_position, - rate: 1, - }); + dispatch({ + type: "increment", + time: estimated_position, + rate: newPlayRate, + }); + } + + // This function established new recording instances when re-entering play mode + const startRecording = async () => { + // It's possible some of these can be removed; not sure which relate to the + // making of the recorded object we don't need and which relate to the + // buffer we send to the backend. + startTimeRef.current = Date.now(); + //create new Media recorder instance using the stream + const media = new MediaRecorder(stream, { mimeType: "audio/webm" }); + //set the MediaRecorder instance to the mediaRecorder ref + mediaRecorder.current = media; + //invokes the start method to start the recording process + mediaRecorder.current.start(); + let localAudioChunks: Blob[] = []; + mediaRecorder.current.ondataavailable = (event) => { + if (typeof event.data === "undefined") return; + if (event.data.size === 0) return; + localAudioChunks.push(event.data); }; + setAudioChunks(localAudioChunks); - // Start polling - setInterval(() => { - if (state.playing) { - getAPIData(); + audioContextRef.current = new window.AudioContext(); + const source = audioContextRef.current.createMediaStreamSource(stream); + analyserRef.current = audioContextRef.current.createAnalyser(); + analyserRef.current.fftSize = 2048; + source.connect(analyserRef.current); + + const bufferLength = analyserRef.current.frequencyBinCount; + dataArrayRef.current = new Uint8Array(bufferLength); + + getAPIData(); // run the first call + }; + + //stops the recording instance + const stopRecording = () => { + mediaRecorder.current.stop(); + audioContextRef.current?.close(); + }; + + // Function to get permission to use browser microphone + const getMicrophonePermission = async () => { + if ("MediaRecorder" in window) { + try { + const streamData = await navigator.mediaDevices.getUserMedia({ + audio: true, + video: false, + }); + setPermission(true); + setStream(streamData); + } catch (err) { + alert((err as Error).message); } - }, UPDATE_INTERVAL); - }, [state.playing, state.timestamp, state.sessionToken]); - // The "could be moved into any subcomponent" comment refers to the above - /////////////////////////////////////////////////////////////////////////////// + } else { + alert("The MediaRecorder API is not supported in your browser."); + } + }; + + ///////////////////////////////////////////// + // Audio-related effects + // Get microphone permission on first time entering play state + useEffect(() => { + if (!permission) getMicrophonePermission(); + }, [state.inPlayMode]); + + // Start and stop recording when player is or isn't playing + useEffect(() => { + if (state.playing) startRecording(); + else stopRecording(); + }, [state.playing]); + + // Keep synchronizing while playing + useEffect(() => { + if (state.playing) setTimeout(getAPIData, UPDATE_INTERVAL); + }, [state.timestamp]) //////////////////////////////////////////////////////////////////////////////// // Render the component's UI @@ -103,41 +185,41 @@ export default function App() { return ( {/* Provides safe area insets for mobile devices */} - - Companion, the digital accompanist - - - - + + - - + { + state.inPlayMode ? + + : + + } + + + { // List of scores, show when not in play mode + state.inPlayMode || } + - {/* Automatically adjusts the status bar style */} @@ -154,65 +236,37 @@ const styles = StyleSheet.create({ justifyContent: "center", // Center children vertically padding: 16, // Add padding around the container }, - title: { - fontSize: 20, // Set the font size for the title - marginBottom: 20, // Add space below the title - }, - label: { - fontSize: 16, - }, - synthesize_button: { - flex: 0.2, - borderColor: "black", - borderRadius: 15, - backgroundColor: "lightblue", - justifyContent: "center", - }, - play_button: { - flex: 0.2, - borderColor: "black", - borderRadius: 15, - backgroundColor: "lightblue", - justifyContent: "center", - }, - reset_button: { - flex: 0.8, + button_format: { borderColor: "black", borderRadius: 15, backgroundColor: "lightblue", + justifyContent: "center" }, button_text: { - fontSize: 20, + fontSize: 24, textAlign: "center", }, - button_wrapper: { - flex: 1, + menu_bar: { + flex: 0, flexDirection: "row", justifyContent: "space-between", - padding: 10, backgroundColor: "lightgray", width: "100%", - minHeight: 72, + minHeight: 100, }, - measure_box: { - flexDirection: "row", - padding: 10, - justifyContent: "space-between", - width: "40%", - flex: 0.4, - height: "80%", - }, - tempo_box: { + main_area: { + flex: 1, flexDirection: "row", - padding: 10, justifyContent: "space-between", - width: "20%", - flex: 0.2, + backgroundColor: "white", + width: "100%", height: "80%", }, - text_input: { + logo: { backgroundColor: "white", - flex: 0.3, + flex: 0.25, + width: "25%", height: "100%", + resizeMode: 'contain' }, }); diff --git a/frontend/companion-app/Dispatch.ts b/frontend/companion-app/Dispatch.ts index 910afb5..80c9385 100644 --- a/frontend/companion-app/Dispatch.ts +++ b/frontend/companion-app/Dispatch.ts @@ -31,6 +31,14 @@ const reducer_function = (state: any, action: any) => { timestamp: action.time as number, }, }; + + case "swap_mode": + return { + ...state, + ...{ + inPlayMode: !state.inPlayMode + } + } // When resetting, move the cursor, then adjust the timestamp accordingly and reset the playback rate case "reset": @@ -38,7 +46,7 @@ const reducer_function = (state: any, action: any) => { var reset_time = (60 * state.time_signature.Numerator * (state.resetMeasure - 1)) / state.synth_tempo; - state.sound?.setPositionAsync(reset_time * 1000); + state.accompanimentSound.setPositionAsync(reset_time * 1000); return { ...state, ...{ playing: false, playRate: 1.0, timestamp: reset_time }, diff --git a/frontend/companion-app/assets/companion.png b/frontend/companion-app/assets/companion.png new file mode 100644 index 0000000..b8cba01 Binary files /dev/null and b/frontend/companion-app/assets/companion.png differ diff --git a/frontend/companion-app/components/AudioPlayer.tsx b/frontend/companion-app/components/AudioPlayer.tsx index 6d22c17..d37b425 100644 --- a/frontend/companion-app/components/AudioPlayer.tsx +++ b/frontend/companion-app/components/AudioPlayer.tsx @@ -26,8 +26,6 @@ export function AudioPlayer({ useEffect(() => { const updateWhetherPlaying = async () => { - console.log("Playing:", state.playing); - console.log("Sound:", state.accompanimentSound); if (state.accompanimentSound) { if (state.playing) { await state.accompanimentSound.playAsync(); diff --git a/frontend/companion-app/components/AudioRecorderStateVersion.tsx b/frontend/companion-app/components/AudioRecorderStateVersion.tsx new file mode 100644 index 0000000..92aa449 --- /dev/null +++ b/frontend/companion-app/components/AudioRecorderStateVersion.tsx @@ -0,0 +1,125 @@ +import { useState, useRef, useEffect, Ref } from "react"; +import { synchronize } from "./Utils"; + +const mimeType = "audio/webm"; + +const AudioRecorder = ({ + state, + dispatch +}: { + state: { playing: boolean, inPlayMode: boolean, timestamp: number, sessionToken: string }; + dispatch: Function +}) => { + /////////////////////////////////// + // States and references + /////////////////////////////////// + // State for whether we have microphone permissions - is set to true on first trip to playmode + const [permission, setPermission] = useState(false); + // Assorted audio-related objects in need of reference + // Tend to be re-created upon starting a recording + const mediaRecorder = useRef( + new MediaRecorder(new MediaStream()), + ); + const [stream, setStream] = useState(new MediaStream()); + const [audioChunks, setAudioChunks] = useState([]); + + const audioContextRef = useRef(null); + const analyserRef = useRef(null); + const dataArrayRef = useRef(null); + const startTimeRef = useRef(null); + + ///////////////////////////////////////////////////////// + // This function sends a synchronization request and updates the state with the result + const UPDATE_INTERVAL = 100; + + const getAPIData = async () => { + analyserRef.current?.getByteTimeDomainData(dataArrayRef.current); + const { + playback_rate: newPlayRate, + estimated_position: estimated_position, + } = await synchronize(state.sessionToken, Array.from(dataArrayRef.current), state.timestamp); + + dispatch({ + type: "increment", + time: estimated_position, + rate: newPlayRate, + }); + } + + // Starts recorder instances + const startRecording = async () => { + startTimeRef.current = Date.now(); + //create new Media recorder instance using the stream + const media = new MediaRecorder(stream, { mimeType: mimeType }); + //set the MediaRecorder instance to the mediaRecorder ref + mediaRecorder.current = media; + //invokes the start method to start the recording process + mediaRecorder.current.start(); + let localAudioChunks: Blob[] = []; + mediaRecorder.current.ondataavailable = (event) => { + if (typeof event.data === "undefined") return; + if (event.data.size === 0) return; + localAudioChunks.push(event.data); + }; + setAudioChunks(localAudioChunks); + + audioContextRef.current = new window.AudioContext(); + const source = audioContextRef.current.createMediaStreamSource(stream); + analyserRef.current = audioContextRef.current.createAnalyser(); + analyserRef.current.fftSize = 2048; + source.connect(analyserRef.current); + + const bufferLength = analyserRef.current.frequencyBinCount; + dataArrayRef.current = new Uint8Array(bufferLength); + + getAPIData(); // run the first call + }; + + //stops the recording instance + const stopRecording = () => { + mediaRecorder.current.stop(); + audioContextRef.current?.close(); + }; + + // Function to get permission to use browser microphone + const getMicrophonePermission = async () => { + if ("MediaRecorder" in window) { + try { + const streamData = await navigator.mediaDevices.getUserMedia({ + audio: true, + video: false, + }); + setPermission(true); + setStream(streamData); + } catch (err) { + alert((err as Error).message); + } + } else { + alert("The MediaRecorder API is not supported in your browser."); + } + }; + + // Get microphone permission on first time entering play state + useEffect(() => { + if (!permission) getMicrophonePermission(); + }, [state.inPlayMode]); + + // Start and stop recording when player is or isn't playing + useEffect(() => { + if (state.playing) startRecording(); + else stopRecording(); + }, [state.playing]); + + useEffect(() => { + if (state.playing) setTimeout(getAPIData, UPDATE_INTERVAL); + }, [state.timestamp]) + + return ( +
+

Audio Recorder

+
+ ); +}; +export default AudioRecorder; diff --git a/frontend/companion-app/components/MeasureSetter.tsx b/frontend/companion-app/components/MeasureSetter.tsx index 1eae242..5702d2e 100644 --- a/frontend/companion-app/components/MeasureSetter.tsx +++ b/frontend/companion-app/components/MeasureSetter.tsx @@ -1,26 +1,19 @@ -import { View, Text } from "react-native"; +import { StyleSheet, View, Text } from "react-native"; import { Pressable, TextInput, TextStyle, ViewStyle } from "react-native"; export function MeasureSetBox({ state, dispatch, - wrapper_style, button_style, - text_input_style, - button_text_style, - label_text_style, + button_text_style }: { state: { resetMeasure: number }; dispatch: Function; - wrapper_style: ViewStyle; button_style: ViewStyle; - text_input_style: ViewStyle; button_text_style: TextStyle; - label_text_style: TextStyle; }) { return ( - - Start measure: + dispatch({ type: "change_reset", measure: text as unknown as number }) @@ -28,10 +21,10 @@ export function MeasureSetBox({ value={String(state.resetMeasure)} placeholder="Enter measure number" inputMode="numeric" - style={text_input_style} + style={styles.measure_input_shape} /> { dispatch({ type: "reset" }); }} @@ -41,3 +34,26 @@ export function MeasureSetBox({ ); } + +const styles = StyleSheet.create({ + measure_button_shape: { + width: "40%", + marginRight: "10%", + height: "50%" + }, + measure_input_shape: { + width: "40%", + marginRight: "10%", + height: "50%", + borderRadius: 15, + backgroundColor: "white" + }, + flexing_box: { + width: "37.5%", + display: "flex", + flexDirection: "row", + justifyContent: "center", + alignContent: "center", + alignItems: "center" + } +}) diff --git a/frontend/companion-app/components/ReturnButton.tsx b/frontend/companion-app/components/ReturnButton.tsx new file mode 100644 index 0000000..3668a35 --- /dev/null +++ b/frontend/companion-app/components/ReturnButton.tsx @@ -0,0 +1,38 @@ +import { StyleSheet, View, Text, TextStyle, ViewStyle, Pressable } from "react-native"; + +export function Return_Button({ + state, + dispatch, + button_format, + text_style, + }: { + state: { inPlayMode: boolean }; + dispatch: Function; + button_format: ViewStyle; + text_style: TextStyle; + }) { + return ( + + { dispatch({ type: "swap_mode" }); }} + > + {"↩️"} + + + ); + } + +const styles = StyleSheet.create({ + button_shape: { + width: "50%", + height: "50%" + }, + flexing_box: { + width: "12.5%", + height: "100%", + justifyContent: "center", + alignContent: "center", + alignItems: "center" + } +}) \ No newline at end of file diff --git a/frontend/companion-app/components/ScoreDisplay.tsx b/frontend/companion-app/components/ScoreDisplay.tsx index 4e28893..47f1ace 100644 --- a/frontend/companion-app/components/ScoreDisplay.tsx +++ b/frontend/companion-app/components/ScoreDisplay.tsx @@ -131,13 +131,8 @@ export default function ScoreDisplay({ state.synth_tempo, }); } - }, [ - dispatch, - state.cursorTimestamp, - state.synth_tempo, - state.time_signature.Denominator, - state.timestamp, - ]); + }, +[state.timestamp]); return (
@@ -151,7 +146,7 @@ export default function ScoreDisplay({ const styles = StyleSheet.create({ scrollContainer: { width: "100%", // Make the scroll container fill the width of the parent - height: "80%", // Set a specific height for scrolling (adjust as needed) + height: "100%", // Set a specific height for scrolling (adjust as needed) overflow: "scroll", // Enable vertical scrolling borderWidth: 1, // Add border to the container borderColor: "black", // Set border color to black diff --git a/frontend/companion-app/components/ScoreSelect.tsx b/frontend/companion-app/components/ScoreSelect.tsx index 4c152b1..56b77c9 100644 --- a/frontend/companion-app/components/ScoreSelect.tsx +++ b/frontend/companion-app/components/ScoreSelect.tsx @@ -1,4 +1,4 @@ -import { View, Text } from "react-native"; +import { StyleSheet, View, Text } from "react-native"; import RNPickerSelect from "react-native-picker-select"; import React, { useEffect } from "react"; @@ -54,7 +54,7 @@ export function Score_Select({ }; return ( - + Select a score: ); } + +const styles = StyleSheet.create({ + tempo_text_shape: { + width: "30%", + height: "100%" + }, + tempo_input_shape: { + width: "40%", + height: "100%", + backgroundColor: "white" + }, + flexing_box: { + width: "25%", + height: "100%", + display: "flex", + padding: "2%", + backgroundColor: "lightgray" + } +}) diff --git a/frontend/companion-app/components/StartButton.tsx b/frontend/companion-app/components/StartButton.tsx index 33e9fe3..c035aca 100644 --- a/frontend/companion-app/components/StartButton.tsx +++ b/frontend/companion-app/components/StartButton.tsx @@ -1,25 +1,68 @@ -import { Text, TextStyle, ViewStyle, Pressable } from "react-native"; +import { StyleSheet, Text, TextStyle, ViewStyle, Pressable, View } from "react-native"; +import { synthesizeAudio } from "./Utils"; +import { Audio } from "expo-av"; export function Start_Stop_Button({ state, dispatch, - button_style, + button_format, text_style, }: { - state: { playing: boolean }; + state: { playing: boolean, inPlayMode: boolean, sessionToken: string, score: string, tempo: number }; dispatch: Function; - button_style: ViewStyle; + button_format: ViewStyle; text_style: TextStyle; }) { + + // Copied from SynthesizeButton.tsx + const synthesize_audio_handler = async () => { + console.log("Synthesizing audio..."); + synthesizeAudio(state.sessionToken, state.score, state.tempo).then( + async (data) => { + console.log("Synthesized audio data:", data); + const { sound: newSound } = await Audio.Sound.createAsync( + { uri: data.uri }, + { shouldPlay: false } + ); + const status = await newSound.getStatusAsync(); + console.log("New sound status:", status); + dispatch({ + type: "new_audio", + sound: newSound, + synth_tempo: state.tempo, + }); + } + ); + }; + return ( - { - console.log("The play button's on_press runs."); - dispatch({ type: "start/stop" }); - }} - > - {state.playing ? "STOP" : "PLAY"} - + + { + if (state.inPlayMode) dispatch({ type: "start/stop" }); + else { + synthesize_audio_handler(); + dispatch({ type: "swap_mode"}); + } + }} + > + {state.inPlayMode? state.playing ? "STOP" : "PLAY" : "SELECT" } + + ); } + +const styles = StyleSheet.create({ + button_shape: { + width: "75%", + height: "75%" + }, + flexing_box: { + width: "25%", + height: "100%", + justifyContent: "center", + alignContent: "center", + alignItems: "center" + } +}) diff --git a/frontend/companion-app/components/TempoBox.tsx b/frontend/companion-app/components/TempoBox.tsx index dc373f7..000c982 100644 --- a/frontend/companion-app/components/TempoBox.tsx +++ b/frontend/companion-app/components/TempoBox.tsx @@ -1,22 +1,20 @@ -import { View, Text } from "react-native"; +import { StyleSheet, View, Text } from "react-native"; import { TextInput, TextStyle, ViewStyle } from "react-native"; export function TempoBox({ state, dispatch, - wrapper_style, - text_input_style, label_text_style, }: { state: { tempo: number }; dispatch: Function; - wrapper_style: ViewStyle; - text_input_style: ViewStyle; label_text_style: TextStyle; }) { return ( - - Tempo (BPM): + + + Tempo: + dispatch({ type: "change_tempo", tempo: text as unknown as number }) @@ -24,8 +22,33 @@ export function TempoBox({ value={String(state.tempo)} placeholder="Enter tempo" inputMode="numeric" - style={text_input_style} + style={styles.tempo_input_shape} /> + + BPM + ); } + +const styles = StyleSheet.create({ + tempo_text_shape: { + width: "30%", + padding: "10%", + justifyContent: "center" + }, + tempo_input_shape: { + width: "40%", + height: "50%", + borderRadius: 15, + backgroundColor: "white" + }, + flexing_box: { + width: "37.5%", + display: "flex", + flexDirection: "row", + justifyContent: "center", + alignContent: "center", + alignItems: "center" + } +})