Merge pull request #10 from nipunaw/sprint-2

Sprint 2 (Beta Build)

Author: leslieharvey

package-lock.json

@@ -4,13 +4,19 @@
   "description": "FPGA and software crossover to bring your microphone clarity and fidelity",
   "main": "public/main.js",
   "dependencies": {
+    "@joe_six/duarte-watanabe-peak-detection": "^0.2.0",
+    "@joe_six/smoothed-z-score-peak-signal-detection": "^0.1.2",
     "@testing-library/jest-dom": "^5.11.4",
     "@testing-library/react": "^11.1.0",
     "@testing-library/user-event": "^12.1.10",
     "@types/jest": "^27.4.0",
     "@types/node": "^17.0.10",
     "@types/react": "^17.0.38",
     "bootstrap": "^5.1.3",
+    "fft-js": "^0.0.12",
+    "fft-windowing": "^0.1.4",
+    "fili": "^2.0.3",
+    "nodeplotlib": "^1.0.0",
     "pitchfinder": "^2.3.2",
     "python-shell": "^3.0.1",
     "react": "^17.0.2",

package.json

@@ -2,7 +2,8 @@
 const { app, BrowserWindow, Menu, ipcMain } = require("electron");
 const fs = require("fs");
 const { resolve } = require("path");
-const { GetPitchValue } = require("./main/audioProcess");
+const { electron } = require("process");
+const { GetPitchValue, fftAnalysis } = require("./main/audioProcess");
 
 function createWindow() {
   // Create the browser window.
@@ -60,13 +61,27 @@ function createWindow() {
     }
   );
 
-  ipcMain.on("process-audio", (event, float32Array) => {
-    const pitch = GetPitchValue(float32Array);
-    console.log(`Value: ${pitch}`);
-    win.webContents.send(
-      "audio-finished",
-      `Sent data over UART if FPGA is connected`
-    );
+
+  ipcMain.on("process-audio", (event, rawRecordedData, sampleRate) => {
+    //Pitch method is deprecated
+    try {
+      const pitch = GetPitchValue(rawRecordedData);
+      const noiseTaps = fftAnalysis(rawRecordedData, sampleRate);
+
+      win.webContents.send(
+        "audio-finished",
+        true,
+        `Sent information over UART if connected`,
+        noiseTaps
+      );
+    } catch {
+      win.webContents.send(
+        "audio-finished",
+        false,
+        `An error has occured.`,
+        []
+      );
+    }
   });
 }
 

public/main.js

@@ -1,12 +1,125 @@
 const { app, BrowserWindow, Menu, ipcMain } = require("electron");
 const Pitchfinder = require("pitchfinder");
+// FIR and FFT Analysis
+const fft = require('fft-js').fft;
+const fftUtil = require('fft-js').util;
+const windowing = require('fft-windowing');
+const Fili = require('fili');
+// Smoothing and Plots
+const smoothed_z_score = require("@joe_six/smoothed-z-score-peak-signal-detection");
+const detect_peaks = require("@joe_six/duarte-watanabe-peak-detection")
+const { plot, Plot } = require("nodeplotlib");
 
-const GetPitchValue = (arrayData) => {
+const fftAnalysis = (rawRecordedData, sampleRate) => {
+  var windowedRecordedData = windowing.hann(rawRecordedData);
+  var phasors = fft(windowedRecordedData.slice(0, 262144)); // 48000 > samples 
+  var frequencies = fftUtil.fftFreq(phasors, sampleRate); // Sample rate and coef is just used for length, and frequency step
+  var magnitudes = fftUtil.fftMag(phasors);
+
+  console.log("Sample Rate: " + sampleRate);
+  console.log("Length of raw data: " + rawRecordedData.length);
+  console.log("Length of windowed data: " + windowedRecordedData.length);
+  console.log("Length of phasors: " + phasors.length);
+  console.log("Length of frequencies: " + frequencies.length);
+  console.log("Length of magnitudes: " + magnitudes.length);
+  console.log("");
+  return firFilterTaps(frequencies, magnitudes, sampleRate);
+}
+
+const firFilterTaps = (frequencies, magnitudes, sampleRate) => {
+  // TO-DO: Continue testing smoothing methods, for now electing Watanabe method
+  //const peaksSmoothed = smoothed_z_score(magnitudes, {lag: 40, threshold: 4.5, influence: 0.2});
+  
+  const noiseTaps = noiseRemoval(frequencies, magnitudes, 101, sampleRate);
+  return noiseTaps;
+}
+
+const identifyPeaks = (magnitudes, mpdVal) => {
+  const magnitudesNoiseDescending = [...magnitudes].sort(function (a, b) {  return a - b;  }).reverse();
+  const magnitudesNoiseThreshold = magnitudesNoiseDescending[15]; // Assumes no more than 15 peaks
+  return detect_peaks(magnitudes, {mpd: mpdVal, mph: magnitudesNoiseThreshold});
+}
+
+// Helper function for graphing/testing
+const graphFrequencies= (frequencies, magnitudes, logScale, plotType) => {
+  let logFrequencies = [];
+  if (logScale == true) {
+    for (let i = 0; i < frequencies.length; i++) {
+      logFrequencies.push(Math.log10(frequencies[i]));
+    }
+  } else {
+    logFrequencies = frequencies;
+  }
+  
+  const data = [
+    {
+      x: logFrequencies,
+      y: magnitudes,
+      type: plotType
+    },
+  ];
+  
+  plot(data);
+}
+
+const bandstopTaps = (filterOrder, sampleRate, lowerFreq, upperFreq, attenuation) => {
+  var firCalculator = new Fili.FirCoeffs();
+  var firFilterCoeffsK = firCalculator.kbFilter({
+    order: filterOrder, // filter order (must be odd)
+    Fs: sampleRate, // sampling frequency
+    Fa: lowerFreq, // rise, 0 for lowpass
+    Fb: upperFreq, // fall, Fs/2 for highpass
+    Att: attenuation // attenuation in dB
+  });
+  return firFilterCoeffsK;
+}
+
+const noiseRemoval = (frequencies, magnitudes, filterOrder, sampleRate) => {
+  const peaksWatanabeIndices = identifyPeaks(magnitudes, 50);
+  let noisyFrequencies = [];
+  for (let i = 0; i < peaksWatanabeIndices.length; i++) {
+    if (frequencies[peaksWatanabeIndices[i]] < 80) {
+      noisyFrequencies.push(Math.round(frequencies[peaksWatanabeIndices[i]]));
+    }
+  }
+
+  
+  let bands = [];
+  for (let i = 0; i < noisyFrequencies.length; i++) {
+    if (noisyFrequencies[i] > 5) {
+      bands.push(bandstopTaps(filterOrder, sampleRate, noisyFrequencies[i]-5, noisyFrequencies[i]+5, 5)); //Attenuate by 5 dB, order of 101
+    } else {
+      bands.push(bandstopTaps(filterOrder, sampleRate, 0, noisyFrequencies[i]+5, 5)); //Attenuate by 5 dB, order of 101
+    }
+  }
+
+  // Safely assume independence of bands
+  var sum = (r, a) => r.map((b, i) => a[i] + b);
+  let noisyTaps = bands.reduce(sum);
+
+  return noisyTaps;
+}
+
+const generalAnalysis = (frequencies, magnitudes) => {
+  const magnitudesDescending = [...magnitudes].sort(function (a, b) {  return a - b;  }).reverse();
+  const magnitudesThreshold = magnitudesDescending[15];
+
+  for (let i = 0; i < frequencies.length; i++) {
+    if (magnitudes[i] > magnitudesThreshold) {
+      console.log("General Frequency (Hz): "+ frequencies[i] + ", Noise Magnitude: "+ magnitudes[i])
+    }
+  }
+  console.log("");
+}
+
+// Deprecated analysis method using pitch values instead of FFT
+const GetPitchValue = (recordedData) => {
   const detectPitch = Pitchfinder.AMDF();
-  const pitch = detectPitch(arrayData);
+  const pitch = detectPitch(recordedData);
   return pitch;
 };
 
 module.exports = {
   GetPitchValue: GetPitchValue,
+  fftAnalysis: fftAnalysis
 };

public/main/audioProcess.js

@@ -2,13 +2,16 @@ import { useEffect, useState } from "react";
 import "../css/MainContent.css";
 import AudioDeviceList from "./AudioDeviceList";
 import { writeSerial } from "functions/serial";
+import Loading from "./Loading";
 const electron = window.require("electron");
 
 enum AudioState {
-  Loading,
-  Recording,
-  Ready,
   Idle,
+  Ready,
+  Recording,
+  Processing,
+  Error,
+  Finished,
 }
 
 interface AudioDevice extends MediaStreamConstraints {
@@ -23,39 +26,68 @@ var constraints: AudioDevice = {
   },
 };
 
+const RECORD_MS_TIME = 5500;
+
+const handleStart = (event: Event) => {
+  setTimeout(function () {
+    let eventRecorder = event.target as MediaRecorder;
+    eventRecorder.stop();
+  }, RECORD_MS_TIME);
+};
+
+const handleStop = (setRecorderState: () => void, updateState: () => void) => {
+  setRecorderState();
+  updateState();
+};
+
 const handleDataAvailable = (event: BlobEvent) => {
   let audioCtx = new AudioContext();
   event.data.arrayBuffer().then((arrayBuf) => {
     audioCtx.decodeAudioData(arrayBuf).then((buffer) => {
-      const float32Array = buffer.getChannelData(0); // get a single channel of sound
-      electron.ipcRenderer.send("process-audio", float32Array);
-      const data = new Uint8Array([104, 101, 108, 108, 111]); // hello
-      writeSerial(data);
+      //sample rate is 48kHz for my device
+      const rawRecordedData = buffer.getChannelData(0); // get a single channel of sound
+      const sampleRate = audioCtx.sampleRate;
+      electron.ipcRenderer.send("process-audio", rawRecordedData, sampleRate);
     });
   });
 };
 
 export default function AudioRecord() {
-  //const audioRef = useRef<HTMLAudioElement>(null);
   const [state, setState] = useState<AudioState>(AudioState.Idle);
-  const [mediaStream, setMediaStream] = useState<MediaStream>();
   const [recorder, setRecorder] = useState<MediaRecorder>();
   const [selectedDevice, setSelectedDevice] = useState<MediaDeviceInfo>();
   const [feedbackMsg, setFeedbackMsg] = useState<string | null>();
 
   useEffect(() => {
-    electron.ipcRenderer.on("audio-finished", (event, message) => {
-      setState(AudioState.Ready);
-      setFeedbackMsg(message);
-    });
+    electron.ipcRenderer.on(
+      "audio-finished",
+      (event, status, message, data) => {
+        console.log(status);
+        if (status === true) {
+          setState(AudioState.Finished);
+          setFeedbackMsg(message);
+          writeSerial(data).then((serialStatus) => {
+            console.log(serialStatus);
+          });
+        } else {
+          setState(AudioState.Error);
+        }
+      }
+    );
   }, []);
 
   function handleSuccess(stream: MediaStream) {
-    setMediaStream(stream);
     const options = { mimeType: "audio/webm" };
     const _recorder = new MediaRecorder(stream, options);
+    _recorder.onstart = handleStart;
+    _recorder.onstop = () =>
+      handleStop(
+        () => setRecorder(undefined),
+        () => setState(AudioState.Processing)
+      );
+    _recorder.ondataavailable = handleDataAvailable;
+    _recorder.start();
     setRecorder(_recorder);
-    //audioRef.current.srcObject = stream;
   }
 
   function handleError(error: Error) {
@@ -73,13 +105,6 @@ export default function AudioRecord() {
     }
   }, [selectedDevice]);
 
-  useEffect(() => {
-    if (recorder) {
-      recorder.ondataavailable = handleDataAvailable;
-      recorder.start();
-    }
-  }, [recorder]);
-
   const handleClick = () => {
     if (state === AudioState.Ready && selectedDevice) {
       setFeedbackMsg(null);
@@ -89,15 +114,6 @@ export default function AudioRecord() {
         .getUserMedia(constraints)
         .then(handleSuccess)
         .catch(handleError);
-    } else if (state === AudioState.Recording) {
-      setState(AudioState.Loading);
-      if (recorder) {
-        recorder.stop();
-        setRecorder(undefined);
-      }
-      if (mediaStream) {
-        setMediaStream(undefined);
-      }
     }
   };
 
@@ -112,18 +128,29 @@ export default function AudioRecord() {
         break;
       }
       case AudioState.Recording: {
-        buttonDisplay = "Click to Stop";
-        isDisabled = false;
+        buttonDisplay = "Recording...";
+        isDisabled = true;
         break;
       }
       case AudioState.Ready: {
         buttonDisplay = "Click to Start";
         isDisabled = false;
         break;
       }
-      case AudioState.Loading: {
+      case AudioState.Processing: {
         buttonDisplay = "Processing...";
         isDisabled = true;
+        break;
+      }
+      case AudioState.Finished: {
+        buttonDisplay = "Finished";
+        isDisabled = true;
+        break;
+      }
+      case AudioState.Error: {
+        buttonDisplay = "Error";
+        isDisabled = true;
+        break;
       }
     }
     return (
@@ -136,10 +163,12 @@ export default function AudioRecord() {
   const getStateMessage = (): string | null => {
     if (state === AudioState.Idle) {
       return "Please select an audio device";
-    } else if (state === AudioState.Loading) {
+    } else if (state === AudioState.Processing) {
       return "Processing...";
     } else if (state === AudioState.Recording) {
       return "Recording...";
+    } else if (state === AudioState.Error) {
+      return "An error occured while performing test. Please try again.";
     }
     return null;
   };
@@ -150,12 +179,11 @@ export default function AudioRecord() {
         {getStateMessage()}
         {feedbackMsg}
       </div>
-      <AudioDeviceList selectDevice={setSelectedDevice} />
-      {/*
-      <div className="media-player">
-        <audio id="gum-local" ref={audioRef} controls autoPlay></audio>
-      </div>
-      */}
+      {state == AudioState.Recording || state == AudioState.Processing ? (
+        <Loading />
+      ) : (
+        <AudioDeviceList selectDevice={setSelectedDevice} />
+      )}
       {getButton()}
     </div>
   );