Feat/migrate boxes box id data sensor

app/lib/measurement-service.server.ts

@@ -1,31 +1,40 @@
 import { decodeMeasurements, hasDecoder } from "~/lib/decoding-service.server";
-import { getDeviceWithoutSensors, getDevice, findAccessToken } from "~/models/device.server";
+import { type DeviceWithoutSensors, getDeviceWithoutSensors, getDevice, findAccessToken } from "~/models/device.server";
 import { saveMeasurements } from "~/models/measurement.server";
-import { getSensorsWithLastMeasurement } from "~/models/sensor.server";
+import { getSensorsWithLastMeasurement, getSensorWithLastMeasurement } from "~/models/sensor.server";
+import { type SensorWithLatestMeasurement } from "~/schema";
+
+export type DeviceWithSensors = DeviceWithoutSensors & {sensors: SensorWithLatestMeasurement[]}
+
+export async function getLatestMeasurementsForSensor(boxId: string, sensorId: string, count?: number):
+  Promise<SensorWithLatestMeasurement | null> {
+
+  const device: DeviceWithoutSensors = await getDeviceWithoutSensors({ id: boxId });
+  if (!device) return null;
+
+  // single sensor, no need for having info about device
+  return await getSensorWithLastMeasurement(device.id, sensorId, count);
+}
 
 /**
  *
  * @param boxId
  * @param sensorId
  * @param count
  */
-export const getLatestMeasurements = async (
+export async function getLatestMeasurements (
   boxId: string,
-  sensorId: string | undefined,
-  count: number | undefined,
-): Promise<any | null> => {
-  const device = await getDeviceWithoutSensors({ id: boxId });
+  count?: number,
+): Promise<DeviceWithSensors | null> {
+  const device: DeviceWithoutSensors = await getDeviceWithoutSensors({ id: boxId });
   if (!device) return null;
 
   const sensorsWithMeasurements = await getSensorsWithLastMeasurement(
-    device.id,
-    sensorId,
-    count,
-  );
-  if (sensorId !== undefined) return sensorsWithMeasurements; // single sensor, no need for having info about device
-
-  (device as any).sensors = sensorsWithMeasurements;
-  return device;
+    device.id, count);
+
+  const deviceWithSensors: DeviceWithSensors = device as DeviceWithSensors
+  deviceWithSensors.sensors = sensorsWithMeasurements;
+  return deviceWithSensors;
 };
 
 interface PostMeasurementsOptions {

app/lib/outlier-transform.ts

@@ -0,0 +1,49 @@
+import { mean, median } from "simple-statistics";
+import { type Measurement } from "~/schema";
+
+export type TransformedMeasurement = Measurement & { isOutlier: boolean };
+
+export function transformOutliers(measurements: Measurement[], window: number, replaceOutlier: boolean) : TransformedMeasurement[] {
+    const res: TransformedMeasurement[] = [];
+    const values: number[] = [];
+
+    for (let i = 0; i < measurements.length; i++) {
+        let current: TransformedMeasurement = measurements[i] as TransformedMeasurement;
+        if (current.value === null)
+            continue;
+        current.isOutlier = false;
+
+        if (values.length === window) {
+            // We only add non-null values, so all previous measurement values must be non-null
+            current.isOutlier = isOutlier(current.value, values);
+            if (current.isOutlier && replaceOutlier)
+                current.value = mean(values);
+
+            values.shift();
+        }
+
+        values.push(current.value);
+        res.push(current);
+    }
+
+    return res;
+}
+
+function isOutlier(measurement: number, otherMeasurements: number[]): boolean {
+    const med = median(otherMeasurements);
+
+    // compute the medianAbsoluteDeviation
+    // The mad of nothing is null
+    const medianAbsoluteDeviations: number[] = [];
+
+    // Make a list of absolute deviations from the median
+    for (let i = 0; i < otherMeasurements.length; i++)
+        medianAbsoluteDeviations.push(Math.abs(otherMeasurements[i] - med));
+
+    // Find the median value of that list
+    const mad = median(medianAbsoluteDeviations),
+        max = med + 3 * mad, //3 times medianAbsoluteDeviation around median best solution to check for outliers (see bachelor thesis Joana Gockel)
+        min = med - 3 * mad;
+
+    return (measurement > max || measurement < min);
+}

app/models/device.server.ts

@@ -3,7 +3,6 @@ import { eq, sql, desc, ilike, arrayContains, and } from 'drizzle-orm'
 import { type Point } from 'geojson'
 import { drizzleClient } from '~/db.server'
 import { device, location, sensor, type Device, type Sensor } from '~/schema'
-import { accessToken } from '~/schema/accessToken'
 
 const BASE_DEVICE_COLUMNS = {
 	id: true,
@@ -92,6 +91,8 @@ export function getDeviceWithoutSensors({ id }: Pick<Device, 'id'>) {
 	})
 }
 
+export type DeviceWithoutSensors = Awaited<ReturnType<typeof getDeviceWithoutSensors>>
+
 export function updateDeviceInfo({
 	id,
 	name,

app/models/measurement.server.ts

@@ -388,23 +388,21 @@ export async function saveMeasurements(
   });
 }
 
-async function insertMeasurements(measurements: any[]): Promise<void> {
+export async function insertMeasurements(measurements: any[]): Promise<void> {
   const measurementInserts = measurements.map(measurement => ({
     sensorId: measurement.sensor_id,
     value: measurement.value,
     time: measurement.createdAt || new Date(),
   }));
 
-
-
   await drizzleClient.insert(measurement).values(measurementInserts);
 }
 
-async function insertMeasurement(measurement: any): Promise<any> {
-  return drizzleClient.insert(measurement).values({
-    sensorId: measurement.sensor_id,
-    value: measurement.value,
-    time: measurement.createdAt
-  });
+export async function deleteMeasurementsForSensor(sensorId: string) {
+  return await drizzleClient.delete(measurement).where(eq(measurement.sensorId, sensorId));
+}
+
+export async function deleteMeasurementsForTime(date: Date) {
+  return await drizzleClient.delete(measurement).where(eq(measurement.time, date));
 }
 

app/models/sensor.server.ts

@@ -1,6 +1,7 @@
 import { eq, sql } from "drizzle-orm";
 import { drizzleClient } from "~/db.server";
 import {
+  type Measurement,
   sensor,
   type Sensor,
   type SensorWithLatestMeasurement,
@@ -69,17 +70,21 @@ export function getSensorsFromDevice(deviceId: Sensor["deviceId"]) {
   });
 }
 
-export async function getSensorsWithLastMeasurement(
+export async function getSensorWithLastMeasurement(
   deviceId: Sensor["deviceId"],
-  sensorId?: Sensor["id"],
-  count?: number,
-): Promise<SensorWithLatestMeasurement | SensorWithLatestMeasurement[]>;
+  sensorId: Sensor["id"],
+  count: number = 1
+): Promise<SensorWithLatestMeasurement> {
+  const allSensors = await getSensorsWithLastMeasurement(deviceId, count);
+  return allSensors.find(
+    (c: any) => c.id === sensorId,
+  ) as SensorWithLatestMeasurement;
+}
 
 export async function getSensorsWithLastMeasurement(
   deviceId: Sensor["deviceId"],
-  sensorId: Sensor["id"] | undefined = undefined,
   count: number = 1,
-): Promise<SensorWithLatestMeasurement | SensorWithLatestMeasurement[]> {
+): Promise<SensorWithLatestMeasurement[]> {
   const result = await drizzleClient.execute(
     sql`SELECT 
         s.id,
@@ -118,11 +123,23 @@ export async function getSensorsWithLastMeasurement(
     } else return { ...r, lastMeasurements: [] } as any;
   }) as any;
 
-  if (sensorId === undefined) return cast as SensorWithLatestMeasurement[];
-  else
-    return cast.find(
-      (c: any) => c.id === sensorId,
-    ) as SensorWithLatestMeasurement;
+  return cast as SensorWithLatestMeasurement[];
+}
+
+export async function getMeasurements(
+  sensorId: Measurement["sensorId"],
+  fromDate: string,
+  toDate: string,
+  count: number = 10000): Promise<Measurement[]> {
+    return await drizzleClient.execute(
+      sql`SELECT *
+          FROM measurement
+          WHERE sensor_id = ${sensorId} AND
+          time >= ${fromDate} AND
+          time <= ${toDate}
+          ORDER BY time DESC
+          LIMIT ${count}`
+    ) as Measurement[];
 }
 
 export async function registerSensor(newSensor: Sensor) {

app/root.tsx

@@ -1,7 +1,6 @@
 import tailwindStylesheetUrl from "/app/tailwind.css?url";
 import appStylesheetUrl from "/app/app.css?url";
 import clsx from "clsx";
-import i18next from "./i18next.server";
 import { useTranslation } from "react-i18next";
 import {
   data,
@@ -17,6 +16,7 @@ import {
 import { useChangeLanguage } from "remix-i18next/react";
 import { Toaster } from "./components/ui/toaster";
 import { i18nCookie } from "./cookies";
+import i18next from "./i18next.server";
 import { getEnv } from "./utils/env.server";
 import { getUser } from "./utils/session.server";