diff --git a/app/lib/user-service.server.ts b/app/lib/user-service.server.ts index 2a342cc7..e05fafcd 100644 --- a/app/lib/user-service.server.ts +++ b/app/lib/user-service.server.ts @@ -1,5 +1,11 @@ import bcrypt from 'bcryptjs' import { eq } from 'drizzle-orm' +import ConfirmEmailAddress, { + subject as ConfirmEmailAddressSubject, +} from 'emails/confirm-email' +import DeleteUserEmail, { + subject as DeleteUserEmailSubject, +} from 'emails/delete-user' import NewUserEmail, { subject as NewUserEmailSubject } from 'emails/new-user' import PasswordResetEmail, { subject as PasswordResetEmailSubject, @@ -33,12 +39,6 @@ import { verifyLogin, } from '~/models/user.server' import { passwordResetRequest, user, type User } from '~/schema' -import ConfirmEmailAddress, { - subject as ConfirmEmailAddressSubject, -} from 'emails/confirm-email' -import DeleteUserEmail, { - subject as DeleteUserEmailSubject, -} from 'emails/delete-user' const ONE_HOUR_MILLIS: number = 60 * 60 * 1000 diff --git a/app/models/measurement.server.ts b/app/models/measurement.server.ts index 249f63b6..5aa9ac9e 100644 --- a/app/models/measurement.server.ts +++ b/app/models/measurement.server.ts @@ -1,7 +1,7 @@ import { and, desc, eq, gte, lte, sql } from "drizzle-orm"; import { drizzleClient } from "~/db.server"; import { - deviceToLocation, + type LastMeasurement, location, measurement, measurements10minView, @@ -9,8 +9,16 @@ import { measurements1hourView, measurements1monthView, measurements1yearView, - sensor, } from "~/schema"; +import { + type MinimalDevice, + type MeasurementWithLocation, + getLocationUpdates, + findOrCreateLocations, + addLocationUpdates, + insertMeasurementsWithLocation, + updateLastMeasurements, +} from '~/utils/measurement-server-helper' // This function retrieves measurements from the database based on the provided parameters. export function getMeasurement( @@ -166,100 +174,17 @@ export function getMeasurement( }); } -interface MeasurementWithLocation { - sensor_id: string; - value: number; - createdAt?: Date; - location?: { - lng: number; - lat: number; - height?: number; - } | null; -} - -interface MeasurementWithLocation { - sensor_id: string; - value: number; - createdAt?: Date; - location?: { - lng: number; - lat: number; - height?: number; - } | null; -} - -/** - * Get the device location that was valid at a specific timestamp - * Returns the most recent location that was set before or at the given timestamp - */ -async function getDeviceLocationAtTime( - tx: any, - deviceId: string, - timestamp: Date -): Promise { - const locationAtTime = await tx - .select({ - locationId: deviceToLocation.locationId, - }) - .from(deviceToLocation) - .where( - and( - eq(deviceToLocation.deviceId, deviceId), - lte(deviceToLocation.time, timestamp) - ) - ) - .orderBy(desc(deviceToLocation.time)) - .limit(1); - - return locationAtTime.length > 0 ? locationAtTime[0].locationId : null; -} - -async function findOrCreateLocation( - tx: any, - lng: number, - lat: number -): Promise { - const existingLocation = await tx - .select({ id: location.id }) - .from(location) - .where( - sql`ST_Equals( - ${location.location}, - ST_SetSRID(ST_MakePoint(${lng}, ${lat}), 4326) - )` - ) - .limit(1); - - - if (existingLocation.length > 0) { - return existingLocation[0].id; - } - - const [newLocation] = await tx - .insert(location) - .values({ - location: sql`ST_SetSRID(ST_MakePoint(${lng}, ${lat}), 4326)`, - }) - .returning(); - - return newLocation.id; -} - export async function saveMeasurements( - device: any, + device: MinimalDevice, measurements: MeasurementWithLocation[] ): Promise { + if (!device) + throw new Error("No device given!") if (!Array.isArray(measurements)) throw new Error("Array expected"); const sensorIds = device.sensors.map((s: any) => s.id); - const lastMeasurements: Record = {}; - - // Track measurements that update device location (those with explicit locations) - const deviceLocationUpdates: Array<{ - location: { lng: number; lat: number; height?: number }; - time: Date; - }> = []; + const lastMeasurements: Record> = {}; // Validate and prepare measurements for (let i = measurements.length - 1; i >= 0; i--) { @@ -286,108 +211,36 @@ export async function saveMeasurements( throw error; } - if (!lastMeasurements[m.sensor_id]) { + if (!lastMeasurements[m.sensor_id] || + lastMeasurements[m.sensor_id].createdAt < measurementTime.toISOString()) { lastMeasurements[m.sensor_id] = { value: m.value, createdAt: measurementTime.toISOString(), sensorId: m.sensor_id, }; } - - // Track measurements with explicit locations for device location updates - if (m.location) { - deviceLocationUpdates.push({ - location: m.location, - time: measurementTime, - }); - } } - // Sort device location updates by time (oldest first) to process in order - deviceLocationUpdates.sort((a, b) => a.time.getTime() - b.time.getTime()); + // Track measurements that update device location (those with explicit locations) + const deviceLocationUpdates = getLocationUpdates(measurements); + const locations = await findOrCreateLocations(deviceLocationUpdates); + + // First, update device locations for all measurements with explicit locations + // This ensures the location history is complete before we infer locations + await addLocationUpdates(deviceLocationUpdates, device.id, locations); + // Note that the insertion of measurements and update of sensors need to be in one + // transaction, since otherwise other updates could get in between and the data would be + // inconsistent. This shouldn't be a problem for the updates above. await drizzleClient.transaction(async (tx) => { - // First, update device locations for all measurements with explicit locations - // This ensures the location history is complete before we infer locations - for (const update of deviceLocationUpdates) { - const locationId = await findOrCreateLocation( - tx, - update.location.lng, - update.location.lat - ); - - // Check if we should add this to device location history - // Only add if it's newer than the current latest location - const currentLatestLocation = await tx - .select({ time: deviceToLocation.time }) - .from(deviceToLocation) - .where(eq(deviceToLocation.deviceId, device.id)) - .orderBy(desc(deviceToLocation.time)) - .limit(1); - - const shouldAdd = - currentLatestLocation.length === 0 || - update.time >= currentLatestLocation[0].time; - - if (shouldAdd) { - await tx - .insert(deviceToLocation) - .values({ - deviceId: device.id, - locationId: locationId, - time: update.time, - }) - .onConflictDoNothing(); - } - } - // Now process each measurement and infer locations if needed - for (const m of measurements) { - const measurementTime = m.createdAt || new Date(); - let locationId: bigint | null = null; - - if (m.location) { - // Measurement has explicit location - locationId = await findOrCreateLocation( - tx, - m.location.lng, - m.location.lat - ); - } else { - // No explicit location - infer from device location history - locationId = await getDeviceLocationAtTime( - tx, - device.id, - measurementTime - ); - } - - // Insert measurement with locationId (may be null for measurements - // without location and before any device location was set) - await tx.insert(measurement).values({ - sensorId: m.sensor_id, - value: m.value, - time: measurementTime, - locationId: locationId, - }).onConflictDoNothing(); - - } - + await insertMeasurementsWithLocation(measurements, locations, device.id, tx); // Update sensor lastMeasurement values - const updatePromises = Object.entries(lastMeasurements).map( - ([sensorId, lastMeasurement]) => - tx - .update(sensor) - .set({ lastMeasurement }) - .where(eq(sensor.id, sensorId)) - ); - - - await Promise.all(updatePromises); - + await updateLastMeasurements(lastMeasurements, tx); }); } + export async function insertMeasurements(measurements: any[]): Promise { const measurementInserts = measurements.map(measurement => ({ sensorId: measurement.sensor_id, diff --git a/app/utils/measurement-server-helper.ts b/app/utils/measurement-server-helper.ts new file mode 100644 index 00000000..afff0850 --- /dev/null +++ b/app/utils/measurement-server-helper.ts @@ -0,0 +1,289 @@ +import { desc, eq, inArray, or, type SQL, sql } from 'drizzle-orm' +import { drizzleClient } from '~/db.server' +import { + deviceToLocation, + type LastMeasurement, + location, + type Measurement, + measurement, + sensor, +} from '~/schema' + +export interface MeasurementWithLocation { + sensor_id: string + value: number + createdAt?: Date + location?: Location | null +} + +export type Location = { + lng: number + lat: number + height?: number +} + +export type LocationWithId = Location & { id: bigint } + +export type DeviceLocationUpdate = { + location: Location + time: Date +} + +export type MinimalDevice = { + id: string + sensors: Array<{ + id: string + }> +} + +/** + * Extracts location updates from measurements (with explicit locations) + * @param measurements The measurements with potential location udpates + * @returns The found location updates, sorted oldest first + */ +export function getLocationUpdates( + measurements: MeasurementWithLocation[], +): DeviceLocationUpdate[] { + return ( + measurements + .filter((measurement) => measurement.location) + .map((measurement) => { + return { + location: measurement.location as Location, + time: new Date(measurement.createdAt || Date.now()), + } + }) + // Sort device location updates by time (oldest first) to process in order + .sort((a, b) => a.time.getTime() - b.time.getTime()) + ) +} + +/** + * Makes sure all locations for the location updates are in the database + * @param locationUpdates The location updates from `getLocationUpdates` + * @returns A map of the IDs of the locations for the location updates + */ +export async function findOrCreateLocations( + locationUpdates: DeviceLocationUpdate[], +): Promise { + const newLocations = locationUpdates.map((update) => update.location) + + let foundLocations: LocationWithId[] = [] + + await drizzleClient.transaction(async (tx) => { + const existingLocations = await tx + .select({ id: location.id, location: location.location }) + .from(location) + .where( + or( + ...newLocations.map( + (newLocation) => + sql`ST_EQUALS( + ${location.location}, + ST_SetSRID(ST_MakePoint(${newLocation.lng}, ${newLocation.lat}), 4326) + )`, + ), + ), + ) + + foundLocations = existingLocations.map((location) => { + return { + lng: location.location.x, + lat: location.location.y, + height: undefined, + id: location.id, + } + }) + + const toInsert = newLocations.filter( + (newLocation) => !foundLocationsContain(foundLocations, newLocation), + ) + + const inserted = + toInsert.length > 0 + ? await tx + .insert(location) + .values( + toInsert.map((newLocation) => { + return { + location: sql`ST_SetSRID(ST_MakePoint(${newLocation.lng}, ${newLocation.lat}), 4326)`, + } + }), + ) + .returning() + : [] + + inserted.forEach((value) => + foundLocations.push({ + lng: value.location.x, + lat: value.location.y, + height: undefined, + id: value.id, + }), + ) + }) + + return foundLocations +} + +export function foundLocationsContain( + foundLocations: LocationWithId[], + location: Location, +): boolean { + return foundLocations.some((found) => foundLocationEquals(found, location)) +} + +export function foundLocationsGet( + foundLocations: LocationWithId[], + location: Location, +): bigint | undefined { + return foundLocations.find((found) => foundLocationEquals(found, location)) + ?.id +} + +function foundLocationEquals( + foundLocation: LocationWithId, + location: Location, +): boolean { + return ( + foundLocation.lat === location.lat && foundLocation.lng === location.lng + ) +} + +/** + * Filters the location updates to not add older updates than the newest already existing one, + * then inserts the filtered location updates + * @param deviceLocationUpdates The updates to add + * @param deviceId The device ID the updates are referring to + * @param locations The found locations with the IDs of the locations already in the database + */ +export async function addLocationUpdates( + deviceLocationUpdates: DeviceLocationUpdate[], + deviceId: string, + locations: LocationWithId[], +) { + await drizzleClient.transaction(async (tx) => { + let filteredUpdates = await filterLocationUpdates( + deviceLocationUpdates, + deviceId, + tx, + ) + + filteredUpdates + .filter((update) => !foundLocationsContain(locations, update.location)) + .forEach((update) => { + throw new Error(`Location ID for location ${update.location} not found, + even though it should've been inserted`) + }) + + if (filteredUpdates.length > 0) + await tx + .insert(deviceToLocation) + .values( + filteredUpdates.map((update) => { + return { + deviceId: deviceId, + locationId: foundLocationsGet( + locations, + update.location, + ) as bigint, + time: update.time, + } + }), + ) + .onConflictDoNothing() + }) +} + +/** + * Filters out location updates that don't need to be inserted because they're older than the latest update + * @param deviceLocationUpdates The device location updates to filter through + */ +export async function filterLocationUpdates( + deviceLocationUpdates: DeviceLocationUpdate[], + deviceId: string, + tx: any, +): Promise { + const currentLatestLocation = await tx + .select({ time: deviceToLocation.time }) + .from(deviceToLocation) + .where(eq(deviceToLocation.deviceId, deviceId)) + .orderBy(desc(deviceToLocation.time)) + .limit(1) + + return deviceLocationUpdates.filter( + (update) => + currentLatestLocation.length === 0 || + update.time >= currentLatestLocation[0].time, + ) +} + +/** + * Inserts measurements with their evaluated locations (either from the explicit location, which is assumed to already be + * in the location map), or from the last device location at the measurement time. + * @param measurements The measurements to insert + * @param locations The locations with the location IDs for the explicit locations + * @param deviceId The devices ID for the measurements + * @param tx The current transaction to run the insert SQLs in + */ +export async function insertMeasurementsWithLocation( + measurements: MeasurementWithLocation[], + locations: LocationWithId[], + deviceId: string, + tx: any, +): Promise { + const measuresWithLocationId = measurements.map((measurement) => { + const measurementTime = measurement.createdAt || new Date() + return { + sensorId: measurement.sensor_id, + value: measurement.value, + time: measurementTime, + locationId: measurement.location + ? foundLocationsGet(locations, measurement.location) + : sql`(select ${deviceToLocation.locationId} + from ${deviceToLocation} + where ${deviceToLocation.deviceId} = ${deviceId} + and ${deviceToLocation.time} <= ${measurementTime.toISOString()} + order by ${deviceToLocation.time} desc + limit 1)`, + } + }) + + // Insert measurements with locationIds (may be null for measurements + // without location and before any device location was set) + return measuresWithLocationId.length > 0 + ? await tx + .insert(measurement) + .values(measuresWithLocationId) + .onConflictDoNothing() + .returning() + : [] +} + +/** + * Updates the last measurement values for all given sensors + * @param lastMeasurements The measurements to update, including the sensor keys as values + * @param tx The current transaction to execute the update in + */ +export async function updateLastMeasurements( + lastMeasurements: Record>, + tx: any, +) { + const sqlChunks: SQL[] = [ + sql`(case`, + ...Object.entries(lastMeasurements) + .map(([sensorId, lastMeasurement]) => [ + sql`when ${sensor.id} = ${sensorId} then`, + sql`${JSON.stringify(lastMeasurement)}::json`, + ]) + .flat(), + sql`end)`, + ] + + const finalSql: SQL = sql.join(sqlChunks, sql.raw(' ')) + + await tx + .update(sensor) + .set({ lastMeasurement: finalSql }) + .where(inArray(sensor.id, Object.keys(lastMeasurements))) +} diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index eb148279..60fc841d 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,7 +1,7 @@ version: "3.7" services: postgres: - image: timescale/timescaledb-ha:pg14-latest + image: timescale/timescaledb-ha:pg18 # Preload pg_cron extension # https://github.com/timescale/timescaledb-docker-ha/issues/293 # Extension is created with prisma diff --git a/docker-compose.yml b/docker-compose.yml index f7f1b967..6916e8d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,6 @@ services: postgres: - # image: timescale/timescaledb:2.10.2-pg14 - image: timescale/timescaledb-ha:pg15 + image: timescale/timescaledb-ha:pg18 # Preload pg_cron extension # https://github.com/timescale/timescaledb-docker-ha/issues/293 # Extension is created with prisma diff --git a/tests/request-parsing.spec.ts b/tests/request-parsing.spec.ts index e4ed0410..840e55db 100644 --- a/tests/request-parsing.spec.ts +++ b/tests/request-parsing.spec.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect } from 'vitest'; import { parseRequestData, parseUserRegistrationData, parseUserSignInData, parseRefreshTokenData } from '~/lib/request-parsing'; describe('parseRequestData', () => { diff --git a/tests/routes/api.boxes.$deviceId.locations.spec.ts b/tests/routes/api.boxes.$deviceId.locations.spec.ts index fb5d8d85..42af89e6 100644 --- a/tests/routes/api.boxes.$deviceId.locations.spec.ts +++ b/tests/routes/api.boxes.$deviceId.locations.spec.ts @@ -1,6 +1,8 @@ +import { or, sql } from 'drizzle-orm' import { type Params, type LoaderFunctionArgs } from 'react-router' import { generateTestUserCredentials } from 'tests/data/generate_test_user' import { BASE_URL } from 'vitest.setup' +import { drizzleClient } from '~/db.server' import { registerUser } from '~/lib/user-service.server' import { createDevice, deleteDevice } from '~/models/device.server' import { @@ -11,7 +13,7 @@ import { import { getSensors } from '~/models/sensor.server' import { deleteUserByEmail } from '~/models/user.server' import { loader } from '~/routes/api.boxes.$deviceId.locations' -import { type Sensor, type Device, type User } from '~/schema' +import { type Sensor, type User, location } from '~/schema' const DEVICE_SENSORS_ID_USER = generateTestUserCredentials() @@ -50,9 +52,9 @@ const MEASUREMENTS = [ createdAt: new Date('1954-06-07 12:00:00+00'), sensor_id: '', location: { - lng: 1, - lat: 2, - height: 3, + lng: -179.65, + lat: 89.76, + height: 3.123, }, }, { @@ -60,8 +62,8 @@ const MEASUREMENTS = [ createdAt: new Date('1988-03-14 1:59:26+00'), sensor_id: '', location: { - lng: 4, - lat: 5, + lng: 56.78123, + lat: 32.198, }, }, { @@ -69,15 +71,15 @@ const MEASUREMENTS = [ createdAt: new Date('2000-05-25 11:11:11+00'), sensor_id: '', location: { - lng: 6, - lat: 7, - height: 8, + lng: 123.456789, + lat: 0.12345, + height: 8.71, }, }, ] describe('openSenseMap API Routes: /api/boxes/:deviceId/locations', () => { - let device: Device + let device let deviceId: string = '' let sensors: Sensor[] @@ -123,10 +125,10 @@ describe('openSenseMap API Routes: /api/boxes/:deviceId/locations', () => { expect(body).toHaveLength(2) expect(body[0].coordinates).toHaveLength(2) expect(body[1].coordinates).toHaveLength(2) - expect(body[0].coordinates[0]).toBe(4) - expect(body[0].coordinates[1]).toBe(5) - expect(body[1].coordinates[0]).toBe(1) - expect(body[1].coordinates[1]).toBe(2) + expect(body[0].coordinates[0]).toBe(56.78123) + expect(body[0].coordinates[1]).toBe(32.198) + expect(body[1].coordinates[0]).toBe(-179.65) + expect(body[1].coordinates[1]).toBe(89.76) expect(body[0].type).toBe('Point') expect(body[1].type).toBe('Point') expect(body[0].timestamp).toBe('1988-03-14T01:59:26.000Z') @@ -160,10 +162,10 @@ describe('openSenseMap API Routes: /api/boxes/:deviceId/locations', () => { expect(body.geometry.coordinates).toHaveLength(2) expect(body.geometry.coordinates[0]).toHaveLength(2) expect(body.geometry.coordinates[1]).toHaveLength(2) - expect(body.geometry.coordinates[0][0]).toBe(4) - expect(body.geometry.coordinates[0][1]).toBe(5) - expect(body.geometry.coordinates[1][0]).toBe(1) - expect(body.geometry.coordinates[1][1]).toBe(2) + expect(body.geometry.coordinates[0][0]).toBe(56.78123) + expect(body.geometry.coordinates[0][1]).toBe(32.198) + expect(body.geometry.coordinates[1][0]).toBe(-179.65) + expect(body.geometry.coordinates[1][1]).toBe(89.76) expect(body.properties.timestamps).toHaveLength(2) expect(body.properties.timestamps[0]).toBe('1988-03-14T01:59:26.000Z') expect(body.properties.timestamps[1]).toBe('1954-06-07T12:00:00.000Z') @@ -183,5 +185,21 @@ describe('openSenseMap API Routes: /api/boxes/:deviceId/locations', () => { await deleteUserByEmail(DEVICE_SENSORS_ID_USER.email) // delete the box await deleteDevice({ id: deviceId }) + // delete created locations + await deleteLocations() }) }) + +async function deleteLocations() { + await drizzleClient.delete(location).where( + or( + ...MEASUREMENTS.map( + (measurement) => + sql`ST_EQUALS( + ${location.location}, + ST_SetSRID(ST_MakePoint(${measurement.location.lng}, ${measurement.location.lat}), 4326) + )`, + ), + ), + ) +} diff --git a/tests/utils/measurement-server-helper.spec.ts b/tests/utils/measurement-server-helper.spec.ts new file mode 100644 index 00000000..8cdf8b02 --- /dev/null +++ b/tests/utils/measurement-server-helper.spec.ts @@ -0,0 +1,351 @@ +import { asc, eq, inArray, or, sql } from 'drizzle-orm' +import { generateTestUserCredentials } from 'tests/data/generate_test_user' +import { drizzleClient } from '~/db.server' +import { registerUser } from '~/lib/user-service.server' +import { createDevice, deleteDevice } from '~/models/device.server' +import { + deleteMeasurementsForSensor, + deleteMeasurementsForTime, +} from '~/models/measurement.server' +import { getSensors } from '~/models/sensor.server' +import { deleteUserByEmail } from '~/models/user.server' +import { deviceToLocation, location } from '~/schema' +import { type LastMeasurement, sensor, type Sensor } from '~/schema/sensor' +import { type User } from '~/schema/user' +import { + addLocationUpdates, + filterLocationUpdates, + findOrCreateLocations, + foundLocationsContain, + foundLocationsGet, + getLocationUpdates, + insertMeasurementsWithLocation, + type Location, + type LocationWithId, + updateLastMeasurements, +} from '~/utils/measurement-server-helper' + +const DEVICE_SENSORS_ID_USER = generateTestUserCredentials() + +const DEVICE_SENSOR_ID_BOX = { + name: `${DEVICE_SENSORS_ID_USER}s Box`, + exposure: 'outdoor', + expiresAt: null, + tags: [], + latitude: 0, + longitude: 0, + model: 'luftdaten.info', + mqttEnabled: false, + ttnEnabled: false, + sensors: [ + { + title: 'Temp', + unit: '°C', + sensorType: 'dummy', + }, + { + title: 'CO2', + unit: 'mol/L', + sensorType: 'dummy', + }, + { + title: 'Air Pressure', + unit: 'kPa', + sensorType: 'dummy', + }, + ], +} +const locations: LocationWithId[] = [ + { + lng: 1, + lat: 2, + height: 3, + id: BigInt(4), + }, + { + lng: 5, + lat: 6, + id: BigInt(7), + }, +] + +const containedLocation: Location = { + lng: 1, + lat: 2, + height: 4, +} + +const otherContainedLocation: Location = { + lng: 5, + lat: 6, +} + +const notContainedLocation: Location = { + lng: 1, + lat: 3, +} + +describe('measurement server helper', () => { + let device + let deviceId: string = '' + let sensors: Sensor[] + + let foundOrCreatedLocations: LocationWithId[] + let locationIds: bigint[] = [] + + const MEASUREMENTS = [ + { + value: 3.14159, + createdAt: new Date('1988-03-14 1:59:26+00'), + sensor_id: '', + location: { + lng: -180, + lat: -90, + }, + }, + { + value: 1589625, + createdAt: new Date('1954-06-07 12:00:00+00'), + sensor_id: '', + location: { + lng: 90, + lat: 45, + height: 3.123, + }, + }, + { + value: 0, + createdAt: new Date('2000-05-25 11:11:11+00'), + sensor_id: '', + }, + ] + + beforeAll(async () => { + const user = await registerUser( + DEVICE_SENSORS_ID_USER.name, + DEVICE_SENSORS_ID_USER.email, + DEVICE_SENSORS_ID_USER.password, + 'en_US', + ) + + device = await createDevice(DEVICE_SENSOR_ID_BOX, (user as User).id) + deviceId = device.id + sensors = await getSensors(deviceId) + + MEASUREMENTS.forEach((meas) => (meas.sensor_id = sensors[0].id)) + }) + + it('should get location updates', async () => { + const result = getLocationUpdates(MEASUREMENTS) + + // Check filtering + expect(result).toHaveLength(2) + + // Check ordering + expect(result[0].time).toEqual(new Date('1954-06-07 12:00:00+00')) + expect(result[1].time).toEqual(new Date('1988-03-14 1:59:26+00')) + }) + + it('should find or create locations', async () => { + // Create one location to already exist + const inserted = await drizzleClient + .insert(location) + .values({ + location: sql`ST_SetSRID(ST_MakePoint(${90}, ${45}), 4326)`, + }) + .returning() + + expect(inserted).toHaveLength(1) + const insertedId = inserted[0].id + locationIds.push(insertedId) + + // Call function + const result = await findOrCreateLocations(getLocationUpdates(MEASUREMENTS)) + foundOrCreatedLocations = result + + // Check locations + expect(result).toHaveLength(2) + expect( + result.some((location) => location.lng == -180 && location.lat == -90), + ).toBeTruthy() + expect( + result.some((location) => location.lng == 90 && location.lat == 45), + ).toBeTruthy() + + // Check that location isn't inserted twice + expect( + result.find((location) => location.lng == 90 && location.lat == 45)?.id, + ).toBe(insertedId) + + // Check that locations are actually inserted + const otherId = result.find( + (location) => location.lng == -180 && location.lat == -90, + )?.id as bigint + locationIds.push(otherId) + const databaseEntry = await drizzleClient + .select() + .from(location) + .where(eq(location.id, otherId)) + + expect(databaseEntry).toHaveLength(1) + expect(databaseEntry[0].location.x).toBe(-180) + expect(databaseEntry[0].location.y).toBe(-90) + }) + + it('should identify if found locations contain', () => { + const result1 = foundLocationsContain(locations, containedLocation) + const result2 = foundLocationsContain(locations, otherContainedLocation) + const result3 = foundLocationsContain(locations, notContainedLocation) + + expect(result1).toBeTruthy() + expect(result2).toBeTruthy() + expect(result3).toBeFalsy() + }) + + it('should get found locations', () => { + const result1 = foundLocationsGet(locations, containedLocation) + const result2 = foundLocationsGet(locations, otherContainedLocation) + const result3 = foundLocationsGet(locations, notContainedLocation) + + expect(result1).toBe(BigInt(4)) + expect(result2).toBe(BigInt(7)) + expect(result3).toBeUndefined() + }) + + it('should filter location updates', async () => { + // Add one location update so that the oldest one gets filtered out + await drizzleClient.insert(deviceToLocation).values({ + deviceId: deviceId, + locationId: locationIds[0], + time: new Date('1970-01-01'), + }) + + await drizzleClient.transaction(async (tx) => { + const result = await filterLocationUpdates( + getLocationUpdates(MEASUREMENTS), + deviceId, + tx, + ) + + expect(result).toHaveLength(1) + // The filtered udpates should only include the newer one + expect(result[0].time).toEqual(new Date('1988-03-14 1:59:26+00')) + }) + }) + + it('should add location updates', async () => { + await addLocationUpdates( + getLocationUpdates(MEASUREMENTS), + deviceId, + foundOrCreatedLocations, + ) + + const inserted = await drizzleClient + .select() + .from(deviceToLocation) + .where(eq(deviceToLocation.deviceId, deviceId)) + .orderBy(asc(deviceToLocation.time)) + + // One location update was already added in the previous test, + // one more should have been added by the tested function + expect(inserted).toHaveLength(2) + expect(inserted[1].locationId).toBe(locationIds[1]) + expect(inserted[1].time).toEqual(new Date('1988-03-14 1:59:26+00')) + }) + + it('should inserted measurements with location', async () => { + await drizzleClient.transaction(async (tx) => { + const result = await insertMeasurementsWithLocation( + MEASUREMENTS, + foundOrCreatedLocations, + deviceId, + tx, + ) + + // All 3 should have been inserted + expect(result).toHaveLength(3) + + // Location IDs should have been fetched + // The locationIds[1] is what was inserted in a previous test + expect(result[0].locationId).toBe(locationIds[1]) + // The locationIds[0] is what was created beforehand for test reasons + expect(result[1].locationId).toBe(locationIds[0]) + // The last measurement didn't have a location, + // the latest location ID is what was created by the previous test + expect(result[2].locationId).toBe(locationIds[1]) + + // The other values should be as expected + expect(result[0].sensorId).toBe(sensors[0].id) + expect(result[1].sensorId).toBe(sensors[0].id) + expect(result[2].sensorId).toBe(sensors[0].id) + expect(result[0].time).toEqual(new Date('1988-03-14 1:59:26+00')) + expect(result[1].time).toEqual(new Date('1954-06-07 12:00:00+00')) + expect(result[2].time).toEqual(new Date('2000-05-25 11:11:11+00')) + expect(result[0].value).toBe(3.14159) + expect(result[1].value).toBe(1589625) + expect(result[2].value).toBe(0) + }) + }) + + it('should update last measurements', async () => { + const lastMeasurements: Record> = {} + // This is true + lastMeasurements[sensors[0].id] = { + value: 0, + createdAt: new Date('2000-05-25 11:11:11+00').toISOString(), + sensorId: sensors[0].id, + } + // This is made up, but should (currently) still work + lastMeasurements[sensors[1].id] = { + value: 42, + createdAt: new Date('1954-06-07 12:00:00+00').toISOString(), + sensorId: sensors[1].id, + } + + await drizzleClient.transaction(async (tx) => { + await updateLastMeasurements(lastMeasurements, tx) + }) + + const results = await drizzleClient + .select() + .from(sensor) + .where(inArray(sensor.id, [sensors[0].id, sensors[1].id])) + + expect(results).toHaveLength(2) + expect( + results.find((sensor) => sensor.id == sensors[0].id)?.lastMeasurement, + ).toEqual(lastMeasurements[sensors[0].id]) + expect( + results.find((sensor) => sensor.id == sensors[1].id)?.lastMeasurement, + ).toEqual(lastMeasurements[sensors[1].id]) + }) + + afterAll(async () => { + //delete measurements + await deleteMeasurementsForSensor(sensors[0].id) + MEASUREMENTS.forEach( + async (measurement) => + await deleteMeasurementsForTime(measurement.createdAt), + ) + // delete the valid test user + await deleteUserByEmail(DEVICE_SENSORS_ID_USER.email) + // delete the box + await deleteDevice({ id: deviceId }) + // delete created locations + await deleteLocations() + }) + + async function deleteLocations() { + await drizzleClient.delete(location).where( + or( + ...MEASUREMENTS.filter((measurement) => measurement.location).map( + (measurement) => + sql`ST_EQUALS( + ${location.location}, + ST_SetSRID(ST_MakePoint(${measurement.location?.lng}, ${measurement.location?.lat}), 4326) + )`, + ), + ), + ) + } +})