diff --git a/server/package.json b/server/package.json index 516626f..6c982cc 100644 --- a/server/package.json +++ b/server/package.json @@ -13,14 +13,14 @@ }, "dependencies": { "async-mutex": "^0.4.0", - "better-sqlite3": "^9.5.0", - "kysely": "^0.26.1", + "better-sqlite3": "^11.10.0", + "kysely": "^0.28.2", "source-map-support": "^0.5.21", "zod": "^3.21.4", "zod-validation-error": "^1.3.1" }, "devDependencies": { - "@types/better-sqlite3": "^7.6.4", + "@types/better-sqlite3": "^7.6.13", "@types/node": "^18.17.4", "dotenv": "^16.0.1", "prettier": "^3.0.1", diff --git a/server/src/Renderer.ts b/server/src/Renderer.ts index 3d6f603..09380b4 100644 --- a/server/src/Renderer.ts +++ b/server/src/Renderer.ts @@ -1,6 +1,6 @@ import { spawn } from "child_process"; import { promisify } from "util"; -import * as database from "./database"; +import * as database from "./db/database"; export async function renderTile( dimension: string, @@ -25,8 +25,8 @@ export async function renderTile( const chunkHeaderBuf = Buffer.allocUnsafe(4 + 4 + 2); // reused. 32+32+16 bit for (const chunk of allChunks) { - chunkHeaderBuf.writeInt32BE(chunk.chunk_x, 0); - chunkHeaderBuf.writeInt32BE(chunk.chunk_z, 4); + chunkHeaderBuf.writeInt32BE(chunk.chunkX, 0); + chunkHeaderBuf.writeInt32BE(chunk.chunkZ, 4); chunkHeaderBuf.writeUInt16BE(chunk.version, 8); await write(chunkHeaderBuf); await write(chunk.data); diff --git a/server/src/database.ts b/server/src/database.ts deleted file mode 100644 index 0b073c0..0000000 --- a/server/src/database.ts +++ /dev/null @@ -1,219 +0,0 @@ -import * as kysely from "kysely"; -import sqlite from "better-sqlite3"; -import { DATA_FOLDER } from "./metadata"; -import { type Pos2D } from "./model"; - -let database: kysely.Kysely | null = null; - -export interface Database { - chunk_data: { - hash: Buffer; - version: number; - data: Buffer; - }; - player_chunk: { - world: string; - chunk_x: number; - chunk_z: number; - uuid: string; - ts: number; - hash: Buffer; - }; -} - -export function get() { - if (!database) { - database = new kysely.Kysely({ - dialect: new kysely.SqliteDialect({ - database: async () => - sqlite( - process.env["SQLITE_PATH"] ?? - `${DATA_FOLDER}/db.sqlite`, - {}, - ), - }), - }); - } - return database; -} - -export async function setup() { - await get() - .schema.createTable("chunk_data") - .ifNotExists() - .addColumn("hash", "blob", (col) => col.notNull().primaryKey()) - .addColumn("version", "integer", (col) => col.notNull()) - .addColumn("data", "blob", (col) => col.notNull()) - .execute(); - await get() - .schema.createTable("player_chunk") - .ifNotExists() - .addColumn("world", "text", (col) => col.notNull()) - .addColumn("chunk_x", "integer", (col) => col.notNull()) - .addColumn("chunk_z", "integer", (col) => col.notNull()) - .addColumn("uuid", "text", (col) => col.notNull()) - .addColumn("ts", "bigint", (col) => col.notNull()) - .addColumn("hash", "blob", (col) => col.notNull()) - .addPrimaryKeyConstraint("PK_coords_and_player", [ - "world", - "chunk_x", - "chunk_z", - "uuid", - ]) - .addForeignKeyConstraint( - "FK_chunk_ref", - ["hash"], - "chunk_data", - ["hash"], - (fk) => fk.onUpdate("no action").onDelete("no action"), - ) - .execute(); -} - -/** - * Converts the entire database of player chunks into regions, with each region - * having the highest (aka newest) timestamp. - */ -export function getRegionTimestamps(dimension: string) { - // computing region coordinates in SQL requires truncating, not rounding - return get() - .selectFrom("player_chunk") - .select([ - (eb) => - kysely.sql`floor(${eb.ref("chunk_x")} / 32.0)`.as( - "regionX", - ), - (eb) => - kysely.sql`floor(${eb.ref("chunk_z")} / 32.0)`.as( - "regionZ", - ), - (eb) => eb.fn.max("ts").as("timestamp"), - ]) - .where("world", "=", dimension) - .groupBy(["regionX", "regionZ"]) - .orderBy("regionX", "desc") - .execute(); -} - -/** - * Converts an array of region coords into an array of timestamped chunk coords. - */ -export async function getChunkTimestamps(dimension: string, regions: Pos2D[]) { - return get() - .with("regions", (db) => - db - .selectFrom("player_chunk") - .select([ - (eb) => - kysely.sql`(cast(floor(${eb.ref( - "chunk_x", - )} / 32.0) as int) || '_' || cast(floor(${eb.ref( - "chunk_z", - )} / 32.0) as int))`.as("region"), - "chunk_x as x", - "chunk_z as z", - (eb) => eb.fn.max("ts").as("timestamp"), - ]) - .where("world", "=", dimension) - .groupBy(["x", "z"]), - ) - .selectFrom("regions") - .select(["x as chunkX", "z as chunkZ", "timestamp"]) - .where( - "region", - "in", - regions.map((region) => region.x + "_" + region.z), - ) - .orderBy("timestamp", "desc") - .execute(); -} - -/** - * Retrieves the data for a given chunk's world, x, z, and timestamp. - * - * TODO: May want to consider making world, x, z, and timestamp a unique in the - * database table... may help performance. - */ -export async function getChunkData( - dimension: string, - chunkX: number, - chunkZ: number, -) { - return get() - .selectFrom("player_chunk") - .innerJoin("chunk_data", "chunk_data.hash", "player_chunk.hash") - .select([ - "chunk_data.hash as hash", - "chunk_data.version as version", - "chunk_data.data as data", - "player_chunk.ts as ts", - ]) - .where("player_chunk.world", "=", dimension) - .where("player_chunk.chunk_x", "=", chunkX) - .where("player_chunk.chunk_z", "=", chunkZ) - .orderBy("player_chunk.ts", "desc") - .limit(1) - .executeTakeFirst(); -} - -/** - * Stores a player's chunk data. - */ -export async function storeChunkData( - dimension: string, - chunkX: number, - chunkZ: number, - uuid: string, - timestamp: number, - version: number, - hash: Buffer, - data: Buffer, -) { - await get() - .insertInto("chunk_data") - .values({ hash, version, data }) - .onConflict((oc) => oc.column("hash").doNothing()) - .execute(); - await get() - .replaceInto("player_chunk") - .values({ - world: dimension, - chunk_x: chunkX, - chunk_z: chunkZ, - uuid, - ts: timestamp, - hash, - }) - .execute(); -} - -/** - * Gets all the [latest] chunks within a region. - */ -export async function getRegionChunks( - dimension: string, - regionX: number, - regionZ: number, -) { - const minChunkX = regionX << 4, - maxChunkX = minChunkX + 16; - const minChunkZ = regionZ << 4, - maxChunkZ = minChunkZ + 16; - return get() - .selectFrom("player_chunk") - .innerJoin("chunk_data", "chunk_data.hash", "player_chunk.hash") - .select([ - "player_chunk.chunk_x as chunk_x", - "player_chunk.chunk_z as chunk_z", - (eb) => eb.fn.max("player_chunk.ts").as("timestamp"), - "chunk_data.version as version", - "chunk_data.data as data", - ]) - .where("player_chunk.world", "=", dimension) - .where("player_chunk.chunk_x", ">=", minChunkX) - .where("player_chunk.chunk_x", "<", maxChunkX) - .where("player_chunk.chunk_z", ">=", minChunkZ) - .where("player_chunk.chunk_z", "<", maxChunkZ) - .orderBy("player_chunk.ts", "desc") - .execute(); -} diff --git a/server/src/db/database.ts b/server/src/db/database.ts new file mode 100644 index 0000000..3e2f8bd --- /dev/null +++ b/server/src/db/database.ts @@ -0,0 +1,195 @@ +import { Kysely, SqliteDialect, type Generated, Migrator } from "kysely"; +import sqlite from "better-sqlite3"; + +import { DATA_FOLDER } from "../metadata"; +import Migrations from "./migrations"; +import { type Pos2D } from "../model"; + +let database: Kysely | null = null; + +export interface Database { + chunk_data: { + hash: Buffer; + version: number; + data: Buffer; + }; + player_chunk: { + world: string; + chunk_x: number; + chunk_z: number; + gen_region_x: Generated; + gen_region_z: Generated; + gen_region_coord: Generated; + uuid: string; + ts: number; + hash: Buffer; + }; +} + +export function get() { + return (database ??= new Kysely({ + dialect: new SqliteDialect({ + database: async () => + sqlite( + process.env["SQLITE_PATH"] ?? `${DATA_FOLDER}/db.sqlite`, + ), + }), + })); +} + +export function getMigrations(): Migrator { + return new Migrator({ + db: get(), + provider: new Migrations(), + }); +} + +/** Convenience function to migrate to latest */ +export async function setup() { + const results = await getMigrations().migrateToLatest(); + for (const result of (results.results ?? [])) { + switch (result.status) { + case "Success": + console.info(`Migration [${result.migrationName}] applied!`); + break; + case "Error": + console.error( + `Migration [${result.migrationName}] failed to apply!`, + ); + break; + case "NotExecuted": + console.warn( + `Migration [${result.migrationName}] was not applied!`, + ); + break; + } + } + if (results.error) { + throw results.error; + } +} + +/** + * Gets the timestamps for ALL regions stored. + */ +export async function getRegionTimestamps(dimension: string) { + return await get() + .selectFrom("player_chunk") + .select([ + "gen_region_x as regionX", + "gen_region_z as regionZ", + (eb) => eb.fn.max("ts").as("timestamp"), + ]) + .where("world", "=", dimension) + .groupBy(["regionX", "regionZ"]) + .orderBy("timestamp", "asc") + .execute(); +} + +export async function getChunkTimestamps(dimension: string, regions: Pos2D[]) { + return await get() + .selectFrom("player_chunk") + .select([ + "chunk_x as chunkX", + "chunk_z as chunkZ", + (eb) => eb.fn.max("ts").as("timestamp"), + ]) + .where( + "gen_region_coord", + "in", + regions.map((region) => region.x + "_" + region.z), + ) + .where("world", "=", dimension) + .groupBy(["chunkX", "chunkZ"]) + .orderBy("timestamp", "desc") + .execute(); +} + +/** + * Retrieves the data for a given chunk's world, x, z, and timestamp. + * + * TODO: May want to consider making world, x, z, and timestamp a unique in the + * database table... may help performance. + */ +export async function getChunkData( + dimension: string, + chunkX: number, + chunkZ: number, +) { + return await get() + .selectFrom("player_chunk") + .innerJoin("chunk_data", "chunk_data.hash", "player_chunk.hash") + .select([ + "chunk_data.hash as hash", + "chunk_data.version as version", + "chunk_data.data as data", + "player_chunk.ts as ts", + ]) + .where("player_chunk.world", "=", dimension) + .where("player_chunk.chunk_x", "=", chunkX) + .where("player_chunk.chunk_z", "=", chunkZ) + .orderBy("player_chunk.ts", "desc") + .limit(1) + .executeTakeFirst(); +} + +/** + * Stores a player's chunk data. + */ +export async function storeChunkData( + dimension: string, + chunkX: number, + chunkZ: number, + uuid: string, + timestamp: number, + version: number, + hash: Buffer, + data: Buffer, +) { + await get() + .transaction() + .execute(async (transaction) => { + await transaction + .insertInto("chunk_data") + .values({ hash, version, data }) + .onConflict((oc) => oc.column("hash").doNothing()) + .execute(); + await transaction + .replaceInto("player_chunk") + .values({ + world: dimension, + chunk_x: chunkX, + chunk_z: chunkZ, + uuid, + ts: timestamp, + hash, + }) + .execute(); + }); +} + +/** + * Gets all the [latest] chunks within a region. + */ +export async function getRegionChunks( + dimension: string, + regionX: number, + regionZ: number, +) { + return await get() + .selectFrom("player_chunk") + .innerJoin("chunk_data", "chunk_data.hash", "player_chunk.hash") + .select([ + "player_chunk.chunk_x as chunkX", + "player_chunk.chunk_z as chunkZ", + (eb) => eb.fn.max("player_chunk.ts").as("timestamp"), + "chunk_data.version as version", + "chunk_data.data as data", + ]) + .where("player_chunk.world", "=", dimension) + .where("player_chunk.gen_region_x", "=", regionX) + .where("player_chunk.gen_region_z", "=", regionZ) + .groupBy(["chunkX", "chunkZ", "version", "data"]) + .orderBy("timestamp", "desc") + .execute(); +} diff --git a/server/src/db/migrations.ts b/server/src/db/migrations.ts new file mode 100644 index 0000000..494a192 --- /dev/null +++ b/server/src/db/migrations.ts @@ -0,0 +1,121 @@ +import { Kysely, sql, type Migration, type MigrationProvider } from "kysely"; + +type MigrationRegistry = Record; +type MigrationClass = { name: string } & (new () => Migration); + +export default class Migrations implements MigrationProvider { + public async getMigrations(): Promise { + return this.generateMigrationRegistry([ + Migration_0001_InitialSetup, + Migration_0002_GenerateRegionCoordColumns, + ]); + } + + private generateMigrationRegistry( + migrations: Array, + ): MigrationRegistry { + const registry: MigrationRegistry = {}; + for (const clazz of migrations) { + registry[clazz.name] = new clazz(); + } + return registry; + } +} + +// ============================================================ +// WARNING FOR WRITING MIGRATIONS! +// +// Kysely does not respect class functions: your "up" and "down" methods MUST +// be fields, not class functions, otherwise your migration will fail! +// +// I've named the migrations like "Migration_0001_InitialSetup" since they are +// applied in order of their names, in alphanumeric order. I'd name them +// "0001_InitialSetup" if I could, but alas. +// ============================================================ + +export class Migration_0001_InitialSetup implements Migration { + public up = async (db: Kysely) => { + await db.transaction().execute(async (transaction) => { + await transaction.schema + .createTable("chunk_data") + .ifNotExists() + .addColumn("hash", "blob", (col) => col.notNull().primaryKey()) + .addColumn("version", "integer", (col) => col.notNull()) + .addColumn("data", "blob", (col) => col.notNull()) + .execute(); + await transaction.schema + .createTable("player_chunk") + .ifNotExists() + .addColumn("world", "text", (col) => col.notNull()) + .addColumn("chunk_x", "integer", (col) => col.notNull()) + .addColumn("chunk_z", "integer", (col) => col.notNull()) + .addColumn("uuid", "text", (col) => col.notNull()) + .addColumn("ts", "bigint", (col) => col.notNull()) + .addColumn("hash", "blob", (col) => col.notNull()) + .addPrimaryKeyConstraint("PK_coords_and_player", [ + "world", + "chunk_x", + "chunk_z", + "uuid", + ]) + .addForeignKeyConstraint( + "FK_chunk_ref", + ["hash"], + "chunk_data", + ["hash"], + (fk) => fk.onUpdate("no action").onDelete("no action"), + ) + .execute(); + }); + }; + // Probably shouldn't define a "down" since that just means an empty db +} + +export class Migration_0002_GenerateRegionCoordColumns implements Migration { + public up = async (db: Kysely) => { + await db.transaction().execute(async (transaction) => { + await transaction.schema + .alterTable("player_chunk") + .addColumn("gen_region_x", "integer", (col) => { + return col + .generatedAlwaysAs(sql`floor(chunk_x / 32.0)`) + .notNull(); + }) + .execute(); + await transaction.schema + .alterTable("player_chunk") + .addColumn("gen_region_z", "integer", (col) => { + return col + .generatedAlwaysAs(sql`floor(chunk_z / 32.0)`) + .notNull(); + }) + .execute(); + await transaction.schema + .alterTable("player_chunk") + .addColumn("gen_region_coord", "text", (col) => { + return col + .generatedAlwaysAs( + sql`gen_region_x || '_' || gen_region_z`, + ) + .notNull(); + }) + .execute(); + }); + }; + public down = async (db: Kysely) => { + await db.transaction().execute(async (transaction) => { + await transaction.schema + .alterTable("player_chunk") + .dropColumn("gen_region_coord") + .execute(); + await transaction.schema + .alterTable("player_chunk") + .dropColumn("gen_region_x") + .execute(); + await transaction.schema + .alterTable("player_chunk") + .dropColumn("gen_region_z") + .execute(); + }); + }; +} diff --git a/server/src/main.ts b/server/src/main.ts index c082e3c..2b9dc05 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -1,5 +1,5 @@ import "./cli"; -import * as database from "./database"; +import * as database from "./db/database"; import * as metadata from "./metadata"; import { ClientPacket } from "./protocol"; import { CatchupRequestPacket } from "./protocol/CatchupRequestPacket"; diff --git a/server/yarn.lock b/server/yarn.lock index db76b2f..cff198b 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -2,10 +2,10 @@ # yarn lockfile v1 -"@types/better-sqlite3@^7.6.4": - version "7.6.4" - resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-7.6.4.tgz#102462611e67aadf950d3ccca10292de91e6f35b" - integrity sha512-dzrRZCYPXIXfSR1/surNbJ/grU3scTaygS0OMzjlGf71i9sc2fGyHPXXiXmEvNIoE0cGwsanEFMVJxPXmco9Eg== +"@types/better-sqlite3@^7.6.13": + version "7.6.13" + resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz#a72387f00d2f53cab699e63f2e2c05453cf953f0" + integrity sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA== dependencies: "@types/node" "*" @@ -31,10 +31,10 @@ base64-js@^1.3.1: resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -better-sqlite3@^9.5.0: - version "9.6.0" - resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-9.6.0.tgz#b01e58ba7c48abcdc0383b8301206ee2ab81d271" - integrity sha512-yR5HATnqeYNVnkaUTf4bOP2dJSnyhP4puJN/QPRyx4YkBEEUxib422n2XzPqDEHjQQqazoYoADdAm5vE15+dAQ== +better-sqlite3@^11.10.0: + version "11.10.0" + resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-11.10.0.tgz#2b1b14c5acd75a43fd84d12cc291ea98cef57d98" + integrity sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ== dependencies: bindings "^1.5.0" prebuild-install "^7.1.1" @@ -137,10 +137,10 @@ ini@~1.3.0: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== -kysely@^0.26.1: - version "0.26.1" - resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.26.1.tgz#2d2fb9316d53f3062596102c98d0d476e4e097b5" - integrity sha512-FVRomkdZofBu3O8SiwAOXrwbhPZZr8mBN5ZeUWyprH29jzvy6Inzqbd0IMmGxpd4rcOCL9HyyBNWBa8FBqDAdg== +kysely@^0.28.2: + version "0.28.2" + resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.28.2.tgz#1a4ff83cedf2b259a151ab9c42ad28bcac6c2f98" + integrity sha512-4YAVLoF0Sf0UTqlhgQMFU9iQECdah7n+13ANkiuVfRvlK+uI0Etbgd7bVP36dKlG+NXWbhGua8vnGt+sdhvT7A== lru-cache@^6.0.0: version "6.0.0"