Feature/get range (#283)

* first try least impacting effort

* did not mean to be abstract

* improve error checking for windows support

* improve error checking for windows support

* improve error checking for windows support

* nits

* nits part 2

Author: CraigglesO

packages/shared/src/storage.ts

@@ -78,6 +78,13 @@ export abstract class Storage {
     skipMetadata: true
   ): Awaitable<StorageListResult<StoredKey>>;
 
+  // Implementations (e.g. storage-file) may override this for efficient range requests
+  async getRangeMaybeExpired?<Meta = unknown>(
+    key: string,
+    start: number,
+    length: number
+  ): Promise<StoredValueMeta<Meta> | undefined>;
+
   // Batch functions, default implementations may be overridden to optimise
 
   async hasMany(keys: string[]): Promise<number> {

packages/storage-file/src/helpers.ts

@@ -1,11 +1,13 @@
 import fs from "fs/promises";
 import path from "path";
 
-function onNotFound<T, V>(promise: Promise<T>, value: V): Promise<T | V> {
-  return promise.catch((e) => {
+async function onNotFound<T, V>(promise: Promise<T>, value: V): Promise<T | V> {
+  try {
+    return await promise;
+  } catch (e: any) {
     if (e.code === "ENOENT") return value;
     throw e;
-  });
+  }
 }
 
 export function readFile(filePath: string): Promise<Buffer | undefined>;
@@ -20,6 +22,26 @@ export function readFile(
   return onNotFound(fs.readFile(filePath, decode && "utf8"), undefined);
 }
 
+export async function readFileRange(
+  filePath: string,
+  start: number,
+  length: number
+): Promise<Buffer | undefined> {
+  let fd: fs.FileHandle | null = null;
+  let res: Buffer;
+  try {
+    fd = await fs.open(filePath, "r");
+    res = Buffer.alloc(length);
+    await fd.read(res, 0, length, start);
+  } catch (e: any) {
+    if (e.code === "ENOENT") return undefined;
+    throw e;
+  } finally {
+    await fd?.close();
+  }
+  return res;
+}
+
 export async function writeFile(
   filePath: string,
   data: Uint8Array | string

packages/storage-file/src/index.ts

@@ -10,7 +10,13 @@ import {
   viewToArray,
 } from "@miniflare/shared";
 import { LocalStorage } from "@miniflare/storage-memory";
-import { deleteFile, readFile, walk, writeFile } from "./helpers";
+import {
+  deleteFile,
+  readFile,
+  readFileRange,
+  walk,
+  writeFile,
+} from "./helpers";
 
 const metaSuffix = ".meta.json";
 
@@ -85,6 +91,33 @@ export class FileStorage extends LocalStorage {
     }
   }
 
+  async getRangeMaybeExpired<Meta = unknown>(
+    key: string,
+    start: number,
+    length: number
+  ): Promise<StoredValueMeta<Meta> | undefined> {
+    const [filePath] = this.keyPath(key);
+    if (!filePath) return;
+
+    try {
+      const value = await readFileRange(filePath, start, length);
+
+      if (value === undefined) return;
+      const meta = await this.meta<Meta>(filePath);
+      return {
+        value: viewToArray(value),
+        expiration: meta.expiration,
+        metadata: meta.metadata,
+      };
+    } catch (e: any) {
+      // We'll get this error if we try to get a namespaced key, where the
+      // namespace itself is also a key (e.g. trying to get "key/sub-key" where
+      // "key" is also a key). In this case, "key/sub-key" doesn't exist.
+      if (e.code === "ENOTDIR") return;
+      throw e;
+    }
+  }
+
   async put<Meta = unknown>(
     key: string,
     { value, expiration, metadata }: StoredValueMeta<Meta>

packages/storage-file/test/index.spec.ts

@@ -67,6 +67,33 @@ test("FileStorage: list: returns original keys if sanitised", async (t) => {
     cursor: "",
   });
 });
+test("FileStorage: getRangeMaybeExpired: returns partial values", async (t) => {
+  const storage = await storageFactory.factory(t, {});
+  await storage.put("key", { value: utf8Encode("123456789") });
+
+  const getFront = await storage.getRangeMaybeExpired?.("key", 0, 3);
+  t.is(utf8Decode(getFront?.value), "123");
+  const getBack = await storage.getRangeMaybeExpired?.("key", 6, 3);
+  t.is(utf8Decode(getBack?.value), "789");
+  const getMiddle = await storage.getRangeMaybeExpired?.("key", 3, 3);
+  t.is(utf8Decode(getMiddle?.value), "456");
+
+  // below 0 start defaults to 0
+  const outside = await storage.getRangeMaybeExpired?.("key", -2, 3);
+  t.is(utf8Decode(outside?.value), "123");
+  // past end adds 0 for each missing byte
+  const outside2 = await storage.getRangeMaybeExpired?.("key", 12, 7);
+  t.is(
+    utf8Decode(outside2?.value),
+    utf8Decode(new Uint8Array([0, 0, 0, 0, 0, 0, 0]))
+  );
+  // length past end just pads with 0s for each missing byte
+  const outside3 = await storage.getRangeMaybeExpired?.("key", 6, 6);
+  t.is(
+    utf8Decode(outside3?.value),
+    "789" + utf8Decode(new Uint8Array([0, 0, 0]))
+  );
+});
 
 async function unsanitisedStorageFactory(
   t: ExecutionContext
@@ -88,6 +115,25 @@ test("FileStorage: get: ignores files outside root", async (t) => {
   t.is(utf8Decode((await storage.get("dir/../key"))?.value), "value");
   t.is(await storage.get("../secrets.txt"), undefined);
 });
+test("FileStorage: getRangeMaybeExpired: ignores files outside root", async (t) => {
+  const storage = await unsanitisedStorageFactory(t);
+  t.is(
+    utf8Decode(
+      (await storage.getRangeMaybeExpired?.("dir/../key", 0, 5))?.value
+    ),
+    "value"
+  );
+  t.is(await storage.getRangeMaybeExpired?.("../secrets.txt", 0, 6), undefined);
+});
+test("FileStorage: getRangeMaybeExpired: non-existant file returns undefined", async (t) => {
+  const storage = await unsanitisedStorageFactory(t);
+  t.is(await storage.getRangeMaybeExpired?.("doesntexist", 0, 6), undefined);
+});
+test("FileStorage: getRangeMaybeExpired: dir that does not exist will return undefined", async (t) => {
+  const storage = await unsanitisedStorageFactory(t);
+  const empty = await storage.getRangeMaybeExpired?.("key/sub-key", 0, 6);
+  t.is(empty, undefined);
+});
 test("FileStorage: put: throws on files outside root", async (t) => {
   const storage = await unsanitisedStorageFactory(t);
   await t.throwsAsync(