feat(checkpoint-validation): migrate CLI to Vitest (#1346)

Author: dqbd

libs/checkpoint-validation/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/langgraph-checkpoint-validation",
-  "version": "0.0.2",
+  "version": "0.1.0",
   "description": "Library for validating LangGraph checkpoint saver implementations.",
   "type": "module",
   "engines": {
@@ -14,7 +14,7 @@
   },
   "scripts": {
     "build": "yarn turbo:command build:internal --filter=@langchain/langgraph-checkpoint-validation",
-    "build:internal": "yarn clean && yarn lc_build --create-entrypoints --pre --tree-shaking",
+    "build:internal": "yarn clean && yarn lc_build --create-entrypoints --pre --tree-shaking && cp src/runner.ts dist/runner.ts",
     "clean": "rm -rf dist/ dist-cjs/ .turbo/",
     "lint:eslint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
     "lint:dpdm": "dpdm --exit-code circular:1 --no-warning --no-tree src/*.ts src/**/*.ts",
@@ -49,6 +49,7 @@
     "@testcontainers/postgresql": "^11.0.3",
     "@tsconfig/recommended": "^1.0.3",
     "@types/uuid": "^10",
+    "@types/yargs": "^17.0.33",
     "@typescript-eslint/eslint-plugin": "^6.12.0",
     "@typescript-eslint/parser": "^6.12.0",
     "better-sqlite3": "^11.7.0",

libs/checkpoint-validation/src/cli.ts

@@ -0,0 +1,71 @@
+import * as url from "node:url";
+import * as path from "node:path";
+import { startVitest } from "vitest/node";
+import yargs from "yargs";
+import {
+  isTestTypeFilter,
+  isTestTypeFilterArray,
+  testTypeFilters,
+} from "./types.js";
+import { resolveImportPath } from "./import_utils.js";
+
+const builder = yargs()
+  .command("* <initializer-import-path> [filters..]", "Validate a checkpointer")
+  .positional("initializerImportPath", {
+    type: "string",
+    describe:
+      "The import path of the CheckpointSaverTestInitializer for the checkpointer (passed to 'import()'). " +
+      "Must be the default export.",
+    demandOption: true,
+  })
+  .positional("filters", {
+    array: true,
+    choices: ["getTuple", "put", "putWrites", "list"],
+    default: [],
+    describe:
+      "Only run the specified suites. Valid values are 'getTuple', 'put', 'putWrites', and 'list'",
+    demandOption: false,
+  })
+  .help()
+  .alias("h", "help")
+  .wrap(yargs().terminalWidth())
+  .strict();
+
+export async function main() {
+  const parsed = await builder.parse(process.argv.slice(2));
+
+  try {
+    resolveImportPath(parsed.initializerImportPath);
+  } catch (e) {
+    console.error(
+      `Failed to resolve import path '${parsed.initializerImportPath}': ${e}`
+    );
+    process.exit(1);
+  }
+
+  if (!isTestTypeFilterArray(parsed.filters)) {
+    console.error(
+      `Invalid filters: '${parsed.filters
+        .filter((f) => !isTestTypeFilter(f))
+        .join("', '")}'. Expected only values from '${testTypeFilters.join(
+        "', '"
+      )}'`
+    );
+    process.exit(1);
+  }
+
+  const rootDir = path.resolve(
+    path.dirname(url.fileURLToPath(import.meta.url)),
+    "..",
+    "dist"
+  );
+  const runner = path.resolve(rootDir, "runner.ts");
+
+  await startVitest("test", [runner], {
+    globals: true,
+    include: [runner],
+    exclude: [],
+    provide: { LANGGRAPH_ARGS: parsed },
+    dir: rootDir,
+  });
+}

libs/checkpoint-validation/src/import_utils.ts

@@ -0,0 +1,50 @@
+import {
+  isAbsolute as pathIsAbsolute,
+  resolve as pathResolve,
+  dirname,
+} from "node:path";
+import { existsSync } from "node:fs";
+import { createRequire } from "node:module";
+
+export function findPackageRoot(path: string): string {
+  const packageJsonPath = pathResolve(path, "package.json");
+  if (existsSync(packageJsonPath)) {
+    return path;
+  }
+
+  if (pathResolve(dirname(path)) === pathResolve(path)) {
+    throw new Error("Could not find package root");
+  }
+
+  return findPackageRoot(pathResolve(dirname(path)));
+}
+
+export function resolveImportPath(path: string) {
+  // absolute path
+  if (pathIsAbsolute(path)) {
+    return path;
+  }
+
+  // relative path
+  if (/^\.\.?(\/|\\)/.test(path)) {
+    return pathResolve(path);
+  } else {
+    const resolvedPath = pathResolve(process.cwd(), path);
+    // try it as a relative path, anyway
+    if (existsSync(resolvedPath)) {
+      return resolvedPath;
+    }
+  }
+
+  // module name
+  const packageRoot = findPackageRoot(process.cwd());
+  if (packageRoot === undefined) {
+    console.error(
+      "Could not find package root to resolve initializer import path."
+    );
+    process.exit(1);
+  }
+
+  const localRequire = createRequire(pathResolve(packageRoot, "package.json"));
+  return localRequire.resolve(path);
+}

libs/checkpoint-validation/src/runner.ts

@@ -0,0 +1,86 @@
+// This file is used by the CLI to dynamically execute tests against the user-provided checkpointer. It's written as a
+// Vitest test file because unfortunately there's no good way to just pass Vitest a test definition function and tell it to
+// run it.
+import { inject } from "vitest";
+import type { BaseCheckpointSaver } from "@langchain/langgraph-checkpoint";
+import { resolve as pathResolve } from "node:path";
+import { createRequire } from "node:module";
+import { readFileSync } from "node:fs";
+import {
+  checkpointerTestInitializerSchema,
+  isTestTypeFilter,
+  isTestTypeFilterArray,
+  testTypeFilters,
+  type CheckpointerTestInitializer,
+} from "./types.js";
+import { specTest } from "./spec/index.js";
+import { findPackageRoot, resolveImportPath } from "./import_utils.js";
+
+declare module "vitest" {
+  interface ProvidedContext {
+    LANGGRAPH_ARGS: { initializerImportPath: string; filters: string[] };
+  }
+}
+
+export function isESM(path: string) {
+  if (path.endsWith(".mjs") || path.endsWith(".mts")) {
+    return true;
+  }
+
+  if (path.endsWith(".cjs") || path.endsWith(".cts")) {
+    return false;
+  }
+
+  const packageJsonPath = pathResolve(findPackageRoot(path), "package.json");
+  const packageConfig = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
+  return packageConfig.type === "module";
+}
+
+async function dynamicImport(modulePath: string) {
+  if (isESM(modulePath)) return import(modulePath);
+  const localRequire = createRequire(pathResolve(modulePath, "package.json"));
+  return localRequire(modulePath);
+}
+
+const { initializerImportPath, filters } = inject("LANGGRAPH_ARGS");
+const resolvedImportPath = resolveImportPath(initializerImportPath);
+
+let initializerExport: unknown;
+try {
+  initializerExport = await dynamicImport(resolvedImportPath);
+} catch (e) {
+  console.error(
+    `Failed to import initializer from import path '${initializerImportPath}' (resolved to '${resolvedImportPath}'): ${e}`
+  );
+  process.exit(1);
+}
+
+let initializer: CheckpointerTestInitializer<BaseCheckpointSaver>;
+try {
+  initializer = checkpointerTestInitializerSchema.parse(
+    (initializerExport as { default?: unknown }).default ?? initializerExport
+  ) as CheckpointerTestInitializer<BaseCheckpointSaver>;
+} catch (e) {
+  console.error(
+    `Initializer imported from '${initializerImportPath}' does not conform to the expected schema. Make sure " +
+      "it is the default export, and that implements the CheckpointSaverTestInitializer interface. Error: ${e}`
+  );
+  process.exit(1);
+}
+
+if (!isTestTypeFilterArray(filters)) {
+  console.error(
+    `Invalid filters: '${filters
+      .filter((f) => !isTestTypeFilter(f))
+      .join("', '")}'. Expected only values from '${testTypeFilters.join(
+      "', '"
+    )}'`
+  );
+  process.exit(1);
+}
+
+if (!initializer) {
+  throw new Error("Test configuration error: initializer is not set.");
+}
+
+specTest(initializer, filters);

libs/checkpoint-validation/src/spec/get_tuple.ts

@@ -1,4 +1,3 @@
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
 import {
   CheckpointTuple,
   PendingWrite,

libs/checkpoint-validation/src/spec/index.ts

@@ -1,4 +1,3 @@
-import { describe, beforeAll, afterAll } from "vitest";
 import { type BaseCheckpointSaver } from "@langchain/langgraph-checkpoint";
 
 import { CheckpointerTestInitializer, TestTypeFilter } from "../types.js";

libs/checkpoint-validation/src/spec/list.ts

@@ -1,4 +1,3 @@
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
 import {
   CheckpointTuple,
   PendingWrite,

libs/checkpoint-validation/src/spec/put.ts

@@ -1,4 +1,3 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
 import {
   Checkpoint,
   CheckpointMetadata,

libs/checkpoint-validation/src/spec/put_writes.ts

@@ -1,4 +1,3 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
 import {
   Checkpoint,
   CheckpointMetadata,

libs/checkpoint-validation/src/test_utils.ts

@@ -1,4 +1,3 @@
-import { it, expect } from "vitest";
 import {
   BaseCheckpointSaver,
   ChannelVersions,