diff --git a/bundle/cli.js b/bundle/cli.js index 7b201714..55d08479 100755 --- a/bundle/cli.js +++ b/bundle/cli.js @@ -17,21 +17,21 @@ __export(index_marker_store_exports, { hasFreshIndexMarker: () => hasFreshIndexMarker, writeIndexMarker: () => writeIndexMarker }); -import { existsSync as existsSync13, mkdirSync as mkdirSync3, readFileSync as readFileSync10, writeFileSync as writeFileSync7 } from "node:fs"; -import { join as join16 } from "node:path"; +import { existsSync as existsSync14, mkdirSync as mkdirSync4, readFileSync as readFileSync12, writeFileSync as writeFileSync8 } from "node:fs"; +import { join as join17 } from "node:path"; import { tmpdir } from "node:os"; function getIndexMarkerDir() { - return process.env.HIVEMIND_INDEX_MARKER_DIR ?? join16(tmpdir(), "hivemind-deeplake-indexes"); + return process.env.HIVEMIND_INDEX_MARKER_DIR ?? join17(tmpdir(), "hivemind-deeplake-indexes"); } function buildIndexMarkerPath(workspaceId, orgId, table, suffix) { const markerKey = [workspaceId, orgId, table, suffix].join("__").replace(/[^a-zA-Z0-9_.-]/g, "_"); - return join16(getIndexMarkerDir(), `${markerKey}.json`); + return join17(getIndexMarkerDir(), `${markerKey}.json`); } function hasFreshIndexMarker(markerPath) { - if (!existsSync13(markerPath)) + if (!existsSync14(markerPath)) return false; try { - const raw = JSON.parse(readFileSync10(markerPath, "utf-8")); + const raw = JSON.parse(readFileSync12(markerPath, "utf-8")); const updatedAt = raw.updatedAt ? new Date(raw.updatedAt).getTime() : NaN; if (!Number.isFinite(updatedAt) || Date.now() - updatedAt > INDEX_MARKER_TTL_MS) return false; @@ -41,8 +41,8 @@ function hasFreshIndexMarker(markerPath) { } } function writeIndexMarker(markerPath) { - mkdirSync3(getIndexMarkerDir(), { recursive: true }); - writeFileSync7(markerPath, JSON.stringify({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }), "utf-8"); + mkdirSync4(getIndexMarkerDir(), { recursive: true }); + writeFileSync8(markerPath, JSON.stringify({ updatedAt: (/* @__PURE__ */ new Date()).toISOString() }), "utf-8"); } var INDEX_MARKER_TTL_MS; var init_index_marker_store = __esm({ @@ -3681,42 +3681,137 @@ function uninstallPi() { } // dist/src/cli/embeddings.js -import { copyFileSync as copyFileSync3, chmodSync, existsSync as existsSync10, lstatSync as lstatSync2, readdirSync, readlinkSync, rmSync as rmSync4, statSync, unlinkSync as unlinkSync5 } from "node:fs"; -import { execFileSync as execFileSync3 } from "node:child_process"; -import { join as join11 } from "node:path"; -var SHARED_DIR = join11(HOME, ".hivemind", "embed-deps"); -var SHARED_NODE_MODULES = join11(SHARED_DIR, "node_modules"); -var SHARED_DAEMON_PATH = join11(SHARED_DIR, "embed-daemon.js"); +import { copyFileSync as copyFileSync3, chmodSync, existsSync as existsSync11, lstatSync as lstatSync2, readdirSync, readFileSync as readFileSync9, readlinkSync, rmSync as rmSync4, statSync, unlinkSync as unlinkSync5 } from "node:fs"; +import { execFileSync as execFileSync3, spawnSync } from "node:child_process"; +import { userInfo } from "node:os"; +import { join as join12 } from "node:path"; + +// dist/src/embeddings/protocol.js +var DEFAULT_SOCKET_DIR = "/tmp"; +var DEFAULT_IDLE_TIMEOUT_MS = 10 * 60 * 1e3; +function socketPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.sock`; +} +function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { + return `${dir}/hivemind-embed-${uid}.pid`; +} + +// dist/src/user-config.js +import { existsSync as existsSync10, mkdirSync as mkdirSync2, readFileSync as readFileSync8, renameSync as renameSync2, writeFileSync as writeFileSync6 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname as dirname2, join as join11 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join11(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync10(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync8(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname2(path); + if (!existsSync10(dir)) + mkdirSync2(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync6(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function setEmbeddingsEnabled(enabled) { + writeUserConfig({ embeddings: { enabled } }); +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/cli/embeddings.js +var SHARED_DIR = join12(HOME, ".hivemind", "embed-deps"); +var SHARED_NODE_MODULES = join12(SHARED_DIR, "node_modules"); +var SHARED_DAEMON_PATH = join12(SHARED_DIR, "embed-daemon.js"); var TRANSFORMERS_PKG = "@huggingface/transformers"; var TRANSFORMERS_RANGE = "^3.0.0"; function findHivemindInstalls(home = HOME) { const out = []; const fixed = [ - { id: "codex", pluginDir: join11(home, ".codex", "hivemind") }, - { id: "cursor", pluginDir: join11(home, ".cursor", "hivemind") }, - { id: "hermes", pluginDir: join11(home, ".hermes", "hivemind") } + { id: "codex", pluginDir: join12(home, ".codex", "hivemind") }, + { id: "cursor", pluginDir: join12(home, ".cursor", "hivemind") }, + { id: "hermes", pluginDir: join12(home, ".hermes", "hivemind") } ]; for (const inst of fixed) { - if (existsSync10(join11(inst.pluginDir, "bundle"))) + if (existsSync11(join12(inst.pluginDir, "bundle"))) out.push(inst); } - const ccCache = join11(home, ".claude", "plugins", "cache", "hivemind", "hivemind"); - if (existsSync10(ccCache)) { + const ccCache = join12(home, ".claude", "plugins", "cache", "hivemind", "hivemind"); + if (existsSync11(ccCache)) { let entries = []; try { entries = readdirSync(ccCache); } catch { } for (const ver of entries) { - const dir = join11(ccCache, ver); + const dir = join12(ccCache, ver); try { if (!statSync(dir).isDirectory()) continue; } catch { continue; } - const candidates = [join11(dir, "bundle"), join11(dir, "claude-code", "bundle")]; - if (candidates.some((p) => existsSync10(p))) { + const candidates = [join12(dir, "bundle"), join12(dir, "claude-code", "bundle")]; + if (candidates.some((p) => existsSync11(p))) { out.push({ id: `claude (${ver})`, pluginDir: dir }); } } @@ -3724,10 +3819,10 @@ function findHivemindInstalls(home = HOME) { return out; } function isSharedDepsInstalled(sharedNodeModules = SHARED_NODE_MODULES) { - return existsSync10(join11(sharedNodeModules, TRANSFORMERS_PKG)); + return existsSync11(join12(sharedNodeModules, TRANSFORMERS_PKG)); } function isSymlinkToSharedDeps(linkPath, sharedNodeModules) { - if (!existsSync10(linkPath)) + if (!existsSync11(linkPath)) return false; try { if (!lstatSync2(linkPath).isSymbolicLink()) @@ -3738,8 +3833,8 @@ function isSymlinkToSharedDeps(linkPath, sharedNodeModules) { } } function linkStateFor(install, sharedNodeModules = SHARED_NODE_MODULES) { - const link = join11(install.pluginDir, "node_modules"); - if (!existsSync10(link) && !isSymbolicLink(link)) + const link = join12(install.pluginDir, "node_modules"); + if (!existsSync11(link) && !isSymbolicLink(link)) return { kind: "no-node-modules" }; try { if (lstatSync2(link).isSymbolicLink()) { @@ -3763,7 +3858,7 @@ function ensureSharedDeps() { log(` Embeddings installing ${TRANSFORMERS_PKG}@${TRANSFORMERS_RANGE} into ${SHARED_DIR}`); log(` (~600 MB; first install only \u2014 every agent will share this)`); ensureDir(SHARED_DIR); - writeJson(join11(SHARED_DIR, "package.json"), { + writeJson(join12(SHARED_DIR, "package.json"), { name: "hivemind-embed-deps", version: "1.0.0", private: true, @@ -3777,8 +3872,8 @@ function ensureSharedDeps() { log(` Embeddings shared deps already present at ${SHARED_DIR}`); } ensureDir(SHARED_DIR); - const src = join11(pkgRoot(), "embeddings", "embed-daemon.js"); - if (existsSync10(src)) { + const src = join12(pkgRoot(), "embeddings", "embed-daemon.js"); + if (existsSync11(src)) { copyFileSync3(src, SHARED_DAEMON_PATH); chmodSync(SHARED_DAEMON_PATH, 493); } else { @@ -3786,40 +3881,115 @@ function ensureSharedDeps() { } } function linkAgent(install) { - const link = join11(install.pluginDir, "node_modules"); + const link = join12(install.pluginDir, "node_modules"); + const state = linkStateFor(install); + if (state.kind === "owns-own-node-modules") { + warn(` Embeddings ${install.id.padEnd(20)} owns its own node_modules \u2014 skipping symlink (status: owns-own-node-modules)`); + return; + } symlinkForce(SHARED_NODE_MODULES, link); log(` Embeddings linked ${install.id.padEnd(20)} -> shared deps`); } -function enableEmbeddings() { +function installEmbeddings() { ensureSharedDeps(); const installs = findHivemindInstalls(); if (installs.length === 0) { warn(" Embeddings no hivemind installs detected \u2014 run `hivemind install` first"); warn(" (the shared deps are in place; subsequent agent installs will pick them up if you re-run `hivemind embeddings install`)"); - return; + } else { + for (const inst of installs) + linkAgent(inst); } - for (const inst of installs) - linkAgent(inst); - log(` Embeddings enabled. Restart your agents to pick up.`); + setEmbeddingsEnabled(true); + log(` Embeddings enabled in ~/.deeplake/config.json`); + log(` Embeddings ready. Restart your agents to pick up.`); } -function disableEmbeddings(opts) { +function enableEmbeddings() { + setEmbeddingsEnabled(true); + log(` Embeddings enabled in ~/.deeplake/config.json`); + if (!isSharedDepsInstalled()) { + warn(` Embeddings shared deps not installed yet \u2014 run \`hivemind embeddings install\` to download them`); + } else { + log(` Embeddings shared deps present \u2014 sessions will start producing embeddings on next restart`); + } +} +function uninstallEmbeddings(opts) { const installs = findHivemindInstalls(); for (const inst of installs) { - const link = join11(inst.pluginDir, "node_modules"); + const link = join12(inst.pluginDir, "node_modules"); if (isSymlinkToSharedDeps(link, SHARED_NODE_MODULES)) { unlinkSync5(link); log(` Embeddings unlinked ${inst.id}`); } } - if (opts?.prune && existsSync10(SHARED_DIR)) { + if (opts?.prune && existsSync11(SHARED_DIR)) { rmSync4(SHARED_DIR, { recursive: true, force: true }); log(` Embeddings pruned ${SHARED_DIR}`); } + setEmbeddingsEnabled(false); + killEmbedDaemon(); + log(` Embeddings disabled in ~/.deeplake/config.json`); +} +function disableEmbeddings() { + setEmbeddingsEnabled(false); + killEmbedDaemon(); + log(` Embeddings disabled in ~/.deeplake/config.json`); + log(` Embeddings daemon terminated; shared deps preserved (run \`hivemind embeddings uninstall\` to remove)`); +} +function killEmbedDaemon(socketDir) { + const uid = typeof process.getuid === "function" ? process.getuid() : userInfo().uid; + const pidPath = pidPathFor(String(uid), socketDir); + const sockPath = socketPathFor(String(uid), socketDir); + let pid = null; + try { + pid = Number.parseInt(readFileSync9(pidPath, "utf-8").trim(), 10); + } catch { + } + if (pid !== null && Number.isFinite(pid) && _isDaemonAliveOnSocket(sockPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log(` Embeddings pidfile present but socket dead \u2014 skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync5(sockPath); + } catch { + } + try { + unlinkSync5(pidPath); + } catch { + } +} +function _isDaemonAliveOnSocket(sockPath, timeoutMs = 200) { + if (!existsSync11(sockPath)) + return false; + try { + const child = spawnSync("node", [ + "-e", + `const n=require("node:net");const s=n.connect(${JSON.stringify(sockPath)});s.once("connect",()=>{s.end();process.exit(0)});s.once("error",()=>process.exit(2));setTimeout(()=>process.exit(3),${timeoutMs});` + ], { timeout: timeoutMs + 1e3, stdio: "ignore" }); + return child.status === 0; + } catch { + return false; + } } function statusEmbeddings() { + const enabled = getEmbeddingsEnabled(); + log(`Config: ~/.deeplake/config.json embeddings.enabled = ${enabled}`); log(`Shared deps: ${SHARED_DIR}`); log(`Installed: ${isSharedDepsInstalled() ? "yes" : "no"}`); - log(`Daemon: ${existsSync10(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : "(not present)"}`); + log(`Daemon: ${existsSync11(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : "(not present)"}`); + if (!enabled) { + log(""); + log(`Embeddings are DISABLED in user config. Run \`hivemind embeddings enable\` to opt in,`); + log(`or \`hivemind embeddings install\` if the shared deps are not yet downloaded.`); + } else if (!isSharedDepsInstalled()) { + log(""); + warn(`Embeddings are enabled in config but shared deps are missing.`); + warn(`Run \`hivemind embeddings install\` to download @huggingface/transformers.`); + } log(""); log(`Agent installs:`); const installs = findHivemindInstalls(); @@ -3835,7 +4005,7 @@ function statusEmbeddings() { label = "\u2713 linked \u2192 shared"; break; case "no-node-modules": - label = "\u2717 not linked (embeddings disabled)"; + label = "\u2717 not linked"; break; case "owns-own-node-modules": label = "\u25B3 has its own node_modules (not shared)"; @@ -3850,8 +4020,8 @@ function statusEmbeddings() { } // dist/src/cli/auth.js -import { existsSync as existsSync11 } from "node:fs"; -import { join as join13 } from "node:path"; +import { existsSync as existsSync12 } from "node:fs"; +import { join as join14 } from "node:path"; // dist/src/commands/auth.js import { execSync } from "node:child_process"; @@ -3866,25 +4036,25 @@ function deeplakeClientHeader() { } // dist/src/commands/auth-creds.js -import { readFileSync as readFileSync8, writeFileSync as writeFileSync6, mkdirSync as mkdirSync2, unlinkSync as unlinkSync6 } from "node:fs"; -import { join as join12 } from "node:path"; -import { homedir as homedir4 } from "node:os"; +import { readFileSync as readFileSync10, writeFileSync as writeFileSync7, mkdirSync as mkdirSync3, unlinkSync as unlinkSync6 } from "node:fs"; +import { join as join13 } from "node:path"; +import { homedir as homedir5 } from "node:os"; function configDir() { - return join12(homedir4(), ".deeplake"); + return join13(homedir5(), ".deeplake"); } function credsPath() { - return join12(configDir(), "credentials.json"); + return join13(configDir(), "credentials.json"); } function loadCredentials() { try { - return JSON.parse(readFileSync8(credsPath(), "utf-8")); + return JSON.parse(readFileSync10(credsPath(), "utf-8")); } catch { return null; } } function saveCredentials(creds) { - mkdirSync2(configDir(), { recursive: true, mode: 448 }); - writeFileSync6(credsPath(), JSON.stringify({ ...creds, savedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2), { mode: 384 }); + mkdirSync3(configDir(), { recursive: true, mode: 448 }); + writeFileSync7(credsPath(), JSON.stringify({ ...creds, savedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2), { mode: 384 }); } function deleteCredentials() { try { @@ -4073,9 +4243,9 @@ Using: ${orgName} } // dist/src/cli/auth.js -var CREDS_PATH = join13(HOME, ".deeplake", "credentials.json"); +var CREDS_PATH = join14(HOME, ".deeplake", "credentials.json"); function isLoggedIn() { - return existsSync11(CREDS_PATH) && loadCredentials() !== null; + return existsSync12(CREDS_PATH) && loadCredentials() !== null; } async function ensureLoggedIn() { if (isLoggedIn()) @@ -4108,16 +4278,16 @@ async function maybeShowOrgChoice() { } // dist/src/config.js -import { readFileSync as readFileSync9, existsSync as existsSync12 } from "node:fs"; -import { join as join14 } from "node:path"; -import { homedir as homedir5, userInfo } from "node:os"; +import { readFileSync as readFileSync11, existsSync as existsSync13 } from "node:fs"; +import { join as join15 } from "node:path"; +import { homedir as homedir6, userInfo as userInfo2 } from "node:os"; function loadConfig() { - const home = homedir5(); - const credPath = join14(home, ".deeplake", "credentials.json"); + const home = homedir6(); + const credPath = join15(home, ".deeplake", "credentials.json"); let creds = null; - if (existsSync12(credPath)) { + if (existsSync13(credPath)) { try { - creds = JSON.parse(readFileSync9(credPath, "utf-8")); + creds = JSON.parse(readFileSync11(credPath, "utf-8")); } catch { return null; } @@ -4130,13 +4300,13 @@ function loadConfig() { token, orgId, orgName: creds?.orgName ?? orgId, - userName: creds?.userName || userInfo().username || "unknown", + userName: creds?.userName || userInfo2().username || "unknown", workspaceId: process.env.HIVEMIND_WORKSPACE_ID ?? creds?.workspaceId ?? "default", apiUrl: process.env.HIVEMIND_API_URL ?? creds?.apiUrl ?? "https://api.deeplake.ai", tableName: process.env.HIVEMIND_TABLE ?? "memory", sessionsTableName: process.env.HIVEMIND_SESSIONS_TABLE ?? "sessions", skillsTableName: process.env.HIVEMIND_SKILLS_TABLE ?? "skills", - memoryPath: process.env.HIVEMIND_MEMORY_PATH ?? join14(home, ".deeplake", "memory") + memoryPath: process.env.HIVEMIND_MEMORY_PATH ?? join15(home, ".deeplake", "memory") }; } @@ -4145,9 +4315,9 @@ import { randomUUID } from "node:crypto"; // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; -import { join as join15 } from "node:path"; -import { homedir as homedir6 } from "node:os"; -var LOG = join15(homedir6(), ".deeplake", "hook-debug.log"); +import { join as join16 } from "node:path"; +import { homedir as homedir7 } from "node:os"; +var LOG = join16(homedir7(), ".deeplake", "hook-debug.log"); function isDebug() { return process.env.HIVEMIND_DEBUG === "1"; } @@ -4915,34 +5085,34 @@ if (process.argv[1] && process.argv[1].endsWith("auth-login.js")) { } // dist/src/commands/skillify.js -import { readdirSync as readdirSync5, existsSync as existsSync25, readFileSync as readFileSync18, mkdirSync as mkdirSync10, renameSync as renameSync5 } from "node:fs"; -import { homedir as homedir18 } from "node:os"; -import { dirname as dirname6, join as join28 } from "node:path"; +import { readdirSync as readdirSync5, existsSync as existsSync26, readFileSync as readFileSync20, mkdirSync as mkdirSync11, renameSync as renameSync6 } from "node:fs"; +import { homedir as homedir19 } from "node:os"; +import { dirname as dirname7, join as join29 } from "node:path"; // dist/src/skillify/scope-config.js -import { existsSync as existsSync15, mkdirSync as mkdirSync4, readFileSync as readFileSync11, writeFileSync as writeFileSync8 } from "node:fs"; -import { homedir as homedir8 } from "node:os"; -import { join as join18 } from "node:path"; +import { existsSync as existsSync16, mkdirSync as mkdirSync5, readFileSync as readFileSync13, writeFileSync as writeFileSync9 } from "node:fs"; +import { homedir as homedir9 } from "node:os"; +import { join as join19 } from "node:path"; // dist/src/skillify/legacy-migration.js -import { existsSync as existsSync14, renameSync as renameSync2 } from "node:fs"; -import { homedir as homedir7 } from "node:os"; -import { join as join17 } from "node:path"; +import { existsSync as existsSync15, renameSync as renameSync3 } from "node:fs"; +import { homedir as homedir8 } from "node:os"; +import { join as join18 } from "node:path"; var dlog = (msg) => log2("skillify-migrate", msg); var attempted = false; function migrateLegacyStateDir() { if (attempted) return; attempted = true; - const root = join17(homedir7(), ".deeplake", "state"); - const legacy = join17(root, "skilify"); - const current = join17(root, "skillify"); - if (!existsSync14(legacy)) + const root = join18(homedir8(), ".deeplake", "state"); + const legacy = join18(root, "skilify"); + const current = join18(root, "skillify"); + if (!existsSync15(legacy)) return; - if (existsSync14(current)) + if (existsSync15(current)) return; try { - renameSync2(legacy, current); + renameSync3(legacy, current); dlog(`migrated ${legacy} -> ${current}`); } catch (err) { const code = err.code; @@ -4955,15 +5125,15 @@ function migrateLegacyStateDir() { } // dist/src/skillify/scope-config.js -var STATE_DIR = join18(homedir8(), ".deeplake", "state", "skillify"); -var CONFIG_PATH2 = join18(STATE_DIR, "config.json"); +var STATE_DIR = join19(homedir9(), ".deeplake", "state", "skillify"); +var CONFIG_PATH2 = join19(STATE_DIR, "config.json"); var DEFAULT = { scope: "me", team: [], install: "project" }; function loadScopeConfig() { migrateLegacyStateDir(); - if (!existsSync15(CONFIG_PATH2)) + if (!existsSync16(CONFIG_PATH2)) return DEFAULT; try { - const raw = JSON.parse(readFileSync11(CONFIG_PATH2, "utf-8")); + const raw = JSON.parse(readFileSync13(CONFIG_PATH2, "utf-8")); const scope = raw.scope === "team" ? "team" : raw.scope === "org" ? "team" : "me"; const team = Array.isArray(raw.team) ? raw.team.filter((s) => typeof s === "string") : []; const install = raw.install === "global" ? "global" : "project"; @@ -4974,19 +5144,19 @@ function loadScopeConfig() { } function saveScopeConfig(cfg) { migrateLegacyStateDir(); - mkdirSync4(STATE_DIR, { recursive: true }); - writeFileSync8(CONFIG_PATH2, JSON.stringify(cfg, null, 2)); + mkdirSync5(STATE_DIR, { recursive: true }); + writeFileSync9(CONFIG_PATH2, JSON.stringify(cfg, null, 2)); } // dist/src/skillify/pull.js -import { existsSync as existsSync19, readFileSync as readFileSync14, writeFileSync as writeFileSync11, mkdirSync as mkdirSync7, renameSync as renameSync4, lstatSync as lstatSync4, readlinkSync as readlinkSync2, symlinkSync as symlinkSync2, unlinkSync as unlinkSync8 } from "node:fs"; -import { homedir as homedir12 } from "node:os"; -import { dirname as dirname3, join as join22 } from "node:path"; +import { existsSync as existsSync20, readFileSync as readFileSync16, writeFileSync as writeFileSync12, mkdirSync as mkdirSync8, renameSync as renameSync5, lstatSync as lstatSync4, readlinkSync as readlinkSync2, symlinkSync as symlinkSync2, unlinkSync as unlinkSync8 } from "node:fs"; +import { homedir as homedir13 } from "node:os"; +import { dirname as dirname4, join as join23 } from "node:path"; // dist/src/skillify/skill-writer.js -import { existsSync as existsSync16, mkdirSync as mkdirSync5, readFileSync as readFileSync12, readdirSync as readdirSync2, statSync as statSync2, writeFileSync as writeFileSync9 } from "node:fs"; -import { homedir as homedir9 } from "node:os"; -import { join as join19 } from "node:path"; +import { existsSync as existsSync17, mkdirSync as mkdirSync6, readFileSync as readFileSync14, readdirSync as readdirSync2, statSync as statSync2, writeFileSync as writeFileSync10 } from "node:fs"; +import { homedir as homedir10 } from "node:os"; +import { join as join20 } from "node:path"; function assertValidSkillName(name) { if (typeof name !== "string" || name.length === 0) { throw new Error(`invalid skill name: empty or non-string`); @@ -5002,10 +5172,10 @@ function assertValidSkillName(name) { } } function skillDir(skillsRoot, name) { - return join19(skillsRoot, name); + return join20(skillsRoot, name); } function skillPath(skillsRoot, name) { - return join19(skillDir(skillsRoot, name), "SKILL.md"); + return join20(skillDir(skillsRoot, name), "SKILL.md"); } function renderFrontmatter(fm) { const lines = ["---"]; @@ -5083,10 +5253,10 @@ function writeNewSkill(args) { assertValidSkillName(args.name); const dir = skillDir(args.skillsRoot, args.name); const path = skillPath(args.skillsRoot, args.name); - if (existsSync16(path)) { + if (existsSync17(path)) { throw new Error(`skill already exists at ${path}; use mergeSkill`); } - mkdirSync5(dir, { recursive: true }); + mkdirSync6(dir, { recursive: true }); const now = (/* @__PURE__ */ new Date()).toISOString(); const author = args.author && args.author.length > 0 ? args.author : void 0; const contributors = author ? [author] : []; @@ -5106,7 +5276,7 @@ function writeNewSkill(args) { ${args.body.trim()} `; - writeFileSync9(path, text); + writeFileSync10(path, text); return { path, action: "created", @@ -5118,41 +5288,41 @@ ${args.body.trim()} }; } function listSkills(skillsRoot) { - if (!existsSync16(skillsRoot)) + if (!existsSync17(skillsRoot)) return []; const out = []; for (const name of readdirSync2(skillsRoot)) { - const skillFile = join19(skillsRoot, name, "SKILL.md"); - if (existsSync16(skillFile) && statSync2(skillFile).isFile()) { - out.push({ name, body: readFileSync12(skillFile, "utf-8") }); + const skillFile = join20(skillsRoot, name, "SKILL.md"); + if (existsSync17(skillFile) && statSync2(skillFile).isFile()) { + out.push({ name, body: readFileSync14(skillFile, "utf-8") }); } } return out; } function resolveSkillsRoot(install, cwd) { if (install === "global") { - return join19(homedir9(), ".claude", "skills"); + return join20(homedir10(), ".claude", "skills"); } - return join19(cwd, ".claude", "skills"); + return join20(cwd, ".claude", "skills"); } // dist/src/skillify/manifest.js -import { existsSync as existsSync17, lstatSync as lstatSync3, mkdirSync as mkdirSync6, readFileSync as readFileSync13, renameSync as renameSync3, unlinkSync as unlinkSync7, writeFileSync as writeFileSync10 } from "node:fs"; -import { homedir as homedir10 } from "node:os"; -import { dirname as dirname2, join as join20 } from "node:path"; +import { existsSync as existsSync18, lstatSync as lstatSync3, mkdirSync as mkdirSync7, readFileSync as readFileSync15, renameSync as renameSync4, unlinkSync as unlinkSync7, writeFileSync as writeFileSync11 } from "node:fs"; +import { homedir as homedir11 } from "node:os"; +import { dirname as dirname3, join as join21 } from "node:path"; function emptyManifest() { return { version: 1, entries: [] }; } function manifestPath() { - return join20(homedir10(), ".deeplake", "state", "skillify", "pulled.json"); + return join21(homedir11(), ".deeplake", "state", "skillify", "pulled.json"); } function loadManifest(path = manifestPath()) { migrateLegacyStateDir(); - if (!existsSync17(path)) + if (!existsSync18(path)) return emptyManifest(); let raw; try { - raw = readFileSync13(path, "utf-8"); + raw = readFileSync15(path, "utf-8"); } catch { return emptyManifest(); } @@ -5199,10 +5369,10 @@ function loadManifest(path = manifestPath()) { } function saveManifest(m, path = manifestPath()) { migrateLegacyStateDir(); - mkdirSync6(dirname2(path), { recursive: true }); + mkdirSync7(dirname3(path), { recursive: true }); const tmp = `${path}.tmp`; - writeFileSync10(tmp, JSON.stringify(m, null, 2) + "\n", { mode: 384 }); - renameSync3(tmp, path); + writeFileSync11(tmp, JSON.stringify(m, null, 2) + "\n", { mode: 384 }); + renameSync4(tmp, path); } function recordPull(entry, path = manifestPath()) { const m = loadManifest(path); @@ -5244,7 +5414,7 @@ function pruneOrphanedEntries(path = manifestPath()) { const live = []; let pruned = 0; for (const e of m.entries) { - if (existsSync17(join20(e.installRoot, e.dirName))) { + if (existsSync18(join21(e.installRoot, e.dirName))) { live.push(e); continue; } @@ -5257,26 +5427,26 @@ function pruneOrphanedEntries(path = manifestPath()) { } // dist/src/skillify/agent-roots.js -import { existsSync as existsSync18 } from "node:fs"; -import { homedir as homedir11 } from "node:os"; -import { join as join21 } from "node:path"; +import { existsSync as existsSync19 } from "node:fs"; +import { homedir as homedir12 } from "node:os"; +import { join as join22 } from "node:path"; function resolveDetected(home) { const out = []; - const codexInstalled = existsSync18(join21(home, ".codex")); - const piInstalled = existsSync18(join21(home, ".pi", "agent")); - const hermesInstalled = existsSync18(join21(home, ".hermes")); + const codexInstalled = existsSync19(join22(home, ".codex")); + const piInstalled = existsSync19(join22(home, ".pi", "agent")); + const hermesInstalled = existsSync19(join22(home, ".hermes")); if (codexInstalled || piInstalled) { - out.push(join21(home, ".agents", "skills")); + out.push(join22(home, ".agents", "skills")); } if (hermesInstalled) { - out.push(join21(home, ".hermes", "skills")); + out.push(join22(home, ".hermes", "skills")); } if (piInstalled) { - out.push(join21(home, ".pi", "agent", "skills")); + out.push(join22(home, ".pi", "agent", "skills")); } return out; } -function detectAgentSkillsRoots(canonicalRoot, home = homedir11()) { +function detectAgentSkillsRoots(canonicalRoot, home = homedir12()) { return resolveDetected(home).filter((p) => p !== canonicalRoot); } @@ -5320,15 +5490,15 @@ function isMissingTableError(message) { } function resolvePullDestination(install, cwd) { if (install === "global") - return join22(homedir12(), ".claude", "skills"); + return join23(homedir13(), ".claude", "skills"); if (!cwd) throw new Error("install=project requires a cwd"); - return join22(cwd, ".claude", "skills"); + return join23(cwd, ".claude", "skills"); } function fanOutSymlinks(canonicalDir, dirName, agentRoots) { const out = []; for (const root of agentRoots) { - const link = join22(root, dirName); + const link = join23(root, dirName); let existing; try { existing = lstatSync4(link); @@ -5356,7 +5526,7 @@ function fanOutSymlinks(canonicalDir, dirName, agentRoots) { } } try { - mkdirSync7(dirname3(link), { recursive: true }); + mkdirSync8(dirname4(link), { recursive: true }); symlinkSync2(canonicalDir, link, "dir"); out.push(link); } catch { @@ -5371,8 +5541,8 @@ function backfillSymlinks(installRoot) { return; const detected = detectAgentSkillsRoots(installRoot); for (const entry of entries) { - const canonical = join22(entry.installRoot, entry.dirName); - if (!existsSync19(canonical)) + const canonical = join23(entry.installRoot, entry.dirName); + if (!existsSync20(canonical)) continue; const fresh = fanOutSymlinks(canonical, entry.dirName, detected); if (sameSorted(fresh, entry.symlinks)) @@ -5482,10 +5652,10 @@ function renderFrontmatter2(fm) { return lines.join("\n"); } function readLocalVersion(path) { - if (!existsSync19(path)) + if (!existsSync20(path)) return null; try { - const text = readFileSync14(path, "utf-8"); + const text = readFileSync16(path, "utf-8"); const parsed = parseFrontmatter(text); if (!parsed) return null; @@ -5580,8 +5750,8 @@ async function runPull(opts) { summary.skipped++; continue; } - const skillDir2 = join22(root, dirName); - const skillFile = join22(skillDir2, "SKILL.md"); + const skillDir2 = join23(root, dirName); + const skillFile = join23(skillDir2, "SKILL.md"); const remoteVersion = Number(row.version ?? 1); const localVersion = readLocalVersion(skillFile); const action = decideAction({ @@ -5592,14 +5762,14 @@ async function runPull(opts) { }); let manifestError; if (action === "wrote") { - mkdirSync7(skillDir2, { recursive: true }); - if (existsSync19(skillFile)) { + mkdirSync8(skillDir2, { recursive: true }); + if (existsSync20(skillFile)) { try { - renameSync4(skillFile, `${skillFile}.bak`); + renameSync5(skillFile, `${skillFile}.bak`); } catch { } } - writeFileSync11(skillFile, renderSkillFile(row)); + writeFileSync12(skillFile, renderSkillFile(row)); const symlinks = opts.install === "global" ? fanOutSymlinks(skillDir2, dirName, detectAgentSkillsRoots(root)) : []; try { recordPull({ @@ -5641,15 +5811,15 @@ async function runPull(opts) { } // dist/src/skillify/unpull.js -import { existsSync as existsSync20, readdirSync as readdirSync3, rmSync as rmSync5, statSync as statSync3 } from "node:fs"; -import { homedir as homedir13 } from "node:os"; -import { join as join23 } from "node:path"; +import { existsSync as existsSync21, readdirSync as readdirSync3, rmSync as rmSync5, statSync as statSync3 } from "node:fs"; +import { homedir as homedir14 } from "node:os"; +import { join as join24 } from "node:path"; function resolveUnpullRoot(install, cwd) { if (install === "global") - return join23(homedir13(), ".claude", "skills"); + return join24(homedir14(), ".claude", "skills"); if (!cwd) throw new Error("cwd required when install === 'project'"); - return join23(cwd, ".claude", "skills"); + return join24(cwd, ".claude", "skills"); } function runUnpull(opts) { const root = resolveUnpullRoot(opts.install, opts.cwd); @@ -5672,8 +5842,8 @@ function runUnpull(opts) { const entries = entriesForRoot(manifest, opts.install, root); for (const entry of entries) { summary.scanned++; - const path = join23(root, entry.dirName); - if (!existsSync20(path)) { + const path = join24(root, entry.dirName); + if (!existsSync21(path)) { if (!opts.dryRun) { unlinkSymlinks(entry.symlinks); removePullEntry(opts.install, entry.installRoot, entry.dirName); @@ -5726,12 +5896,12 @@ function runUnpull(opts) { } summary.entries.push(result); } - if (existsSync20(root) && (opts.all || opts.legacyCleanup)) { + if (existsSync21(root) && (opts.all || opts.legacyCleanup)) { const manifestDirNames = new Set(entries.map((e) => e.dirName)); for (const dirName of readdirSync3(root)) { if (manifestDirNames.has(dirName)) continue; - const path = join23(root, dirName); + const path = join24(root, dirName); let st; try { st = statSync3(path); @@ -5810,30 +5980,30 @@ function decideTargetForManifestEntry(entry, opts, userFilter, haveUserFilter) { // dist/src/commands/mine-local.js import { spawn } from "node:child_process"; -import { existsSync as existsSync24, mkdirSync as mkdirSync9, readFileSync as readFileSync17, writeFileSync as writeFileSync13 } from "node:fs"; -import { homedir as homedir17 } from "node:os"; -import { basename, dirname as dirname5, join as join27 } from "node:path"; +import { existsSync as existsSync25, mkdirSync as mkdirSync10, readFileSync as readFileSync19, writeFileSync as writeFileSync14 } from "node:fs"; +import { homedir as homedir18 } from "node:os"; +import { basename, dirname as dirname6, join as join28 } from "node:path"; // dist/src/skillify/local-source.js -import { readdirSync as readdirSync4, readFileSync as readFileSync15, existsSync as existsSync21, statSync as statSync4 } from "node:fs"; -import { homedir as homedir14 } from "node:os"; -import { join as join24 } from "node:path"; -var HOME2 = homedir14(); +import { readdirSync as readdirSync4, readFileSync as readFileSync17, existsSync as existsSync22, statSync as statSync4 } from "node:fs"; +import { homedir as homedir15 } from "node:os"; +import { join as join25 } from "node:path"; +var HOME2 = homedir15(); function encodeCwdClaudeCode(cwd) { return cwd.replace(/[/_]/g, "-"); } function detectInstalledAgents() { const installs = []; - const claudeRoot = join24(HOME2, ".claude", "projects"); - if (existsSync21(claudeRoot)) { + const claudeRoot = join25(HOME2, ".claude", "projects"); + if (existsSync22(claudeRoot)) { installs.push({ agent: "claude_code", sessionRoot: claudeRoot, encodeCwd: encodeCwdClaudeCode }); } - const codexRoot = join24(HOME2, ".codex", "sessions"); - if (existsSync21(codexRoot)) { + const codexRoot = join25(HOME2, ".codex", "sessions"); + if (existsSync22(codexRoot)) { installs.push({ agent: "codex", sessionRoot: codexRoot, @@ -5860,7 +6030,7 @@ function listLocalSessions(installs, cwd) { continue; } for (const sub of subdirs) { - const subdirPath = join24(install.sessionRoot, sub); + const subdirPath = join25(install.sessionRoot, sub); try { if (!statSync4(subdirPath).isDirectory()) continue; @@ -5877,7 +6047,7 @@ function listLocalSessions(installs, cwd) { for (const f of files) { if (!f.endsWith(".jsonl")) continue; - const fullPath = join24(subdirPath, f); + const fullPath = join25(subdirPath, f); let stats; try { stats = statSync4(fullPath); @@ -5938,7 +6108,7 @@ function pickSessions(candidates, opts) { function nativeJsonlToRows(filePath, sessionId, agent) { let raw; try { - raw = readFileSync15(filePath, "utf-8"); + raw = readFileSync17(filePath, "utf-8"); } catch { return []; } @@ -6028,22 +6198,22 @@ function extractPairs(rows) { } // dist/src/skillify/gate-runner.js -import { existsSync as existsSync22 } from "node:fs"; +import { existsSync as existsSync23 } from "node:fs"; import { createRequire } from "node:module"; -import { homedir as homedir15 } from "node:os"; -import { join as join25 } from "node:path"; +import { homedir as homedir16 } from "node:os"; +import { join as join26 } from "node:path"; var requireForCp = createRequire(import.meta.url); var { execFileSync: runChildProcess } = requireForCp("node:child_process"); var inheritedEnv = process; function firstExistingPath(candidates) { for (const c of candidates) { - if (existsSync22(c)) + if (existsSync23(c)) return c; } return null; } function findAgentBin(agent) { - const home = homedir15(); + const home = homedir16(); switch (agent) { // /usr/bin/ is included in every candidate list — that's the // common Linux package-manager install path (apt, dnf, pacman). Old @@ -6052,45 +6222,45 @@ function findAgentBin(agent) { // #170 caught the gap. case "claude_code": return firstExistingPath([ - join25(home, ".claude", "local", "claude"), + join26(home, ".claude", "local", "claude"), "/usr/local/bin/claude", "/usr/bin/claude", - join25(home, ".npm-global", "bin", "claude"), - join25(home, ".local", "bin", "claude"), + join26(home, ".npm-global", "bin", "claude"), + join26(home, ".local", "bin", "claude"), "/opt/homebrew/bin/claude" - ]) ?? join25(home, ".claude", "local", "claude"); + ]) ?? join26(home, ".claude", "local", "claude"); case "codex": return firstExistingPath([ "/usr/local/bin/codex", "/usr/bin/codex", - join25(home, ".npm-global", "bin", "codex"), - join25(home, ".local", "bin", "codex"), + join26(home, ".npm-global", "bin", "codex"), + join26(home, ".local", "bin", "codex"), "/opt/homebrew/bin/codex" ]) ?? "/usr/local/bin/codex"; case "cursor": return firstExistingPath([ "/usr/local/bin/cursor-agent", "/usr/bin/cursor-agent", - join25(home, ".npm-global", "bin", "cursor-agent"), - join25(home, ".local", "bin", "cursor-agent"), + join26(home, ".npm-global", "bin", "cursor-agent"), + join26(home, ".local", "bin", "cursor-agent"), "/opt/homebrew/bin/cursor-agent" ]) ?? "/usr/local/bin/cursor-agent"; case "hermes": return firstExistingPath([ - join25(home, ".local", "bin", "hermes"), + join26(home, ".local", "bin", "hermes"), "/usr/local/bin/hermes", "/usr/bin/hermes", - join25(home, ".npm-global", "bin", "hermes"), + join26(home, ".npm-global", "bin", "hermes"), "/opt/homebrew/bin/hermes" - ]) ?? join25(home, ".local", "bin", "hermes"); + ]) ?? join26(home, ".local", "bin", "hermes"); case "pi": return firstExistingPath([ - join25(home, ".local", "bin", "pi"), + join26(home, ".local", "bin", "pi"), "/usr/local/bin/pi", "/usr/bin/pi", - join25(home, ".npm-global", "bin", "pi"), + join26(home, ".npm-global", "bin", "pi"), "/opt/homebrew/bin/pi" - ]) ?? join25(home, ".local", "bin", "pi"); + ]) ?? join26(home, ".local", "bin", "pi"); } } @@ -6120,23 +6290,23 @@ function extractJsonBlock(s) { } // dist/src/skillify/local-manifest.js -import { existsSync as existsSync23, mkdirSync as mkdirSync8, readFileSync as readFileSync16, writeFileSync as writeFileSync12 } from "node:fs"; -import { homedir as homedir16 } from "node:os"; -import { dirname as dirname4, join as join26 } from "node:path"; -var LOCAL_MANIFEST_PATH = join26(homedir16(), ".claude", "hivemind", "local-mined.json"); -var LOCAL_MINE_LOCK_PATH = join26(homedir16(), ".claude", "hivemind", "local-mined.lock"); +import { existsSync as existsSync24, mkdirSync as mkdirSync9, readFileSync as readFileSync18, writeFileSync as writeFileSync13 } from "node:fs"; +import { homedir as homedir17 } from "node:os"; +import { dirname as dirname5, join as join27 } from "node:path"; +var LOCAL_MANIFEST_PATH = join27(homedir17(), ".claude", "hivemind", "local-mined.json"); +var LOCAL_MINE_LOCK_PATH = join27(homedir17(), ".claude", "hivemind", "local-mined.lock"); function readLocalManifest(path = LOCAL_MANIFEST_PATH) { - if (!existsSync23(path)) + if (!existsSync24(path)) return null; try { - return JSON.parse(readFileSync16(path, "utf-8")); + return JSON.parse(readFileSync18(path, "utf-8")); } catch { return null; } } function writeLocalManifest(m, path = LOCAL_MANIFEST_PATH) { - mkdirSync8(dirname4(path), { recursive: true }); - writeFileSync12(path, JSON.stringify(m, null, 2)); + mkdirSync9(dirname5(path), { recursive: true }); + writeFileSync13(path, JSON.stringify(m, null, 2)); } // dist/src/commands/mine-local.js @@ -6161,7 +6331,7 @@ function runGateViaStdin(opts) { }); return; } - if (!existsSync24(opts.bin)) { + if (!existsSync25(opts.bin)) { resolve({ stdout: "", stderr: "", @@ -6506,8 +6676,8 @@ async function runMineLocalImpl(args) { console.log(`Dry-run: would invoke ${gateAgent} gate on ${picked.length} session(s) in parallel (concurrency=${GATE_CONCURRENCY}).`); return; } - const tmpDir = join27(homedir17(), ".claude", "hivemind", `mine-local-${Date.now()}`); - mkdirSync9(tmpDir, { recursive: true }); + const tmpDir = join28(homedir18(), ".claude", "hivemind", `mine-local-${Date.now()}`); + mkdirSync10(tmpDir, { recursive: true }); console.log(`Running ${picked.length} gate call(s) in parallel (concurrency=${GATE_CONCURRENCY}, timeout=${GATE_TIMEOUT_MS / 1e3}s each)...`); const results = await parallelMap(picked, GATE_CONCURRENCY, async (s) => { const shortId = s.sessionId.slice(0, 8); @@ -6518,23 +6688,23 @@ async function runMineLocalImpl(args) { return { session: s, skills: [], reason: "no pairs", error: null }; } const tail = pairs2.slice(-PER_SESSION_PAIR_CAP); - const sessionTmp = join27(tmpDir, `s-${shortId}`); - mkdirSync9(sessionTmp, { recursive: true }); - const verdictPath = join27(sessionTmp, "verdict.json"); + const sessionTmp = join28(tmpDir, `s-${shortId}`); + mkdirSync10(sessionTmp, { recursive: true }); + const verdictPath = join28(sessionTmp, "verdict.json"); const prompt = buildSessionPrompt(tail, s, verdictPath); - writeFileSync13(join27(sessionTmp, "prompt.txt"), prompt); + writeFileSync14(join28(sessionTmp, "prompt.txt"), prompt); const gate = await runGateViaStdin({ agent: gateAgent, bin: gateBin, prompt, timeoutMs: GATE_TIMEOUT_MS }); try { - writeFileSync13(join27(sessionTmp, "gate-stdout.txt"), gate.stdout); + writeFileSync14(join28(sessionTmp, "gate-stdout.txt"), gate.stdout); if (gate.stderr) - writeFileSync13(join27(sessionTmp, "gate-stderr.txt"), gate.stderr); + writeFileSync14(join28(sessionTmp, "gate-stderr.txt"), gate.stderr); } catch { } if (gate.errored) { console.log(` [${shortId}] gate failed: ${gate.errorMessage}`); return { session: s, skills: [], reason: null, error: gate.errorMessage ?? "gate failed" }; } - const verdictText = existsSync24(verdictPath) ? readFileSync17(verdictPath, "utf-8") : gate.stdout; + const verdictText = existsSync25(verdictPath) ? readFileSync19(verdictPath, "utf-8") : gate.stdout; const mv = parseMultiVerdict(verdictText); if (!mv) { console.log(` [${shortId}] unparseable verdict (kept at ${sessionTmp})`); @@ -6586,7 +6756,7 @@ async function runMineLocalImpl(args) { sourceSessions: [session.sessionId], agent: gateAgent }); - const canonicalDir = dirname5(result.path); + const canonicalDir = dirname6(result.path); const symlinks = fanOutRoots.length > 0 ? fanOutSymlinks(canonicalDir, basename(canonicalDir), fanOutRoots) : []; const symlinkSuffix = symlinks.length > 0 ? `, fan-out \u2192 ${symlinks.length} root(s)` : ""; console.log(` wrote ${skill.name} \u2190 session ${session.sessionId.slice(0, 8)} (${session.agent}${symlinkSuffix})`); @@ -6750,7 +6920,7 @@ function wrapAt(s, max) { // dist/src/commands/skillify.js function stateDir() { - return join28(homedir18(), ".deeplake", "state", "skillify"); + return join29(homedir19(), ".deeplake", "state", "skillify"); } function showStatus() { const cfg = loadScopeConfig(); @@ -6758,7 +6928,7 @@ function showStatus() { console.log(`team: ${cfg.team.length === 0 ? "(empty)" : cfg.team.join(", ")}`); console.log(`install: ${cfg.install} (${cfg.install === "global" ? "~/.claude/skills/" : "/.claude/skills/"})`); const dir = stateDir(); - if (!existsSync25(dir)) { + if (!existsSync26(dir)) { console.log(`state: (no projects tracked yet)`); return; } @@ -6770,7 +6940,7 @@ function showStatus() { console.log(`state: ${files.length} project(s) tracked`); for (const f of files) { try { - const s = JSON.parse(readFileSync18(join28(dir, f), "utf-8")); + const s = JSON.parse(readFileSync20(join29(dir, f), "utf-8")); const last = typeof s.updatedAt === "number" ? new Date(s.updatedAt).toISOString() : s.lastDate ?? "never"; const skills = Array.isArray(s.skillsGenerated) && s.skillsGenerated.length > 0 ? s.skillsGenerated.join(", ") : "none"; console.log(` - ${s.project} (counter=${s.counter}, last=${last}, skills=${skills})`); @@ -6797,7 +6967,7 @@ function setInstall(loc) { } const cfg = loadScopeConfig(); saveScopeConfig({ ...cfg, install: loc }); - const path = loc === "global" ? join28(homedir18(), ".claude", "skills") : "/.claude/skills"; + const path = loc === "global" ? join29(homedir19(), ".claude", "skills") : "/.claude/skills"; console.log(`Install location set to '${loc}'. New skills will be written to ${path}//SKILL.md.`); } function promoteSkill(name, cwd) { @@ -6805,18 +6975,18 @@ function promoteSkill(name, cwd) { console.error("Usage: hivemind skillify promote "); process.exit(1); } - const projectPath = join28(cwd, ".claude", "skills", name); - const globalPath = join28(homedir18(), ".claude", "skills", name); - if (!existsSync25(join28(projectPath, "SKILL.md"))) { + const projectPath = join29(cwd, ".claude", "skills", name); + const globalPath = join29(homedir19(), ".claude", "skills", name); + if (!existsSync26(join29(projectPath, "SKILL.md"))) { console.error(`Skill '${name}' not found at ${projectPath}/SKILL.md`); process.exit(1); } - if (existsSync25(join28(globalPath, "SKILL.md"))) { + if (existsSync26(join29(globalPath, "SKILL.md"))) { console.error(`Skill '${name}' already exists at ${globalPath}/SKILL.md \u2014 refusing to overwrite. Remove it first or rename the project skill.`); process.exit(1); } - mkdirSync10(dirname6(globalPath), { recursive: true }); - renameSync5(projectPath, globalPath); + mkdirSync11(dirname7(globalPath), { recursive: true }); + renameSync6(projectPath, globalPath); console.log(`Promoted '${name}' from ${projectPath} \u2192 ${globalPath}.`); } function teamAdd(name) { @@ -6922,7 +7092,7 @@ async function pullSkills(args) { console.error(`pull failed: ${e?.message ?? e}`); process.exit(1); } - const dest = toRaw === "global" ? join28(homedir18(), ".claude", "skills") : `${process.cwd()}/.claude/skills`; + const dest = toRaw === "global" ? join29(homedir19(), ".claude", "skills") : `${process.cwd()}/.claude/skills`; const filterDesc = users.length === 0 ? "all users" : users.join(", "); console.log(`Destination: ${dest}`); console.log(`Filter: ${filterDesc}${skillName ? ` \xB7 skill='${skillName}'` : ""}${dryRun ? " \xB7 dry-run" : ""}${force ? " \xB7 force" : ""}`); @@ -6972,7 +7142,7 @@ async function unpullSkills(args) { all, legacyCleanup }); - const dest = toRaw === "global" ? join28(homedir18(), ".claude", "skills") : `${process.cwd()}/.claude/skills`; + const dest = toRaw === "global" ? join29(homedir19(), ".claude", "skills") : `${process.cwd()}/.claude/skills`; const filterParts = []; if (users.length > 0) filterParts.push(`users=${users.join(",")}`); @@ -7068,13 +7238,13 @@ if (process.argv[1] && process.argv[1].endsWith("skillify.js")) { // dist/src/cli/update.js import { execFileSync as execFileSync4 } from "node:child_process"; -import { existsSync as existsSync26, readFileSync as readFileSync20, realpathSync } from "node:fs"; -import { dirname as dirname8, sep } from "node:path"; +import { existsSync as existsSync27, readFileSync as readFileSync22, realpathSync } from "node:fs"; +import { dirname as dirname9, sep } from "node:path"; import { fileURLToPath as fileURLToPath2 } from "node:url"; // dist/src/utils/version-check.js -import { readFileSync as readFileSync19 } from "node:fs"; -import { dirname as dirname7, join as join29 } from "node:path"; +import { readFileSync as readFileSync21 } from "node:fs"; +import { dirname as dirname8, join as join30 } from "node:path"; function isNewer(latest, current) { const parse = (v) => v.split(".").map(Number); const [la, lb, lc] = parse(latest); @@ -7093,24 +7263,24 @@ function detectInstallKind(argv1) { return argv1 ?? process.argv[1] ?? fileURLToPath2(import.meta.url); } })(); - let dir = dirname8(realArgv1); + let dir = dirname9(realArgv1); let installDir = null; for (let i = 0; i < 10; i++) { const pkgPath = `${dir}${sep}package.json`; try { - const pkg = JSON.parse(readFileSync20(pkgPath, "utf-8")); + const pkg = JSON.parse(readFileSync22(pkgPath, "utf-8")); if (pkg.name === PKG_NAME || pkg.name === "hivemind") { installDir = dir; break; } } catch { } - const parent = dirname8(dir); + const parent = dirname9(dir); if (parent === dir) break; dir = parent; } - installDir ??= dirname8(realArgv1); + installDir ??= dirname9(realArgv1); if (realArgv1.includes(`${sep}_npx${sep}`) || realArgv1.includes(`${sep}.npx${sep}`)) { return { kind: "npx", installDir }; } @@ -7119,10 +7289,10 @@ function detectInstallKind(argv1) { } let gitDir = installDir; for (let i = 0; i < 6; i++) { - if (existsSync26(`${gitDir}${sep}.git`)) { + if (existsSync27(`${gitDir}${sep}.git`)) { return { kind: "local-dev", installDir }; } - const parent = dirname8(gitDir); + const parent = dirname9(gitDir); if (parent === gitDir) break; gitDir = parent; @@ -7265,12 +7435,28 @@ Usage: Semantic search (embeddings): hivemind embeddings install Download @huggingface/transformers - once (~600 MB) into a shared dir - and symlink every detected agent - plugin to it. Idempotent. - hivemind embeddings uninstall [--prune] Remove the per-agent symlinks. - --prune also deletes the shared dir. - hivemind embeddings status Show shared-deps + per-agent state. + once (~600 MB) into a shared dir, + symlink every detected agent + plugin to it, and set + embeddings.enabled = true in + ~/.deeplake/config.json. Idempotent. + hivemind embeddings enable Light opt-in: flip + embeddings.enabled = true in + ~/.deeplake/config.json. Use this + after \`disable\` to turn back on + without re-running install. + hivemind embeddings disable Light opt-out: flip + embeddings.enabled = false and + SIGTERM the running daemon. Shared + deps stay on disk. + hivemind embeddings uninstall [--prune] Full opt-out: remove the per-agent + symlinks, flip + embeddings.enabled = false, and + SIGTERM the daemon. --prune also + deletes the shared dir to reclaim + ~600 MB. + hivemind embeddings status Show config + shared-deps + per- + agent state. Add --with-embeddings to "hivemind install" (or "hivemind install") to run "embeddings install" automatically after installing the agent(s). @@ -7340,7 +7526,7 @@ async function runInstallAll(args) { runSingleInstall(id); if (withEmbeddings) { log(""); - enableEmbeddings(); + installEmbeddings(); } await maybeShowOrgChoice(); log(""); @@ -7433,19 +7619,27 @@ async function main() { } if (cmd === "embeddings") { const sub = args[1]; - if (sub === "install" || sub === "enable") { + if (sub === "install") { + installEmbeddings(); + return; + } + if (sub === "enable") { enableEmbeddings(); return; } - if (sub === "uninstall" || sub === "disable") { - disableEmbeddings({ prune: hasFlag(args.slice(2), "--prune") }); + if (sub === "disable") { + disableEmbeddings(); + return; + } + if (sub === "uninstall") { + uninstallEmbeddings({ prune: hasFlag(args.slice(2), "--prune") }); return; } if (sub === "status") { statusEmbeddings(); return; } - warn("Usage: hivemind embeddings install | uninstall [--prune] | status"); + warn("Usage: hivemind embeddings install | enable | disable | uninstall [--prune] | status"); process.exit(1); } if (AUTH_SUBCOMMANDS.has(cmd)) { @@ -7459,7 +7653,7 @@ async function main() { runSingleInstall(cmd); if (hasFlag(args.slice(2), "--with-embeddings")) { log(""); - enableEmbeddings(); + installEmbeddings(); } } else if (sub === "uninstall") runSingleUninstall(cmd); diff --git a/claude-code/bundle/capture.js b/claude-code/bundle/capture.js index 0288b2f5..f867621f 100755 --- a/claude-code/bundle/capture.js +++ b/claude-code/bundle/capture.js @@ -54,13 +54,13 @@ var init_index_marker_store = __esm({ // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -176,7 +176,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -206,7 +206,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -1272,9 +1272,9 @@ function tryStopCounterTrigger(opts) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn as spawn3 } from "node:child_process"; -import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync3, unlinkSync as unlinkSync3, existsSync as existsSync8, readFileSync as readFileSync7 } from "node:fs"; -import { homedir as homedir10 } from "node:os"; -import { join as join13 } from "node:path"; +import { openSync as openSync4, closeSync as closeSync4, writeSync as writeSync3, unlinkSync as unlinkSync4, existsSync as existsSync9, readFileSync as readFileSync9 } from "node:fs"; +import { homedir as homedir13 } from "node:os"; +import { join as join16 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -1287,13 +1287,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync7, writeFileSync as writeFileSync7, renameSync as renameSync4, mkdirSync as mkdirSync8, openSync as openSync3, closeSync as closeSync3, unlinkSync as unlinkSync3, statSync } from "node:fs"; +import { join as join13, resolve } from "node:path"; +import { homedir as homedir10 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join13(homedir10(), ".deeplake", "notifications-queue.json"); +} +function lockPath3() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync7(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir10()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync8(join13(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync7(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync4(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath3(); + mkdirSync8(join13(homedir10(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync3(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync3(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync3(fd); + } catch { + } + try { + unlinkSync3(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire as createRequire2 } from "node:module"; +import { homedir as homedir12 } from "node:os"; +import { join as join15 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync8, mkdirSync as mkdirSync9, readFileSync as readFileSync8, renameSync as renameSync5, writeFileSync as writeFileSync8 } from "node:fs"; +import { homedir as homedir11 } from "node:os"; +import { dirname as dirname4, join as join14 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join14(homedir11(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync8(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync8(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname4(path); + if (!existsSync8(dir)) + mkdirSync9(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync8(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync5(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join15(homedir12(), ".hivemind", "embed-deps"); + try { + createRequire2(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire2(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join13(homedir10(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join16(homedir13(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -1302,13 +1523,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync8(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync9(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -1318,8 +1540,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -1329,17 +1576,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -1348,6 +1603,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync9(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync9(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync9(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync9(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync4(this.socketPath); + } catch { + } + try { + unlinkSync4(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -1371,7 +1759,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -1379,7 +1767,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -1390,16 +1778,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync3(this.pidPath, "wx", 384); + fd = openSync4(this.pidPath, "wx", 384); writeSync3(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync3(this.pidPath); + unlinkSync4(this.pidPath); } catch { } try { - fd = openSync3(this.pidPath, "wx", 384); + fd = openSync4(this.pidPath, "wx", 384); writeSync3(fd, String(process.pid)); } catch { return; @@ -1408,11 +1796,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync8(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync9(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync3(fd); - unlinkSync3(this.pidPath); + closeSync4(fd); + unlinkSync4(this.pidPath); } catch { } return; @@ -1424,14 +1812,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync3(fd); + closeSync4(fd); } } isPidFileStale() { try { - const raw = readFileSync7(this.pidPath, "utf-8").trim(); + const raw = readFileSync9(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -1449,9 +1837,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync8(this.socketPath)) + if (!existsSync9(this.socketPath)) continue; try { return await this.connectOnce(); @@ -1461,7 +1849,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -1476,7 +1864,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -1493,9 +1881,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -1510,51 +1903,86 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire as createRequire2 } from "node:module"; -import { homedir as homedir11 } from "node:os"; -import { join as join14 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { +// dist/src/embeddings/self-heal.js +import { existsSync as existsSync10, lstatSync, mkdirSync as mkdirSync10, readlinkSync, renameSync as renameSync6, rmSync, symlinkSync, statSync as statSync2 } from "node:fs"; +import { homedir as homedir14 } from "node:os"; +import { basename as basename2, dirname as dirname5, join as join17 } from "node:path"; +function ensurePluginNodeModulesLink(opts) { + if (basename2(opts.bundleDir) !== "bundle") { + return { kind: "not-bundle-layout", bundleDir: opts.bundleDir }; + } + const target = opts.sharedNodeModules ?? join17(homedir14(), ".hivemind", "embed-deps", "node_modules"); + const pluginDir = dirname5(opts.bundleDir); + const link = join17(pluginDir, "node_modules"); + if (!existsSync10(target)) { + return { kind: "shared-deps-missing", target }; + } + let linkStat; try { - createRequire2(import.meta.url).resolve("@huggingface/transformers"); - return; + linkStat = lstatSync(link); } catch { + return createSymlinkAtomic(target, link); + } + if (linkStat.isSymbolicLink()) { + let existingTarget; + try { + existingTarget = readlinkSync(link); + } catch (e) { + return { kind: "error", detail: `readlink failed: ${e instanceof Error ? e.message : String(e)}` }; + } + if (existingTarget === target) { + return { kind: "already-linked", target, link }; + } + try { + statSync2(link); + return { kind: "linked-elsewhere", link, existingTarget }; + } catch { + try { + rmSync(link); + } catch { + } + const recreated = createSymlinkAtomic(target, link); + if (recreated.kind === "linked") { + return { kind: "stale-link-removed", link, danglingTarget: existingTarget }; + } + return recreated; + } } - const sharedDir = join14(homedir11(), ".hivemind", "embed-deps"); - createRequire2(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return { kind: "plugin-owns-node-modules", link }; } -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; +function createSymlinkAtomic(target, link) { try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; + const parent = dirname5(link); + if (!existsSync10(parent)) + mkdirSync10(parent, { recursive: true }); + const tmp = `${link}.tmp.${process.pid}`; + try { + rmSync(tmp, { force: true }); + } catch { + } + symlinkSync(target, tmp); + renameSync6(tmp, link); + return { kind: "linked", target, link }; + } catch (e) { + return { kind: "error", detail: e instanceof Error ? e.message : String(e) }; } } -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} // dist/src/hooks/capture.js import { fileURLToPath as fileURLToPath3 } from "node:url"; -import { dirname as dirname4, join as join15 } from "node:path"; -var log4 = (msg) => log("capture", msg); +import { dirname as dirname6, join as join18 } from "node:path"; +var log5 = (msg) => log("capture", msg); function resolveEmbedDaemonPath() { - return join15(dirname4(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); + return join18(dirname6(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); } -var __bundleDir = dirname4(fileURLToPath3(import.meta.url)); +var __bundleDir = dirname6(fileURLToPath3(import.meta.url)); var PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".claude-plugin") ?? ""; +if (!embeddingsDisabled()) { + try { + ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); + } catch { + } +} var CAPTURE = process.env.HIVEMIND_CAPTURE !== "false"; async function main() { if (!CAPTURE) @@ -1562,7 +1990,7 @@ async function main() { const input = await readStdin(); const config = loadConfig(); if (!config) { - log4("no config"); + log5("no config"); return; } const sessionsTable = config.sessionsTableName; @@ -1580,7 +2008,7 @@ async function main() { }; let entry; if (input.prompt !== void 0) { - log4(`user session=${input.session_id}`); + log5(`user session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1588,7 +2016,7 @@ async function main() { content: input.prompt }; } else if (input.tool_name !== void 0) { - log4(`tool=${input.tool_name} session=${input.session_id}`); + log5(`tool=${input.tool_name} session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1599,7 +2027,7 @@ async function main() { tool_response: JSON.stringify(input.tool_response) }; } else if (input.last_assistant_message !== void 0) { - log4(`assistant session=${input.session_id}`); + log5(`assistant session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1608,12 +2036,12 @@ async function main() { ...input.agent_transcript_path ? { agent_transcript_path: input.agent_transcript_path } : {} }; } else { - log4("unknown event, skipping"); + log5("unknown event, skipping"); return; } const sessionPath = buildSessionPath(config, input.session_id); const line = JSON.stringify(entry); - log4(`writing to ${sessionPath}`); + log5(`writing to ${sessionPath}`); const projectName = (input.cwd ?? "").split("/").pop() || "unknown"; const filename = sessionPath.split("/").pop() ?? ""; const jsonForSql = line.replace(/'/g, "''"); @@ -1624,14 +2052,14 @@ async function main() { await api.query(insertSql); } catch (e) { if (e.message?.includes("permission denied") || e.message?.includes("does not exist")) { - log4("table missing, creating and retrying"); + log5("table missing, creating and retrying"); await api.ensureSessionsTable(sessionsTable); await api.query(insertSql); } else { throw e; } } - log4("capture ok \u2192 cloud"); + log5("capture ok \u2192 cloud"); maybeTriggerPeriodicSummary(input.session_id, input.cwd ?? "", config); if (input.hook_event_name === "Stop") { if (process.env.HIVEMIND_WIKI_WORKER === "1") @@ -1654,7 +2082,7 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { if (!shouldTrigger(state, cfg)) return; if (!tryAcquireLock(sessionId)) { - log4(`periodic trigger suppressed (lock held) session=${sessionId}`); + log5(`periodic trigger suppressed (lock held) session=${sessionId}`); return; } wikiLog(`Periodic: threshold hit (total=${state.totalCount}, since=${state.totalCount - state.lastSummaryCount}, N=${cfg.everyNMessages}, hours=${cfg.everyHours})`); @@ -1667,19 +2095,19 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { reason: "Periodic" }); } catch (e) { - log4(`periodic spawn failed: ${e.message}`); + log5(`periodic spawn failed: ${e.message}`); try { releaseLock(sessionId); } catch (releaseErr) { - log4(`releaseLock after periodic spawn failure also failed: ${releaseErr.message}`); + log5(`releaseLock after periodic spawn failure also failed: ${releaseErr.message}`); } throw e; } } catch (e) { - log4(`periodic trigger error: ${e.message}`); + log5(`periodic trigger error: ${e.message}`); } } main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); diff --git a/claude-code/bundle/embeddings/embed-daemon.js b/claude-code/bundle/embeddings/embed-daemon.js index 5f711a3c..a81ffbd1 100755 --- a/claude-code/bundle/embeddings/embed-daemon.js +++ b/claude-code/bundle/embeddings/embed-daemon.js @@ -4,7 +4,14 @@ import { createServer } from "node:net"; import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +// dist/src/embeddings/nomic.js +import { createRequire } from "node:module"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + // dist/src/embeddings/protocol.js +var PROTOCOL_VERSION = 1; var DEFAULT_SOCKET_DIR = "/tmp"; var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; var DEFAULT_DTYPE = "q8"; @@ -20,6 +27,39 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { } // dist/src/embeddings/nomic.js +async function _importFromCanonicalSharedDeps(sharedDir = join(homedir(), ".hivemind", "embed-deps")) { + const base = pathToFileURL(`${sharedDir}/`).href; + const absMain = createRequire(base).resolve("@huggingface/transformers"); + const mod = await import(pathToFileURL(absMain).href); + return _normalizeTransformersModule(mod); +} +async function _importFromBareSpecifier() { + const mod = await import("@huggingface/transformers"); + return _normalizeTransformersModule(mod); +} +function _normalizeTransformersModule(mod) { + const m = mod; + if (m.default && typeof m.default === "object" && "pipeline" in m.default) { + return m.default; + } + return m; +} +async function defaultImportTransformers(canonical = _importFromCanonicalSharedDeps, bare = _importFromBareSpecifier) { + let canonicalErr; + try { + return await canonical(); + } catch (err) { + canonicalErr = err; + } + try { + return await bare(); + } catch (bareErr) { + const detail = bareErr instanceof Error ? bareErr.message : String(bareErr); + const canonicalDetail = canonicalErr instanceof Error ? canonicalErr.message : String(canonicalErr); + throw new Error(`@huggingface/transformers is not installed anywhere reachable. Run \`hivemind embeddings install\` to install it. (canonical: ${canonicalDetail}; bare: ${detail})`); + } +} +var _importTransformers = defaultImportTransformers; var NomicEmbedder = class { pipeline = null; loading = null; @@ -37,7 +77,7 @@ var NomicEmbedder = class { if (this.loading) return this.loading; this.loading = (async () => { - const mod = await import("@huggingface/transformers"); + const mod = await _importTransformers(); mod.env.allowLocalModels = false; mod.env.useFSCache = true; this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype }); @@ -94,9 +134,9 @@ var NomicEmbedder = class { // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; -var LOG = join(homedir(), ".deeplake", "hook-debug.log"); +import { join as join2 } from "node:path"; +import { homedir as homedir2 } from "node:os"; +var LOG = join2(homedir2(), ".deeplake", "hook-debug.log"); function isDebug() { return process.env.HIVEMIND_DEBUG === "1"; } @@ -120,6 +160,7 @@ var EmbedDaemon = class { pidPath; idleTimeoutMs; idleTimer = null; + daemonPath; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; @@ -127,6 +168,7 @@ var EmbedDaemon = class { this.pidPath = pidPathFor(uid, dir); this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + this.daemonPath = opts.daemonPath ?? process.argv[1] ?? ""; } async start() { mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); @@ -218,6 +260,15 @@ var EmbedDaemon = class { } } async dispatch(req) { + if (req.op === "hello") { + const h = req; + return { + id: h.id, + daemonPath: this.daemonPath, + pid: process.pid, + protocolVersion: PROTOCOL_VERSION + }; + } if (req.op === "ping") { const p = req; return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; diff --git a/claude-code/bundle/pre-tool-use.js b/claude-code/bundle/pre-tool-use.js index 227bc13b..563e1eac 100755 --- a/claude-code/bundle/pre-tool-use.js +++ b/claude-code/bundle/pre-tool-use.js @@ -53,20 +53,20 @@ var init_index_marker_store = __esm({ }); // dist/src/hooks/pre-tool-use.js -import { existsSync as existsSync4, mkdirSync as mkdirSync3, writeFileSync as writeFileSync3 } from "node:fs"; -import { homedir as homedir7 } from "node:os"; -import { join as join9, dirname as dirname2, sep } from "node:path"; +import { existsSync as existsSync5, mkdirSync as mkdirSync5, writeFileSync as writeFileSync5 } from "node:fs"; +import { homedir as homedir9 } from "node:os"; +import { join as join11, dirname as dirname3, sep } from "node:path"; import { fileURLToPath as fileURLToPath3 } from "node:url"; // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve2, reject) => { + return new Promise((resolve3, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve2(JSON.parse(data)); + resolve3(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -182,7 +182,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve2) => setTimeout(resolve2, ms)); + return new Promise((resolve3) => setTimeout(resolve3, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -212,7 +212,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve2) => this.waiting.push(resolve2)); + await new Promise((resolve3) => this.waiting.push(resolve3)); } release() { this.active--; @@ -1063,9 +1063,9 @@ function capOutputForClaude(output, options = {}) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join4 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -1078,13 +1078,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync } from "node:fs"; +import { join as join4, resolve as resolve2 } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join4(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve2(path); + const h = resolve2(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve2(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join4(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync2(join4(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join6 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join5 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join5(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join6(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join4(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join7(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -1093,13 +1314,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -1109,8 +1331,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -1120,17 +1367,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -1139,6 +1394,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -1162,7 +1550,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve2, reject) => { + return new Promise((resolve3, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -1170,7 +1558,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve2(sock); + resolve3(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -1181,16 +1569,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -1199,11 +1587,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -1215,14 +1603,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -1240,9 +1628,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -1252,7 +1640,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve2, reject) => { + return new Promise((resolve3, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -1267,7 +1655,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve2(JSON.parse(line)); + resolve3(JSON.parse(line)); } catch (e) { reject(e); } @@ -1284,53 +1672,22 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join5(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/hooks/grep-direct.js import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname, join as join6 } from "node:path"; +import { dirname as dirname2, join as join8 } from "node:path"; var SEMANTIC_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveDaemonPath() { - return join6(dirname(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join8(dirname2(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedEmbedClient = null; function getEmbedClient() { @@ -2333,20 +2690,20 @@ async function executeCompiledBashCommand(api, memoryTable, sessionsTable, cmd, } // dist/src/hooks/query-cache.js -import { mkdirSync as mkdirSync2, readFileSync as readFileSync4, rmSync, writeFileSync as writeFileSync2 } from "node:fs"; -import { join as join7 } from "node:path"; -import { homedir as homedir5 } from "node:os"; -var log4 = (msg) => log("query-cache", msg); -var DEFAULT_CACHE_ROOT = join7(homedir5(), ".deeplake", "query-cache"); +import { mkdirSync as mkdirSync4, readFileSync as readFileSync6, rmSync, writeFileSync as writeFileSync4 } from "node:fs"; +import { join as join9 } from "node:path"; +import { homedir as homedir7 } from "node:os"; +var log5 = (msg) => log("query-cache", msg); +var DEFAULT_CACHE_ROOT = join9(homedir7(), ".deeplake", "query-cache"); var INDEX_CACHE_FILE = "index.md"; function getSessionQueryCacheDir(sessionId, deps = {}) { const { cacheRoot = DEFAULT_CACHE_ROOT } = deps; - return join7(cacheRoot, sessionId); + return join9(cacheRoot, sessionId); } function readCachedIndexContent(sessionId, deps = {}) { - const { logFn = log4 } = deps; + const { logFn = log5 } = deps; try { - return readFileSync4(join7(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); + return readFileSync6(join9(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); } catch (e) { if (e?.code === "ENOENT") return null; @@ -2355,20 +2712,20 @@ function readCachedIndexContent(sessionId, deps = {}) { } } function writeCachedIndexContent(sessionId, content, deps = {}) { - const { logFn = log4 } = deps; + const { logFn = log5 } = deps; try { const dir = getSessionQueryCacheDir(sessionId, deps); - mkdirSync2(dir, { recursive: true }); - writeFileSync2(join7(dir, INDEX_CACHE_FILE), content, "utf-8"); + mkdirSync4(dir, { recursive: true }); + writeFileSync4(join9(dir, INDEX_CACHE_FILE), content, "utf-8"); } catch (e) { logFn(`write failed for session=${sessionId}: ${e.message}`); } } // dist/src/hooks/memory-path-utils.js -import { homedir as homedir6 } from "node:os"; -import { join as join8 } from "node:path"; -var MEMORY_PATH = join8(homedir6(), ".deeplake", "memory"); +import { homedir as homedir8 } from "node:os"; +import { join as join10 } from "node:path"; +var MEMORY_PATH = join10(homedir8(), ".deeplake", "memory"); var TILDE_PATH = "~/.deeplake/memory"; var HOME_VAR_PATH = "$HOME/.deeplake/memory"; var SAFE_BUILTINS = /* @__PURE__ */ new Set([ @@ -2484,21 +2841,21 @@ function rewritePaths(cmd) { } // dist/src/hooks/pre-tool-use.js -var log5 = (msg) => log("pre", msg); -var __bundleDir = dirname2(fileURLToPath3(import.meta.url)); -var SHELL_BUNDLE = existsSync4(join9(__bundleDir, "shell", "deeplake-shell.js")) ? join9(__bundleDir, "shell", "deeplake-shell.js") : join9(__bundleDir, "..", "shell", "deeplake-shell.js"); -var READ_CACHE_ROOT = join9(homedir7(), ".deeplake", "query-cache"); +var log6 = (msg) => log("pre", msg); +var __bundleDir = dirname3(fileURLToPath3(import.meta.url)); +var SHELL_BUNDLE = existsSync5(join11(__bundleDir, "shell", "deeplake-shell.js")) ? join11(__bundleDir, "shell", "deeplake-shell.js") : join11(__bundleDir, "..", "shell", "deeplake-shell.js"); +var READ_CACHE_ROOT = join11(homedir9(), ".deeplake", "query-cache"); function writeReadCacheFile(sessionId, virtualPath, content, deps = {}) { const { cacheRoot = READ_CACHE_ROOT } = deps; const safeSessionId = sessionId.replace(/[^a-zA-Z0-9._-]/g, "_") || "unknown"; const rel = virtualPath.replace(/^\/+/, "") || "content"; - const expectedRoot = join9(cacheRoot, safeSessionId, "read"); - const absPath = join9(expectedRoot, rel); + const expectedRoot = join11(cacheRoot, safeSessionId, "read"); + const absPath = join11(expectedRoot, rel); if (absPath !== expectedRoot && !absPath.startsWith(expectedRoot + sep)) { throw new Error(`writeReadCacheFile: path escapes cache root: ${absPath}`); } - mkdirSync3(dirname2(absPath), { recursive: true }); - writeFileSync3(absPath, content, "utf-8"); + mkdirSync5(dirname3(absPath), { recursive: true }); + writeFileSync5(absPath, content, "utf-8"); return absPath; } function buildReadDecision(file_path, description) { @@ -2544,7 +2901,7 @@ function getShellCommand(toolName, toolInput) { break; const rewritten = rewritePaths(cmd); if (!isSafe(rewritten)) { - log5(`unsafe command blocked: ${rewritten}`); + log6(`unsafe command blocked: ${rewritten}`); return null; } return rewritten; @@ -2584,7 +2941,7 @@ function buildFallbackDecision(shellCmd, shellBundle = SHELL_BUNDLE) { return buildAllowDecision(`node "${shellBundle}" -c "${shellCmd.replace(/"/g, '\\"')}"`, `[DeepLake shell] ${shellCmd}`); } async function processPreToolUse(input, deps = {}) { - const { config = loadConfig(), createApi = (table2, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table2), executeCompiledBashCommandFn = executeCompiledBashCommand, handleGrepDirectFn = handleGrepDirect, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, writeReadCacheFileFn = writeReadCacheFile, shellBundle = SHELL_BUNDLE, logFn = log5 } = deps; + const { config = loadConfig(), createApi = (table2, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table2), executeCompiledBashCommandFn = executeCompiledBashCommand, handleGrepDirectFn = handleGrepDirect, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, writeReadCacheFileFn = writeReadCacheFile, shellBundle = SHELL_BUNDLE, logFn = log6 } = deps; const cmd = input.tool_input.command ?? ""; const shellCmd = getShellCommand(input.tool_name, input.tool_input); const toolPath = getReadTargetPath(input.tool_input) ?? input.tool_input.path ?? ""; @@ -2807,7 +3164,7 @@ async function main() { } if (isDirectRun(import.meta.url)) { main().catch((e) => { - log5(`fatal: ${e.message}`); + log6(`fatal: ${e.message}`); process.exit(0); }); } diff --git a/claude-code/bundle/session-notifications.js b/claude-code/bundle/session-notifications.js index 2c6526f8..421b68bf 100755 --- a/claude-code/bundle/session-notifications.js +++ b/claude-code/bundle/session-notifications.js @@ -59,9 +59,10 @@ function evaluateRules(trigger, ctx) { } // dist/src/notifications/queue.js -import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2 } from "node:fs"; +import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync as unlinkSync2, statSync } from "node:fs"; import { join as join3, resolve } from "node:path"; import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; @@ -96,10 +97,15 @@ function readQueue() { return { queue: [] }; } } +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} function writeQueue(q) { const path = queuePath(); const home = resolve(homedir3()); - if (!resolve(path).startsWith(home + "/") && resolve(path) !== home) { + if (!_isQueuePathInsideHome(path, home)) { throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); } mkdirSync2(join3(home, ".deeplake"), { recursive: true, mode: 448 }); @@ -109,7 +115,7 @@ function writeQueue(q) { } // dist/src/notifications/state.js -import { closeSync, mkdirSync as mkdirSync3, openSync, readFileSync as readFileSync3, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { closeSync as closeSync2, mkdirSync as mkdirSync3, openSync as openSync2, readFileSync as readFileSync3, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; import { createHash } from "node:crypto"; import { join as join4, resolve as resolve2 } from "node:path"; import { homedir as homedir4 } from "node:os"; @@ -168,8 +174,8 @@ function tryClaim(n) { const safeId = n.id.replace(/[^a-zA-Z0-9_.:-]/g, "_"); const claimPath = join4(claimsDir, `${safeId}-${keyHash}`); try { - const fd = openSync(claimPath, "wx", 384); - closeSync(fd); + const fd = openSync2(claimPath, "wx", 384); + closeSync2(fd); return true; } catch (e) { if (e?.code === "EEXIST") diff --git a/claude-code/bundle/session-start-setup.js b/claude-code/bundle/session-start-setup.js index fde4efe4..7dfbd49a 100755 --- a/claude-code/bundle/session-start-setup.js +++ b/claude-code/bundle/session-start-setup.js @@ -54,8 +54,8 @@ var init_index_marker_store = __esm({ // dist/src/hooks/session-start-setup.js import { fileURLToPath } from "node:url"; -import { dirname, join as join9 } from "node:path"; -import { homedir as homedir6 } from "node:os"; +import { dirname as dirname2, join as join11 } from "node:path"; +import { homedir as homedir8 } from "node:os"; // dist/src/commands/auth.js import { execSync } from "node:child_process"; @@ -189,7 +189,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -219,7 +219,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -576,13 +576,13 @@ var DeeplakeApi = class { // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -612,9 +612,9 @@ function makeWikiLogger(hooksDir, filename = "deeplake-wiki.log") { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync3, readFileSync as readFileSync4 } from "node:fs"; -import { homedir as homedir4 } from "node:os"; -import { join as join6 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync3, existsSync as existsSync4, readFileSync as readFileSync6 } from "node:fs"; +import { homedir as homedir7 } from "node:os"; +import { join as join9 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -627,13 +627,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync4, writeFileSync as writeFileSync3, renameSync, mkdirSync as mkdirSync4, openSync, closeSync, unlinkSync as unlinkSync2, statSync } from "node:fs"; +import { join as join6, resolve } from "node:path"; +import { homedir as homedir4 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join6(homedir4(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync4(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir4()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync4(join6(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync3(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync4(join6(homedir4(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync2(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync2(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir6 } from "node:os"; +import { join as join8 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync5, readFileSync as readFileSync5, renameSync as renameSync2, writeFileSync as writeFileSync4 } from "node:fs"; +import { homedir as homedir5 } from "node:os"; +import { dirname, join as join7 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join7(homedir5(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync5(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync5(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync4(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join8(homedir6(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join6(homedir4(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join9(homedir7(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -642,13 +863,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -658,8 +880,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -669,17 +916,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -688,6 +943,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync6(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync3(this.socketPath); + } catch { + } + try { + unlinkSync3(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -711,7 +1099,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -719,7 +1107,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -730,16 +1118,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync2(this.pidPath); + unlinkSync3(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -748,11 +1136,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync2(this.pidPath); + closeSync2(fd); + unlinkSync3(this.pidPath); } catch { } return; @@ -764,14 +1152,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync4(this.pidPath, "utf-8").trim(); + const raw = readFileSync6(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -789,9 +1177,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -801,7 +1189,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -816,7 +1204,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -833,51 +1221,20 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir5 } from "node:os"; -import { join as join7 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join7(homedir5(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/hooks/shared/autoupdate.js import { spawn as spawn2 } from "node:child_process"; -import { existsSync as existsSync4 } from "node:fs"; -import { join as join8 } from "node:path"; -var log4 = (msg) => log("autoupdate", msg); +import { existsSync as existsSync5 } from "node:fs"; +import { join as join10 } from "node:path"; +var log5 = (msg) => log("autoupdate", msg); var defaultSpawn = (cmd, args) => { const child = spawn2(cmd, args, { detached: true, @@ -892,51 +1249,51 @@ function findHivemindOnPath() { const PATH = process.env.PATH ?? ""; const dirs = PATH.split(":").filter(Boolean); for (const dir of dirs) { - const candidate = join8(dir, "hivemind"); - if (existsSync4(candidate)) + const candidate = join10(dir, "hivemind"); + if (existsSync5(candidate)) return candidate; } return null; } async function autoUpdate(creds, opts) { const t0 = Date.now(); - log4(`agent=${opts.agent} entered`); + log5(`agent=${opts.agent} entered`); if (!creds?.token) { - log4(`agent=${opts.agent} skip: no creds.token (${Date.now() - t0}ms)`); + log5(`agent=${opts.agent} skip: no creds.token (${Date.now() - t0}ms)`); return; } if (creds.autoupdate === false) { - log4(`agent=${opts.agent} skip: autoupdate=false (${Date.now() - t0}ms)`); + log5(`agent=${opts.agent} skip: autoupdate=false (${Date.now() - t0}ms)`); return; } const binaryPath = opts.hivemindBinaryPath !== void 0 ? opts.hivemindBinaryPath : findHivemindOnPath(); if (!binaryPath) { - log4(`agent=${opts.agent} skip: hivemind binary not on PATH (${Date.now() - t0}ms)`); + log5(`agent=${opts.agent} skip: hivemind binary not on PATH (${Date.now() - t0}ms)`); return; } - log4(`agent=${opts.agent} binary=${binaryPath} \u2192 dispatching detached update`); + log5(`agent=${opts.agent} binary=${binaryPath} \u2192 dispatching detached update`); const spawnFn = opts.spawn ?? defaultSpawn; let pid; try { pid = spawnFn(binaryPath, ["update"]).pid; } catch (e) { - log4(`agent=${opts.agent} dispatch threw: ${e?.message ?? e} (${Date.now() - t0}ms)`); + log5(`agent=${opts.agent} dispatch threw: ${e?.message ?? e} (${Date.now() - t0}ms)`); return; } - log4(`agent=${opts.agent} dispatched (pid=${pid ?? "?"}) (${Date.now() - t0}ms total)`); + log5(`agent=${opts.agent} dispatched (pid=${pid ?? "?"}) (${Date.now() - t0}ms total)`); } // dist/src/hooks/session-start-setup.js -var log5 = (msg) => log("session-setup", msg); -var __bundleDir = dirname(fileURLToPath(import.meta.url)); -var { log: wikiLog } = makeWikiLogger(join9(homedir6(), ".claude", "hooks")); +var log6 = (msg) => log("session-setup", msg); +var __bundleDir = dirname2(fileURLToPath(import.meta.url)); +var { log: wikiLog } = makeWikiLogger(join11(homedir8(), ".claude", "hooks")); async function main() { if (process.env.HIVEMIND_WIKI_WORKER === "1") return; const input = await readStdin(); const creds = loadCredentials(); if (!creds?.token) { - log5("no credentials"); + log6("no credentials"); return; } if (!creds.userName) { @@ -944,7 +1301,7 @@ async function main() { const { userInfo: userInfo2 } = await import("node:os"); creds.userName = userInfo2().username ?? "unknown"; saveCredentials(creds); - log5(`backfilled userName: ${creds.userName}`); + log6(`backfilled userName: ${creds.userName}`); } catch { } } @@ -956,31 +1313,31 @@ async function main() { const api = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.tableName); await api.ensureTable(); await api.ensureSessionsTable(config.sessionsTableName); - log5("setup complete"); + log6("setup complete"); } } catch (e) { - log5(`setup failed: ${e.message}`); + log6(`setup failed: ${e.message}`); wikiLog(`SessionSetup: failed for ${input.session_id}: ${e.message}`); } } if (embeddingsDisabled()) { const status = embeddingsStatus(); - const reason = status === "no-transformers" ? "@huggingface/transformers not installed (see README to enable embeddings)" : "HIVEMIND_EMBEDDINGS=false"; - log5(`embed daemon warmup skipped: ${reason}`); + const reason = status === "no-transformers" ? "@huggingface/transformers not installed (run `hivemind embeddings install` to enable)" : "embeddings disabled in ~/.deeplake/config.json (run `hivemind embeddings enable` to opt in)"; + log6(`embed daemon warmup skipped: ${reason}`); } else if (process.env.HIVEMIND_EMBED_WARMUP !== "false") { try { - const daemonEntry = join9(__bundleDir, "embeddings", "embed-daemon.js"); + const daemonEntry = join11(__bundleDir, "embeddings", "embed-daemon.js"); const client = new EmbedClient({ daemonEntry, timeoutMs: 300, spawnWaitMs: 5e3 }); const ok = await client.warmup(); - log5(`embed daemon warmup: ${ok ? "ok" : "failed"}`); + log6(`embed daemon warmup: ${ok ? "ok" : "failed"}`); } catch (e) { - log5(`embed daemon warmup threw: ${e.message}`); + log6(`embed daemon warmup threw: ${e.message}`); } } else { - log5("embed daemon warmup skipped via HIVEMIND_EMBED_WARMUP=false"); + log6("embed daemon warmup skipped via HIVEMIND_EMBED_WARMUP=false"); } } main().catch((e) => { - log5(`fatal: ${e.message}`); + log6(`fatal: ${e.message}`); process.exit(0); }); diff --git a/claude-code/bundle/session-start.js b/claude-code/bundle/session-start.js index 74467e78..f3171e58 100755 --- a/claude-code/bundle/session-start.js +++ b/claude-code/bundle/session-start.js @@ -1552,6 +1552,13 @@ Organization management \u2014 each argument is SEPARATE (do NOT quote subcomman Skill management (mine + share reusable Claude skills across the org): ${renderSkillifyCommands()} +Embeddings (semantic memory search) \u2014 opt-in, persisted in ~/.deeplake/config.json: +- hivemind embeddings install \u2014 download deps (~600MB), symlink agents, set enabled:true +- hivemind embeddings enable \u2014 flip enabled:true (run install first if deps missing) +- hivemind embeddings disable \u2014 flip enabled:false + SIGTERM daemon (deps stay on disk) +- hivemind embeddings uninstall [--prune] \u2014 remove agent symlinks + disable; --prune wipes deps too +- hivemind embeddings status \u2014 show config + deps + per-agent link state + IMPORTANT: Only use bash commands (cat, ls, grep, echo, jq, head, tail, etc.) to interact with ~/.deeplake/memory/. Do NOT use python, python3, node, curl, or other interpreters \u2014 they are not available in the memory filesystem. Avoid bash brace expansions like \`{1..10}\` (not fully supported); spell out paths explicitly. Bash output is capped at 10MB total \u2014 avoid \`for f in *.json; do cat $f\` style loops on the whole sessions dir. LIMITS: Do NOT spawn subagents to read deeplake memory. If a file returns empty after 2 attempts, skip it and move on. Report what you found rather than exhaustively retrying. diff --git a/claude-code/bundle/shell/deeplake-shell.js b/claude-code/bundle/shell/deeplake-shell.js index e3f8c1d6..402fb0a3 100755 --- a/claude-code/bundle/shell/deeplake-shell.js +++ b/claude-code/bundle/shell/deeplake-shell.js @@ -46081,14 +46081,14 @@ var require_turndown_cjs = __commonJS({ } else if (node.nodeType === 1) { replacement = replacementForNode.call(self2, node); } - return join11(output, replacement); + return join13(output, replacement); }, ""); } function postProcess(output) { var self2 = this; this.rules.forEach(function(rule) { if (typeof rule.append === "function") { - output = join11(output, rule.append(self2.options)); + output = join13(output, rule.append(self2.options)); } }); return output.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, ""); @@ -46100,7 +46100,7 @@ var require_turndown_cjs = __commonJS({ if (whitespace.leading || whitespace.trailing) content = content.trim(); return whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing; } - function join11(output, replacement) { + function join13(output, replacement) { var s12 = trimTrailingNewlines(output); var s22 = trimLeadingNewlines(replacement); var nls = Math.max(output.length - s12.length, replacement.length - s22.length); @@ -66870,7 +66870,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms3) { - return new Promise((resolve5) => setTimeout(resolve5, ms3)); + return new Promise((resolve6) => setTimeout(resolve6, ms3)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -66900,7 +66900,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve5) => this.waiting.push(resolve5)); + await new Promise((resolve6) => this.waiting.push(resolve6)); } release() { this.active--; @@ -67259,7 +67259,7 @@ var DeeplakeApi = class { import { basename as basename4, posix } from "node:path"; import { randomUUID as randomUUID2 } from "node:crypto"; import { fileURLToPath } from "node:url"; -import { dirname as dirname4, join as join9 } from "node:path"; +import { dirname as dirname5, join as join11 } from "node:path"; // dist/src/shell/grep-core.js var TOOL_INPUT_FIELDS = [ @@ -67689,9 +67689,9 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join7 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync5, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join10 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -67704,13 +67704,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync as statSync2 } from "node:fs"; +import { join as join7, resolve as resolve4 } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join7(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path2, home) { + const r10 = resolve4(path2); + const h18 = resolve4(home); + return r10.startsWith(h18 + "/") || r10 === h18; +} +function writeQueue(q17) { + const path2 = queuePath(); + const home = resolve4(homedir3()); + if (!_isQueuePathInsideHome(path2, home)) { + throw new Error(`notifications-queue write blocked: ${path2} is outside ${home}`); + } + mkdirSync2(join7(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path2}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q17, null, 2), { mode: 384 }); + renameSync(tmp, path2); +} +async function withQueueLock(fn4) { + const path2 = lockPath(); + mkdirSync2(join7(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path2, "wx", 384); + break; + } catch (e6) { + const code = e6.code; + if (code !== "EEXIST") + throw e6; + try { + const age = Date.now() - statSync2(path2).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path2); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn4(); + } + try { + return fn4(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path2); + } catch { + } + } +} +function sameDedupKey(a15, b26) { + if (a15.id !== b26.id) + return false; + return JSON.stringify(a15.dedupKey) === JSON.stringify(b26.dedupKey); +} +async function enqueueNotification(n24) { + await withQueueLock(() => { + const q17 = readQueue(); + if (q17.queue.some((existing) => sameDedupKey(existing, n24))) { + return; + } + q17.queue.push(n24); + writeQueue(q17); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join9 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname as dirname4, join as join8 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join8(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path2 = _configPath(); + if (!existsSync4(path2)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path2, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path2 = _configPath(); + const dir = dirname4(path2); + if (!existsSync4(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path2}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path2); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join9(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join7(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m26) => log("embed-client", m26); +var SHARED_DAEMON_PATH = join10(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m26) => log("embed-client", m26); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -67719,13 +67940,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync5(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -67735,8 +67957,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v27 = await this.embedAttempt(text, kind); + if (v27 !== "recycled") + return v27; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -67746,17 +67993,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e6) { const err = e6 instanceof Error ? e6.message : String(e6); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -67765,6 +68020,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync5(this.socketPath)) + return; + await new Promise((r10) => setTimeout(r10, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e6) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e6 instanceof Error ? e6.message : String(e6)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync5(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e6) => { + log4(`enqueue embed-deps-missing failed: ${e6 instanceof Error ? e6.message : String(e6)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync5(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -67788,7 +68176,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { const sock = connect(this.socketPath); const to3 = setTimeout(() => { sock.destroy(); @@ -67796,7 +68184,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to3); - resolve5(sock); + resolve6(sock); }); sock.once("error", (e6) => { clearTimeout(to3); @@ -67807,16 +68195,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e6) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -67825,11 +68213,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync5(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -67841,14 +68229,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -67866,9 +68254,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync4(this.socketPath)) + if (!existsSync5(this.socketPath)) continue; try { return await this.connectOnce(); @@ -67878,7 +68266,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { let buf = ""; const to3 = setTimeout(() => { sock.destroy(); @@ -67893,7 +68281,7 @@ var EmbedClient = class { const line = buf.slice(0, nl3); clearTimeout(to3); try { - resolve5(JSON.parse(line)); + resolve6(JSON.parse(line)); } catch (e6) { reject(e6); } @@ -67910,9 +68298,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms3) { +function sleep3(ms3) { return new Promise((r10) => setTimeout(r10, ms3)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -67927,42 +68320,6 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join8 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join8(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} - // dist/src/hooks/virtual-table-query.js var INDEX_LIMIT_PER_SECTION = 50; function buildVirtualIndexContent(summaryRows, sessionRows = [], opts = {}) { @@ -68055,7 +68412,7 @@ function normalizeSessionMessage(path2, message) { return normalizeContent(path2, raw); } function resolveEmbedDaemonPath() { - return join9(dirname4(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + return join11(dirname5(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } function joinSessionMessages(path2, messages) { return messages.map((message) => normalizeSessionMessage(path2, message)).join("\n"); @@ -68621,7 +68978,7 @@ var DeeplakeFs = class _DeeplakeFs { // node_modules/yargs-parser/build/lib/index.js import { format } from "util"; -import { normalize, resolve as resolve4 } from "path"; +import { normalize, resolve as resolve5 } from "path"; // node_modules/yargs-parser/build/lib/string-utils.js function camelCase2(str) { @@ -69559,7 +69916,7 @@ function stripQuotes(val) { } // node_modules/yargs-parser/build/lib/index.js -import { readFileSync as readFileSync4 } from "fs"; +import { readFileSync as readFileSync6 } from "fs"; import { createRequire as createRequire2 } from "node:module"; var _a3; var _b; @@ -69581,12 +69938,12 @@ var parser = new YargsParser({ }, format, normalize, - resolve: resolve4, + resolve: resolve5, require: (path2) => { if (typeof require2 !== "undefined") { return require2(path2); } else if (path2.match(/\.json$/)) { - return JSON.parse(readFileSync4(path2, "utf8")); + return JSON.parse(readFileSync6(path2, "utf8")); } else { throw Error("only .json config files are supported in ESM"); } @@ -69606,11 +69963,11 @@ var lib_default = yargsParser; // dist/src/shell/grep-interceptor.js import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname as dirname5, join as join10 } from "node:path"; +import { dirname as dirname6, join as join12 } from "node:path"; var SEMANTIC_SEARCH_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_EMBED_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveGrepEmbedDaemonPath() { - return join10(dirname5(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join12(dirname6(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedGrepEmbedClient = null; function getGrepEmbedClient() { diff --git a/claude-code/bundle/wiki-worker.js b/claude-code/bundle/wiki-worker.js index cb6348a1..dca7a1aa 100755 --- a/claude-code/bundle/wiki-worker.js +++ b/claude-code/bundle/wiki-worker.js @@ -1,9 +1,9 @@ #!/usr/bin/env node // dist/src/hooks/wiki-worker.js -import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync3, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; +import { readFileSync as readFileSync5, writeFileSync as writeFileSync4, existsSync as existsSync4, appendFileSync as appendFileSync2, mkdirSync as mkdirSync4, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { dirname, join as join5 } from "node:path"; +import { dirname as dirname2, join as join7 } from "node:path"; import { fileURLToPath } from "node:url"; // dist/src/utils/debug.js @@ -164,9 +164,9 @@ async function uploadSummary(query2, params) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join3 } from "node:path"; +import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync2, unlinkSync as unlinkSync3, existsSync as existsSync3, readFileSync as readFileSync4 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join6 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -179,13 +179,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync as renameSync2, mkdirSync as mkdirSync2, openSync as openSync2, closeSync as closeSync2, unlinkSync as unlinkSync2, statSync } from "node:fs"; +import { join as join3, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; +var log2 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join3(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath2() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync2(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log2(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join3(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync2(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath2(); + mkdirSync2(join3(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync2(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync2(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep(delay); + } + } + if (fd === null) { + log2(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync2(fd); + } catch { + } + try { + unlinkSync2(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join5 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join4 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join4(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync2(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync3(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync2(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync3(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg2 = readUserConfig(); + if (cfg2.embeddings && typeof cfg2.embeddings.enabled === "boolean") { + return cfg2.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg2 ?? {}, embeddings: { ...cfg2?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join5(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join3(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log2 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join6(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log3 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -194,13 +415,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync2(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -210,8 +432,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -221,17 +468,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log2(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log3(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log2(`embed failed: ${err}`); + log3(`embed failed: ${err}`); return null; } finally { try { @@ -240,6 +495,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync3(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log3(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log3(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync3(hello.daemonPath)) { + _recycledStuckDaemon = true; + log3(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log3(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync4(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync3(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log3(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync3(this.socketPath); + } catch { + } + try { + unlinkSync3(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -263,7 +651,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -271,7 +659,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -282,16 +670,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync2(this.pidPath); + unlinkSync3(this.pidPath); } catch { } try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch { return; @@ -300,11 +688,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync2(this.daemonEntry)) { - log2(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync2(fd); - unlinkSync2(this.pidPath); + closeSync3(fd); + unlinkSync3(this.pidPath); } catch { } return; @@ -316,14 +704,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log2(`spawned daemon pid=${child.pid}`); + log3(`spawned daemon pid=${child.pid}`); } finally { - closeSync2(fd); + closeSync3(fd); } } isPidFileStale() { try { - const raw = readFileSync2(this.pidPath, "utf-8").trim(); + const raw = readFileSync4(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -341,9 +729,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep(delay); + await sleep2(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync2(this.socketPath)) + if (!existsSync3(this.socketPath)) continue; try { return await this.connectOnce(); @@ -353,7 +741,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -368,7 +756,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -385,55 +773,24 @@ var EmbedClient = class { }); } }; -function sleep(ms) { +function sleep2(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join4 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join4(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/hooks/wiki-worker.js var dlog2 = (msg) => log("wiki-worker", msg); -var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8")); +var cfg = JSON.parse(readFileSync5(process.argv[2], "utf-8")); var tmpDir = cfg.tmpDir; -var tmpJsonl = join5(tmpDir, "session.jsonl"); -var tmpSummary = join5(tmpDir, "summary.md"); +var tmpJsonl = join7(tmpDir, "session.jsonl"); +var tmpSummary = join7(tmpDir, "summary.md"); function wlog(msg) { try { - mkdirSync2(cfg.hooksDir, { recursive: true }); + mkdirSync4(cfg.hooksDir, { recursive: true }); appendFileSync2(cfg.wikiLog, `[${utcTimestamp()}] wiki-worker(${cfg.sessionId}): ${msg} `); } catch { @@ -465,7 +822,7 @@ async function query(sql, retries = 4) { const base = Math.min(3e4, 2e3 * Math.pow(2, attempt)); const delay = base + Math.floor(Math.random() * 1e3); wlog(`API ${r.status}, retrying in ${delay}ms (attempt ${attempt + 1}/${retries})`); - await new Promise((resolve) => setTimeout(resolve, delay)); + await new Promise((resolve2) => setTimeout(resolve2, delay)); continue; } throw new Error(`API ${r.status}: ${(await r.text()).slice(0, 200)}`); @@ -491,7 +848,7 @@ async function main() { const jsonlLines = rows.length; const pathRows = await query(`SELECT DISTINCT path FROM "${cfg.sessionsTable}" WHERE path LIKE '${esc2(`/sessions/%${cfg.sessionId}%`)}' LIMIT 1`); const jsonlServerPath = pathRows.length > 0 ? pathRows[0].path : `/sessions/unknown/${cfg.sessionId}.jsonl`; - writeFileSync2(tmpJsonl, jsonlContent); + writeFileSync4(tmpJsonl, jsonlContent); wlog(`found ${jsonlLines} events at ${jsonlServerPath}`); let prevOffset = 0; try { @@ -501,7 +858,7 @@ async function main() { const match = existing.match(/\*\*JSONL offset\*\*:\s*(\d+)/); if (match) prevOffset = parseInt(match[1], 10); - writeFileSync2(tmpSummary, existing); + writeFileSync4(tmpSummary, existing); wlog(`existing summary found, offset=${prevOffset}`); } } catch { @@ -526,15 +883,15 @@ async function main() { } catch (e) { wlog(`claude -p failed: ${e.status ?? e.message}`); } - if (existsSync3(tmpSummary)) { - const text = readFileSync3(tmpSummary, "utf-8"); + if (existsSync4(tmpSummary)) { + const text = readFileSync5(tmpSummary, "utf-8"); if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; let embedding = null; if (!embeddingsDisabled()) { try { - const daemonEntry = join5(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + const daemonEntry = join7(dirname2(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); } catch (e) { wlog(`summary embedding failed, writing NULL: ${e.message}`); diff --git a/codex/bundle/capture.js b/codex/bundle/capture.js index e116bfed..026f64bf 100755 --- a/codex/bundle/capture.js +++ b/codex/bundle/capture.js @@ -54,13 +54,13 @@ var init_index_marker_store = __esm({ // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -176,7 +176,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -206,7 +206,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -570,9 +570,9 @@ function buildSessionPath(config, sessionId) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join4 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -585,13 +585,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync } from "node:fs"; +import { join as join4, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join4(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join4(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync2(join4(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join6 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join5 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join5(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join6(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join4(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join7(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -600,13 +821,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -616,8 +838,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -627,17 +874,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -646,6 +901,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -669,7 +1057,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -677,7 +1065,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -688,16 +1076,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -706,11 +1094,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -722,14 +1110,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -747,9 +1135,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -759,7 +1147,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -774,7 +1162,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -791,9 +1179,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -808,91 +1201,120 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { +// dist/src/embeddings/self-heal.js +import { existsSync as existsSync5, lstatSync, mkdirSync as mkdirSync4, readlinkSync, renameSync as renameSync3, rmSync, symlinkSync, statSync as statSync2 } from "node:fs"; +import { homedir as homedir7 } from "node:os"; +import { basename, dirname as dirname2, join as join8 } from "node:path"; +function ensurePluginNodeModulesLink(opts) { + if (basename(opts.bundleDir) !== "bundle") { + return { kind: "not-bundle-layout", bundleDir: opts.bundleDir }; + } + const target = opts.sharedNodeModules ?? join8(homedir7(), ".hivemind", "embed-deps", "node_modules"); + const pluginDir = dirname2(opts.bundleDir); + const link = join8(pluginDir, "node_modules"); + if (!existsSync5(target)) { + return { kind: "shared-deps-missing", target }; + } + let linkStat; try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; + linkStat = lstatSync(link); } catch { + return createSymlinkAtomic(target, link); } - const sharedDir = join5(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + if (linkStat.isSymbolicLink()) { + let existingTarget; + try { + existingTarget = readlinkSync(link); + } catch (e) { + return { kind: "error", detail: `readlink failed: ${e instanceof Error ? e.message : String(e)}` }; + } + if (existingTarget === target) { + return { kind: "already-linked", target, link }; + } + try { + statSync2(link); + return { kind: "linked-elsewhere", link, existingTarget }; + } catch { + try { + rmSync(link); + } catch { + } + const recreated = createSymlinkAtomic(target, link); + if (recreated.kind === "linked") { + return { kind: "stale-link-removed", link, danglingTarget: existingTarget }; + } + return recreated; + } + } + return { kind: "plugin-owns-node-modules", link }; } -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; +function createSymlinkAtomic(target, link) { try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; + const parent = dirname2(link); + if (!existsSync5(parent)) + mkdirSync4(parent, { recursive: true }); + const tmp = `${link}.tmp.${process.pid}`; + try { + rmSync(tmp, { force: true }); + } catch { + } + symlinkSync(target, tmp); + renameSync3(tmp, link); + return { kind: "linked", target, link }; + } catch (e) { + return { kind: "error", detail: e instanceof Error ? e.message : String(e) }; } } -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} // dist/src/hooks/codex/capture.js import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname as dirname3, join as join10 } from "node:path"; +import { dirname as dirname5, join as join13 } from "node:path"; // dist/src/hooks/summary-state.js -import { readFileSync as readFileSync4, writeFileSync as writeFileSync2, writeSync as writeSync2, mkdirSync as mkdirSync2, renameSync, existsSync as existsSync4, unlinkSync as unlinkSync2, openSync as openSync2, closeSync as closeSync2 } from "node:fs"; -import { homedir as homedir5 } from "node:os"; -import { join as join6 } from "node:path"; +import { readFileSync as readFileSync6, writeFileSync as writeFileSync4, writeSync as writeSync2, mkdirSync as mkdirSync5, renameSync as renameSync4, existsSync as existsSync6, unlinkSync as unlinkSync3, openSync as openSync3, closeSync as closeSync3 } from "node:fs"; +import { homedir as homedir8 } from "node:os"; +import { join as join9 } from "node:path"; var dlog = (msg) => log("summary-state", msg); -var STATE_DIR = join6(homedir5(), ".claude", "hooks", "summary-state"); +var STATE_DIR = join9(homedir8(), ".claude", "hooks", "summary-state"); var YIELD_BUF = new Int32Array(new SharedArrayBuffer(4)); function statePath(sessionId) { - return join6(STATE_DIR, `${sessionId}.json`); + return join9(STATE_DIR, `${sessionId}.json`); } -function lockPath(sessionId) { - return join6(STATE_DIR, `${sessionId}.lock`); +function lockPath2(sessionId) { + return join9(STATE_DIR, `${sessionId}.lock`); } function readState(sessionId) { const p = statePath(sessionId); - if (!existsSync4(p)) + if (!existsSync6(p)) return null; try { - return JSON.parse(readFileSync4(p, "utf-8")); + return JSON.parse(readFileSync6(p, "utf-8")); } catch { return null; } } function writeState(sessionId, state) { - mkdirSync2(STATE_DIR, { recursive: true }); + mkdirSync5(STATE_DIR, { recursive: true }); const p = statePath(sessionId); const tmp = `${p}.${process.pid}.${Date.now()}.tmp`; - writeFileSync2(tmp, JSON.stringify(state)); - renameSync(tmp, p); + writeFileSync4(tmp, JSON.stringify(state)); + renameSync4(tmp, p); } function withRmwLock(sessionId, fn) { - mkdirSync2(STATE_DIR, { recursive: true }); + mkdirSync5(STATE_DIR, { recursive: true }); const rmwLock = statePath(sessionId) + ".rmw"; const deadline = Date.now() + 2e3; let fd = null; while (fd === null) { try { - fd = openSync2(rmwLock, "wx"); + fd = openSync3(rmwLock, "wx"); } catch (e) { if (e.code !== "EEXIST") throw e; if (Date.now() > deadline) { dlog(`rmw lock deadline exceeded for ${sessionId}, reclaiming stale lock`); try { - unlinkSync2(rmwLock); + unlinkSync3(rmwLock); } catch (unlinkErr) { dlog(`stale rmw lock unlink failed for ${sessionId}: ${unlinkErr.message}`); } @@ -904,9 +1326,9 @@ function withRmwLock(sessionId, fn) { try { return fn(); } finally { - closeSync2(fd); + closeSync3(fd); try { - unlinkSync2(rmwLock); + unlinkSync3(rmwLock); } catch (unlinkErr) { dlog(`rmw lock cleanup failed for ${sessionId}: ${unlinkErr.message}`); } @@ -941,29 +1363,29 @@ function shouldTrigger(state, cfg, now = Date.now()) { return false; } function tryAcquireLock(sessionId, maxAgeMs = 10 * 60 * 1e3) { - mkdirSync2(STATE_DIR, { recursive: true }); - const p = lockPath(sessionId); - if (existsSync4(p)) { + mkdirSync5(STATE_DIR, { recursive: true }); + const p = lockPath2(sessionId); + if (existsSync6(p)) { try { - const ageMs = Date.now() - parseInt(readFileSync4(p, "utf-8"), 10); + const ageMs = Date.now() - parseInt(readFileSync6(p, "utf-8"), 10); if (Number.isFinite(ageMs) && ageMs < maxAgeMs) return false; } catch (readErr) { dlog(`lock file unreadable for ${sessionId}, treating as stale: ${readErr.message}`); } try { - unlinkSync2(p); + unlinkSync3(p); } catch (unlinkErr) { dlog(`could not unlink stale lock for ${sessionId}: ${unlinkErr.message}`); return false; } } try { - const fd = openSync2(p, "wx"); + const fd = openSync3(p, "wx"); try { writeSync2(fd, String(Date.now())); } finally { - closeSync2(fd); + closeSync3(fd); } return true; } catch (e) { @@ -974,7 +1396,7 @@ function tryAcquireLock(sessionId, maxAgeMs = 10 * 60 * 1e3) { } function releaseLock(sessionId) { try { - unlinkSync2(lockPath(sessionId)); + unlinkSync3(lockPath2(sessionId)); } catch (e) { if (e?.code !== "ENOENT") { dlog(`releaseLock unlink failed for ${sessionId}: ${e.message}`); @@ -985,20 +1407,20 @@ function releaseLock(sessionId) { // dist/src/hooks/codex/spawn-wiki-worker.js import { spawn as spawn2, execSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { dirname as dirname2, join as join9 } from "node:path"; -import { writeFileSync as writeFileSync3, mkdirSync as mkdirSync4 } from "node:fs"; -import { homedir as homedir6, tmpdir as tmpdir2 } from "node:os"; +import { dirname as dirname4, join as join12 } from "node:path"; +import { writeFileSync as writeFileSync5, mkdirSync as mkdirSync7 } from "node:fs"; +import { homedir as homedir9, tmpdir as tmpdir2 } from "node:os"; // dist/src/utils/wiki-log.js -import { mkdirSync as mkdirSync3, appendFileSync as appendFileSync2 } from "node:fs"; -import { join as join7 } from "node:path"; +import { mkdirSync as mkdirSync6, appendFileSync as appendFileSync2 } from "node:fs"; +import { join as join10 } from "node:path"; function makeWikiLogger(hooksDir, filename = "deeplake-wiki.log") { - const path = join7(hooksDir, filename); + const path = join10(hooksDir, filename); return { path, log(msg) { try { - mkdirSync3(hooksDir, { recursive: true }); + mkdirSync6(hooksDir, { recursive: true }); appendFileSync2(path, `[${utcTimestamp()}] ${msg} `); } catch { @@ -1008,18 +1430,18 @@ function makeWikiLogger(hooksDir, filename = "deeplake-wiki.log") { } // dist/src/utils/version-check.js -import { readFileSync as readFileSync5 } from "node:fs"; -import { dirname, join as join8 } from "node:path"; +import { readFileSync as readFileSync7 } from "node:fs"; +import { dirname as dirname3, join as join11 } from "node:path"; function getInstalledVersion(bundleDir, pluginManifestDir) { try { - const pluginJson = join8(bundleDir, "..", pluginManifestDir, "plugin.json"); - const plugin = JSON.parse(readFileSync5(pluginJson, "utf-8")); + const pluginJson = join11(bundleDir, "..", pluginManifestDir, "plugin.json"); + const plugin = JSON.parse(readFileSync7(pluginJson, "utf-8")); if (plugin.version) return plugin.version; } catch { } try { - const stamp = readFileSync5(join8(bundleDir, "..", ".hivemind_version"), "utf-8").trim(); + const stamp = readFileSync7(join11(bundleDir, "..", ".hivemind_version"), "utf-8").trim(); if (stamp) return stamp; } catch { @@ -1034,14 +1456,14 @@ function getInstalledVersion(bundleDir, pluginManifestDir) { ]); let dir = bundleDir; for (let i = 0; i < 5; i++) { - const candidate = join8(dir, "package.json"); + const candidate = join11(dir, "package.json"); try { - const pkg = JSON.parse(readFileSync5(candidate, "utf-8")); + const pkg = JSON.parse(readFileSync7(candidate, "utf-8")); if (HIVEMIND_PKG_NAMES.has(pkg.name) && pkg.version) return pkg.version; } catch { } - const parent = dirname(dir); + const parent = dirname3(dir); if (parent === dir) break; dir = parent; @@ -1050,8 +1472,8 @@ function getInstalledVersion(bundleDir, pluginManifestDir) { } // dist/src/hooks/codex/spawn-wiki-worker.js -var HOME = homedir6(); -var wikiLogger = makeWikiLogger(join9(HOME, ".codex", "hooks")); +var HOME = homedir9(); +var wikiLogger = makeWikiLogger(join12(HOME, ".codex", "hooks")); var WIKI_LOG = wikiLogger.path; var WIKI_PROMPT_TEMPLATE = `You are building a personal wiki from a coding session. Your goal is to extract every piece of knowledge \u2014 entities, decisions, relationships, and facts \u2014 into a structured, searchable wiki entry. @@ -1113,11 +1535,11 @@ function findCodexBin() { function spawnCodexWikiWorker(opts) { const { config, sessionId, cwd, bundleDir, reason } = opts; const projectName = cwd.split("/").pop() || "unknown"; - const tmpDir = join9(tmpdir2(), `deeplake-wiki-${sessionId}-${Date.now()}`); - mkdirSync4(tmpDir, { recursive: true }); + const tmpDir = join12(tmpdir2(), `deeplake-wiki-${sessionId}-${Date.now()}`); + mkdirSync7(tmpDir, { recursive: true }); const pluginVersion = getInstalledVersion(bundleDir, ".codex-plugin") ?? ""; - const configFile = join9(tmpDir, "config.json"); - writeFileSync3(configFile, JSON.stringify({ + const configFile = join12(tmpDir, "config.json"); + writeFileSync5(configFile, JSON.stringify({ apiUrl: config.apiUrl, token: config.token, orgId: config.orgId, @@ -1131,11 +1553,11 @@ function spawnCodexWikiWorker(opts) { tmpDir, codexBin: findCodexBin(), wikiLog: WIKI_LOG, - hooksDir: join9(HOME, ".codex", "hooks"), + hooksDir: join12(HOME, ".codex", "hooks"), promptTemplate: WIKI_PROMPT_TEMPLATE })); wikiLog(`${reason}: spawning summary worker for ${sessionId}`); - const workerPath = join9(bundleDir, "wiki-worker.js"); + const workerPath = join12(bundleDir, "wiki-worker.js"); spawn2("nohup", ["node", workerPath, configFile], { detached: true, stdio: ["ignore", "ignore", "ignore"] @@ -1143,16 +1565,22 @@ function spawnCodexWikiWorker(opts) { wikiLog(`${reason}: spawned summary worker for ${sessionId}`); } function bundleDirFromImportMeta(importMetaUrl) { - return dirname2(fileURLToPath(importMetaUrl)); + return dirname4(fileURLToPath(importMetaUrl)); } // dist/src/hooks/codex/capture.js -var log4 = (msg) => log("codex-capture", msg); +var log5 = (msg) => log("codex-capture", msg); function resolveEmbedDaemonPath() { - return join10(dirname3(fileURLToPath2(import.meta.url)), "embeddings", "embed-daemon.js"); + return join13(dirname5(fileURLToPath2(import.meta.url)), "embeddings", "embed-daemon.js"); } -var __bundleDir = dirname3(fileURLToPath2(import.meta.url)); +var __bundleDir = dirname5(fileURLToPath2(import.meta.url)); var PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".codex-plugin") ?? ""; +if (!embeddingsDisabled()) { + try { + ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); + } catch { + } +} var CAPTURE = process.env.HIVEMIND_CAPTURE !== "false"; async function main() { if (!CAPTURE) @@ -1160,7 +1588,7 @@ async function main() { const input = await readStdin(); const config = loadConfig(); if (!config) { - log4("no config"); + log5("no config"); return; } const sessionsTable = config.sessionsTableName; @@ -1177,7 +1605,7 @@ async function main() { }; let entry; if (input.hook_event_name === "UserPromptSubmit" && input.prompt !== void 0) { - log4(`user session=${input.session_id}`); + log5(`user session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1185,7 +1613,7 @@ async function main() { content: input.prompt }; } else if (input.hook_event_name === "PostToolUse" && input.tool_name !== void 0) { - log4(`tool=${input.tool_name} session=${input.session_id}`); + log5(`tool=${input.tool_name} session=${input.session_id}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1196,12 +1624,12 @@ async function main() { tool_response: JSON.stringify(input.tool_response) }; } else { - log4(`unknown event: ${input.hook_event_name}, skipping`); + log5(`unknown event: ${input.hook_event_name}, skipping`); return; } const sessionPath = buildSessionPath(config, input.session_id); const line = JSON.stringify(entry); - log4(`writing to ${sessionPath}`); + log5(`writing to ${sessionPath}`); const projectName = (input.cwd ?? "").split("/").pop() || "unknown"; const filename = sessionPath.split("/").pop() ?? ""; const jsonForSql = line.replace(/'/g, "''"); @@ -1212,14 +1640,14 @@ async function main() { await api.query(insertSql); } catch (e) { if (e.message?.includes("permission denied") || e.message?.includes("does not exist")) { - log4("table missing, creating and retrying"); + log5("table missing, creating and retrying"); await api.ensureSessionsTable(sessionsTable); await api.query(insertSql); } else { throw e; } } - log4("capture ok"); + log5("capture ok"); maybeTriggerPeriodicSummary(input.session_id, input.cwd ?? "", config); } function maybeTriggerPeriodicSummary(sessionId, cwd, config) { @@ -1231,7 +1659,7 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { if (!shouldTrigger(state, cfg)) return; if (!tryAcquireLock(sessionId)) { - log4(`periodic trigger suppressed (lock held) session=${sessionId}`); + log5(`periodic trigger suppressed (lock held) session=${sessionId}`); return; } wikiLog(`Periodic: threshold hit (total=${state.totalCount}, since=${state.totalCount - state.lastSummaryCount}, N=${cfg.everyNMessages}, hours=${cfg.everyHours})`); @@ -1244,19 +1672,19 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { reason: "Periodic" }); } catch (e) { - log4(`periodic spawn failed: ${e.message}`); + log5(`periodic spawn failed: ${e.message}`); try { releaseLock(sessionId); } catch (releaseErr) { - log4(`releaseLock after periodic spawn failure also failed: ${releaseErr.message}`); + log5(`releaseLock after periodic spawn failure also failed: ${releaseErr.message}`); } throw e; } } catch (e) { - log4(`periodic trigger error: ${e.message}`); + log5(`periodic trigger error: ${e.message}`); } } main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); diff --git a/codex/bundle/embeddings/embed-daemon.js b/codex/bundle/embeddings/embed-daemon.js index 5f711a3c..a81ffbd1 100755 --- a/codex/bundle/embeddings/embed-daemon.js +++ b/codex/bundle/embeddings/embed-daemon.js @@ -4,7 +4,14 @@ import { createServer } from "node:net"; import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +// dist/src/embeddings/nomic.js +import { createRequire } from "node:module"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + // dist/src/embeddings/protocol.js +var PROTOCOL_VERSION = 1; var DEFAULT_SOCKET_DIR = "/tmp"; var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; var DEFAULT_DTYPE = "q8"; @@ -20,6 +27,39 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { } // dist/src/embeddings/nomic.js +async function _importFromCanonicalSharedDeps(sharedDir = join(homedir(), ".hivemind", "embed-deps")) { + const base = pathToFileURL(`${sharedDir}/`).href; + const absMain = createRequire(base).resolve("@huggingface/transformers"); + const mod = await import(pathToFileURL(absMain).href); + return _normalizeTransformersModule(mod); +} +async function _importFromBareSpecifier() { + const mod = await import("@huggingface/transformers"); + return _normalizeTransformersModule(mod); +} +function _normalizeTransformersModule(mod) { + const m = mod; + if (m.default && typeof m.default === "object" && "pipeline" in m.default) { + return m.default; + } + return m; +} +async function defaultImportTransformers(canonical = _importFromCanonicalSharedDeps, bare = _importFromBareSpecifier) { + let canonicalErr; + try { + return await canonical(); + } catch (err) { + canonicalErr = err; + } + try { + return await bare(); + } catch (bareErr) { + const detail = bareErr instanceof Error ? bareErr.message : String(bareErr); + const canonicalDetail = canonicalErr instanceof Error ? canonicalErr.message : String(canonicalErr); + throw new Error(`@huggingface/transformers is not installed anywhere reachable. Run \`hivemind embeddings install\` to install it. (canonical: ${canonicalDetail}; bare: ${detail})`); + } +} +var _importTransformers = defaultImportTransformers; var NomicEmbedder = class { pipeline = null; loading = null; @@ -37,7 +77,7 @@ var NomicEmbedder = class { if (this.loading) return this.loading; this.loading = (async () => { - const mod = await import("@huggingface/transformers"); + const mod = await _importTransformers(); mod.env.allowLocalModels = false; mod.env.useFSCache = true; this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype }); @@ -94,9 +134,9 @@ var NomicEmbedder = class { // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; -var LOG = join(homedir(), ".deeplake", "hook-debug.log"); +import { join as join2 } from "node:path"; +import { homedir as homedir2 } from "node:os"; +var LOG = join2(homedir2(), ".deeplake", "hook-debug.log"); function isDebug() { return process.env.HIVEMIND_DEBUG === "1"; } @@ -120,6 +160,7 @@ var EmbedDaemon = class { pidPath; idleTimeoutMs; idleTimer = null; + daemonPath; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; @@ -127,6 +168,7 @@ var EmbedDaemon = class { this.pidPath = pidPathFor(uid, dir); this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + this.daemonPath = opts.daemonPath ?? process.argv[1] ?? ""; } async start() { mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); @@ -218,6 +260,15 @@ var EmbedDaemon = class { } } async dispatch(req) { + if (req.op === "hello") { + const h = req; + return { + id: h.id, + daemonPath: this.daemonPath, + pid: process.pid, + protocolVersion: PROTOCOL_VERSION + }; + } if (req.op === "ping") { const p = req; return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; diff --git a/codex/bundle/pre-tool-use.js b/codex/bundle/pre-tool-use.js index d46bf68b..907d679c 100755 --- a/codex/bundle/pre-tool-use.js +++ b/codex/bundle/pre-tool-use.js @@ -54,19 +54,19 @@ var init_index_marker_store = __esm({ // dist/src/hooks/codex/pre-tool-use.js import { execFileSync } from "node:child_process"; -import { existsSync as existsSync4 } from "node:fs"; -import { join as join9, dirname as dirname2 } from "node:path"; +import { existsSync as existsSync5 } from "node:fs"; +import { join as join11, dirname as dirname3 } from "node:path"; import { fileURLToPath as fileURLToPath3 } from "node:url"; // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve2, reject) => { + return new Promise((resolve3, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve2(JSON.parse(data)); + resolve3(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -182,7 +182,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve2) => setTimeout(resolve2, ms)); + return new Promise((resolve3) => setTimeout(resolve3, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -212,7 +212,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve2) => this.waiting.push(resolve2)); + await new Promise((resolve3) => this.waiting.push(resolve3)); } release() { this.active--; @@ -1049,9 +1049,9 @@ function capOutputForClaude(output, options = {}) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join4 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -1064,13 +1064,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync } from "node:fs"; +import { join as join4, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join4(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join4(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync2(join4(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join6 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join5 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join5(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join6(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join4(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join7(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -1079,13 +1300,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -1095,8 +1317,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -1106,17 +1353,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -1125,6 +1380,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -1148,7 +1536,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve2, reject) => { + return new Promise((resolve3, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -1156,7 +1544,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve2(sock); + resolve3(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -1167,16 +1555,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -1185,11 +1573,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -1201,14 +1589,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -1226,9 +1614,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -1238,7 +1626,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve2, reject) => { + return new Promise((resolve3, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -1253,7 +1641,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve2(JSON.parse(line)); + resolve3(JSON.parse(line)); } catch (e) { reject(e); } @@ -1270,53 +1658,22 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join5(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/hooks/grep-direct.js import { fileURLToPath } from "node:url"; -import { dirname, join as join6 } from "node:path"; +import { dirname as dirname2, join as join8 } from "node:path"; var SEMANTIC_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveDaemonPath() { - return join6(dirname(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join8(dirname2(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedEmbedClient = null; function getEmbedClient() { @@ -2319,20 +2676,20 @@ async function executeCompiledBashCommand(api, memoryTable, sessionsTable, cmd, } // dist/src/hooks/query-cache.js -import { mkdirSync as mkdirSync2, readFileSync as readFileSync4, rmSync, writeFileSync as writeFileSync2 } from "node:fs"; -import { join as join7 } from "node:path"; -import { homedir as homedir5 } from "node:os"; -var log4 = (msg) => log("query-cache", msg); -var DEFAULT_CACHE_ROOT = join7(homedir5(), ".deeplake", "query-cache"); +import { mkdirSync as mkdirSync4, readFileSync as readFileSync6, rmSync, writeFileSync as writeFileSync4 } from "node:fs"; +import { join as join9 } from "node:path"; +import { homedir as homedir7 } from "node:os"; +var log5 = (msg) => log("query-cache", msg); +var DEFAULT_CACHE_ROOT = join9(homedir7(), ".deeplake", "query-cache"); var INDEX_CACHE_FILE = "index.md"; function getSessionQueryCacheDir(sessionId, deps = {}) { const { cacheRoot = DEFAULT_CACHE_ROOT } = deps; - return join7(cacheRoot, sessionId); + return join9(cacheRoot, sessionId); } function readCachedIndexContent(sessionId, deps = {}) { - const { logFn = log4 } = deps; + const { logFn = log5 } = deps; try { - return readFileSync4(join7(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); + return readFileSync6(join9(getSessionQueryCacheDir(sessionId, deps), INDEX_CACHE_FILE), "utf-8"); } catch (e) { if (e?.code === "ENOENT") return null; @@ -2341,34 +2698,34 @@ function readCachedIndexContent(sessionId, deps = {}) { } } function writeCachedIndexContent(sessionId, content, deps = {}) { - const { logFn = log4 } = deps; + const { logFn = log5 } = deps; try { const dir = getSessionQueryCacheDir(sessionId, deps); - mkdirSync2(dir, { recursive: true }); - writeFileSync2(join7(dir, INDEX_CACHE_FILE), content, "utf-8"); + mkdirSync4(dir, { recursive: true }); + writeFileSync4(join9(dir, INDEX_CACHE_FILE), content, "utf-8"); } catch (e) { logFn(`write failed for session=${sessionId}: ${e.message}`); } } // dist/src/utils/direct-run.js -import { resolve } from "node:path"; +import { resolve as resolve2 } from "node:path"; import { fileURLToPath as fileURLToPath2 } from "node:url"; function isDirectRun(metaUrl) { const entry = process.argv[1]; if (!entry) return false; try { - return resolve(fileURLToPath2(metaUrl)) === resolve(entry); + return resolve2(fileURLToPath2(metaUrl)) === resolve2(entry); } catch { return false; } } // dist/src/hooks/memory-path-utils.js -import { homedir as homedir6 } from "node:os"; -import { join as join8 } from "node:path"; -var MEMORY_PATH = join8(homedir6(), ".deeplake", "memory"); +import { homedir as homedir8 } from "node:os"; +import { join as join10 } from "node:path"; +var MEMORY_PATH = join10(homedir8(), ".deeplake", "memory"); var TILDE_PATH = "~/.deeplake/memory"; var HOME_VAR_PATH = "$HOME/.deeplake/memory"; var SAFE_BUILTINS = /* @__PURE__ */ new Set([ @@ -2484,13 +2841,13 @@ function rewritePaths(cmd) { } // dist/src/hooks/codex/pre-tool-use.js -var log5 = (msg) => log("codex-pre", msg); -var __bundleDir = dirname2(fileURLToPath3(import.meta.url)); -var SHELL_BUNDLE = existsSync4(join9(__bundleDir, "shell", "deeplake-shell.js")) ? join9(__bundleDir, "shell", "deeplake-shell.js") : join9(__bundleDir, "..", "shell", "deeplake-shell.js"); +var log6 = (msg) => log("codex-pre", msg); +var __bundleDir = dirname3(fileURLToPath3(import.meta.url)); +var SHELL_BUNDLE = existsSync5(join11(__bundleDir, "shell", "deeplake-shell.js")) ? join11(__bundleDir, "shell", "deeplake-shell.js") : join11(__bundleDir, "..", "shell", "deeplake-shell.js"); function buildUnsupportedGuidance() { return "This command is not supported for ~/.deeplake/memory/ operations. Only bash builtins are available: cat, ls, grep, echo, jq, head, tail, sed, awk, wc, sort, find, etc. Do NOT use python, python3, node, curl, or other interpreters. Rewrite your command using only bash tools and retry."; } -function runVirtualShell(cmd, shellBundle = SHELL_BUNDLE, logFn = log5) { +function runVirtualShell(cmd, shellBundle = SHELL_BUNDLE, logFn = log6) { try { return execFileSync("node", [shellBundle, "-c", cmd], { encoding: "utf-8", @@ -2515,7 +2872,7 @@ function buildIndexContent(rows) { return lines.join("\n"); } async function processCodexPreToolUse(input, deps = {}) { - const { config = loadConfig(), createApi = (table, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table), executeCompiledBashCommandFn = executeCompiledBashCommand, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, handleGrepDirectFn = handleGrepDirect, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, runVirtualShellFn = runVirtualShell, shellBundle = SHELL_BUNDLE, logFn = log5 } = deps; + const { config = loadConfig(), createApi = (table, activeConfig) => new DeeplakeApi(activeConfig.token, activeConfig.apiUrl, activeConfig.orgId, activeConfig.workspaceId, table), executeCompiledBashCommandFn = executeCompiledBashCommand, readVirtualPathContentsFn = readVirtualPathContents, readVirtualPathContentFn = readVirtualPathContent, listVirtualPathRowsFn = listVirtualPathRows, findVirtualPathsFn = findVirtualPaths, handleGrepDirectFn = handleGrepDirect, readCachedIndexContentFn = readCachedIndexContent, writeCachedIndexContentFn = writeCachedIndexContent, runVirtualShellFn = runVirtualShell, shellBundle = SHELL_BUNDLE, logFn = log6 } = deps; const cmd = input.tool_input?.command ?? ""; logFn(`hook fired: cmd=${cmd}`); if (!touchesMemory(cmd)) @@ -2725,7 +3082,7 @@ async function main() { } if (isDirectRun(import.meta.url)) { main().catch((e) => { - log5(`fatal: ${e.message}`); + log6(`fatal: ${e.message}`); process.exit(0); }); } diff --git a/codex/bundle/shell/deeplake-shell.js b/codex/bundle/shell/deeplake-shell.js index e3f8c1d6..402fb0a3 100755 --- a/codex/bundle/shell/deeplake-shell.js +++ b/codex/bundle/shell/deeplake-shell.js @@ -46081,14 +46081,14 @@ var require_turndown_cjs = __commonJS({ } else if (node.nodeType === 1) { replacement = replacementForNode.call(self2, node); } - return join11(output, replacement); + return join13(output, replacement); }, ""); } function postProcess(output) { var self2 = this; this.rules.forEach(function(rule) { if (typeof rule.append === "function") { - output = join11(output, rule.append(self2.options)); + output = join13(output, rule.append(self2.options)); } }); return output.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, ""); @@ -46100,7 +46100,7 @@ var require_turndown_cjs = __commonJS({ if (whitespace.leading || whitespace.trailing) content = content.trim(); return whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing; } - function join11(output, replacement) { + function join13(output, replacement) { var s12 = trimTrailingNewlines(output); var s22 = trimLeadingNewlines(replacement); var nls = Math.max(output.length - s12.length, replacement.length - s22.length); @@ -66870,7 +66870,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms3) { - return new Promise((resolve5) => setTimeout(resolve5, ms3)); + return new Promise((resolve6) => setTimeout(resolve6, ms3)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -66900,7 +66900,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve5) => this.waiting.push(resolve5)); + await new Promise((resolve6) => this.waiting.push(resolve6)); } release() { this.active--; @@ -67259,7 +67259,7 @@ var DeeplakeApi = class { import { basename as basename4, posix } from "node:path"; import { randomUUID as randomUUID2 } from "node:crypto"; import { fileURLToPath } from "node:url"; -import { dirname as dirname4, join as join9 } from "node:path"; +import { dirname as dirname5, join as join11 } from "node:path"; // dist/src/shell/grep-core.js var TOOL_INPUT_FIELDS = [ @@ -67689,9 +67689,9 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join7 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync5, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join10 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -67704,13 +67704,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync as statSync2 } from "node:fs"; +import { join as join7, resolve as resolve4 } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join7(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path2, home) { + const r10 = resolve4(path2); + const h18 = resolve4(home); + return r10.startsWith(h18 + "/") || r10 === h18; +} +function writeQueue(q17) { + const path2 = queuePath(); + const home = resolve4(homedir3()); + if (!_isQueuePathInsideHome(path2, home)) { + throw new Error(`notifications-queue write blocked: ${path2} is outside ${home}`); + } + mkdirSync2(join7(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path2}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q17, null, 2), { mode: 384 }); + renameSync(tmp, path2); +} +async function withQueueLock(fn4) { + const path2 = lockPath(); + mkdirSync2(join7(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path2, "wx", 384); + break; + } catch (e6) { + const code = e6.code; + if (code !== "EEXIST") + throw e6; + try { + const age = Date.now() - statSync2(path2).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path2); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn4(); + } + try { + return fn4(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path2); + } catch { + } + } +} +function sameDedupKey(a15, b26) { + if (a15.id !== b26.id) + return false; + return JSON.stringify(a15.dedupKey) === JSON.stringify(b26.dedupKey); +} +async function enqueueNotification(n24) { + await withQueueLock(() => { + const q17 = readQueue(); + if (q17.queue.some((existing) => sameDedupKey(existing, n24))) { + return; + } + q17.queue.push(n24); + writeQueue(q17); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join9 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname as dirname4, join as join8 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join8(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path2 = _configPath(); + if (!existsSync4(path2)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path2, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path2 = _configPath(); + const dir = dirname4(path2); + if (!existsSync4(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path2}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path2); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join9(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join7(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m26) => log("embed-client", m26); +var SHARED_DAEMON_PATH = join10(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m26) => log("embed-client", m26); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -67719,13 +67940,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync5(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -67735,8 +67957,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v27 = await this.embedAttempt(text, kind); + if (v27 !== "recycled") + return v27; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -67746,17 +67993,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e6) { const err = e6 instanceof Error ? e6.message : String(e6); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -67765,6 +68020,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync5(this.socketPath)) + return; + await new Promise((r10) => setTimeout(r10, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e6) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e6 instanceof Error ? e6.message : String(e6)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync5(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e6) => { + log4(`enqueue embed-deps-missing failed: ${e6 instanceof Error ? e6.message : String(e6)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync5(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -67788,7 +68176,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { const sock = connect(this.socketPath); const to3 = setTimeout(() => { sock.destroy(); @@ -67796,7 +68184,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to3); - resolve5(sock); + resolve6(sock); }); sock.once("error", (e6) => { clearTimeout(to3); @@ -67807,16 +68195,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e6) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -67825,11 +68213,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync5(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -67841,14 +68229,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -67866,9 +68254,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync4(this.socketPath)) + if (!existsSync5(this.socketPath)) continue; try { return await this.connectOnce(); @@ -67878,7 +68266,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { let buf = ""; const to3 = setTimeout(() => { sock.destroy(); @@ -67893,7 +68281,7 @@ var EmbedClient = class { const line = buf.slice(0, nl3); clearTimeout(to3); try { - resolve5(JSON.parse(line)); + resolve6(JSON.parse(line)); } catch (e6) { reject(e6); } @@ -67910,9 +68298,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms3) { +function sleep3(ms3) { return new Promise((r10) => setTimeout(r10, ms3)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -67927,42 +68320,6 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join8 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join8(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} - // dist/src/hooks/virtual-table-query.js var INDEX_LIMIT_PER_SECTION = 50; function buildVirtualIndexContent(summaryRows, sessionRows = [], opts = {}) { @@ -68055,7 +68412,7 @@ function normalizeSessionMessage(path2, message) { return normalizeContent(path2, raw); } function resolveEmbedDaemonPath() { - return join9(dirname4(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + return join11(dirname5(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } function joinSessionMessages(path2, messages) { return messages.map((message) => normalizeSessionMessage(path2, message)).join("\n"); @@ -68621,7 +68978,7 @@ var DeeplakeFs = class _DeeplakeFs { // node_modules/yargs-parser/build/lib/index.js import { format } from "util"; -import { normalize, resolve as resolve4 } from "path"; +import { normalize, resolve as resolve5 } from "path"; // node_modules/yargs-parser/build/lib/string-utils.js function camelCase2(str) { @@ -69559,7 +69916,7 @@ function stripQuotes(val) { } // node_modules/yargs-parser/build/lib/index.js -import { readFileSync as readFileSync4 } from "fs"; +import { readFileSync as readFileSync6 } from "fs"; import { createRequire as createRequire2 } from "node:module"; var _a3; var _b; @@ -69581,12 +69938,12 @@ var parser = new YargsParser({ }, format, normalize, - resolve: resolve4, + resolve: resolve5, require: (path2) => { if (typeof require2 !== "undefined") { return require2(path2); } else if (path2.match(/\.json$/)) { - return JSON.parse(readFileSync4(path2, "utf8")); + return JSON.parse(readFileSync6(path2, "utf8")); } else { throw Error("only .json config files are supported in ESM"); } @@ -69606,11 +69963,11 @@ var lib_default = yargsParser; // dist/src/shell/grep-interceptor.js import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname as dirname5, join as join10 } from "node:path"; +import { dirname as dirname6, join as join12 } from "node:path"; var SEMANTIC_SEARCH_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_EMBED_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveGrepEmbedDaemonPath() { - return join10(dirname5(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join12(dirname6(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedGrepEmbedClient = null; function getGrepEmbedClient() { diff --git a/codex/bundle/stop.js b/codex/bundle/stop.js index ce2080b8..38196601 100755 --- a/codex/bundle/stop.js +++ b/codex/bundle/stop.js @@ -53,19 +53,19 @@ var init_index_marker_store = __esm({ }); // dist/src/hooks/codex/stop.js -import { readFileSync as readFileSync8, existsSync as existsSync9 } from "node:fs"; +import { readFileSync as readFileSync10, existsSync as existsSync10 } from "node:fs"; import { fileURLToPath as fileURLToPath3 } from "node:url"; -import { dirname as dirname4, join as join15 } from "node:path"; +import { dirname as dirname5, join as join17 } from "node:path"; // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -181,7 +181,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -211,7 +211,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -1175,9 +1175,9 @@ function buildSessionPath(config, sessionId) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn as spawn3 } from "node:child_process"; -import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync3, unlinkSync as unlinkSync3, existsSync as existsSync8, readFileSync as readFileSync7 } from "node:fs"; -import { homedir as homedir10 } from "node:os"; -import { join as join13 } from "node:path"; +import { openSync as openSync4, closeSync as closeSync4, writeSync as writeSync3, unlinkSync as unlinkSync4, existsSync as existsSync9, readFileSync as readFileSync9 } from "node:fs"; +import { homedir as homedir13 } from "node:os"; +import { join as join16 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -1190,13 +1190,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync7, writeFileSync as writeFileSync7, renameSync as renameSync4, mkdirSync as mkdirSync8, openSync as openSync3, closeSync as closeSync3, unlinkSync as unlinkSync3, statSync } from "node:fs"; +import { join as join13, resolve } from "node:path"; +import { homedir as homedir10 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join13(homedir10(), ".deeplake", "notifications-queue.json"); +} +function lockPath3() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync7(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir10()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync8(join13(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync7(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync4(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath3(); + mkdirSync8(join13(homedir10(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync3(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync3(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync3(fd); + } catch { + } + try { + unlinkSync3(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire as createRequire2 } from "node:module"; +import { homedir as homedir12 } from "node:os"; +import { join as join15 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync8, mkdirSync as mkdirSync9, readFileSync as readFileSync8, renameSync as renameSync5, writeFileSync as writeFileSync8 } from "node:fs"; +import { homedir as homedir11 } from "node:os"; +import { dirname as dirname4, join as join14 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join14(homedir11(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync8(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync8(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname4(path); + if (!existsSync8(dir)) + mkdirSync9(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync8(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync5(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join15(homedir12(), ".hivemind", "embed-deps"); + try { + createRequire2(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire2(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join13(homedir10(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join16(homedir13(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -1205,13 +1426,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync8(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync9(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -1221,8 +1443,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -1232,17 +1479,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -1251,6 +1506,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync9(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync9(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync9(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync9(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync4(this.socketPath); + } catch { + } + try { + unlinkSync4(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -1274,7 +1662,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -1282,7 +1670,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -1293,16 +1681,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync3(this.pidPath, "wx", 384); + fd = openSync4(this.pidPath, "wx", 384); writeSync3(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync3(this.pidPath); + unlinkSync4(this.pidPath); } catch { } try { - fd = openSync3(this.pidPath, "wx", 384); + fd = openSync4(this.pidPath, "wx", 384); writeSync3(fd, String(process.pid)); } catch { return; @@ -1311,11 +1699,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync8(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync9(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync3(fd); - unlinkSync3(this.pidPath); + closeSync4(fd); + unlinkSync4(this.pidPath); } catch { } return; @@ -1327,14 +1715,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync3(fd); + closeSync4(fd); } } isPidFileStale() { try { - const raw = readFileSync7(this.pidPath, "utf-8").trim(); + const raw = readFileSync9(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -1352,9 +1740,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync8(this.socketPath)) + if (!existsSync9(this.socketPath)) continue; try { return await this.connectOnce(); @@ -1364,7 +1752,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -1379,7 +1767,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -1396,9 +1784,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -1413,48 +1806,12 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire as createRequire2 } from "node:module"; -import { homedir as homedir11 } from "node:os"; -import { join as join14 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire2(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join14(homedir11(), ".hivemind", "embed-deps"); - createRequire2(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} - // dist/src/hooks/codex/stop.js -var log4 = (msg) => log("codex-stop", msg); +var log5 = (msg) => log("codex-stop", msg); function resolveEmbedDaemonPath() { - return join15(dirname4(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); + return join17(dirname5(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); } -var __bundleDir = dirname4(fileURLToPath3(import.meta.url)); +var __bundleDir = dirname5(fileURLToPath3(import.meta.url)); var PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".codex-plugin") ?? ""; var CAPTURE = process.env.HIVEMIND_CAPTURE !== "false"; async function main() { @@ -1466,7 +1823,7 @@ async function main() { return; const config = loadConfig(); if (!config) { - log4("no config"); + log5("no config"); return; } if (CAPTURE) { @@ -1478,8 +1835,8 @@ async function main() { if (input.transcript_path) { try { const transcriptPath = input.transcript_path; - if (existsSync9(transcriptPath)) { - const transcript = readFileSync8(transcriptPath, "utf-8"); + if (existsSync10(transcriptPath)) { + const transcript = readFileSync10(transcriptPath, "utf-8"); const lines = transcript.trim().split("\n").reverse(); for (const line2 of lines) { try { @@ -1496,10 +1853,10 @@ async function main() { } } if (lastAssistantMessage) - log4(`extracted assistant message from transcript (${lastAssistantMessage.length} chars)`); + log5(`extracted assistant message from transcript (${lastAssistantMessage.length} chars)`); } } catch (e) { - log4(`transcript read failed: ${e.message}`); + log5(`transcript read failed: ${e.message}`); } } const entry = { @@ -1522,9 +1879,9 @@ async function main() { const embeddingSql = embeddingSqlLiteral(embedding); const insertSql = `INSERT INTO "${sessionsTable}" (id, path, filename, message, message_embedding, author, size_bytes, project, description, agent, plugin_version, creation_date, last_update_date) VALUES ('${crypto.randomUUID()}', '${sqlStr(sessionPath)}', '${sqlStr(filename)}', '${jsonForSql}'::jsonb, ${embeddingSql}, '${sqlStr(config.userName)}', ${Buffer.byteLength(line, "utf-8")}, '${sqlStr(projectName)}', 'Stop', 'codex', '${sqlStr(PLUGIN_VERSION)}', '${ts}', '${ts}')`; await api.query(insertSql); - log4("stop event captured"); + log5("stop event captured"); } catch (e) { - log4(`capture failed: ${e.message}`); + log5(`capture failed: ${e.message}`); } } if (!CAPTURE) @@ -1543,11 +1900,11 @@ async function main() { reason: "Stop" }); } catch (e) { - log4(`spawn failed: ${e.message}`); + log5(`spawn failed: ${e.message}`); try { releaseLock(sessionId); } catch (releaseErr) { - log4(`releaseLock after spawn failure also failed: ${releaseErr.message}`); + log5(`releaseLock after spawn failure also failed: ${releaseErr.message}`); } throw e; } @@ -1560,6 +1917,6 @@ async function main() { }); } main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); diff --git a/codex/bundle/wiki-worker.js b/codex/bundle/wiki-worker.js index ae4ac88c..2027e9c5 100755 --- a/codex/bundle/wiki-worker.js +++ b/codex/bundle/wiki-worker.js @@ -1,9 +1,9 @@ #!/usr/bin/env node // dist/src/hooks/codex/wiki-worker.js -import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync3, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; +import { readFileSync as readFileSync5, writeFileSync as writeFileSync4, existsSync as existsSync4, appendFileSync as appendFileSync2, mkdirSync as mkdirSync4, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { dirname, join as join5 } from "node:path"; +import { dirname as dirname2, join as join7 } from "node:path"; import { fileURLToPath } from "node:url"; // dist/src/hooks/summary-state.js @@ -154,9 +154,9 @@ async function uploadSummary(query2, params) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join3 } from "node:path"; +import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync2, unlinkSync as unlinkSync3, existsSync as existsSync3, readFileSync as readFileSync4 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join6 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -169,13 +169,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync as renameSync2, mkdirSync as mkdirSync2, openSync as openSync2, closeSync as closeSync2, unlinkSync as unlinkSync2, statSync } from "node:fs"; +import { join as join3, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; +var log2 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join3(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath2() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync2(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log2(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join3(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync2(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath2(); + mkdirSync2(join3(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync2(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync2(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep(delay); + } + } + if (fd === null) { + log2(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync2(fd); + } catch { + } + try { + unlinkSync2(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join5 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join4 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join4(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync2(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync3(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync2(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync3(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg2 = readUserConfig(); + if (cfg2.embeddings && typeof cfg2.embeddings.enabled === "boolean") { + return cfg2.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg2 ?? {}, embeddings: { ...cfg2?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join5(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join3(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log2 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join6(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log3 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -184,13 +405,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync2(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -200,8 +422,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -211,17 +458,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log2(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log3(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log2(`embed failed: ${err}`); + log3(`embed failed: ${err}`); return null; } finally { try { @@ -230,6 +485,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync3(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log3(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log3(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync3(hello.daemonPath)) { + _recycledStuckDaemon = true; + log3(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log3(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync4(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync3(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log3(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync3(this.socketPath); + } catch { + } + try { + unlinkSync3(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -253,7 +641,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -261,7 +649,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -272,16 +660,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync2(this.pidPath); + unlinkSync3(this.pidPath); } catch { } try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch { return; @@ -290,11 +678,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync2(this.daemonEntry)) { - log2(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync2(fd); - unlinkSync2(this.pidPath); + closeSync3(fd); + unlinkSync3(this.pidPath); } catch { } return; @@ -306,14 +694,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log2(`spawned daemon pid=${child.pid}`); + log3(`spawned daemon pid=${child.pid}`); } finally { - closeSync2(fd); + closeSync3(fd); } } isPidFileStale() { try { - const raw = readFileSync2(this.pidPath, "utf-8").trim(); + const raw = readFileSync4(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -331,9 +719,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep(delay); + await sleep2(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync2(this.socketPath)) + if (!existsSync3(this.socketPath)) continue; try { return await this.connectOnce(); @@ -343,7 +731,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -358,7 +746,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -375,44 +763,13 @@ var EmbedClient = class { }); } }; -function sleep(ms) { +function sleep2(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join4 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join4(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/utils/client-header.js @@ -426,13 +783,13 @@ function deeplakeClientHeader() { // dist/src/hooks/codex/wiki-worker.js var dlog2 = (msg) => log("codex-wiki-worker", msg); -var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8")); +var cfg = JSON.parse(readFileSync5(process.argv[2], "utf-8")); var tmpDir = cfg.tmpDir; -var tmpJsonl = join5(tmpDir, "session.jsonl"); -var tmpSummary = join5(tmpDir, "summary.md"); +var tmpJsonl = join7(tmpDir, "session.jsonl"); +var tmpSummary = join7(tmpDir, "summary.md"); function wlog(msg) { try { - mkdirSync2(cfg.hooksDir, { recursive: true }); + mkdirSync4(cfg.hooksDir, { recursive: true }); appendFileSync2(cfg.wikiLog, `[${(/* @__PURE__ */ new Date()).toISOString().replace("T", " ").slice(0, 19)}] wiki-worker(${cfg.sessionId}): ${msg} `); } catch { @@ -464,7 +821,7 @@ async function query(sql, retries = 4) { const base = Math.min(3e4, 2e3 * Math.pow(2, attempt)); const delay = base + Math.floor(Math.random() * 1e3); wlog(`API ${r.status}, retrying in ${delay}ms (attempt ${attempt + 1}/${retries})`); - await new Promise((resolve) => setTimeout(resolve, delay)); + await new Promise((resolve2) => setTimeout(resolve2, delay)); continue; } throw new Error(`API ${r.status}: ${(await r.text()).slice(0, 200)}`); @@ -490,7 +847,7 @@ async function main() { const jsonlLines = rows.length; const pathRows = await query(`SELECT DISTINCT path FROM "${cfg.sessionsTable}" WHERE path LIKE '${esc2(`/sessions/%${cfg.sessionId}%`)}' LIMIT 1`); const jsonlServerPath = pathRows.length > 0 ? pathRows[0].path : `/sessions/unknown/${cfg.sessionId}.jsonl`; - writeFileSync2(tmpJsonl, jsonlContent); + writeFileSync4(tmpJsonl, jsonlContent); wlog(`found ${jsonlLines} events at ${jsonlServerPath}`); let prevOffset = 0; try { @@ -500,7 +857,7 @@ async function main() { const match = existing.match(/\*\*JSONL offset\*\*:\s*(\d+)/); if (match) prevOffset = parseInt(match[1], 10); - writeFileSync2(tmpSummary, existing); + writeFileSync4(tmpSummary, existing); wlog(`existing summary found, offset=${prevOffset}`); } } catch { @@ -521,15 +878,15 @@ async function main() { } catch (e) { wlog(`codex exec failed: ${e.status ?? e.message}`); } - if (existsSync3(tmpSummary)) { - const text = readFileSync3(tmpSummary, "utf-8"); + if (existsSync4(tmpSummary)) { + const text = readFileSync5(tmpSummary, "utf-8"); if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; let embedding = null; if (!embeddingsDisabled()) { try { - const daemonEntry = join5(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + const daemonEntry = join7(dirname2(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); } catch (e) { wlog(`summary embedding failed, writing NULL: ${e.message}`); diff --git a/cursor/bundle/capture.js b/cursor/bundle/capture.js index 2969946a..15a2f20d 100755 --- a/cursor/bundle/capture.js +++ b/cursor/bundle/capture.js @@ -54,13 +54,13 @@ var init_index_marker_store = __esm({ // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -176,7 +176,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -206,7 +206,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -570,9 +570,9 @@ function buildSessionPath(config, sessionId) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join4 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -585,13 +585,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync } from "node:fs"; +import { join as join4, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join4(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join4(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync2(join4(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join6 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join5 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join5(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join6(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join4(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join7(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -600,13 +821,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -616,8 +838,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -627,17 +874,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -646,6 +901,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -669,7 +1057,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -677,7 +1065,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -688,16 +1076,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -706,11 +1094,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -722,14 +1110,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -747,9 +1135,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -759,7 +1147,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -774,7 +1162,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -791,9 +1179,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -808,91 +1201,120 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { +// dist/src/embeddings/self-heal.js +import { existsSync as existsSync5, lstatSync, mkdirSync as mkdirSync4, readlinkSync, renameSync as renameSync3, rmSync, symlinkSync, statSync as statSync2 } from "node:fs"; +import { homedir as homedir7 } from "node:os"; +import { basename, dirname as dirname2, join as join8 } from "node:path"; +function ensurePluginNodeModulesLink(opts) { + if (basename(opts.bundleDir) !== "bundle") { + return { kind: "not-bundle-layout", bundleDir: opts.bundleDir }; + } + const target = opts.sharedNodeModules ?? join8(homedir7(), ".hivemind", "embed-deps", "node_modules"); + const pluginDir = dirname2(opts.bundleDir); + const link = join8(pluginDir, "node_modules"); + if (!existsSync5(target)) { + return { kind: "shared-deps-missing", target }; + } + let linkStat; try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; + linkStat = lstatSync(link); } catch { + return createSymlinkAtomic(target, link); } - const sharedDir = join5(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + if (linkStat.isSymbolicLink()) { + let existingTarget; + try { + existingTarget = readlinkSync(link); + } catch (e) { + return { kind: "error", detail: `readlink failed: ${e instanceof Error ? e.message : String(e)}` }; + } + if (existingTarget === target) { + return { kind: "already-linked", target, link }; + } + try { + statSync2(link); + return { kind: "linked-elsewhere", link, existingTarget }; + } catch { + try { + rmSync(link); + } catch { + } + const recreated = createSymlinkAtomic(target, link); + if (recreated.kind === "linked") { + return { kind: "stale-link-removed", link, danglingTarget: existingTarget }; + } + return recreated; + } + } + return { kind: "plugin-owns-node-modules", link }; } -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; +function createSymlinkAtomic(target, link) { try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; + const parent = dirname2(link); + if (!existsSync5(parent)) + mkdirSync4(parent, { recursive: true }); + const tmp = `${link}.tmp.${process.pid}`; + try { + rmSync(tmp, { force: true }); + } catch { + } + symlinkSync(target, tmp); + renameSync3(tmp, link); + return { kind: "linked", target, link }; + } catch (e) { + return { kind: "error", detail: e instanceof Error ? e.message : String(e) }; } } -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} // dist/src/hooks/cursor/capture.js import { fileURLToPath as fileURLToPath3 } from "node:url"; -import { dirname as dirname4, join as join15 } from "node:path"; +import { dirname as dirname6, join as join18 } from "node:path"; // dist/src/hooks/summary-state.js -import { readFileSync as readFileSync4, writeFileSync as writeFileSync2, writeSync as writeSync2, mkdirSync as mkdirSync2, renameSync, existsSync as existsSync4, unlinkSync as unlinkSync2, openSync as openSync2, closeSync as closeSync2 } from "node:fs"; -import { homedir as homedir5 } from "node:os"; -import { join as join6 } from "node:path"; +import { readFileSync as readFileSync6, writeFileSync as writeFileSync4, writeSync as writeSync2, mkdirSync as mkdirSync5, renameSync as renameSync4, existsSync as existsSync6, unlinkSync as unlinkSync3, openSync as openSync3, closeSync as closeSync3 } from "node:fs"; +import { homedir as homedir8 } from "node:os"; +import { join as join9 } from "node:path"; var dlog = (msg) => log("summary-state", msg); -var STATE_DIR = join6(homedir5(), ".claude", "hooks", "summary-state"); +var STATE_DIR = join9(homedir8(), ".claude", "hooks", "summary-state"); var YIELD_BUF = new Int32Array(new SharedArrayBuffer(4)); function statePath(sessionId) { - return join6(STATE_DIR, `${sessionId}.json`); + return join9(STATE_DIR, `${sessionId}.json`); } -function lockPath(sessionId) { - return join6(STATE_DIR, `${sessionId}.lock`); +function lockPath2(sessionId) { + return join9(STATE_DIR, `${sessionId}.lock`); } function readState(sessionId) { const p = statePath(sessionId); - if (!existsSync4(p)) + if (!existsSync6(p)) return null; try { - return JSON.parse(readFileSync4(p, "utf-8")); + return JSON.parse(readFileSync6(p, "utf-8")); } catch { return null; } } function writeState(sessionId, state) { - mkdirSync2(STATE_DIR, { recursive: true }); + mkdirSync5(STATE_DIR, { recursive: true }); const p = statePath(sessionId); const tmp = `${p}.${process.pid}.${Date.now()}.tmp`; - writeFileSync2(tmp, JSON.stringify(state)); - renameSync(tmp, p); + writeFileSync4(tmp, JSON.stringify(state)); + renameSync4(tmp, p); } function withRmwLock(sessionId, fn) { - mkdirSync2(STATE_DIR, { recursive: true }); + mkdirSync5(STATE_DIR, { recursive: true }); const rmwLock = statePath(sessionId) + ".rmw"; const deadline = Date.now() + 2e3; let fd = null; while (fd === null) { try { - fd = openSync2(rmwLock, "wx"); + fd = openSync3(rmwLock, "wx"); } catch (e) { if (e.code !== "EEXIST") throw e; if (Date.now() > deadline) { dlog(`rmw lock deadline exceeded for ${sessionId}, reclaiming stale lock`); try { - unlinkSync2(rmwLock); + unlinkSync3(rmwLock); } catch (unlinkErr) { dlog(`stale rmw lock unlink failed for ${sessionId}: ${unlinkErr.message}`); } @@ -904,9 +1326,9 @@ function withRmwLock(sessionId, fn) { try { return fn(); } finally { - closeSync2(fd); + closeSync3(fd); try { - unlinkSync2(rmwLock); + unlinkSync3(rmwLock); } catch (unlinkErr) { dlog(`rmw lock cleanup failed for ${sessionId}: ${unlinkErr.message}`); } @@ -941,29 +1363,29 @@ function shouldTrigger(state, cfg, now = Date.now()) { return false; } function tryAcquireLock(sessionId, maxAgeMs = 10 * 60 * 1e3) { - mkdirSync2(STATE_DIR, { recursive: true }); - const p = lockPath(sessionId); - if (existsSync4(p)) { + mkdirSync5(STATE_DIR, { recursive: true }); + const p = lockPath2(sessionId); + if (existsSync6(p)) { try { - const ageMs = Date.now() - parseInt(readFileSync4(p, "utf-8"), 10); + const ageMs = Date.now() - parseInt(readFileSync6(p, "utf-8"), 10); if (Number.isFinite(ageMs) && ageMs < maxAgeMs) return false; } catch (readErr) { dlog(`lock file unreadable for ${sessionId}, treating as stale: ${readErr.message}`); } try { - unlinkSync2(p); + unlinkSync3(p); } catch (unlinkErr) { dlog(`could not unlink stale lock for ${sessionId}: ${unlinkErr.message}`); return false; } } try { - const fd = openSync2(p, "wx"); + const fd = openSync3(p, "wx"); try { writeSync2(fd, String(Date.now())); } finally { - closeSync2(fd); + closeSync3(fd); } return true; } catch (e) { @@ -974,7 +1396,7 @@ function tryAcquireLock(sessionId, maxAgeMs = 10 * 60 * 1e3) { } function releaseLock(sessionId) { try { - unlinkSync2(lockPath(sessionId)); + unlinkSync3(lockPath2(sessionId)); } catch (e) { if (e?.code !== "ENOENT") { dlog(`releaseLock unlink failed for ${sessionId}: ${e.message}`); @@ -985,20 +1407,20 @@ function releaseLock(sessionId) { // dist/src/hooks/cursor/spawn-wiki-worker.js import { spawn as spawn2, execSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { dirname as dirname2, join as join9 } from "node:path"; -import { writeFileSync as writeFileSync3, mkdirSync as mkdirSync4 } from "node:fs"; -import { homedir as homedir6, tmpdir as tmpdir2 } from "node:os"; +import { dirname as dirname4, join as join12 } from "node:path"; +import { writeFileSync as writeFileSync5, mkdirSync as mkdirSync7 } from "node:fs"; +import { homedir as homedir9, tmpdir as tmpdir2 } from "node:os"; // dist/src/utils/wiki-log.js -import { mkdirSync as mkdirSync3, appendFileSync as appendFileSync2 } from "node:fs"; -import { join as join7 } from "node:path"; +import { mkdirSync as mkdirSync6, appendFileSync as appendFileSync2 } from "node:fs"; +import { join as join10 } from "node:path"; function makeWikiLogger(hooksDir, filename = "deeplake-wiki.log") { - const path = join7(hooksDir, filename); + const path = join10(hooksDir, filename); return { path, log(msg) { try { - mkdirSync3(hooksDir, { recursive: true }); + mkdirSync6(hooksDir, { recursive: true }); appendFileSync2(path, `[${utcTimestamp()}] ${msg} `); } catch { @@ -1008,18 +1430,18 @@ function makeWikiLogger(hooksDir, filename = "deeplake-wiki.log") { } // dist/src/utils/version-check.js -import { readFileSync as readFileSync5 } from "node:fs"; -import { dirname, join as join8 } from "node:path"; +import { readFileSync as readFileSync7 } from "node:fs"; +import { dirname as dirname3, join as join11 } from "node:path"; function getInstalledVersion(bundleDir, pluginManifestDir) { try { - const pluginJson = join8(bundleDir, "..", pluginManifestDir, "plugin.json"); - const plugin = JSON.parse(readFileSync5(pluginJson, "utf-8")); + const pluginJson = join11(bundleDir, "..", pluginManifestDir, "plugin.json"); + const plugin = JSON.parse(readFileSync7(pluginJson, "utf-8")); if (plugin.version) return plugin.version; } catch { } try { - const stamp = readFileSync5(join8(bundleDir, "..", ".hivemind_version"), "utf-8").trim(); + const stamp = readFileSync7(join11(bundleDir, "..", ".hivemind_version"), "utf-8").trim(); if (stamp) return stamp; } catch { @@ -1034,14 +1456,14 @@ function getInstalledVersion(bundleDir, pluginManifestDir) { ]); let dir = bundleDir; for (let i = 0; i < 5; i++) { - const candidate = join8(dir, "package.json"); + const candidate = join11(dir, "package.json"); try { - const pkg = JSON.parse(readFileSync5(candidate, "utf-8")); + const pkg = JSON.parse(readFileSync7(candidate, "utf-8")); if (HIVEMIND_PKG_NAMES.has(pkg.name) && pkg.version) return pkg.version; } catch { } - const parent = dirname(dir); + const parent = dirname3(dir); if (parent === dir) break; dir = parent; @@ -1050,8 +1472,8 @@ function getInstalledVersion(bundleDir, pluginManifestDir) { } // dist/src/hooks/cursor/spawn-wiki-worker.js -var HOME = homedir6(); -var wikiLogger = makeWikiLogger(join9(HOME, ".cursor", "hooks")); +var HOME = homedir9(); +var wikiLogger = makeWikiLogger(join12(HOME, ".cursor", "hooks")); var WIKI_LOG = wikiLogger.path; var WIKI_PROMPT_TEMPLATE = `You are building a personal wiki from a coding session. Your goal is to extract every piece of knowledge \u2014 entities, decisions, relationships, and facts \u2014 into a structured, searchable wiki entry. @@ -1113,11 +1535,11 @@ function findCursorBin() { function spawnCursorWikiWorker(opts) { const { config, sessionId, cwd, bundleDir, reason } = opts; const projectName = cwd.split("/").pop() || "unknown"; - const tmpDir = join9(tmpdir2(), `deeplake-wiki-${sessionId}-${Date.now()}`); - mkdirSync4(tmpDir, { recursive: true }); + const tmpDir = join12(tmpdir2(), `deeplake-wiki-${sessionId}-${Date.now()}`); + mkdirSync7(tmpDir, { recursive: true }); const pluginVersion = getInstalledVersion(bundleDir, ".claude-plugin") ?? ""; - const configFile = join9(tmpDir, "config.json"); - writeFileSync3(configFile, JSON.stringify({ + const configFile = join12(tmpDir, "config.json"); + writeFileSync5(configFile, JSON.stringify({ apiUrl: config.apiUrl, token: config.token, orgId: config.orgId, @@ -1132,11 +1554,11 @@ function spawnCursorWikiWorker(opts) { cursorBin: findCursorBin(), cursorModel: process.env.HIVEMIND_CURSOR_MODEL ?? "auto", wikiLog: WIKI_LOG, - hooksDir: join9(HOME, ".cursor", "hooks"), + hooksDir: join12(HOME, ".cursor", "hooks"), promptTemplate: WIKI_PROMPT_TEMPLATE })); wikiLog(`${reason}: spawning summary worker for ${sessionId}`); - const workerPath = join9(bundleDir, "wiki-worker.js"); + const workerPath = join12(bundleDir, "wiki-worker.js"); spawn2("nohup", ["node", workerPath, configFile], { detached: true, stdio: ["ignore", "ignore", "ignore"] @@ -1144,33 +1566,33 @@ function spawnCursorWikiWorker(opts) { wikiLog(`${reason}: spawned summary worker for ${sessionId}`); } function bundleDirFromImportMeta(importMetaUrl) { - return dirname2(fileURLToPath(importMetaUrl)); + return dirname4(fileURLToPath(importMetaUrl)); } // dist/src/skillify/spawn-skillify-worker.js import { spawn as spawn3 } from "node:child_process"; import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname as dirname3, join as join11 } from "node:path"; -import { writeFileSync as writeFileSync4, mkdirSync as mkdirSync5, appendFileSync as appendFileSync3, chmodSync } from "node:fs"; -import { homedir as homedir8, tmpdir as tmpdir3 } from "node:os"; +import { dirname as dirname5, join as join14 } from "node:path"; +import { writeFileSync as writeFileSync6, mkdirSync as mkdirSync8, appendFileSync as appendFileSync3, chmodSync } from "node:fs"; +import { homedir as homedir11, tmpdir as tmpdir3 } from "node:os"; // dist/src/skillify/gate-runner.js -import { existsSync as existsSync5 } from "node:fs"; +import { existsSync as existsSync7 } from "node:fs"; import { createRequire as createRequire2 } from "node:module"; -import { homedir as homedir7 } from "node:os"; -import { join as join10 } from "node:path"; +import { homedir as homedir10 } from "node:os"; +import { join as join13 } from "node:path"; var requireForCp = createRequire2(import.meta.url); var { execFileSync: runChildProcess } = requireForCp("node:child_process"); var inheritedEnv = process; function firstExistingPath(candidates) { for (const c of candidates) { - if (existsSync5(c)) + if (existsSync7(c)) return c; } return null; } function findAgentBin(agent) { - const home = homedir7(); + const home = homedir10(); switch (agent) { // /usr/bin/ is included in every candidate list — that's the // common Linux package-manager install path (apt, dnf, pacman). Old @@ -1179,54 +1601,54 @@ function findAgentBin(agent) { // #170 caught the gap. case "claude_code": return firstExistingPath([ - join10(home, ".claude", "local", "claude"), + join13(home, ".claude", "local", "claude"), "/usr/local/bin/claude", "/usr/bin/claude", - join10(home, ".npm-global", "bin", "claude"), - join10(home, ".local", "bin", "claude"), + join13(home, ".npm-global", "bin", "claude"), + join13(home, ".local", "bin", "claude"), "/opt/homebrew/bin/claude" - ]) ?? join10(home, ".claude", "local", "claude"); + ]) ?? join13(home, ".claude", "local", "claude"); case "codex": return firstExistingPath([ "/usr/local/bin/codex", "/usr/bin/codex", - join10(home, ".npm-global", "bin", "codex"), - join10(home, ".local", "bin", "codex"), + join13(home, ".npm-global", "bin", "codex"), + join13(home, ".local", "bin", "codex"), "/opt/homebrew/bin/codex" ]) ?? "/usr/local/bin/codex"; case "cursor": return firstExistingPath([ "/usr/local/bin/cursor-agent", "/usr/bin/cursor-agent", - join10(home, ".npm-global", "bin", "cursor-agent"), - join10(home, ".local", "bin", "cursor-agent"), + join13(home, ".npm-global", "bin", "cursor-agent"), + join13(home, ".local", "bin", "cursor-agent"), "/opt/homebrew/bin/cursor-agent" ]) ?? "/usr/local/bin/cursor-agent"; case "hermes": return firstExistingPath([ - join10(home, ".local", "bin", "hermes"), + join13(home, ".local", "bin", "hermes"), "/usr/local/bin/hermes", "/usr/bin/hermes", - join10(home, ".npm-global", "bin", "hermes"), + join13(home, ".npm-global", "bin", "hermes"), "/opt/homebrew/bin/hermes" - ]) ?? join10(home, ".local", "bin", "hermes"); + ]) ?? join13(home, ".local", "bin", "hermes"); case "pi": return firstExistingPath([ - join10(home, ".local", "bin", "pi"), + join13(home, ".local", "bin", "pi"), "/usr/local/bin/pi", "/usr/bin/pi", - join10(home, ".npm-global", "bin", "pi"), + join13(home, ".npm-global", "bin", "pi"), "/opt/homebrew/bin/pi" - ]) ?? join10(home, ".local", "bin", "pi"); + ]) ?? join13(home, ".local", "bin", "pi"); } } // dist/src/skillify/spawn-skillify-worker.js -var HOME2 = homedir8(); -var SKILLIFY_LOG = join11(HOME2, ".claude", "hooks", "skillify.log"); +var HOME2 = homedir11(); +var SKILLIFY_LOG = join14(HOME2, ".claude", "hooks", "skillify.log"); function skillifyLog(msg) { try { - mkdirSync5(dirname3(SKILLIFY_LOG), { recursive: true }); + mkdirSync8(dirname5(SKILLIFY_LOG), { recursive: true }); appendFileSync3(SKILLIFY_LOG, `[${utcTimestamp()}] ${msg} `); } catch { @@ -1234,11 +1656,11 @@ function skillifyLog(msg) { } function spawnSkillifyWorker(opts) { const { config, cwd, projectKey, project, bundleDir, agent, scopeConfig, currentSessionId, reason } = opts; - const tmpDir = join11(tmpdir3(), `deeplake-skillify-${projectKey}-${Date.now()}`); - mkdirSync5(tmpDir, { recursive: true, mode: 448 }); + const tmpDir = join14(tmpdir3(), `deeplake-skillify-${projectKey}-${Date.now()}`); + mkdirSync8(tmpDir, { recursive: true, mode: 448 }); const gateBin = findAgentBin(agent); - const configFile = join11(tmpDir, "config.json"); - writeFileSync4(configFile, JSON.stringify({ + const configFile = join14(tmpDir, "config.json"); + writeFileSync6(configFile, JSON.stringify({ apiUrl: config.apiUrl, token: config.token, orgId: config.orgId, @@ -1268,7 +1690,7 @@ function spawnSkillifyWorker(opts) { } catch { } skillifyLog(`${reason}: spawning skillify worker for project=${project} key=${projectKey}`); - const workerPath = join11(bundleDir, "skillify-worker.js"); + const workerPath = join14(bundleDir, "skillify-worker.js"); spawn3("nohup", ["node", workerPath, configFile], { detached: true, stdio: ["ignore", "ignore", "ignore"] @@ -1277,31 +1699,31 @@ function spawnSkillifyWorker(opts) { } // dist/src/skillify/state.js -import { readFileSync as readFileSync6, writeFileSync as writeFileSync5, writeSync as writeSync3, mkdirSync as mkdirSync6, renameSync as renameSync3, existsSync as existsSync7, unlinkSync as unlinkSync3, openSync as openSync3, closeSync as closeSync3 } from "node:fs"; +import { readFileSync as readFileSync8, writeFileSync as writeFileSync7, writeSync as writeSync3, mkdirSync as mkdirSync9, renameSync as renameSync6, existsSync as existsSync9, unlinkSync as unlinkSync4, openSync as openSync4, closeSync as closeSync4 } from "node:fs"; import { execSync as execSync2 } from "node:child_process"; -import { homedir as homedir10 } from "node:os"; +import { homedir as homedir13 } from "node:os"; import { createHash } from "node:crypto"; -import { join as join13, basename } from "node:path"; +import { join as join16, basename as basename2 } from "node:path"; // dist/src/skillify/legacy-migration.js -import { existsSync as existsSync6, renameSync as renameSync2 } from "node:fs"; -import { homedir as homedir9 } from "node:os"; -import { join as join12 } from "node:path"; +import { existsSync as existsSync8, renameSync as renameSync5 } from "node:fs"; +import { homedir as homedir12 } from "node:os"; +import { join as join15 } from "node:path"; var dlog2 = (msg) => log("skillify-migrate", msg); var attempted = false; function migrateLegacyStateDir() { if (attempted) return; attempted = true; - const root = join12(homedir9(), ".deeplake", "state"); - const legacy = join12(root, "skilify"); - const current = join12(root, "skillify"); - if (!existsSync6(legacy)) + const root = join15(homedir12(), ".deeplake", "state"); + const legacy = join15(root, "skilify"); + const current = join15(root, "skillify"); + if (!existsSync8(legacy)) return; - if (existsSync6(current)) + if (existsSync8(current)) return; try { - renameSync2(legacy, current); + renameSync5(legacy, current); dlog2(`migrated ${legacy} -> ${current}`); } catch (err) { const code = err.code; @@ -1315,17 +1737,17 @@ function migrateLegacyStateDir() { // dist/src/skillify/state.js var dlog3 = (msg) => log("skillify-state", msg); -var STATE_DIR2 = join13(homedir10(), ".deeplake", "state", "skillify"); +var STATE_DIR2 = join16(homedir13(), ".deeplake", "state", "skillify"); var YIELD_BUF2 = new Int32Array(new SharedArrayBuffer(4)); var TRIGGER_THRESHOLD = (() => { const n = Number(process.env.HIVEMIND_SKILLIFY_EVERY_N_TURNS ?? ""); return Number.isInteger(n) && n > 0 ? n : 20; })(); function statePath2(projectKey) { - return join13(STATE_DIR2, `${projectKey}.json`); + return join16(STATE_DIR2, `${projectKey}.json`); } -function lockPath2(projectKey) { - return join13(STATE_DIR2, `${projectKey}.lock`); +function lockPath3(projectKey) { + return join16(STATE_DIR2, `${projectKey}.lock`); } var DEFAULT_PORTS = { http: "80", @@ -1353,7 +1775,7 @@ function normalizeGitRemoteUrl(url) { return s.toLowerCase(); } function deriveProjectKey(cwd) { - const project = basename(cwd) || "unknown"; + const project = basename2(cwd) || "unknown"; let signature = null; try { const raw = execSync2("git config --get remote.origin.url", { @@ -1371,38 +1793,38 @@ function deriveProjectKey(cwd) { function readState2(projectKey) { migrateLegacyStateDir(); const p = statePath2(projectKey); - if (!existsSync7(p)) + if (!existsSync9(p)) return null; try { - return JSON.parse(readFileSync6(p, "utf-8")); + return JSON.parse(readFileSync8(p, "utf-8")); } catch { return null; } } function writeState2(projectKey, state) { migrateLegacyStateDir(); - mkdirSync6(STATE_DIR2, { recursive: true }); + mkdirSync9(STATE_DIR2, { recursive: true }); const p = statePath2(projectKey); const tmp = `${p}.${process.pid}.${Date.now()}.tmp`; - writeFileSync5(tmp, JSON.stringify(state, null, 2)); - renameSync3(tmp, p); + writeFileSync7(tmp, JSON.stringify(state, null, 2)); + renameSync6(tmp, p); } function withRmwLock2(projectKey, fn) { migrateLegacyStateDir(); - mkdirSync6(STATE_DIR2, { recursive: true }); - const rmw = lockPath2(projectKey) + ".rmw"; + mkdirSync9(STATE_DIR2, { recursive: true }); + const rmw = lockPath3(projectKey) + ".rmw"; const deadline = Date.now() + 2e3; let fd = null; while (fd === null) { try { - fd = openSync3(rmw, "wx"); + fd = openSync4(rmw, "wx"); } catch (e) { if (e.code !== "EEXIST") throw e; if (Date.now() > deadline) { dlog3(`rmw lock deadline exceeded for ${projectKey}, reclaiming stale lock`); try { - unlinkSync3(rmw); + unlinkSync4(rmw); } catch (unlinkErr) { dlog3(`stale rmw lock unlink failed for ${projectKey}: ${unlinkErr.message}`); } @@ -1414,9 +1836,9 @@ function withRmwLock2(projectKey, fn) { try { return fn(); } finally { - closeSync3(fd); + closeSync4(fd); try { - unlinkSync3(rmw); + unlinkSync4(rmw); } catch (unlinkErr) { dlog3(`rmw lock cleanup failed for ${projectKey}: ${unlinkErr.message}`); } @@ -1449,29 +1871,29 @@ function resetCounter(projectKey) { } function tryAcquireWorkerLock(projectKey, maxAgeMs = 10 * 60 * 1e3) { migrateLegacyStateDir(); - mkdirSync6(STATE_DIR2, { recursive: true }); - const p = lockPath2(projectKey); - if (existsSync7(p)) { + mkdirSync9(STATE_DIR2, { recursive: true }); + const p = lockPath3(projectKey); + if (existsSync9(p)) { try { - const ageMs = Date.now() - parseInt(readFileSync6(p, "utf-8"), 10); + const ageMs = Date.now() - parseInt(readFileSync8(p, "utf-8"), 10); if (Number.isFinite(ageMs) && ageMs < maxAgeMs) return false; } catch (readErr) { dlog3(`worker lock unreadable for ${projectKey}, treating as stale: ${readErr.message}`); } try { - unlinkSync3(p); + unlinkSync4(p); } catch (unlinkErr) { dlog3(`could not unlink stale worker lock for ${projectKey}: ${unlinkErr.message}`); return false; } } try { - const fd = openSync3(p, "wx"); + const fd = openSync4(p, "wx"); try { writeSync3(fd, String(Date.now())); } finally { - closeSync3(fd); + closeSync4(fd); } return true; } catch { @@ -1479,26 +1901,26 @@ function tryAcquireWorkerLock(projectKey, maxAgeMs = 10 * 60 * 1e3) { } } function releaseWorkerLock(projectKey) { - const p = lockPath2(projectKey); + const p = lockPath3(projectKey); try { - unlinkSync3(p); + unlinkSync4(p); } catch { } } // dist/src/skillify/scope-config.js -import { existsSync as existsSync8, mkdirSync as mkdirSync7, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs"; -import { homedir as homedir11 } from "node:os"; -import { join as join14 } from "node:path"; -var STATE_DIR3 = join14(homedir11(), ".deeplake", "state", "skillify"); -var CONFIG_PATH = join14(STATE_DIR3, "config.json"); +import { existsSync as existsSync10, mkdirSync as mkdirSync10, readFileSync as readFileSync9, writeFileSync as writeFileSync8 } from "node:fs"; +import { homedir as homedir14 } from "node:os"; +import { join as join17 } from "node:path"; +var STATE_DIR3 = join17(homedir14(), ".deeplake", "state", "skillify"); +var CONFIG_PATH = join17(STATE_DIR3, "config.json"); var DEFAULT = { scope: "me", team: [], install: "project" }; function loadScopeConfig() { migrateLegacyStateDir(); - if (!existsSync8(CONFIG_PATH)) + if (!existsSync10(CONFIG_PATH)) return DEFAULT; try { - const raw = JSON.parse(readFileSync7(CONFIG_PATH, "utf-8")); + const raw = JSON.parse(readFileSync9(CONFIG_PATH, "utf-8")); const scope = raw.scope === "team" ? "team" : raw.scope === "org" ? "team" : "me"; const team = Array.isArray(raw.team) ? raw.team.filter((s) => typeof s === "string") : []; const install = raw.install === "global" ? "global" : "project"; @@ -1549,12 +1971,18 @@ function tryStopCounterTrigger(opts) { } // dist/src/hooks/cursor/capture.js -var log4 = (msg) => log("cursor-capture", msg); +var log5 = (msg) => log("cursor-capture", msg); function resolveEmbedDaemonPath() { - return join15(dirname4(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); + return join18(dirname6(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); } -var __bundleDir = dirname4(fileURLToPath3(import.meta.url)); +var __bundleDir = dirname6(fileURLToPath3(import.meta.url)); var PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".claude-plugin") ?? ""; +if (!embeddingsDisabled()) { + try { + ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); + } catch { + } +} var CAPTURE = process.env.HIVEMIND_CAPTURE !== "false"; function resolveCwd(input) { if (typeof input.cwd === "string" && input.cwd) @@ -1570,7 +1998,7 @@ async function main() { const input = await readStdin(); const config = loadConfig(); if (!config) { - log4("no config"); + log5("no config"); return; } const sessionId = input.conversation_id ?? `cursor-${Date.now()}`; @@ -1589,10 +2017,10 @@ async function main() { }; let entry = null; if (event === "beforeSubmitPrompt" && typeof input.prompt === "string") { - log4(`user session=${sessionId}`); + log5(`user session=${sessionId}`); entry = { id: crypto.randomUUID(), ...meta, type: "user_message", content: input.prompt }; } else if (event === "postToolUse" && typeof input.tool_name === "string") { - log4(`tool=${input.tool_name} session=${sessionId}`); + log5(`tool=${input.tool_name} session=${sessionId}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1604,10 +2032,10 @@ async function main() { tool_response: typeof input.tool_output === "string" ? input.tool_output : JSON.stringify(input.tool_output) }; } else if (event === "afterAgentResponse" && typeof input.text === "string") { - log4(`assistant session=${sessionId}`); + log5(`assistant session=${sessionId}`); entry = { id: crypto.randomUUID(), ...meta, type: "assistant_message", content: input.text }; } else if (event === "stop") { - log4(`stop session=${sessionId} status=${input.status ?? "unknown"}`); + log5(`stop session=${sessionId} status=${input.status ?? "unknown"}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1616,12 +2044,12 @@ async function main() { loop_count: input.loop_count }; } else { - log4(`unknown event: ${event}, skipping`); + log5(`unknown event: ${event}, skipping`); return; } const sessionPath = buildSessionPath(config, sessionId); const line = JSON.stringify(entry); - log4(`writing to ${sessionPath}`); + log5(`writing to ${sessionPath}`); const projectName = cwd.split("/").pop() || "unknown"; const filename = sessionPath.split("/").pop() ?? ""; const jsonForSql = line.replace(/'/g, "''"); @@ -1632,14 +2060,14 @@ async function main() { await api.query(insertSql); } catch (e) { if (e.message?.includes("permission denied") || e.message?.includes("does not exist")) { - log4("table missing, creating and retrying"); + log5("table missing, creating and retrying"); await api.ensureSessionsTable(sessionsTable); await api.query(insertSql); } else { throw e; } } - log4("capture ok \u2192 cloud"); + log5("capture ok \u2192 cloud"); maybeTriggerPeriodicSummary(sessionId, cwd, config); if (event === "afterAgentResponse" && process.env.HIVEMIND_WIKI_WORKER !== "1" && process.env.HIVEMIND_SKILLIFY_WORKER !== "1") { tryStopCounterTrigger({ @@ -1660,7 +2088,7 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { if (!shouldTrigger(state, cfg)) return; if (!tryAcquireLock(sessionId)) { - log4(`periodic trigger suppressed (lock held) session=${sessionId}`); + log5(`periodic trigger suppressed (lock held) session=${sessionId}`); return; } wikiLog(`Periodic: threshold hit (total=${state.totalCount}, since=${state.totalCount - state.lastSummaryCount}, N=${cfg.everyNMessages}, hours=${cfg.everyHours})`); @@ -1673,17 +2101,17 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { reason: "Periodic" }); } catch (e) { - log4(`periodic spawn failed: ${e.message}`); + log5(`periodic spawn failed: ${e.message}`); try { releaseLock(sessionId); } catch { } } } catch (e) { - log4(`periodic trigger error: ${e.message}`); + log5(`periodic trigger error: ${e.message}`); } } main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); diff --git a/cursor/bundle/embeddings/embed-daemon.js b/cursor/bundle/embeddings/embed-daemon.js index 5f711a3c..a81ffbd1 100755 --- a/cursor/bundle/embeddings/embed-daemon.js +++ b/cursor/bundle/embeddings/embed-daemon.js @@ -4,7 +4,14 @@ import { createServer } from "node:net"; import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +// dist/src/embeddings/nomic.js +import { createRequire } from "node:module"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + // dist/src/embeddings/protocol.js +var PROTOCOL_VERSION = 1; var DEFAULT_SOCKET_DIR = "/tmp"; var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; var DEFAULT_DTYPE = "q8"; @@ -20,6 +27,39 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { } // dist/src/embeddings/nomic.js +async function _importFromCanonicalSharedDeps(sharedDir = join(homedir(), ".hivemind", "embed-deps")) { + const base = pathToFileURL(`${sharedDir}/`).href; + const absMain = createRequire(base).resolve("@huggingface/transformers"); + const mod = await import(pathToFileURL(absMain).href); + return _normalizeTransformersModule(mod); +} +async function _importFromBareSpecifier() { + const mod = await import("@huggingface/transformers"); + return _normalizeTransformersModule(mod); +} +function _normalizeTransformersModule(mod) { + const m = mod; + if (m.default && typeof m.default === "object" && "pipeline" in m.default) { + return m.default; + } + return m; +} +async function defaultImportTransformers(canonical = _importFromCanonicalSharedDeps, bare = _importFromBareSpecifier) { + let canonicalErr; + try { + return await canonical(); + } catch (err) { + canonicalErr = err; + } + try { + return await bare(); + } catch (bareErr) { + const detail = bareErr instanceof Error ? bareErr.message : String(bareErr); + const canonicalDetail = canonicalErr instanceof Error ? canonicalErr.message : String(canonicalErr); + throw new Error(`@huggingface/transformers is not installed anywhere reachable. Run \`hivemind embeddings install\` to install it. (canonical: ${canonicalDetail}; bare: ${detail})`); + } +} +var _importTransformers = defaultImportTransformers; var NomicEmbedder = class { pipeline = null; loading = null; @@ -37,7 +77,7 @@ var NomicEmbedder = class { if (this.loading) return this.loading; this.loading = (async () => { - const mod = await import("@huggingface/transformers"); + const mod = await _importTransformers(); mod.env.allowLocalModels = false; mod.env.useFSCache = true; this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype }); @@ -94,9 +134,9 @@ var NomicEmbedder = class { // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; -var LOG = join(homedir(), ".deeplake", "hook-debug.log"); +import { join as join2 } from "node:path"; +import { homedir as homedir2 } from "node:os"; +var LOG = join2(homedir2(), ".deeplake", "hook-debug.log"); function isDebug() { return process.env.HIVEMIND_DEBUG === "1"; } @@ -120,6 +160,7 @@ var EmbedDaemon = class { pidPath; idleTimeoutMs; idleTimer = null; + daemonPath; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; @@ -127,6 +168,7 @@ var EmbedDaemon = class { this.pidPath = pidPathFor(uid, dir); this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + this.daemonPath = opts.daemonPath ?? process.argv[1] ?? ""; } async start() { mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); @@ -218,6 +260,15 @@ var EmbedDaemon = class { } } async dispatch(req) { + if (req.op === "hello") { + const h = req; + return { + id: h.id, + daemonPath: this.daemonPath, + pid: process.pid, + protocolVersion: PROTOCOL_VERSION + }; + } if (req.op === "ping") { const p = req; return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; diff --git a/cursor/bundle/pre-tool-use.js b/cursor/bundle/pre-tool-use.js index 0b2f8df6..afc8d838 100755 --- a/cursor/bundle/pre-tool-use.js +++ b/cursor/bundle/pre-tool-use.js @@ -53,13 +53,13 @@ var init_index_marker_store = __esm({ // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -175,7 +175,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -205,7 +205,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -1042,9 +1042,9 @@ function capOutputForClaude(output, options = {}) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join4 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -1057,13 +1057,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync } from "node:fs"; +import { join as join4, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join4(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join4(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync2(join4(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join6 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join5 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join5(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join6(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join4(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join7(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -1072,13 +1293,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -1088,8 +1310,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -1099,17 +1346,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -1118,6 +1373,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -1141,7 +1529,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -1149,7 +1537,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -1160,16 +1548,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -1178,11 +1566,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -1194,14 +1582,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -1219,9 +1607,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -1231,7 +1619,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -1246,7 +1634,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -1263,53 +1651,22 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join5(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/hooks/grep-direct.js import { fileURLToPath } from "node:url"; -import { dirname, join as join6 } from "node:path"; +import { dirname as dirname2, join as join8 } from "node:path"; var SEMANTIC_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveDaemonPath() { - return join6(dirname(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join8(dirname2(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedEmbedClient = null; function getEmbedClient() { @@ -1662,9 +2019,9 @@ async function handleGrepDirect(api, table, sessionsTable, params) { } // dist/src/hooks/memory-path-utils.js -import { homedir as homedir5 } from "node:os"; -import { join as join7 } from "node:path"; -var MEMORY_PATH = join7(homedir5(), ".deeplake", "memory"); +import { homedir as homedir7 } from "node:os"; +import { join as join9 } from "node:path"; +var MEMORY_PATH = join9(homedir7(), ".deeplake", "memory"); var TILDE_PATH = "~/.deeplake/memory"; var HOME_VAR_PATH = "$HOME/.deeplake/memory"; function touchesMemory(p) { @@ -1675,7 +2032,7 @@ function rewritePaths(cmd) { } // dist/src/hooks/cursor/pre-tool-use.js -var log4 = (msg) => log("cursor-pre-tool-use", msg); +var log5 = (msg) => log("cursor-pre-tool-use", msg); async function main() { const input = await readStdin(); if (input.tool_name !== "Shell") @@ -1691,17 +2048,17 @@ async function main() { return; const config = loadConfig(); if (!config) { - log4("no config \u2014 falling through to Cursor's bash"); + log5("no config \u2014 falling through to Cursor's bash"); return; } const api = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.tableName); try { const result = await handleGrepDirect(api, config.tableName, config.sessionsTableName, grepParams); if (result === null) { - log4(`fallthrough \u2014 handleGrepDirect returned null for "${grepParams.pattern}"`); + log5(`fallthrough \u2014 handleGrepDirect returned null for "${grepParams.pattern}"`); return; } - log4(`intercepted ${command.slice(0, 80)} \u2192 ${result.length} chars from SQL fast-path`); + log5(`intercepted ${command.slice(0, 80)} \u2192 ${result.length} chars from SQL fast-path`); const echoCmd = `cat <<'__HIVEMIND_RESULT__' ${result} __HIVEMIND_RESULT__`; @@ -1712,10 +2069,10 @@ __HIVEMIND_RESULT__`; })); } catch (err) { const msg = err instanceof Error ? err.message : String(err); - log4(`fast-path failed, falling through: ${msg}`); + log5(`fast-path failed, falling through: ${msg}`); } } main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); diff --git a/cursor/bundle/session-start.js b/cursor/bundle/session-start.js index 61096eab..882acb60 100755 --- a/cursor/bundle/session-start.js +++ b/cursor/bundle/session-start.js @@ -1511,7 +1511,14 @@ Organization management \u2014 each argument is SEPARATE (do NOT quote subcomman - hivemind remove \u2014 remove member SKILLS (skillify) \u2014 mine + share reusable skills across the org: -${renderSkillifyCommands()}`; +${renderSkillifyCommands()} + +Embeddings (semantic memory search) \u2014 opt-in, persisted in ~/.deeplake/config.json: +- hivemind embeddings install \u2014 download deps (~600MB), symlink agents, set enabled:true +- hivemind embeddings enable \u2014 flip enabled:true (run install first if deps missing) +- hivemind embeddings disable \u2014 flip enabled:false + SIGTERM daemon (deps stay on disk) +- hivemind embeddings uninstall [--prune] \u2014 remove agent symlinks + disable; --prune wipes deps too +- hivemind embeddings status \u2014 show config + deps + per-agent link state`; function resolveSessionId(input) { return input.session_id ?? input.conversation_id ?? `cursor-${Date.now()}`; } diff --git a/cursor/bundle/shell/deeplake-shell.js b/cursor/bundle/shell/deeplake-shell.js index e3f8c1d6..402fb0a3 100755 --- a/cursor/bundle/shell/deeplake-shell.js +++ b/cursor/bundle/shell/deeplake-shell.js @@ -46081,14 +46081,14 @@ var require_turndown_cjs = __commonJS({ } else if (node.nodeType === 1) { replacement = replacementForNode.call(self2, node); } - return join11(output, replacement); + return join13(output, replacement); }, ""); } function postProcess(output) { var self2 = this; this.rules.forEach(function(rule) { if (typeof rule.append === "function") { - output = join11(output, rule.append(self2.options)); + output = join13(output, rule.append(self2.options)); } }); return output.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, ""); @@ -46100,7 +46100,7 @@ var require_turndown_cjs = __commonJS({ if (whitespace.leading || whitespace.trailing) content = content.trim(); return whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing; } - function join11(output, replacement) { + function join13(output, replacement) { var s12 = trimTrailingNewlines(output); var s22 = trimLeadingNewlines(replacement); var nls = Math.max(output.length - s12.length, replacement.length - s22.length); @@ -66870,7 +66870,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms3) { - return new Promise((resolve5) => setTimeout(resolve5, ms3)); + return new Promise((resolve6) => setTimeout(resolve6, ms3)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -66900,7 +66900,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve5) => this.waiting.push(resolve5)); + await new Promise((resolve6) => this.waiting.push(resolve6)); } release() { this.active--; @@ -67259,7 +67259,7 @@ var DeeplakeApi = class { import { basename as basename4, posix } from "node:path"; import { randomUUID as randomUUID2 } from "node:crypto"; import { fileURLToPath } from "node:url"; -import { dirname as dirname4, join as join9 } from "node:path"; +import { dirname as dirname5, join as join11 } from "node:path"; // dist/src/shell/grep-core.js var TOOL_INPUT_FIELDS = [ @@ -67689,9 +67689,9 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join7 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync5, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join10 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -67704,13 +67704,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync as statSync2 } from "node:fs"; +import { join as join7, resolve as resolve4 } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join7(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path2, home) { + const r10 = resolve4(path2); + const h18 = resolve4(home); + return r10.startsWith(h18 + "/") || r10 === h18; +} +function writeQueue(q17) { + const path2 = queuePath(); + const home = resolve4(homedir3()); + if (!_isQueuePathInsideHome(path2, home)) { + throw new Error(`notifications-queue write blocked: ${path2} is outside ${home}`); + } + mkdirSync2(join7(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path2}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q17, null, 2), { mode: 384 }); + renameSync(tmp, path2); +} +async function withQueueLock(fn4) { + const path2 = lockPath(); + mkdirSync2(join7(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path2, "wx", 384); + break; + } catch (e6) { + const code = e6.code; + if (code !== "EEXIST") + throw e6; + try { + const age = Date.now() - statSync2(path2).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path2); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn4(); + } + try { + return fn4(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path2); + } catch { + } + } +} +function sameDedupKey(a15, b26) { + if (a15.id !== b26.id) + return false; + return JSON.stringify(a15.dedupKey) === JSON.stringify(b26.dedupKey); +} +async function enqueueNotification(n24) { + await withQueueLock(() => { + const q17 = readQueue(); + if (q17.queue.some((existing) => sameDedupKey(existing, n24))) { + return; + } + q17.queue.push(n24); + writeQueue(q17); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join9 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname as dirname4, join as join8 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join8(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path2 = _configPath(); + if (!existsSync4(path2)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path2, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path2 = _configPath(); + const dir = dirname4(path2); + if (!existsSync4(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path2}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path2); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join9(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join7(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m26) => log("embed-client", m26); +var SHARED_DAEMON_PATH = join10(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m26) => log("embed-client", m26); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -67719,13 +67940,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync5(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -67735,8 +67957,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v27 = await this.embedAttempt(text, kind); + if (v27 !== "recycled") + return v27; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -67746,17 +67993,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e6) { const err = e6 instanceof Error ? e6.message : String(e6); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -67765,6 +68020,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync5(this.socketPath)) + return; + await new Promise((r10) => setTimeout(r10, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e6) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e6 instanceof Error ? e6.message : String(e6)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync5(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e6) => { + log4(`enqueue embed-deps-missing failed: ${e6 instanceof Error ? e6.message : String(e6)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync5(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -67788,7 +68176,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { const sock = connect(this.socketPath); const to3 = setTimeout(() => { sock.destroy(); @@ -67796,7 +68184,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to3); - resolve5(sock); + resolve6(sock); }); sock.once("error", (e6) => { clearTimeout(to3); @@ -67807,16 +68195,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e6) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -67825,11 +68213,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync5(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -67841,14 +68229,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -67866,9 +68254,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync4(this.socketPath)) + if (!existsSync5(this.socketPath)) continue; try { return await this.connectOnce(); @@ -67878,7 +68266,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { let buf = ""; const to3 = setTimeout(() => { sock.destroy(); @@ -67893,7 +68281,7 @@ var EmbedClient = class { const line = buf.slice(0, nl3); clearTimeout(to3); try { - resolve5(JSON.parse(line)); + resolve6(JSON.parse(line)); } catch (e6) { reject(e6); } @@ -67910,9 +68298,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms3) { +function sleep3(ms3) { return new Promise((r10) => setTimeout(r10, ms3)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -67927,42 +68320,6 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join8 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join8(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} - // dist/src/hooks/virtual-table-query.js var INDEX_LIMIT_PER_SECTION = 50; function buildVirtualIndexContent(summaryRows, sessionRows = [], opts = {}) { @@ -68055,7 +68412,7 @@ function normalizeSessionMessage(path2, message) { return normalizeContent(path2, raw); } function resolveEmbedDaemonPath() { - return join9(dirname4(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + return join11(dirname5(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } function joinSessionMessages(path2, messages) { return messages.map((message) => normalizeSessionMessage(path2, message)).join("\n"); @@ -68621,7 +68978,7 @@ var DeeplakeFs = class _DeeplakeFs { // node_modules/yargs-parser/build/lib/index.js import { format } from "util"; -import { normalize, resolve as resolve4 } from "path"; +import { normalize, resolve as resolve5 } from "path"; // node_modules/yargs-parser/build/lib/string-utils.js function camelCase2(str) { @@ -69559,7 +69916,7 @@ function stripQuotes(val) { } // node_modules/yargs-parser/build/lib/index.js -import { readFileSync as readFileSync4 } from "fs"; +import { readFileSync as readFileSync6 } from "fs"; import { createRequire as createRequire2 } from "node:module"; var _a3; var _b; @@ -69581,12 +69938,12 @@ var parser = new YargsParser({ }, format, normalize, - resolve: resolve4, + resolve: resolve5, require: (path2) => { if (typeof require2 !== "undefined") { return require2(path2); } else if (path2.match(/\.json$/)) { - return JSON.parse(readFileSync4(path2, "utf8")); + return JSON.parse(readFileSync6(path2, "utf8")); } else { throw Error("only .json config files are supported in ESM"); } @@ -69606,11 +69963,11 @@ var lib_default = yargsParser; // dist/src/shell/grep-interceptor.js import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname as dirname5, join as join10 } from "node:path"; +import { dirname as dirname6, join as join12 } from "node:path"; var SEMANTIC_SEARCH_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_EMBED_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveGrepEmbedDaemonPath() { - return join10(dirname5(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join12(dirname6(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedGrepEmbedClient = null; function getGrepEmbedClient() { diff --git a/cursor/bundle/wiki-worker.js b/cursor/bundle/wiki-worker.js index 2730597f..dc3a1a7c 100755 --- a/cursor/bundle/wiki-worker.js +++ b/cursor/bundle/wiki-worker.js @@ -1,9 +1,9 @@ #!/usr/bin/env node // dist/src/hooks/cursor/wiki-worker.js -import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync3, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; +import { readFileSync as readFileSync5, writeFileSync as writeFileSync4, existsSync as existsSync4, appendFileSync as appendFileSync2, mkdirSync as mkdirSync4, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { dirname, join as join5 } from "node:path"; +import { dirname as dirname2, join as join7 } from "node:path"; import { fileURLToPath } from "node:url"; // dist/src/hooks/summary-state.js @@ -154,9 +154,9 @@ async function uploadSummary(query2, params) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join3 } from "node:path"; +import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync2, unlinkSync as unlinkSync3, existsSync as existsSync3, readFileSync as readFileSync4 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join6 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -169,13 +169,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync as renameSync2, mkdirSync as mkdirSync2, openSync as openSync2, closeSync as closeSync2, unlinkSync as unlinkSync2, statSync } from "node:fs"; +import { join as join3, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; +var log2 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join3(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath2() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync2(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log2(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join3(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync2(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath2(); + mkdirSync2(join3(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync2(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync2(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep(delay); + } + } + if (fd === null) { + log2(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync2(fd); + } catch { + } + try { + unlinkSync2(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join5 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join4 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join4(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync2(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync3(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync2(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync3(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg2 = readUserConfig(); + if (cfg2.embeddings && typeof cfg2.embeddings.enabled === "boolean") { + return cfg2.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg2 ?? {}, embeddings: { ...cfg2?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join5(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join3(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log2 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join6(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log3 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -184,13 +405,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync2(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -200,8 +422,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -211,17 +458,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log2(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log3(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log2(`embed failed: ${err}`); + log3(`embed failed: ${err}`); return null; } finally { try { @@ -230,6 +485,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync3(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log3(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log3(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync3(hello.daemonPath)) { + _recycledStuckDaemon = true; + log3(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log3(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync4(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync3(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log3(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync3(this.socketPath); + } catch { + } + try { + unlinkSync3(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -253,7 +641,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -261,7 +649,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -272,16 +660,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync2(this.pidPath); + unlinkSync3(this.pidPath); } catch { } try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch { return; @@ -290,11 +678,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync2(this.daemonEntry)) { - log2(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync2(fd); - unlinkSync2(this.pidPath); + closeSync3(fd); + unlinkSync3(this.pidPath); } catch { } return; @@ -306,14 +694,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log2(`spawned daemon pid=${child.pid}`); + log3(`spawned daemon pid=${child.pid}`); } finally { - closeSync2(fd); + closeSync3(fd); } } isPidFileStale() { try { - const raw = readFileSync2(this.pidPath, "utf-8").trim(); + const raw = readFileSync4(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -331,9 +719,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep(delay); + await sleep2(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync2(this.socketPath)) + if (!existsSync3(this.socketPath)) continue; try { return await this.connectOnce(); @@ -343,7 +731,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -358,7 +746,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -375,44 +763,13 @@ var EmbedClient = class { }); } }; -function sleep(ms) { +function sleep2(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join4 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join4(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/utils/client-header.js @@ -426,13 +783,13 @@ function deeplakeClientHeader() { // dist/src/hooks/cursor/wiki-worker.js var dlog2 = (msg) => log("cursor-wiki-worker", msg); -var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8")); +var cfg = JSON.parse(readFileSync5(process.argv[2], "utf-8")); var tmpDir = cfg.tmpDir; -var tmpJsonl = join5(tmpDir, "session.jsonl"); -var tmpSummary = join5(tmpDir, "summary.md"); +var tmpJsonl = join7(tmpDir, "session.jsonl"); +var tmpSummary = join7(tmpDir, "summary.md"); function wlog(msg) { try { - mkdirSync2(cfg.hooksDir, { recursive: true }); + mkdirSync4(cfg.hooksDir, { recursive: true }); appendFileSync2(cfg.wikiLog, `[${(/* @__PURE__ */ new Date()).toISOString().replace("T", " ").slice(0, 19)}] wiki-worker(${cfg.sessionId}): ${msg} `); } catch { @@ -464,7 +821,7 @@ async function query(sql, retries = 4) { const base = Math.min(3e4, 2e3 * Math.pow(2, attempt)); const delay = base + Math.floor(Math.random() * 1e3); wlog(`API ${r.status}, retrying in ${delay}ms (attempt ${attempt + 1}/${retries})`); - await new Promise((resolve) => setTimeout(resolve, delay)); + await new Promise((resolve2) => setTimeout(resolve2, delay)); continue; } throw new Error(`API ${r.status}: ${(await r.text()).slice(0, 200)}`); @@ -490,7 +847,7 @@ async function main() { const jsonlLines = rows.length; const pathRows = await query(`SELECT DISTINCT path FROM "${cfg.sessionsTable}" WHERE path LIKE '${esc2(`/sessions/%${cfg.sessionId}%`)}' LIMIT 1`); const jsonlServerPath = pathRows.length > 0 ? pathRows[0].path : `/sessions/unknown/${cfg.sessionId}.jsonl`; - writeFileSync2(tmpJsonl, jsonlContent); + writeFileSync4(tmpJsonl, jsonlContent); wlog(`found ${jsonlLines} events at ${jsonlServerPath}`); let prevOffset = 0; try { @@ -500,7 +857,7 @@ async function main() { const match = existing.match(/\*\*JSONL offset\*\*:\s*(\d+)/); if (match) prevOffset = parseInt(match[1], 10); - writeFileSync2(tmpSummary, existing); + writeFileSync4(tmpSummary, existing); wlog(`existing summary found, offset=${prevOffset}`); } } catch { @@ -525,15 +882,15 @@ async function main() { } catch (e) { wlog(`cursor-agent --print failed: ${e.status ?? e.message}`); } - if (existsSync3(tmpSummary)) { - const text = readFileSync3(tmpSummary, "utf-8"); + if (existsSync4(tmpSummary)) { + const text = readFileSync5(tmpSummary, "utf-8"); if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; let embedding = null; if (!embeddingsDisabled()) { try { - const daemonEntry = join5(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + const daemonEntry = join7(dirname2(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); } catch (e) { wlog(`summary embedding failed, writing NULL: ${e.message}`); diff --git a/embeddings/embed-daemon.js b/embeddings/embed-daemon.js index 5f711a3c..a81ffbd1 100755 --- a/embeddings/embed-daemon.js +++ b/embeddings/embed-daemon.js @@ -4,7 +4,14 @@ import { createServer } from "node:net"; import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +// dist/src/embeddings/nomic.js +import { createRequire } from "node:module"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + // dist/src/embeddings/protocol.js +var PROTOCOL_VERSION = 1; var DEFAULT_SOCKET_DIR = "/tmp"; var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; var DEFAULT_DTYPE = "q8"; @@ -20,6 +27,39 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { } // dist/src/embeddings/nomic.js +async function _importFromCanonicalSharedDeps(sharedDir = join(homedir(), ".hivemind", "embed-deps")) { + const base = pathToFileURL(`${sharedDir}/`).href; + const absMain = createRequire(base).resolve("@huggingface/transformers"); + const mod = await import(pathToFileURL(absMain).href); + return _normalizeTransformersModule(mod); +} +async function _importFromBareSpecifier() { + const mod = await import("@huggingface/transformers"); + return _normalizeTransformersModule(mod); +} +function _normalizeTransformersModule(mod) { + const m = mod; + if (m.default && typeof m.default === "object" && "pipeline" in m.default) { + return m.default; + } + return m; +} +async function defaultImportTransformers(canonical = _importFromCanonicalSharedDeps, bare = _importFromBareSpecifier) { + let canonicalErr; + try { + return await canonical(); + } catch (err) { + canonicalErr = err; + } + try { + return await bare(); + } catch (bareErr) { + const detail = bareErr instanceof Error ? bareErr.message : String(bareErr); + const canonicalDetail = canonicalErr instanceof Error ? canonicalErr.message : String(canonicalErr); + throw new Error(`@huggingface/transformers is not installed anywhere reachable. Run \`hivemind embeddings install\` to install it. (canonical: ${canonicalDetail}; bare: ${detail})`); + } +} +var _importTransformers = defaultImportTransformers; var NomicEmbedder = class { pipeline = null; loading = null; @@ -37,7 +77,7 @@ var NomicEmbedder = class { if (this.loading) return this.loading; this.loading = (async () => { - const mod = await import("@huggingface/transformers"); + const mod = await _importTransformers(); mod.env.allowLocalModels = false; mod.env.useFSCache = true; this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype }); @@ -94,9 +134,9 @@ var NomicEmbedder = class { // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; -var LOG = join(homedir(), ".deeplake", "hook-debug.log"); +import { join as join2 } from "node:path"; +import { homedir as homedir2 } from "node:os"; +var LOG = join2(homedir2(), ".deeplake", "hook-debug.log"); function isDebug() { return process.env.HIVEMIND_DEBUG === "1"; } @@ -120,6 +160,7 @@ var EmbedDaemon = class { pidPath; idleTimeoutMs; idleTimer = null; + daemonPath; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; @@ -127,6 +168,7 @@ var EmbedDaemon = class { this.pidPath = pidPathFor(uid, dir); this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + this.daemonPath = opts.daemonPath ?? process.argv[1] ?? ""; } async start() { mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); @@ -218,6 +260,15 @@ var EmbedDaemon = class { } } async dispatch(req) { + if (req.op === "hello") { + const h = req; + return { + id: h.id, + daemonPath: this.daemonPath, + pid: process.pid, + protocolVersion: PROTOCOL_VERSION + }; + } if (req.op === "ping") { const p = req; return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; diff --git a/hermes/bundle/capture.js b/hermes/bundle/capture.js index 905a8eac..12b4bb4e 100755 --- a/hermes/bundle/capture.js +++ b/hermes/bundle/capture.js @@ -53,13 +53,13 @@ var init_index_marker_store = __esm({ // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -175,7 +175,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -205,7 +205,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -569,9 +569,9 @@ function buildSessionPath(config, sessionId) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join4 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -584,13 +584,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync } from "node:fs"; +import { join as join4, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join4(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join4(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync2(join4(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join6 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join5 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join5(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join6(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join4(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join7(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -599,13 +820,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -615,8 +837,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -626,17 +873,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -645,6 +900,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -668,7 +1056,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -676,7 +1064,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -687,16 +1075,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -705,11 +1093,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -721,14 +1109,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -746,9 +1134,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -758,7 +1146,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -773,7 +1161,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -790,9 +1178,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -807,91 +1200,120 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { +// dist/src/embeddings/self-heal.js +import { existsSync as existsSync5, lstatSync, mkdirSync as mkdirSync4, readlinkSync, renameSync as renameSync3, rmSync, symlinkSync, statSync as statSync2 } from "node:fs"; +import { homedir as homedir7 } from "node:os"; +import { basename, dirname as dirname2, join as join8 } from "node:path"; +function ensurePluginNodeModulesLink(opts) { + if (basename(opts.bundleDir) !== "bundle") { + return { kind: "not-bundle-layout", bundleDir: opts.bundleDir }; + } + const target = opts.sharedNodeModules ?? join8(homedir7(), ".hivemind", "embed-deps", "node_modules"); + const pluginDir = dirname2(opts.bundleDir); + const link = join8(pluginDir, "node_modules"); + if (!existsSync5(target)) { + return { kind: "shared-deps-missing", target }; + } + let linkStat; try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; + linkStat = lstatSync(link); } catch { + return createSymlinkAtomic(target, link); } - const sharedDir = join5(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + if (linkStat.isSymbolicLink()) { + let existingTarget; + try { + existingTarget = readlinkSync(link); + } catch (e) { + return { kind: "error", detail: `readlink failed: ${e instanceof Error ? e.message : String(e)}` }; + } + if (existingTarget === target) { + return { kind: "already-linked", target, link }; + } + try { + statSync2(link); + return { kind: "linked-elsewhere", link, existingTarget }; + } catch { + try { + rmSync(link); + } catch { + } + const recreated = createSymlinkAtomic(target, link); + if (recreated.kind === "linked") { + return { kind: "stale-link-removed", link, danglingTarget: existingTarget }; + } + return recreated; + } + } + return { kind: "plugin-owns-node-modules", link }; } -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; +function createSymlinkAtomic(target, link) { try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; + const parent = dirname2(link); + if (!existsSync5(parent)) + mkdirSync4(parent, { recursive: true }); + const tmp = `${link}.tmp.${process.pid}`; + try { + rmSync(tmp, { force: true }); + } catch { + } + symlinkSync(target, tmp); + renameSync3(tmp, link); + return { kind: "linked", target, link }; + } catch (e) { + return { kind: "error", detail: e instanceof Error ? e.message : String(e) }; } } -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} // dist/src/hooks/hermes/capture.js import { fileURLToPath as fileURLToPath3 } from "node:url"; -import { dirname as dirname4, join as join15 } from "node:path"; +import { dirname as dirname6, join as join18 } from "node:path"; // dist/src/hooks/summary-state.js -import { readFileSync as readFileSync4, writeFileSync as writeFileSync2, writeSync as writeSync2, mkdirSync as mkdirSync2, renameSync, existsSync as existsSync4, unlinkSync as unlinkSync2, openSync as openSync2, closeSync as closeSync2 } from "node:fs"; -import { homedir as homedir5 } from "node:os"; -import { join as join6 } from "node:path"; +import { readFileSync as readFileSync6, writeFileSync as writeFileSync4, writeSync as writeSync2, mkdirSync as mkdirSync5, renameSync as renameSync4, existsSync as existsSync6, unlinkSync as unlinkSync3, openSync as openSync3, closeSync as closeSync3 } from "node:fs"; +import { homedir as homedir8 } from "node:os"; +import { join as join9 } from "node:path"; var dlog = (msg) => log("summary-state", msg); -var STATE_DIR = join6(homedir5(), ".claude", "hooks", "summary-state"); +var STATE_DIR = join9(homedir8(), ".claude", "hooks", "summary-state"); var YIELD_BUF = new Int32Array(new SharedArrayBuffer(4)); function statePath(sessionId) { - return join6(STATE_DIR, `${sessionId}.json`); + return join9(STATE_DIR, `${sessionId}.json`); } -function lockPath(sessionId) { - return join6(STATE_DIR, `${sessionId}.lock`); +function lockPath2(sessionId) { + return join9(STATE_DIR, `${sessionId}.lock`); } function readState(sessionId) { const p = statePath(sessionId); - if (!existsSync4(p)) + if (!existsSync6(p)) return null; try { - return JSON.parse(readFileSync4(p, "utf-8")); + return JSON.parse(readFileSync6(p, "utf-8")); } catch { return null; } } function writeState(sessionId, state) { - mkdirSync2(STATE_DIR, { recursive: true }); + mkdirSync5(STATE_DIR, { recursive: true }); const p = statePath(sessionId); const tmp = `${p}.${process.pid}.${Date.now()}.tmp`; - writeFileSync2(tmp, JSON.stringify(state)); - renameSync(tmp, p); + writeFileSync4(tmp, JSON.stringify(state)); + renameSync4(tmp, p); } function withRmwLock(sessionId, fn) { - mkdirSync2(STATE_DIR, { recursive: true }); + mkdirSync5(STATE_DIR, { recursive: true }); const rmwLock = statePath(sessionId) + ".rmw"; const deadline = Date.now() + 2e3; let fd = null; while (fd === null) { try { - fd = openSync2(rmwLock, "wx"); + fd = openSync3(rmwLock, "wx"); } catch (e) { if (e.code !== "EEXIST") throw e; if (Date.now() > deadline) { dlog(`rmw lock deadline exceeded for ${sessionId}, reclaiming stale lock`); try { - unlinkSync2(rmwLock); + unlinkSync3(rmwLock); } catch (unlinkErr) { dlog(`stale rmw lock unlink failed for ${sessionId}: ${unlinkErr.message}`); } @@ -903,9 +1325,9 @@ function withRmwLock(sessionId, fn) { try { return fn(); } finally { - closeSync2(fd); + closeSync3(fd); try { - unlinkSync2(rmwLock); + unlinkSync3(rmwLock); } catch (unlinkErr) { dlog(`rmw lock cleanup failed for ${sessionId}: ${unlinkErr.message}`); } @@ -940,29 +1362,29 @@ function shouldTrigger(state, cfg, now = Date.now()) { return false; } function tryAcquireLock(sessionId, maxAgeMs = 10 * 60 * 1e3) { - mkdirSync2(STATE_DIR, { recursive: true }); - const p = lockPath(sessionId); - if (existsSync4(p)) { + mkdirSync5(STATE_DIR, { recursive: true }); + const p = lockPath2(sessionId); + if (existsSync6(p)) { try { - const ageMs = Date.now() - parseInt(readFileSync4(p, "utf-8"), 10); + const ageMs = Date.now() - parseInt(readFileSync6(p, "utf-8"), 10); if (Number.isFinite(ageMs) && ageMs < maxAgeMs) return false; } catch (readErr) { dlog(`lock file unreadable for ${sessionId}, treating as stale: ${readErr.message}`); } try { - unlinkSync2(p); + unlinkSync3(p); } catch (unlinkErr) { dlog(`could not unlink stale lock for ${sessionId}: ${unlinkErr.message}`); return false; } } try { - const fd = openSync2(p, "wx"); + const fd = openSync3(p, "wx"); try { writeSync2(fd, String(Date.now())); } finally { - closeSync2(fd); + closeSync3(fd); } return true; } catch (e) { @@ -973,7 +1395,7 @@ function tryAcquireLock(sessionId, maxAgeMs = 10 * 60 * 1e3) { } function releaseLock(sessionId) { try { - unlinkSync2(lockPath(sessionId)); + unlinkSync3(lockPath2(sessionId)); } catch (e) { if (e?.code !== "ENOENT") { dlog(`releaseLock unlink failed for ${sessionId}: ${e.message}`); @@ -984,20 +1406,20 @@ function releaseLock(sessionId) { // dist/src/hooks/hermes/spawn-wiki-worker.js import { spawn as spawn2, execSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { dirname as dirname2, join as join9 } from "node:path"; -import { writeFileSync as writeFileSync3, mkdirSync as mkdirSync4 } from "node:fs"; -import { homedir as homedir6, tmpdir as tmpdir2 } from "node:os"; +import { dirname as dirname4, join as join12 } from "node:path"; +import { writeFileSync as writeFileSync5, mkdirSync as mkdirSync7 } from "node:fs"; +import { homedir as homedir9, tmpdir as tmpdir2 } from "node:os"; // dist/src/utils/wiki-log.js -import { mkdirSync as mkdirSync3, appendFileSync as appendFileSync2 } from "node:fs"; -import { join as join7 } from "node:path"; +import { mkdirSync as mkdirSync6, appendFileSync as appendFileSync2 } from "node:fs"; +import { join as join10 } from "node:path"; function makeWikiLogger(hooksDir, filename = "deeplake-wiki.log") { - const path = join7(hooksDir, filename); + const path = join10(hooksDir, filename); return { path, log(msg) { try { - mkdirSync3(hooksDir, { recursive: true }); + mkdirSync6(hooksDir, { recursive: true }); appendFileSync2(path, `[${utcTimestamp()}] ${msg} `); } catch { @@ -1007,18 +1429,18 @@ function makeWikiLogger(hooksDir, filename = "deeplake-wiki.log") { } // dist/src/utils/version-check.js -import { readFileSync as readFileSync5 } from "node:fs"; -import { dirname, join as join8 } from "node:path"; +import { readFileSync as readFileSync7 } from "node:fs"; +import { dirname as dirname3, join as join11 } from "node:path"; function getInstalledVersion(bundleDir, pluginManifestDir) { try { - const pluginJson = join8(bundleDir, "..", pluginManifestDir, "plugin.json"); - const plugin = JSON.parse(readFileSync5(pluginJson, "utf-8")); + const pluginJson = join11(bundleDir, "..", pluginManifestDir, "plugin.json"); + const plugin = JSON.parse(readFileSync7(pluginJson, "utf-8")); if (plugin.version) return plugin.version; } catch { } try { - const stamp = readFileSync5(join8(bundleDir, "..", ".hivemind_version"), "utf-8").trim(); + const stamp = readFileSync7(join11(bundleDir, "..", ".hivemind_version"), "utf-8").trim(); if (stamp) return stamp; } catch { @@ -1033,14 +1455,14 @@ function getInstalledVersion(bundleDir, pluginManifestDir) { ]); let dir = bundleDir; for (let i = 0; i < 5; i++) { - const candidate = join8(dir, "package.json"); + const candidate = join11(dir, "package.json"); try { - const pkg = JSON.parse(readFileSync5(candidate, "utf-8")); + const pkg = JSON.parse(readFileSync7(candidate, "utf-8")); if (HIVEMIND_PKG_NAMES.has(pkg.name) && pkg.version) return pkg.version; } catch { } - const parent = dirname(dir); + const parent = dirname3(dir); if (parent === dir) break; dir = parent; @@ -1049,8 +1471,8 @@ function getInstalledVersion(bundleDir, pluginManifestDir) { } // dist/src/hooks/hermes/spawn-wiki-worker.js -var HOME = homedir6(); -var wikiLogger = makeWikiLogger(join9(HOME, ".hermes", "hooks")); +var HOME = homedir9(); +var wikiLogger = makeWikiLogger(join12(HOME, ".hermes", "hooks")); var WIKI_LOG = wikiLogger.path; var WIKI_PROMPT_TEMPLATE = `You are building a personal wiki from a coding session. Your goal is to extract every piece of knowledge \u2014 entities, decisions, relationships, and facts \u2014 into a structured, searchable wiki entry. @@ -1112,11 +1534,11 @@ function findHermesBin() { function spawnHermesWikiWorker(opts) { const { config, sessionId, cwd, bundleDir, reason } = opts; const projectName = cwd.split("/").pop() || "unknown"; - const tmpDir = join9(tmpdir2(), `deeplake-wiki-${sessionId}-${Date.now()}`); - mkdirSync4(tmpDir, { recursive: true }); + const tmpDir = join12(tmpdir2(), `deeplake-wiki-${sessionId}-${Date.now()}`); + mkdirSync7(tmpDir, { recursive: true }); const pluginVersion = getInstalledVersion(bundleDir, ".claude-plugin") ?? ""; - const configFile = join9(tmpDir, "config.json"); - writeFileSync3(configFile, JSON.stringify({ + const configFile = join12(tmpDir, "config.json"); + writeFileSync5(configFile, JSON.stringify({ apiUrl: config.apiUrl, token: config.token, orgId: config.orgId, @@ -1132,11 +1554,11 @@ function spawnHermesWikiWorker(opts) { hermesProvider: process.env.HIVEMIND_HERMES_PROVIDER ?? "openrouter", hermesModel: process.env.HIVEMIND_HERMES_MODEL ?? "anthropic/claude-haiku-4-5", wikiLog: WIKI_LOG, - hooksDir: join9(HOME, ".hermes", "hooks"), + hooksDir: join12(HOME, ".hermes", "hooks"), promptTemplate: WIKI_PROMPT_TEMPLATE })); wikiLog(`${reason}: spawning summary worker for ${sessionId}`); - const workerPath = join9(bundleDir, "wiki-worker.js"); + const workerPath = join12(bundleDir, "wiki-worker.js"); spawn2("nohup", ["node", workerPath, configFile], { detached: true, stdio: ["ignore", "ignore", "ignore"] @@ -1144,33 +1566,33 @@ function spawnHermesWikiWorker(opts) { wikiLog(`${reason}: spawned summary worker for ${sessionId}`); } function bundleDirFromImportMeta(importMetaUrl) { - return dirname2(fileURLToPath(importMetaUrl)); + return dirname4(fileURLToPath(importMetaUrl)); } // dist/src/skillify/spawn-skillify-worker.js import { spawn as spawn3 } from "node:child_process"; import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname as dirname3, join as join11 } from "node:path"; -import { writeFileSync as writeFileSync4, mkdirSync as mkdirSync5, appendFileSync as appendFileSync3, chmodSync } from "node:fs"; -import { homedir as homedir8, tmpdir as tmpdir3 } from "node:os"; +import { dirname as dirname5, join as join14 } from "node:path"; +import { writeFileSync as writeFileSync6, mkdirSync as mkdirSync8, appendFileSync as appendFileSync3, chmodSync } from "node:fs"; +import { homedir as homedir11, tmpdir as tmpdir3 } from "node:os"; // dist/src/skillify/gate-runner.js -import { existsSync as existsSync5 } from "node:fs"; +import { existsSync as existsSync7 } from "node:fs"; import { createRequire as createRequire2 } from "node:module"; -import { homedir as homedir7 } from "node:os"; -import { join as join10 } from "node:path"; +import { homedir as homedir10 } from "node:os"; +import { join as join13 } from "node:path"; var requireForCp = createRequire2(import.meta.url); var { execFileSync: runChildProcess } = requireForCp("node:child_process"); var inheritedEnv = process; function firstExistingPath(candidates) { for (const c of candidates) { - if (existsSync5(c)) + if (existsSync7(c)) return c; } return null; } function findAgentBin(agent) { - const home = homedir7(); + const home = homedir10(); switch (agent) { // /usr/bin/ is included in every candidate list — that's the // common Linux package-manager install path (apt, dnf, pacman). Old @@ -1179,54 +1601,54 @@ function findAgentBin(agent) { // #170 caught the gap. case "claude_code": return firstExistingPath([ - join10(home, ".claude", "local", "claude"), + join13(home, ".claude", "local", "claude"), "/usr/local/bin/claude", "/usr/bin/claude", - join10(home, ".npm-global", "bin", "claude"), - join10(home, ".local", "bin", "claude"), + join13(home, ".npm-global", "bin", "claude"), + join13(home, ".local", "bin", "claude"), "/opt/homebrew/bin/claude" - ]) ?? join10(home, ".claude", "local", "claude"); + ]) ?? join13(home, ".claude", "local", "claude"); case "codex": return firstExistingPath([ "/usr/local/bin/codex", "/usr/bin/codex", - join10(home, ".npm-global", "bin", "codex"), - join10(home, ".local", "bin", "codex"), + join13(home, ".npm-global", "bin", "codex"), + join13(home, ".local", "bin", "codex"), "/opt/homebrew/bin/codex" ]) ?? "/usr/local/bin/codex"; case "cursor": return firstExistingPath([ "/usr/local/bin/cursor-agent", "/usr/bin/cursor-agent", - join10(home, ".npm-global", "bin", "cursor-agent"), - join10(home, ".local", "bin", "cursor-agent"), + join13(home, ".npm-global", "bin", "cursor-agent"), + join13(home, ".local", "bin", "cursor-agent"), "/opt/homebrew/bin/cursor-agent" ]) ?? "/usr/local/bin/cursor-agent"; case "hermes": return firstExistingPath([ - join10(home, ".local", "bin", "hermes"), + join13(home, ".local", "bin", "hermes"), "/usr/local/bin/hermes", "/usr/bin/hermes", - join10(home, ".npm-global", "bin", "hermes"), + join13(home, ".npm-global", "bin", "hermes"), "/opt/homebrew/bin/hermes" - ]) ?? join10(home, ".local", "bin", "hermes"); + ]) ?? join13(home, ".local", "bin", "hermes"); case "pi": return firstExistingPath([ - join10(home, ".local", "bin", "pi"), + join13(home, ".local", "bin", "pi"), "/usr/local/bin/pi", "/usr/bin/pi", - join10(home, ".npm-global", "bin", "pi"), + join13(home, ".npm-global", "bin", "pi"), "/opt/homebrew/bin/pi" - ]) ?? join10(home, ".local", "bin", "pi"); + ]) ?? join13(home, ".local", "bin", "pi"); } } // dist/src/skillify/spawn-skillify-worker.js -var HOME2 = homedir8(); -var SKILLIFY_LOG = join11(HOME2, ".claude", "hooks", "skillify.log"); +var HOME2 = homedir11(); +var SKILLIFY_LOG = join14(HOME2, ".claude", "hooks", "skillify.log"); function skillifyLog(msg) { try { - mkdirSync5(dirname3(SKILLIFY_LOG), { recursive: true }); + mkdirSync8(dirname5(SKILLIFY_LOG), { recursive: true }); appendFileSync3(SKILLIFY_LOG, `[${utcTimestamp()}] ${msg} `); } catch { @@ -1234,11 +1656,11 @@ function skillifyLog(msg) { } function spawnSkillifyWorker(opts) { const { config, cwd, projectKey, project, bundleDir, agent, scopeConfig, currentSessionId, reason } = opts; - const tmpDir = join11(tmpdir3(), `deeplake-skillify-${projectKey}-${Date.now()}`); - mkdirSync5(tmpDir, { recursive: true, mode: 448 }); + const tmpDir = join14(tmpdir3(), `deeplake-skillify-${projectKey}-${Date.now()}`); + mkdirSync8(tmpDir, { recursive: true, mode: 448 }); const gateBin = findAgentBin(agent); - const configFile = join11(tmpDir, "config.json"); - writeFileSync4(configFile, JSON.stringify({ + const configFile = join14(tmpDir, "config.json"); + writeFileSync6(configFile, JSON.stringify({ apiUrl: config.apiUrl, token: config.token, orgId: config.orgId, @@ -1268,7 +1690,7 @@ function spawnSkillifyWorker(opts) { } catch { } skillifyLog(`${reason}: spawning skillify worker for project=${project} key=${projectKey}`); - const workerPath = join11(bundleDir, "skillify-worker.js"); + const workerPath = join14(bundleDir, "skillify-worker.js"); spawn3("nohup", ["node", workerPath, configFile], { detached: true, stdio: ["ignore", "ignore", "ignore"] @@ -1277,31 +1699,31 @@ function spawnSkillifyWorker(opts) { } // dist/src/skillify/state.js -import { readFileSync as readFileSync6, writeFileSync as writeFileSync5, writeSync as writeSync3, mkdirSync as mkdirSync6, renameSync as renameSync3, existsSync as existsSync7, unlinkSync as unlinkSync3, openSync as openSync3, closeSync as closeSync3 } from "node:fs"; +import { readFileSync as readFileSync8, writeFileSync as writeFileSync7, writeSync as writeSync3, mkdirSync as mkdirSync9, renameSync as renameSync6, existsSync as existsSync9, unlinkSync as unlinkSync4, openSync as openSync4, closeSync as closeSync4 } from "node:fs"; import { execSync as execSync2 } from "node:child_process"; -import { homedir as homedir10 } from "node:os"; +import { homedir as homedir13 } from "node:os"; import { createHash } from "node:crypto"; -import { join as join13, basename } from "node:path"; +import { join as join16, basename as basename2 } from "node:path"; // dist/src/skillify/legacy-migration.js -import { existsSync as existsSync6, renameSync as renameSync2 } from "node:fs"; -import { homedir as homedir9 } from "node:os"; -import { join as join12 } from "node:path"; +import { existsSync as existsSync8, renameSync as renameSync5 } from "node:fs"; +import { homedir as homedir12 } from "node:os"; +import { join as join15 } from "node:path"; var dlog2 = (msg) => log("skillify-migrate", msg); var attempted = false; function migrateLegacyStateDir() { if (attempted) return; attempted = true; - const root = join12(homedir9(), ".deeplake", "state"); - const legacy = join12(root, "skilify"); - const current = join12(root, "skillify"); - if (!existsSync6(legacy)) + const root = join15(homedir12(), ".deeplake", "state"); + const legacy = join15(root, "skilify"); + const current = join15(root, "skillify"); + if (!existsSync8(legacy)) return; - if (existsSync6(current)) + if (existsSync8(current)) return; try { - renameSync2(legacy, current); + renameSync5(legacy, current); dlog2(`migrated ${legacy} -> ${current}`); } catch (err) { const code = err.code; @@ -1315,17 +1737,17 @@ function migrateLegacyStateDir() { // dist/src/skillify/state.js var dlog3 = (msg) => log("skillify-state", msg); -var STATE_DIR2 = join13(homedir10(), ".deeplake", "state", "skillify"); +var STATE_DIR2 = join16(homedir13(), ".deeplake", "state", "skillify"); var YIELD_BUF2 = new Int32Array(new SharedArrayBuffer(4)); var TRIGGER_THRESHOLD = (() => { const n = Number(process.env.HIVEMIND_SKILLIFY_EVERY_N_TURNS ?? ""); return Number.isInteger(n) && n > 0 ? n : 20; })(); function statePath2(projectKey) { - return join13(STATE_DIR2, `${projectKey}.json`); + return join16(STATE_DIR2, `${projectKey}.json`); } -function lockPath2(projectKey) { - return join13(STATE_DIR2, `${projectKey}.lock`); +function lockPath3(projectKey) { + return join16(STATE_DIR2, `${projectKey}.lock`); } var DEFAULT_PORTS = { http: "80", @@ -1353,7 +1775,7 @@ function normalizeGitRemoteUrl(url) { return s.toLowerCase(); } function deriveProjectKey(cwd) { - const project = basename(cwd) || "unknown"; + const project = basename2(cwd) || "unknown"; let signature = null; try { const raw = execSync2("git config --get remote.origin.url", { @@ -1371,38 +1793,38 @@ function deriveProjectKey(cwd) { function readState2(projectKey) { migrateLegacyStateDir(); const p = statePath2(projectKey); - if (!existsSync7(p)) + if (!existsSync9(p)) return null; try { - return JSON.parse(readFileSync6(p, "utf-8")); + return JSON.parse(readFileSync8(p, "utf-8")); } catch { return null; } } function writeState2(projectKey, state) { migrateLegacyStateDir(); - mkdirSync6(STATE_DIR2, { recursive: true }); + mkdirSync9(STATE_DIR2, { recursive: true }); const p = statePath2(projectKey); const tmp = `${p}.${process.pid}.${Date.now()}.tmp`; - writeFileSync5(tmp, JSON.stringify(state, null, 2)); - renameSync3(tmp, p); + writeFileSync7(tmp, JSON.stringify(state, null, 2)); + renameSync6(tmp, p); } function withRmwLock2(projectKey, fn) { migrateLegacyStateDir(); - mkdirSync6(STATE_DIR2, { recursive: true }); - const rmw = lockPath2(projectKey) + ".rmw"; + mkdirSync9(STATE_DIR2, { recursive: true }); + const rmw = lockPath3(projectKey) + ".rmw"; const deadline = Date.now() + 2e3; let fd = null; while (fd === null) { try { - fd = openSync3(rmw, "wx"); + fd = openSync4(rmw, "wx"); } catch (e) { if (e.code !== "EEXIST") throw e; if (Date.now() > deadline) { dlog3(`rmw lock deadline exceeded for ${projectKey}, reclaiming stale lock`); try { - unlinkSync3(rmw); + unlinkSync4(rmw); } catch (unlinkErr) { dlog3(`stale rmw lock unlink failed for ${projectKey}: ${unlinkErr.message}`); } @@ -1414,9 +1836,9 @@ function withRmwLock2(projectKey, fn) { try { return fn(); } finally { - closeSync3(fd); + closeSync4(fd); try { - unlinkSync3(rmw); + unlinkSync4(rmw); } catch (unlinkErr) { dlog3(`rmw lock cleanup failed for ${projectKey}: ${unlinkErr.message}`); } @@ -1449,29 +1871,29 @@ function resetCounter(projectKey) { } function tryAcquireWorkerLock(projectKey, maxAgeMs = 10 * 60 * 1e3) { migrateLegacyStateDir(); - mkdirSync6(STATE_DIR2, { recursive: true }); - const p = lockPath2(projectKey); - if (existsSync7(p)) { + mkdirSync9(STATE_DIR2, { recursive: true }); + const p = lockPath3(projectKey); + if (existsSync9(p)) { try { - const ageMs = Date.now() - parseInt(readFileSync6(p, "utf-8"), 10); + const ageMs = Date.now() - parseInt(readFileSync8(p, "utf-8"), 10); if (Number.isFinite(ageMs) && ageMs < maxAgeMs) return false; } catch (readErr) { dlog3(`worker lock unreadable for ${projectKey}, treating as stale: ${readErr.message}`); } try { - unlinkSync3(p); + unlinkSync4(p); } catch (unlinkErr) { dlog3(`could not unlink stale worker lock for ${projectKey}: ${unlinkErr.message}`); return false; } } try { - const fd = openSync3(p, "wx"); + const fd = openSync4(p, "wx"); try { writeSync3(fd, String(Date.now())); } finally { - closeSync3(fd); + closeSync4(fd); } return true; } catch { @@ -1479,26 +1901,26 @@ function tryAcquireWorkerLock(projectKey, maxAgeMs = 10 * 60 * 1e3) { } } function releaseWorkerLock(projectKey) { - const p = lockPath2(projectKey); + const p = lockPath3(projectKey); try { - unlinkSync3(p); + unlinkSync4(p); } catch { } } // dist/src/skillify/scope-config.js -import { existsSync as existsSync8, mkdirSync as mkdirSync7, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "node:fs"; -import { homedir as homedir11 } from "node:os"; -import { join as join14 } from "node:path"; -var STATE_DIR3 = join14(homedir11(), ".deeplake", "state", "skillify"); -var CONFIG_PATH = join14(STATE_DIR3, "config.json"); +import { existsSync as existsSync10, mkdirSync as mkdirSync10, readFileSync as readFileSync9, writeFileSync as writeFileSync8 } from "node:fs"; +import { homedir as homedir14 } from "node:os"; +import { join as join17 } from "node:path"; +var STATE_DIR3 = join17(homedir14(), ".deeplake", "state", "skillify"); +var CONFIG_PATH = join17(STATE_DIR3, "config.json"); var DEFAULT = { scope: "me", team: [], install: "project" }; function loadScopeConfig() { migrateLegacyStateDir(); - if (!existsSync8(CONFIG_PATH)) + if (!existsSync10(CONFIG_PATH)) return DEFAULT; try { - const raw = JSON.parse(readFileSync7(CONFIG_PATH, "utf-8")); + const raw = JSON.parse(readFileSync9(CONFIG_PATH, "utf-8")); const scope = raw.scope === "team" ? "team" : raw.scope === "org" ? "team" : "me"; const team = Array.isArray(raw.team) ? raw.team.filter((s) => typeof s === "string") : []; const install = raw.install === "global" ? "global" : "project"; @@ -1549,12 +1971,18 @@ function tryStopCounterTrigger(opts) { } // dist/src/hooks/hermes/capture.js -var log4 = (msg) => log("hermes-capture", msg); +var log5 = (msg) => log("hermes-capture", msg); function resolveEmbedDaemonPath() { - return join15(dirname4(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); + return join18(dirname6(fileURLToPath3(import.meta.url)), "embeddings", "embed-daemon.js"); } -var __bundleDir = dirname4(fileURLToPath3(import.meta.url)); +var __bundleDir = dirname6(fileURLToPath3(import.meta.url)); var PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".claude-plugin") ?? ""; +if (!embeddingsDisabled()) { + try { + ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); + } catch { + } +} var CAPTURE = process.env.HIVEMIND_CAPTURE !== "false"; function pickString(...candidates) { for (const c of candidates) { @@ -1569,7 +1997,7 @@ async function main() { const input = await readStdin(); const config = loadConfig(); if (!config) { - log4("no config"); + log5("no config"); return; } const sessionId = input.session_id ?? `hermes-${Date.now()}`; @@ -1589,14 +2017,14 @@ async function main() { if (event === "pre_llm_call") { const prompt = pickString(extra.prompt, extra.user_message, extra.message?.content); if (!prompt) { - log4(`pre_llm_call: no prompt found in extra`); + log5(`pre_llm_call: no prompt found in extra`); return; } - log4(`user session=${sessionId}`); + log5(`user session=${sessionId}`); entry = { id: crypto.randomUUID(), ...meta, type: "user_message", content: prompt }; } else if (event === "post_tool_call" && typeof input.tool_name === "string") { const toolResponse = extra.tool_result ?? extra.tool_output ?? extra.result ?? extra.output; - log4(`tool=${input.tool_name} session=${sessionId}`); + log5(`tool=${input.tool_name} session=${sessionId}`); entry = { id: crypto.randomUUID(), ...meta, @@ -1608,18 +2036,18 @@ async function main() { } else if (event === "post_llm_call") { const text = pickString(extra.response, extra.assistant_message, extra.message?.content); if (!text) { - log4(`post_llm_call: no response found in extra`); + log5(`post_llm_call: no response found in extra`); return; } - log4(`assistant session=${sessionId}`); + log5(`assistant session=${sessionId}`); entry = { id: crypto.randomUUID(), ...meta, type: "assistant_message", content: text }; } else { - log4(`unknown/unhandled event: ${event}, skipping`); + log5(`unknown/unhandled event: ${event}, skipping`); return; } const sessionPath = buildSessionPath(config, sessionId); const line = JSON.stringify(entry); - log4(`writing to ${sessionPath}`); + log5(`writing to ${sessionPath}`); const projectName = cwd.split("/").pop() || "unknown"; const filename = sessionPath.split("/").pop() ?? ""; const jsonForSql = line.replace(/'/g, "''"); @@ -1630,14 +2058,14 @@ async function main() { await api.query(insertSql); } catch (e) { if (e.message?.includes("permission denied") || e.message?.includes("does not exist")) { - log4("table missing, creating and retrying"); + log5("table missing, creating and retrying"); await api.ensureSessionsTable(sessionsTable); await api.query(insertSql); } else { throw e; } } - log4("capture ok \u2192 cloud"); + log5("capture ok \u2192 cloud"); maybeTriggerPeriodicSummary(sessionId, cwd, config); if (event === "post_llm_call" && process.env.HIVEMIND_WIKI_WORKER !== "1" && process.env.HIVEMIND_SKILLIFY_WORKER !== "1") { tryStopCounterTrigger({ @@ -1658,7 +2086,7 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { if (!shouldTrigger(state, cfg)) return; if (!tryAcquireLock(sessionId)) { - log4(`periodic trigger suppressed (lock held) session=${sessionId}`); + log5(`periodic trigger suppressed (lock held) session=${sessionId}`); return; } wikiLog(`Periodic: threshold hit (total=${state.totalCount}, since=${state.totalCount - state.lastSummaryCount}, N=${cfg.everyNMessages}, hours=${cfg.everyHours})`); @@ -1671,17 +2099,17 @@ function maybeTriggerPeriodicSummary(sessionId, cwd, config) { reason: "Periodic" }); } catch (e) { - log4(`periodic spawn failed: ${e.message}`); + log5(`periodic spawn failed: ${e.message}`); try { releaseLock(sessionId); } catch { } } } catch (e) { - log4(`periodic trigger error: ${e.message}`); + log5(`periodic trigger error: ${e.message}`); } } main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); diff --git a/hermes/bundle/embeddings/embed-daemon.js b/hermes/bundle/embeddings/embed-daemon.js index 5f711a3c..a81ffbd1 100755 --- a/hermes/bundle/embeddings/embed-daemon.js +++ b/hermes/bundle/embeddings/embed-daemon.js @@ -4,7 +4,14 @@ import { createServer } from "node:net"; import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs"; +// dist/src/embeddings/nomic.js +import { createRequire } from "node:module"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + // dist/src/embeddings/protocol.js +var PROTOCOL_VERSION = 1; var DEFAULT_SOCKET_DIR = "/tmp"; var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; var DEFAULT_DTYPE = "q8"; @@ -20,6 +27,39 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { } // dist/src/embeddings/nomic.js +async function _importFromCanonicalSharedDeps(sharedDir = join(homedir(), ".hivemind", "embed-deps")) { + const base = pathToFileURL(`${sharedDir}/`).href; + const absMain = createRequire(base).resolve("@huggingface/transformers"); + const mod = await import(pathToFileURL(absMain).href); + return _normalizeTransformersModule(mod); +} +async function _importFromBareSpecifier() { + const mod = await import("@huggingface/transformers"); + return _normalizeTransformersModule(mod); +} +function _normalizeTransformersModule(mod) { + const m = mod; + if (m.default && typeof m.default === "object" && "pipeline" in m.default) { + return m.default; + } + return m; +} +async function defaultImportTransformers(canonical = _importFromCanonicalSharedDeps, bare = _importFromBareSpecifier) { + let canonicalErr; + try { + return await canonical(); + } catch (err) { + canonicalErr = err; + } + try { + return await bare(); + } catch (bareErr) { + const detail = bareErr instanceof Error ? bareErr.message : String(bareErr); + const canonicalDetail = canonicalErr instanceof Error ? canonicalErr.message : String(canonicalErr); + throw new Error(`@huggingface/transformers is not installed anywhere reachable. Run \`hivemind embeddings install\` to install it. (canonical: ${canonicalDetail}; bare: ${detail})`); + } +} +var _importTransformers = defaultImportTransformers; var NomicEmbedder = class { pipeline = null; loading = null; @@ -37,7 +77,7 @@ var NomicEmbedder = class { if (this.loading) return this.loading; this.loading = (async () => { - const mod = await import("@huggingface/transformers"); + const mod = await _importTransformers(); mod.env.allowLocalModels = false; mod.env.useFSCache = true; this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype }); @@ -94,9 +134,9 @@ var NomicEmbedder = class { // dist/src/utils/debug.js import { appendFileSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; -var LOG = join(homedir(), ".deeplake", "hook-debug.log"); +import { join as join2 } from "node:path"; +import { homedir as homedir2 } from "node:os"; +var LOG = join2(homedir2(), ".deeplake", "hook-debug.log"); function isDebug() { return process.env.HIVEMIND_DEBUG === "1"; } @@ -120,6 +160,7 @@ var EmbedDaemon = class { pidPath; idleTimeoutMs; idleTimer = null; + daemonPath; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; @@ -127,6 +168,7 @@ var EmbedDaemon = class { this.pidPath = pidPathFor(uid, dir); this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + this.daemonPath = opts.daemonPath ?? process.argv[1] ?? ""; } async start() { mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true }); @@ -218,6 +260,15 @@ var EmbedDaemon = class { } } async dispatch(req) { + if (req.op === "hello") { + const h = req; + return { + id: h.id, + daemonPath: this.daemonPath, + pid: process.pid, + protocolVersion: PROTOCOL_VERSION + }; + } if (req.op === "ping") { const p = req; return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; diff --git a/hermes/bundle/pre-tool-use.js b/hermes/bundle/pre-tool-use.js index 9508b396..394e7325 100755 --- a/hermes/bundle/pre-tool-use.js +++ b/hermes/bundle/pre-tool-use.js @@ -53,13 +53,13 @@ var init_index_marker_store = __esm({ // dist/src/utils/stdin.js function readStdin() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let data = ""; process.stdin.setEncoding("utf-8"); process.stdin.on("data", (chunk) => data += chunk); process.stdin.on("end", () => { try { - resolve(JSON.parse(data)); + resolve2(JSON.parse(data)); } catch (err) { reject(new Error(`Failed to parse hook input: ${err}`)); } @@ -175,7 +175,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); + return new Promise((resolve2) => setTimeout(resolve2, ms)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -205,7 +205,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve) => this.waiting.push(resolve)); + await new Promise((resolve2) => this.waiting.push(resolve2)); } release() { this.active--; @@ -1042,9 +1042,9 @@ function capOutputForClaude(output, options = {}) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join4 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync4, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -1057,13 +1057,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync } from "node:fs"; +import { join as join4, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join4(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join4(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath(); + mkdirSync2(join4(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join6 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join5 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join5(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync3(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync3(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join6(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join4(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join7(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -1072,13 +1293,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -1088,8 +1310,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -1099,17 +1346,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -1118,6 +1373,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync4(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync4(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log4(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync4(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -1141,7 +1529,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -1149,7 +1537,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -1160,16 +1548,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -1178,11 +1566,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -1194,14 +1582,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -1219,9 +1607,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync3(this.socketPath)) + if (!existsSync4(this.socketPath)) continue; try { return await this.connectOnce(); @@ -1231,7 +1619,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -1246,7 +1634,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -1263,53 +1651,22 @@ var EmbedClient = class { }); } }; -function sleep2(ms) { +function sleep3(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join5 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join5(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/hooks/grep-direct.js import { fileURLToPath } from "node:url"; -import { dirname, join as join6 } from "node:path"; +import { dirname as dirname2, join as join8 } from "node:path"; var SEMANTIC_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveDaemonPath() { - return join6(dirname(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join8(dirname2(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedEmbedClient = null; function getEmbedClient() { @@ -1662,9 +2019,9 @@ async function handleGrepDirect(api, table, sessionsTable, params) { } // dist/src/hooks/memory-path-utils.js -import { homedir as homedir5 } from "node:os"; -import { join as join7 } from "node:path"; -var MEMORY_PATH = join7(homedir5(), ".deeplake", "memory"); +import { homedir as homedir7 } from "node:os"; +import { join as join9 } from "node:path"; +var MEMORY_PATH = join9(homedir7(), ".deeplake", "memory"); var TILDE_PATH = "~/.deeplake/memory"; var HOME_VAR_PATH = "$HOME/.deeplake/memory"; function touchesMemory(p) { @@ -1675,7 +2032,7 @@ function rewritePaths(cmd) { } // dist/src/hooks/hermes/pre-tool-use.js -var log4 = (msg) => log("hermes-pre-tool-use", msg); +var log5 = (msg) => log("hermes-pre-tool-use", msg); async function main() { const input = await readStdin(); if (input.tool_name !== "terminal") @@ -1692,7 +2049,7 @@ async function main() { return; const config = loadConfig(); if (!config) { - log4("no config \u2014 falling through to Hermes"); + log5("no config \u2014 falling through to Hermes"); return; } const api = new DeeplakeApi(config.token, config.apiUrl, config.orgId, config.workspaceId, config.tableName); @@ -1700,7 +2057,7 @@ async function main() { const result = await handleGrepDirect(api, config.tableName, config.sessionsTableName, grepParams); if (result === null) return; - log4(`intercepted ${command.slice(0, 80)} \u2192 ${result.length} chars from SQL fast-path`); + log5(`intercepted ${command.slice(0, 80)} \u2192 ${result.length} chars from SQL fast-path`); const message = [ result, "", @@ -1709,10 +2066,10 @@ async function main() { process.stdout.write(JSON.stringify({ action: "block", message })); } catch (err) { const msg = err instanceof Error ? err.message : String(err); - log4(`fast-path failed, falling through: ${msg}`); + log5(`fast-path failed, falling through: ${msg}`); } } main().catch((e) => { - log4(`fatal: ${e.message}`); + log5(`fatal: ${e.message}`); process.exit(0); }); diff --git a/hermes/bundle/session-start.js b/hermes/bundle/session-start.js index 817b3e6e..28326c2b 100755 --- a/hermes/bundle/session-start.js +++ b/hermes/bundle/session-start.js @@ -1511,7 +1511,14 @@ Organization management \u2014 each argument is SEPARATE (do NOT quote subcomman - hivemind remove \u2014 remove member SKILLS (skillify) \u2014 mine + share reusable skills across the org: -${renderSkillifyCommands()}`; +${renderSkillifyCommands()} + +Embeddings (semantic memory search) \u2014 opt-in, persisted in ~/.deeplake/config.json: +- hivemind embeddings install \u2014 download deps (~600MB), symlink agents, set enabled:true +- hivemind embeddings enable \u2014 flip enabled:true (run install first if deps missing) +- hivemind embeddings disable \u2014 flip enabled:false + SIGTERM daemon (deps stay on disk) +- hivemind embeddings uninstall [--prune] \u2014 remove agent symlinks + disable; --prune wipes deps too +- hivemind embeddings status \u2014 show config + deps + per-agent link state`; async function createPlaceholder(api, table, sessionId, cwd, userName, orgName, workspaceId, pluginVersion) { const summaryPath = `/summaries/${userName}/${sessionId}.md`; const existing = await api.query(`SELECT path FROM "${table}" WHERE path = '${sqlStr(summaryPath)}' LIMIT 1`); diff --git a/hermes/bundle/shell/deeplake-shell.js b/hermes/bundle/shell/deeplake-shell.js index e3f8c1d6..402fb0a3 100755 --- a/hermes/bundle/shell/deeplake-shell.js +++ b/hermes/bundle/shell/deeplake-shell.js @@ -46081,14 +46081,14 @@ var require_turndown_cjs = __commonJS({ } else if (node.nodeType === 1) { replacement = replacementForNode.call(self2, node); } - return join11(output, replacement); + return join13(output, replacement); }, ""); } function postProcess(output) { var self2 = this; this.rules.forEach(function(rule) { if (typeof rule.append === "function") { - output = join11(output, rule.append(self2.options)); + output = join13(output, rule.append(self2.options)); } }); return output.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, ""); @@ -46100,7 +46100,7 @@ var require_turndown_cjs = __commonJS({ if (whitespace.leading || whitespace.trailing) content = content.trim(); return whitespace.leading + rule.replacement(content, node, this.options) + whitespace.trailing; } - function join11(output, replacement) { + function join13(output, replacement) { var s12 = trimTrailingNewlines(output); var s22 = trimLeadingNewlines(replacement); var nls = Math.max(output.length - s12.length, replacement.length - s22.length); @@ -66870,7 +66870,7 @@ function getQueryTimeoutMs() { return Number(process.env.HIVEMIND_QUERY_TIMEOUT_MS ?? 1e4); } function sleep(ms3) { - return new Promise((resolve5) => setTimeout(resolve5, ms3)); + return new Promise((resolve6) => setTimeout(resolve6, ms3)); } function isTimeoutError(error) { const name = error instanceof Error ? error.name.toLowerCase() : ""; @@ -66900,7 +66900,7 @@ var Semaphore = class { this.active++; return; } - await new Promise((resolve5) => this.waiting.push(resolve5)); + await new Promise((resolve6) => this.waiting.push(resolve6)); } release() { this.active--; @@ -67259,7 +67259,7 @@ var DeeplakeApi = class { import { basename as basename4, posix } from "node:path"; import { randomUUID as randomUUID2 } from "node:crypto"; import { fileURLToPath } from "node:url"; -import { dirname as dirname4, join as join9 } from "node:path"; +import { dirname as dirname5, join as join11 } from "node:path"; // dist/src/shell/grep-core.js var TOOL_INPUT_FIELDS = [ @@ -67689,9 +67689,9 @@ function refineGrepMatches(rows, params, forceMultiFilePrefix) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync, closeSync, writeSync, unlinkSync, existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join7 } from "node:path"; +import { openSync as openSync2, closeSync as closeSync2, writeSync, unlinkSync as unlinkSync2, existsSync as existsSync5, readFileSync as readFileSync5 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join10 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -67704,13 +67704,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, renameSync, mkdirSync as mkdirSync2, openSync, closeSync, unlinkSync, statSync as statSync2 } from "node:fs"; +import { join as join7, resolve as resolve4 } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep2 } from "node:timers/promises"; +var log3 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join7(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync3(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log3(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path2, home) { + const r10 = resolve4(path2); + const h18 = resolve4(home); + return r10.startsWith(h18 + "/") || r10 === h18; +} +function writeQueue(q17) { + const path2 = queuePath(); + const home = resolve4(homedir3()); + if (!_isQueuePathInsideHome(path2, home)) { + throw new Error(`notifications-queue write blocked: ${path2} is outside ${home}`); + } + mkdirSync2(join7(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path2}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q17, null, 2), { mode: 384 }); + renameSync(tmp, path2); +} +async function withQueueLock(fn4) { + const path2 = lockPath(); + mkdirSync2(join7(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path2, "wx", 384); + break; + } catch (e6) { + const code = e6.code; + if (code !== "EEXIST") + throw e6; + try { + const age = Date.now() - statSync2(path2).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path2); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep2(delay); + } + } + if (fd === null) { + log3(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn4(); + } + try { + return fn4(); + } finally { + try { + closeSync(fd); + } catch { + } + try { + unlinkSync(path2); + } catch { + } + } +} +function sameDedupKey(a15, b26) { + if (a15.id !== b26.id) + return false; + return JSON.stringify(a15.dedupKey) === JSON.stringify(b26.dedupKey); +} +async function enqueueNotification(n24) { + await withQueueLock(() => { + const q17 = readQueue(); + if (q17.queue.some((existing) => sameDedupKey(existing, n24))) { + return; + } + q17.queue.push(n24); + writeQueue(q17); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join9 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname as dirname4, join as join8 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join8(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path2 = _configPath(); + if (!existsSync4(path2)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync4(path2, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path2 = _configPath(); + const dir = dirname4(path2); + if (!existsSync4(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path2}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync2(tmp, path2); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg ?? {}, embeddings: { ...cfg?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join9(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join7(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log3 = (m26) => log("embed-client", m26); +var SHARED_DAEMON_PATH = join10(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log4 = (m26) => log("embed-client", m26); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -67719,13 +67940,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync4(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync5(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -67735,8 +67957,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v27 = await this.embedAttempt(text, kind); + if (v27 !== "recycled") + return v27; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -67746,17 +67993,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log3(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log4(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e6) { const err = e6 instanceof Error ? e6.message : String(e6); - log3(`embed failed: ${err}`); + log4(`embed failed: ${err}`); return null; } finally { try { @@ -67765,6 +68020,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync5(this.socketPath)) + return; + await new Promise((r10) => setTimeout(r10, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e6) { + log4(`hello probe failed (inconclusive, will retry next connect): ${e6 instanceof Error ? e6.message : String(e6)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log4(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync5(hello.daemonPath)) { + _recycledStuckDaemon = true; + log4(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e6) => { + log4(`enqueue embed-deps-missing failed: ${e6 instanceof Error ? e6.message : String(e6)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync5(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync5(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log4(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync2(this.socketPath); + } catch { + } + try { + unlinkSync2(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -67788,7 +68176,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { const sock = connect(this.socketPath); const to3 = setTimeout(() => { sock.destroy(); @@ -67796,7 +68184,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to3); - resolve5(sock); + resolve6(sock); }); sock.once("error", (e6) => { clearTimeout(to3); @@ -67807,16 +68195,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch (e6) { if (this.isPidFileStale()) { try { - unlinkSync(this.pidPath); + unlinkSync2(this.pidPath); } catch { } try { - fd = openSync(this.pidPath, "wx", 384); + fd = openSync2(this.pidPath, "wx", 384); writeSync(fd, String(process.pid)); } catch { return; @@ -67825,11 +68213,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync4(this.daemonEntry)) { - log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync5(this.daemonEntry)) { + log4(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync(fd); - unlinkSync(this.pidPath); + closeSync2(fd); + unlinkSync2(this.pidPath); } catch { } return; @@ -67841,14 +68229,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log3(`spawned daemon pid=${child.pid}`); + log4(`spawned daemon pid=${child.pid}`); } finally { - closeSync(fd); + closeSync2(fd); } } isPidFileStale() { try { - const raw = readFileSync3(this.pidPath, "utf-8").trim(); + const raw = readFileSync5(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -67866,9 +68254,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep2(delay); + await sleep3(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync4(this.socketPath)) + if (!existsSync5(this.socketPath)) continue; try { return await this.connectOnce(); @@ -67878,7 +68266,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve5, reject) => { + return new Promise((resolve6, reject) => { let buf = ""; const to3 = setTimeout(() => { sock.destroy(); @@ -67893,7 +68281,7 @@ var EmbedClient = class { const line = buf.slice(0, nl3); clearTimeout(to3); try { - resolve5(JSON.parse(line)); + resolve6(JSON.parse(line)); } catch (e6) { reject(e6); } @@ -67910,9 +68298,14 @@ var EmbedClient = class { }); } }; -function sleep2(ms3) { +function sleep3(ms3) { return new Promise((r10) => setTimeout(r10, ms3)); } +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); +} // dist/src/embeddings/sql.js function embeddingSqlLiteral(vec) { @@ -67927,42 +68320,6 @@ function embeddingSqlLiteral(vec) { return `ARRAY[${parts.join(",")}]::float4[]`; } -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join8 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join8(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; -} - // dist/src/hooks/virtual-table-query.js var INDEX_LIMIT_PER_SECTION = 50; function buildVirtualIndexContent(summaryRows, sessionRows = [], opts = {}) { @@ -68055,7 +68412,7 @@ function normalizeSessionMessage(path2, message) { return normalizeContent(path2, raw); } function resolveEmbedDaemonPath() { - return join9(dirname4(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + return join11(dirname5(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } function joinSessionMessages(path2, messages) { return messages.map((message) => normalizeSessionMessage(path2, message)).join("\n"); @@ -68621,7 +68978,7 @@ var DeeplakeFs = class _DeeplakeFs { // node_modules/yargs-parser/build/lib/index.js import { format } from "util"; -import { normalize, resolve as resolve4 } from "path"; +import { normalize, resolve as resolve5 } from "path"; // node_modules/yargs-parser/build/lib/string-utils.js function camelCase2(str) { @@ -69559,7 +69916,7 @@ function stripQuotes(val) { } // node_modules/yargs-parser/build/lib/index.js -import { readFileSync as readFileSync4 } from "fs"; +import { readFileSync as readFileSync6 } from "fs"; import { createRequire as createRequire2 } from "node:module"; var _a3; var _b; @@ -69581,12 +69938,12 @@ var parser = new YargsParser({ }, format, normalize, - resolve: resolve4, + resolve: resolve5, require: (path2) => { if (typeof require2 !== "undefined") { return require2(path2); } else if (path2.match(/\.json$/)) { - return JSON.parse(readFileSync4(path2, "utf8")); + return JSON.parse(readFileSync6(path2, "utf8")); } else { throw Error("only .json config files are supported in ESM"); } @@ -69606,11 +69963,11 @@ var lib_default = yargsParser; // dist/src/shell/grep-interceptor.js import { fileURLToPath as fileURLToPath2 } from "node:url"; -import { dirname as dirname5, join as join10 } from "node:path"; +import { dirname as dirname6, join as join12 } from "node:path"; var SEMANTIC_SEARCH_ENABLED = process.env.HIVEMIND_SEMANTIC_SEARCH !== "false" && !embeddingsDisabled(); var SEMANTIC_EMBED_TIMEOUT_MS = Number(process.env.HIVEMIND_SEMANTIC_EMBED_TIMEOUT_MS ?? "500"); function resolveGrepEmbedDaemonPath() { - return join10(dirname5(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); + return join12(dirname6(fileURLToPath2(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } var sharedGrepEmbedClient = null; function getGrepEmbedClient() { diff --git a/hermes/bundle/wiki-worker.js b/hermes/bundle/wiki-worker.js index 045a6348..90707b51 100755 --- a/hermes/bundle/wiki-worker.js +++ b/hermes/bundle/wiki-worker.js @@ -1,9 +1,9 @@ #!/usr/bin/env node // dist/src/hooks/hermes/wiki-worker.js -import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync3, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; +import { readFileSync as readFileSync5, writeFileSync as writeFileSync4, existsSync as existsSync4, appendFileSync as appendFileSync2, mkdirSync as mkdirSync4, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { dirname, join as join5 } from "node:path"; +import { dirname as dirname2, join as join7 } from "node:path"; import { fileURLToPath } from "node:url"; // dist/src/hooks/summary-state.js @@ -154,9 +154,9 @@ async function uploadSummary(query2, params) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join3 } from "node:path"; +import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync2, unlinkSync as unlinkSync3, existsSync as existsSync3, readFileSync as readFileSync4 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join6 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -169,13 +169,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync as renameSync2, mkdirSync as mkdirSync2, openSync as openSync2, closeSync as closeSync2, unlinkSync as unlinkSync2, statSync } from "node:fs"; +import { join as join3, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; +var log2 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join3(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath2() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync2(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log2(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join3(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync2(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath2(); + mkdirSync2(join3(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync2(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync2(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep(delay); + } + } + if (fd === null) { + log2(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync2(fd); + } catch { + } + try { + unlinkSync2(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join5 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join4 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join4(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync2(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync3(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync2(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync3(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg2 = readUserConfig(); + if (cfg2.embeddings && typeof cfg2.embeddings.enabled === "boolean") { + return cfg2.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg2 ?? {}, embeddings: { ...cfg2?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join5(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join3(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log2 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join6(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log3 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -184,13 +405,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync2(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -200,8 +422,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -211,17 +458,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log2(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log3(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log2(`embed failed: ${err}`); + log3(`embed failed: ${err}`); return null; } finally { try { @@ -230,6 +485,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync3(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log3(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log3(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync3(hello.daemonPath)) { + _recycledStuckDaemon = true; + log3(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log3(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync4(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync3(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log3(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync3(this.socketPath); + } catch { + } + try { + unlinkSync3(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -253,7 +641,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -261,7 +649,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -272,16 +660,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync2(this.pidPath); + unlinkSync3(this.pidPath); } catch { } try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch { return; @@ -290,11 +678,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync2(this.daemonEntry)) { - log2(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync2(fd); - unlinkSync2(this.pidPath); + closeSync3(fd); + unlinkSync3(this.pidPath); } catch { } return; @@ -306,14 +694,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log2(`spawned daemon pid=${child.pid}`); + log3(`spawned daemon pid=${child.pid}`); } finally { - closeSync2(fd); + closeSync3(fd); } } isPidFileStale() { try { - const raw = readFileSync2(this.pidPath, "utf-8").trim(); + const raw = readFileSync4(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -331,9 +719,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep(delay); + await sleep2(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync2(this.socketPath)) + if (!existsSync3(this.socketPath)) continue; try { return await this.connectOnce(); @@ -343,7 +731,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -358,7 +746,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -375,44 +763,13 @@ var EmbedClient = class { }); } }; -function sleep(ms) { +function sleep2(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join4 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join4(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/utils/client-header.js @@ -426,13 +783,13 @@ function deeplakeClientHeader() { // dist/src/hooks/hermes/wiki-worker.js var dlog2 = (msg) => log("hermes-wiki-worker", msg); -var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8")); +var cfg = JSON.parse(readFileSync5(process.argv[2], "utf-8")); var tmpDir = cfg.tmpDir; -var tmpJsonl = join5(tmpDir, "session.jsonl"); -var tmpSummary = join5(tmpDir, "summary.md"); +var tmpJsonl = join7(tmpDir, "session.jsonl"); +var tmpSummary = join7(tmpDir, "summary.md"); function wlog(msg) { try { - mkdirSync2(cfg.hooksDir, { recursive: true }); + mkdirSync4(cfg.hooksDir, { recursive: true }); appendFileSync2(cfg.wikiLog, `[${(/* @__PURE__ */ new Date()).toISOString().replace("T", " ").slice(0, 19)}] wiki-worker(${cfg.sessionId}): ${msg} `); } catch { @@ -464,7 +821,7 @@ async function query(sql, retries = 4) { const base = Math.min(3e4, 2e3 * Math.pow(2, attempt)); const delay = base + Math.floor(Math.random() * 1e3); wlog(`API ${r.status}, retrying in ${delay}ms (attempt ${attempt + 1}/${retries})`); - await new Promise((resolve) => setTimeout(resolve, delay)); + await new Promise((resolve2) => setTimeout(resolve2, delay)); continue; } throw new Error(`API ${r.status}: ${(await r.text()).slice(0, 200)}`); @@ -490,7 +847,7 @@ async function main() { const jsonlLines = rows.length; const pathRows = await query(`SELECT DISTINCT path FROM "${cfg.sessionsTable}" WHERE path LIKE '${esc2(`/sessions/%${cfg.sessionId}%`)}' LIMIT 1`); const jsonlServerPath = pathRows.length > 0 ? pathRows[0].path : `/sessions/unknown/${cfg.sessionId}.jsonl`; - writeFileSync2(tmpJsonl, jsonlContent); + writeFileSync4(tmpJsonl, jsonlContent); wlog(`found ${jsonlLines} events at ${jsonlServerPath}`); let prevOffset = 0; try { @@ -500,7 +857,7 @@ async function main() { const match = existing.match(/\*\*JSONL offset\*\*:\s*(\d+)/); if (match) prevOffset = parseInt(match[1], 10); - writeFileSync2(tmpSummary, existing); + writeFileSync4(tmpSummary, existing); wlog(`existing summary found, offset=${prevOffset}`); } } catch { @@ -526,15 +883,15 @@ async function main() { } catch (e) { wlog(`hermes -z failed: ${e.status ?? e.message}`); } - if (existsSync3(tmpSummary)) { - const text = readFileSync3(tmpSummary, "utf-8"); + if (existsSync4(tmpSummary)) { + const text = readFileSync5(tmpSummary, "utf-8"); if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; let embedding = null; if (!embeddingsDisabled()) { try { - const daemonEntry = join5(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + const daemonEntry = join7(dirname2(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); embedding = await new EmbedClient({ daemonEntry }).embed(text, "document"); } catch (e) { wlog(`summary embedding failed, writing NULL: ${e.message}`); diff --git a/pi/bundle/wiki-worker.js b/pi/bundle/wiki-worker.js index 0088ea56..92cfc315 100755 --- a/pi/bundle/wiki-worker.js +++ b/pi/bundle/wiki-worker.js @@ -1,9 +1,9 @@ #!/usr/bin/env node // dist/src/hooks/pi/wiki-worker.js -import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, existsSync as existsSync3, appendFileSync as appendFileSync2, mkdirSync as mkdirSync2, rmSync } from "node:fs"; +import { readFileSync as readFileSync5, writeFileSync as writeFileSync4, existsSync as existsSync4, appendFileSync as appendFileSync2, mkdirSync as mkdirSync4, rmSync } from "node:fs"; import { execFileSync } from "node:child_process"; -import { join as join5 } from "node:path"; +import { join as join7 } from "node:path"; // dist/src/hooks/summary-state.js import { readFileSync, writeFileSync, writeSync, mkdirSync, renameSync, existsSync, unlinkSync, openSync, closeSync } from "node:fs"; @@ -153,9 +153,9 @@ async function uploadSummary(query2, params) { // dist/src/embeddings/client.js import { connect } from "node:net"; import { spawn } from "node:child_process"; -import { openSync as openSync2, closeSync as closeSync2, writeSync as writeSync2, unlinkSync as unlinkSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "node:fs"; -import { homedir as homedir3 } from "node:os"; -import { join as join3 } from "node:path"; +import { openSync as openSync3, closeSync as closeSync3, writeSync as writeSync2, unlinkSync as unlinkSync3, existsSync as existsSync3, readFileSync as readFileSync4 } from "node:fs"; +import { homedir as homedir6 } from "node:os"; +import { join as join6 } from "node:path"; // dist/src/embeddings/protocol.js var DEFAULT_SOCKET_DIR = "/tmp"; @@ -168,13 +168,234 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) { return `${dir}/hivemind-embed-${uid}.pid`; } +// dist/src/notifications/queue.js +import { readFileSync as readFileSync2, writeFileSync as writeFileSync2, renameSync as renameSync2, mkdirSync as mkdirSync2, openSync as openSync2, closeSync as closeSync2, unlinkSync as unlinkSync2, statSync } from "node:fs"; +import { join as join3, resolve } from "node:path"; +import { homedir as homedir3 } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; +var log2 = (msg) => log("notifications-queue", msg); +var LOCK_RETRY_MAX = 50; +var LOCK_RETRY_BASE_MS = 5; +var LOCK_STALE_MS = 5e3; +function queuePath() { + return join3(homedir3(), ".deeplake", "notifications-queue.json"); +} +function lockPath2() { + return `${queuePath()}.lock`; +} +function readQueue() { + try { + const raw = readFileSync2(queuePath(), "utf-8"); + const parsed = JSON.parse(raw); + if (!parsed || !Array.isArray(parsed.queue)) { + log2(`queue malformed \u2192 treating as empty`); + return { queue: [] }; + } + return { queue: parsed.queue }; + } catch { + return { queue: [] }; + } +} +function _isQueuePathInsideHome(path, home) { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} +function writeQueue(q) { + const path = queuePath(); + const home = resolve(homedir3()); + if (!_isQueuePathInsideHome(path, home)) { + throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); + } + mkdirSync2(join3(home, ".deeplake"), { recursive: true, mode: 448 }); + const tmp = `${path}.${process.pid}.tmp`; + writeFileSync2(tmp, JSON.stringify(q, null, 2), { mode: 384 }); + renameSync2(tmp, path); +} +async function withQueueLock(fn) { + const path = lockPath2(); + mkdirSync2(join3(homedir3(), ".deeplake"), { recursive: true, mode: 448 }); + let fd = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync2(path, "wx", 384); + break; + } catch (e) { + const code = e.code; + if (code !== "EEXIST") + throw e; + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync2(path); + continue; + } + } catch { + } + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep(delay); + } + } + if (fd === null) { + log2(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts \u2014 proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { + closeSync2(fd); + } catch { + } + try { + unlinkSync2(path); + } catch { + } + } +} +function sameDedupKey(a, b) { + if (a.id !== b.id) + return false; + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} +async function enqueueNotification(n) { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some((existing) => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); +} + +// dist/src/embeddings/disable.js +import { createRequire } from "node:module"; +import { homedir as homedir5 } from "node:os"; +import { join as join5 } from "node:path"; +import { pathToFileURL } from "node:url"; + +// dist/src/user-config.js +import { existsSync as existsSync2, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync as renameSync3, writeFileSync as writeFileSync3 } from "node:fs"; +import { homedir as homedir4 } from "node:os"; +import { dirname, join as join4 } from "node:path"; +var _configPath = () => process.env.HIVEMIND_CONFIG_PATH ?? join4(homedir4(), ".deeplake", "config.json"); +var _cache = null; +var _migrated = false; +function readUserConfig() { + if (_cache !== null) + return _cache; + const path = _configPath(); + if (!existsSync2(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync3(path, "utf-8"); + const parsed = JSON.parse(raw); + _cache = isPlainObject(parsed) ? parsed : {}; + } catch { + _cache = {}; + } + return _cache; +} +function writeUserConfig(patch) { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync2(dir)) + mkdirSync3(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync3(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync3(tmp, path); + _cache = merged; + return merged; +} +function getEmbeddingsEnabled() { + const cfg2 = readUserConfig(); + if (cfg2.embeddings && typeof cfg2.embeddings.enabled === "boolean") { + return cfg2.embeddings.enabled; + } + if (_migrated) { + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + _cache = { ...cfg2 ?? {}, embeddings: { ...cfg2?.embeddings ?? {}, enabled } }; + } + return enabled; +} +function migrationValueFromEnv() { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === void 0) + return false; + if (raw === "false") + return false; + return true; +} +function isPlainObject(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function deepMerge(base, patch) { + const out = { ...base }; + for (const key of Object.keys(patch)) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + out[key] = { ...baseVal, ...patchVal }; + } else if (patchVal !== void 0) { + out[key] = patchVal; + } + } + return out; +} + +// dist/src/embeddings/disable.js +var cachedStatus = null; +function defaultResolveTransformers() { + const sharedDir = join5(homedir5(), ".hivemind", "embed-deps"); + try { + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + return; + } catch { + } + createRequire(import.meta.url).resolve("@huggingface/transformers"); +} +var _resolve = defaultResolveTransformers; +var _readEnabled = getEmbeddingsEnabled; +function detectStatus() { + if (!_readEnabled()) + return "user-disabled"; + try { + _resolve(); + return "enabled"; + } catch { + return "no-transformers"; + } +} +function embeddingsStatus() { + if (cachedStatus !== null) + return cachedStatus; + cachedStatus = detectStatus(); + return cachedStatus; +} +function embeddingsDisabled() { + return embeddingsStatus() !== "enabled"; +} + // dist/src/embeddings/client.js -var SHARED_DAEMON_PATH = join3(homedir3(), ".hivemind", "embed-deps", "embed-daemon.js"); -var log2 = (m) => log("embed-client", m); +var SHARED_DAEMON_PATH = join6(homedir6(), ".hivemind", "embed-deps", "embed-daemon.js"); +var log3 = (m) => log("embed-client", m); function getUid() { const uid = typeof process.getuid === "function" ? process.getuid() : void 0; return uid !== void 0 ? String(uid) : process.env.USER ?? "default"; } +var _signalledMissingDeps = false; +var _recycledStuckDaemon = false; var EmbedClient = class { socketPath; pidPath; @@ -183,13 +404,14 @@ var EmbedClient = class { autoSpawn; spawnWaitMs; nextId = 0; + helloVerified = false; constructor(opts = {}) { const uid = getUid(); const dir = opts.socketDir ?? "/tmp"; this.socketPath = socketPathFor(uid, dir); this.pidPath = pidPathFor(uid, dir); this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLIENT_TIMEOUT_MS; - this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync2(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); + this.daemonEntry = opts.daemonEntry ?? process.env.HIVEMIND_EMBED_DAEMON ?? (existsSync3(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : void 0); this.autoSpawn = opts.autoSpawn ?? true; this.spawnWaitMs = opts.spawnWaitMs ?? 5e3; } @@ -199,8 +421,33 @@ var EmbedClient = class { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text, kind = "document") { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") + return v; + if (!this.autoSpawn) + return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + async embedAttempt(text, kind) { let sock; try { sock = await this.connectOnce(); @@ -210,17 +457,25 @@ var EmbedClient = class { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + return "recycled"; + } const id = String(++this.nextId); const req = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log2(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log3(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; } catch (e) { const err = e instanceof Error ? e.message : String(e); - log2(`embed failed: ${err}`); + log3(`embed failed: ${err}`); return null; } finally { try { @@ -229,6 +484,139 @@ var EmbedClient = class { } } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + async waitForDaemonReady() { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync3(this.socketPath)) + return; + await new Promise((r) => setTimeout(r, 50)); + } + } + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + async verifyDaemonOnce(sock) { + if (this.helloVerified) + return false; + if (!this.daemonEntry) { + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req = { op: "hello", id }; + let resp; + try { + resp = await this.sendAndWait(sock, req); + } catch (e) { + log3(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp; + if (_recycledStuckDaemon) { + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log3(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync3(hello.daemonPath)) { + _recycledStuckDaemon = true; + log3(`daemon path no longer on disk \u2014 running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + this.helloVerified = true; + return false; + } + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + handleTransformersMissing(detail) { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) + return; + _signalledMissingDeps = true; + let status; + try { + status = embeddingsStatus(); + } catch { + status = "enabled"; + } + if (status === "user-disabled") + return; + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled \u2014 deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) } + }).catch((e) => { + log3(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + recycleDaemon(reportedPid) { + let pid = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync4(this.pidPath, "utf-8").trim(), 10); + } catch { + } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync3(this.socketPath)) { + try { + process.kill(pid, "SIGTERM"); + } catch { + } + } else if (pid !== null) { + log3(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { + unlinkSync3(this.socketPath); + } catch { + } + try { + unlinkSync3(this.pidPath); + } catch { + } + } /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -252,7 +640,7 @@ var EmbedClient = class { } } connectOnce() { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { const sock = connect(this.socketPath); const to = setTimeout(() => { sock.destroy(); @@ -260,7 +648,7 @@ var EmbedClient = class { }, this.timeoutMs); sock.once("connect", () => { clearTimeout(to); - resolve(sock); + resolve2(sock); }); sock.once("error", (e) => { clearTimeout(to); @@ -271,16 +659,16 @@ var EmbedClient = class { trySpawnDaemon() { let fd; try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch (e) { if (this.isPidFileStale()) { try { - unlinkSync2(this.pidPath); + unlinkSync3(this.pidPath); } catch { } try { - fd = openSync2(this.pidPath, "wx", 384); + fd = openSync3(this.pidPath, "wx", 384); writeSync2(fd, String(process.pid)); } catch { return; @@ -289,11 +677,11 @@ var EmbedClient = class { return; } } - if (!this.daemonEntry || !existsSync2(this.daemonEntry)) { - log2(`daemonEntry not configured or missing: ${this.daemonEntry}`); + if (!this.daemonEntry || !existsSync3(this.daemonEntry)) { + log3(`daemonEntry not configured or missing: ${this.daemonEntry}`); try { - closeSync2(fd); - unlinkSync2(this.pidPath); + closeSync3(fd); + unlinkSync3(this.pidPath); } catch { } return; @@ -305,14 +693,14 @@ var EmbedClient = class { env: process.env }); child.unref(); - log2(`spawned daemon pid=${child.pid}`); + log3(`spawned daemon pid=${child.pid}`); } finally { - closeSync2(fd); + closeSync3(fd); } } isPidFileStale() { try { - const raw = readFileSync2(this.pidPath, "utf-8").trim(); + const raw = readFileSync4(this.pidPath, "utf-8").trim(); const pid = Number(raw); if (!pid || Number.isNaN(pid)) return true; @@ -330,9 +718,9 @@ var EmbedClient = class { const deadline = Date.now() + this.spawnWaitMs; let delay = 30; while (Date.now() < deadline) { - await sleep(delay); + await sleep2(delay); delay = Math.min(delay * 1.5, 300); - if (!existsSync2(this.socketPath)) + if (!existsSync3(this.socketPath)) continue; try { return await this.connectOnce(); @@ -342,7 +730,7 @@ var EmbedClient = class { throw new Error("daemon did not become ready within spawnWaitMs"); } sendAndWait(sock, req) { - return new Promise((resolve, reject) => { + return new Promise((resolve2, reject) => { let buf = ""; const to = setTimeout(() => { sock.destroy(); @@ -357,7 +745,7 @@ var EmbedClient = class { const line = buf.slice(0, nl); clearTimeout(to); try { - resolve(JSON.parse(line)); + resolve2(JSON.parse(line)); } catch (e) { reject(e); } @@ -374,44 +762,13 @@ var EmbedClient = class { }); } }; -function sleep(ms) { +function sleep2(ms) { return new Promise((r) => setTimeout(r, ms)); } - -// dist/src/embeddings/disable.js -import { createRequire } from "node:module"; -import { homedir as homedir4 } from "node:os"; -import { join as join4 } from "node:path"; -import { pathToFileURL } from "node:url"; -var cachedStatus = null; -function defaultResolveTransformers() { - try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); - return; - } catch { - } - const sharedDir = join4(homedir4(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); -} -var _resolve = defaultResolveTransformers; -function detectStatus() { - if (process.env.HIVEMIND_EMBEDDINGS === "false") - return "env-disabled"; - try { - _resolve(); - return "enabled"; - } catch { - return "no-transformers"; - } -} -function embeddingsStatus() { - if (cachedStatus !== null) - return cachedStatus; - cachedStatus = detectStatus(); - return cachedStatus; -} -function embeddingsDisabled() { - return embeddingsStatus() !== "enabled"; +function isTransformersMissingError(err) { + if (/hivemind embeddings install/i.test(err)) + return true; + return /@huggingface\/transformers/i.test(err); } // dist/src/utils/client-header.js @@ -425,13 +782,13 @@ function deeplakeClientHeader() { // dist/src/hooks/pi/wiki-worker.js var dlog2 = (msg) => log("pi-wiki-worker", msg); -var cfg = JSON.parse(readFileSync3(process.argv[2], "utf-8")); +var cfg = JSON.parse(readFileSync5(process.argv[2], "utf-8")); var tmpDir = cfg.tmpDir; -var tmpJsonl = join5(tmpDir, "session.jsonl"); -var tmpSummary = join5(tmpDir, "summary.md"); +var tmpJsonl = join7(tmpDir, "session.jsonl"); +var tmpSummary = join7(tmpDir, "summary.md"); function wlog(msg) { try { - mkdirSync2(cfg.hooksDir, { recursive: true }); + mkdirSync4(cfg.hooksDir, { recursive: true }); appendFileSync2(cfg.wikiLog, `[${(/* @__PURE__ */ new Date()).toISOString().replace("T", " ").slice(0, 19)}] wiki-worker(${cfg.sessionId}): ${msg} `); } catch { @@ -463,7 +820,7 @@ async function query(sql, retries = 4) { const base = Math.min(3e4, 2e3 * Math.pow(2, attempt)); const delay = base + Math.floor(Math.random() * 1e3); wlog(`API ${r.status}, retrying in ${delay}ms (attempt ${attempt + 1}/${retries})`); - await new Promise((resolve) => setTimeout(resolve, delay)); + await new Promise((resolve2) => setTimeout(resolve2, delay)); continue; } throw new Error(`API ${r.status}: ${(await r.text()).slice(0, 200)}`); @@ -489,7 +846,7 @@ async function main() { const jsonlLines = rows.length; const pathRows = await query(`SELECT DISTINCT path FROM "${cfg.sessionsTable}" WHERE path LIKE '${esc2(`/sessions/%${cfg.sessionId}%`)}' LIMIT 1`); const jsonlServerPath = pathRows.length > 0 ? pathRows[0].path : `/sessions/unknown/${cfg.sessionId}.jsonl`; - writeFileSync2(tmpJsonl, jsonlContent); + writeFileSync4(tmpJsonl, jsonlContent); wlog(`found ${jsonlLines} events at ${jsonlServerPath}`); let prevOffset = 0; try { @@ -499,7 +856,7 @@ async function main() { const match = existing.match(/\*\*JSONL offset\*\*:\s*(\d+)/); if (match) prevOffset = parseInt(match[1], 10); - writeFileSync2(tmpSummary, existing); + writeFileSync4(tmpSummary, existing); wlog(`existing summary found, offset=${prevOffset}`); } } catch { @@ -523,8 +880,8 @@ async function main() { } catch (e) { wlog(`pi --print failed: ${e.status ?? e.message}`); } - if (existsSync3(tmpSummary)) { - const text = readFileSync3(tmpSummary, "utf-8"); + if (existsSync4(tmpSummary)) { + const text = readFileSync5(tmpSummary, "utf-8"); if (text.trim()) { const fname = `${cfg.sessionId}.md`; const vpath = `/summaries/${cfg.userName}/${fname}`; diff --git a/src/cli/embeddings.ts b/src/cli/embeddings.ts index d5f61782..9f1ed3f6 100644 --- a/src/cli/embeddings.ts +++ b/src/cli/embeddings.ts @@ -1,7 +1,10 @@ -import { copyFileSync, chmodSync, existsSync, lstatSync, readdirSync, readlinkSync, rmSync, statSync, unlinkSync } from "node:fs"; -import { execFileSync } from "node:child_process"; +import { copyFileSync, chmodSync, existsSync, lstatSync, readdirSync, readFileSync, readlinkSync, rmSync, statSync, unlinkSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { userInfo } from "node:os"; import { join } from "node:path"; import { HOME, ensureDir, log, pkgRoot, symlinkForce, warn, writeJson } from "./util.js"; +import { pidPathFor, socketPathFor } from "../embeddings/protocol.js"; +import { getEmbeddingsEnabled, setEmbeddingsEnabled } from "../user-config.js"; /** * Shared-deps location for the embedding daemon's runtime dependencies. @@ -131,34 +134,77 @@ function ensureSharedDeps(): void { } } +export function _linkAgentForTesting(install: AgentInstall): void { + return linkAgent(install); +} + function linkAgent(install: AgentInstall): void { const link = join(install.pluginDir, "node_modules"); + // Don't try to overwrite a real `node_modules` directory: `symlinkForce` + // calls `unlinkSync` first, which throws EISDIR on directories and would + // abort `hivemind embeddings install` partway through, leaving some + // agents linked and others not. Defer to whatever the user/marketplace + // installed there — the same state `status` already surfaces as + // `owns-own-node-modules`. (Symlinks at this path, including stale ones + // pointing at a defunct shared-deps target, still go through + // `symlinkForce` so we replace them with the canonical target.) + const state = linkStateFor(install); + if (state.kind === "owns-own-node-modules") { + warn(` Embeddings ${install.id.padEnd(20)} owns its own node_modules — skipping symlink (status: owns-own-node-modules)`); + return; + } symlinkForce(SHARED_NODE_MODULES, link); log(` Embeddings linked ${install.id.padEnd(20)} -> shared deps`); } /** - * Install shared embedding deps if missing, then symlink every detected - * hivemind plugin install to them. Idempotent: re-runs after installing - * a new agent just add the missing symlink and skip the npm install. + * Heavy "install" path: install shared embedding deps if missing, then + * symlink every detected hivemind plugin install to them, then flip the + * user-config flag to enabled. Idempotent: re-runs after installing a new + * agent just add the missing symlink and skip the npm install. + * + * Running `install` is the canonical way to opt in to embeddings. After + * this finishes, `embeddings.enabled` in `~/.deeplake/config.json` is + * `true`, regardless of any prior value (running install overrides a + * prior `disable`). */ -export function enableEmbeddings(): void { +export function installEmbeddings(): void { ensureSharedDeps(); const installs = findHivemindInstalls(); if (installs.length === 0) { warn(" Embeddings no hivemind installs detected — run `hivemind install` first"); warn(" (the shared deps are in place; subsequent agent installs will pick them up if you re-run `hivemind embeddings install`)"); - return; + } else { + for (const inst of installs) linkAgent(inst); } - for (const inst of installs) linkAgent(inst); - log(` Embeddings enabled. Restart your agents to pick up.`); + setEmbeddingsEnabled(true); + log(` Embeddings enabled in ~/.deeplake/config.json`); + log(` Embeddings ready. Restart your agents to pick up.`); } /** - * Remove the symlink each agent's plugin dir has into the shared deps. - * Optionally prune the shared dir itself if `prune` is set. + * Lightweight opt-in: flip the config flag to enabled. Does NOT install + * deps — use `install` for that. Warns if shared deps are missing so the + * user knows to run install before sessions will actually generate + * embeddings. */ -export function disableEmbeddings(opts?: { prune?: boolean }): void { +export function enableEmbeddings(): void { + setEmbeddingsEnabled(true); + log(` Embeddings enabled in ~/.deeplake/config.json`); + if (!isSharedDepsInstalled()) { + warn(` Embeddings shared deps not installed yet — run \`hivemind embeddings install\` to download them`); + } else { + log(` Embeddings shared deps present — sessions will start producing embeddings on next restart`); + } +} + +/** + * Heavy "uninstall" path: remove every agent's node_modules symlink into + * the shared deps, optionally prune the shared dir itself, flip the + * config flag off, and kill any running daemon so the change takes + * effect immediately. Counterpart to `install`. + */ +export function uninstallEmbeddings(opts?: { prune?: boolean }): void { const installs = findHivemindInstalls(); for (const inst of installs) { const link = join(inst.pluginDir, "node_modules"); @@ -171,12 +217,101 @@ export function disableEmbeddings(opts?: { prune?: boolean }): void { rmSync(SHARED_DIR, { recursive: true, force: true }); log(` Embeddings pruned ${SHARED_DIR}`); } + setEmbeddingsEnabled(false); + killEmbedDaemon(); + log(` Embeddings disabled in ~/.deeplake/config.json`); +} + +/** + * Lightweight opt-out: flip the config flag off and kill the running + * daemon (if any) so the change takes effect immediately. Does NOT + * remove the shared deps or per-agent symlinks — use `uninstall` to + * reclaim disk space too. + */ +export function disableEmbeddings(): void { + setEmbeddingsEnabled(false); + killEmbedDaemon(); + log(` Embeddings disabled in ~/.deeplake/config.json`); + log(` Embeddings daemon terminated; shared deps preserved (run \`hivemind embeddings uninstall\` to remove)`); +} + +/** + * Best-effort SIGTERM on the running embed daemon for this UID, then + * unlink its socket and pidfile. Tolerant of any combination of missing + * pidfile, missing socket, dead PID, or insufficient permissions. + * + * Identity check: before sending SIGTERM, we probe the socket the PID + * is claimed to own. If the socket doesn't exist OR a short connect + * fails, the daemon is already dead and the PID in the file is stale — + * the OS may have recycled it to a totally unrelated process, so + * SIGTERMing it would be a `disable` killing the user's text editor. + * In that case we skip the kill and only clean up the file artifacts. + */ +export function killEmbedDaemon(socketDir?: string): void { + const uid = typeof process.getuid === "function" ? process.getuid() : userInfo().uid; + // socketDir override is for tests only — production always lives in /tmp + // (the protocol default). Tests pass mkdtemp dirs so they don't collide + // with any real daemon for the same uid on the same machine. + const pidPath = pidPathFor(String(uid), socketDir); + const sockPath = socketPathFor(String(uid), socketDir); + let pid: number | null = null; + try { + pid = Number.parseInt(readFileSync(pidPath, "utf-8").trim(), 10); + } catch { /* no pidfile */ } + + if (pid !== null && Number.isFinite(pid) && _isDaemonAliveOnSocket(sockPath)) { + try { process.kill(pid, "SIGTERM"); } catch { /* already dead */ } + } else if (pid !== null) { + // Pidfile present but socket isn't live — daemon crashed; the PID + // value may now belong to an unrelated process. Skip the kill. + log(` Embeddings pidfile present but socket dead — skipping SIGTERM on possibly-stale pid ${pid}`); + } + + try { unlinkSync(sockPath); } catch { /* not present */ } + try { unlinkSync(pidPath); } catch { /* not present */ } +} + +/** + * Probe whether the embed daemon socket is alive: try to connect with a + * short timeout. Doesn't send any payload — a successful TCP/UDS handshake + * is proof that some process is listening on this UDS path, and since + * UDS paths are filesystem-rooted (not PID-rooted), the listener is + * almost certainly the daemon whose pidfile sits next to it. Anything + * else (file missing, ECONNREFUSED, ENOENT, timeout) means the daemon + * isn't actually there. + */ +export function _isDaemonAliveOnSocket(sockPath: string, timeoutMs: number = 200): boolean { + if (!existsSync(sockPath)) return false; + try { + const child = spawnSync("node", [ + "-e", + `const n=require("node:net");` + + `const s=n.connect(${JSON.stringify(sockPath)});` + + `s.once("connect",()=>{s.end();process.exit(0)});` + + `s.once("error",()=>process.exit(2));` + + `setTimeout(()=>process.exit(3),${timeoutMs});`, + ], { timeout: timeoutMs + 1000, stdio: "ignore" }); + return child.status === 0; + } catch { + return false; + } } export function statusEmbeddings(): void { + const enabled = getEmbeddingsEnabled(); + log(`Config: ~/.deeplake/config.json embeddings.enabled = ${enabled}`); log(`Shared deps: ${SHARED_DIR}`); log(`Installed: ${isSharedDepsInstalled() ? "yes" : "no"}`); log(`Daemon: ${existsSync(SHARED_DAEMON_PATH) ? SHARED_DAEMON_PATH : "(not present)"}`); + if (!enabled) { + log(""); + log(`Embeddings are DISABLED in user config. Run \`hivemind embeddings enable\` to opt in,`); + log(`or \`hivemind embeddings install\` if the shared deps are not yet downloaded.`); + } else if (!isSharedDepsInstalled()) { + log(""); + warn(`Embeddings are enabled in config but shared deps are missing.`); + warn(`Run \`hivemind embeddings install\` to download @huggingface/transformers.`); + } log(""); log(`Agent installs:`); const installs = findHivemindInstalls(); @@ -189,7 +324,7 @@ export function statusEmbeddings(): void { let label: string; switch (state.kind) { case "linked-to-shared": label = "✓ linked → shared"; break; - case "no-node-modules": label = "✗ not linked (embeddings disabled)"; break; + case "no-node-modules": label = "✗ not linked"; break; case "owns-own-node-modules": label = "△ has its own node_modules (not shared)"; break; case "linked-elsewhere": label = `△ linked → ${state.target}`; break; } diff --git a/src/cli/index.ts b/src/cli/index.ts index 44bcd336..32b285d2 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -4,7 +4,13 @@ import { installOpenclaw, uninstallOpenclaw } from "./install-openclaw.js"; import { installCursor, uninstallCursor } from "./install-cursor.js"; import { installHermes, uninstallHermes } from "./install-hermes.js"; import { installPi, uninstallPi } from "./install-pi.js"; -import { enableEmbeddings, disableEmbeddings, statusEmbeddings } from "./embeddings.js"; +import { + disableEmbeddings, + enableEmbeddings, + installEmbeddings, + statusEmbeddings, + uninstallEmbeddings, +} from "./embeddings.js"; import { ensureLoggedIn, isLoggedIn, maybeShowOrgChoice } from "./auth.js"; import { runAuthCommand } from "../commands/auth-login.js"; import { runSkillifyCommand } from "../commands/skillify.js"; @@ -54,12 +60,28 @@ Usage: Semantic search (embeddings): hivemind embeddings install Download @huggingface/transformers - once (~600 MB) into a shared dir - and symlink every detected agent - plugin to it. Idempotent. - hivemind embeddings uninstall [--prune] Remove the per-agent symlinks. - --prune also deletes the shared dir. - hivemind embeddings status Show shared-deps + per-agent state. + once (~600 MB) into a shared dir, + symlink every detected agent + plugin to it, and set + embeddings.enabled = true in + ~/.deeplake/config.json. Idempotent. + hivemind embeddings enable Light opt-in: flip + embeddings.enabled = true in + ~/.deeplake/config.json. Use this + after \`disable\` to turn back on + without re-running install. + hivemind embeddings disable Light opt-out: flip + embeddings.enabled = false and + SIGTERM the running daemon. Shared + deps stay on disk. + hivemind embeddings uninstall [--prune] Full opt-out: remove the per-agent + symlinks, flip + embeddings.enabled = false, and + SIGTERM the daemon. --prune also + deletes the shared dir to reclaim + ~600 MB. + hivemind embeddings status Show config + shared-deps + per- + agent state. Add --with-embeddings to "hivemind install" (or "hivemind install") to run "embeddings install" automatically after installing the agent(s). @@ -135,7 +157,7 @@ async function runInstallAll(args: string[]): Promise { if (withEmbeddings) { log(""); - enableEmbeddings(); + installEmbeddings(); } await maybeShowOrgChoice(); @@ -215,13 +237,15 @@ async function main(): Promise { if (cmd === "embeddings") { const sub = args[1]; - if (sub === "install" || sub === "enable") { enableEmbeddings(); return; } - if (sub === "uninstall" || sub === "disable") { - disableEmbeddings({ prune: hasFlag(args.slice(2), "--prune") }); + if (sub === "install") { installEmbeddings(); return; } + if (sub === "enable") { enableEmbeddings(); return; } + if (sub === "disable") { disableEmbeddings(); return; } + if (sub === "uninstall") { + uninstallEmbeddings({ prune: hasFlag(args.slice(2), "--prune") }); return; } if (sub === "status") { statusEmbeddings(); return; } - warn("Usage: hivemind embeddings install | uninstall [--prune] | status"); + warn("Usage: hivemind embeddings install | enable | disable | uninstall [--prune] | status"); process.exit(1); } @@ -238,7 +262,7 @@ async function main(): Promise { runSingleInstall(cmd as PlatformId); if (hasFlag(args.slice(2), "--with-embeddings")) { log(""); - enableEmbeddings(); + installEmbeddings(); } } else if (sub === "uninstall") runSingleUninstall(cmd as PlatformId); diff --git a/src/embeddings/client.ts b/src/embeddings/client.ts index b9a0c6b3..ffb8751e 100644 --- a/src/embeddings/client.ts +++ b/src/embeddings/client.ts @@ -14,8 +14,12 @@ import { type DaemonResponse, type EmbedKind, type EmbedRequest, + type HelloRequest, + type HelloResponse, } from "./protocol.js"; import { log as _log } from "../utils/debug.js"; +import { enqueueNotification } from "../notifications/queue.js"; +import { embeddingsStatus } from "./disable.js"; // Canonical location for the standalone daemon bundle, deposited by // `hivemind embeddings install`. Used as the auto-spawn fallback when @@ -39,6 +43,15 @@ export interface ClientOptions { spawnWaitMs?: number; } +// Process-local flags so an embed-deps-missing notification fires at most +// once per process AND the stuck-daemon kill+recycle path runs at most once +// per process (it's idempotent but the SIGTERM is wasted on every retry). +let _signalledMissingDeps = false; +let _recycledStuckDaemon = false; +// Hello handshake runs at most once per (process, EmbedClient instance). +// Stored on the instance, not module-global, because tests construct +// many clients and each one needs its own verification cycle. + export class EmbedClient { private socketPath: string; private pidPath: string; @@ -47,6 +60,7 @@ export class EmbedClient { private autoSpawn: boolean; private spawnWaitMs: number; private nextId = 0; + private helloVerified = false; constructor(opts: ClientOptions = {}) { const uid = getUid(); @@ -72,8 +86,42 @@ export class EmbedClient { * * Fire-and-forget spawn on miss: if the daemon isn't up, this call returns * null AND kicks off a background spawn. The next call finds a ready daemon. + * + * Stuck-daemon recycle: if the daemon returns a transformers-missing + * error (typical after a marketplace upgrade left an older daemon process + * alive but with no node_modules accessible from its bundle path), we + * SIGTERM it and clear its sock/pid so the very next call spawns a fresh + * daemon from the current bundle. Without this, the stuck daemon would + * keep poisoning every session until its 10-minute idle-out fires. */ async embed(text: string, kind: EmbedKind = "document"): Promise { + const v = await this.embedAttempt(text, kind); + if (v !== "recycled") return v; + // The probe killed the old daemon mid-call. With autoSpawn enabled, + // spawn a fresh one and retry once. Without autoSpawn (tests, pi's + // fallback that relies on the canonical shared daemon already being + // up) we have no way to bring the daemon back, so just return null — + // the caller treats it the same as any other transient miss. + // + // The retry path skips verifyDaemonOnce internally because + // `helloVerified` is still false (we never reached the compatible + // branch) but `_recycledStuckDaemon` is now true, so the second probe + // early-returns instead of triggering another kill. + if (!this.autoSpawn) return null; + this.trySpawnDaemon(); + await this.waitForDaemonReady(); + const retry = await this.embedAttempt(text, kind); + return retry === "recycled" ? null : retry; + } + + /** + * One round-trip: connect → verify → embed. Returns: + * - number[] : embedding vector (happy path) + * - null : timeout / daemon error / transformers-missing + * - "recycled": verifyDaemonOnce killed the daemon mid-call; + * caller should respawn and retry once. + */ + private async embedAttempt(text: string, kind: EmbedKind): Promise { let sock: Socket; try { sock = await this.connectOnce(); @@ -82,11 +130,22 @@ export class EmbedClient { return null; } try { + const recycled = await this.verifyDaemonOnce(sock); + if (recycled) { + // The verify step killed the daemon + cleared the sock. Don't + // send the embed on this now-dead connection — signal "recycled" + // to the caller so it can spawn fresh and retry. + return "recycled"; + } const id = String(++this.nextId); const req: EmbedRequest = { op: "embed", id, kind, text }; const resp = await this.sendAndWait(sock, req); if (resp.error || !("embedding" in resp) || !resp.embedding) { - log(`embed err: ${resp.error ?? "no embedding"}`); + const err = resp.error ?? "no embedding"; + log(`embed err: ${err}`); + if (isTransformersMissingError(err)) { + this.handleTransformersMissing(err); + } return null; } return resp.embedding; @@ -99,6 +158,160 @@ export class EmbedClient { } } + /** + * Poll for the sock file to come back after `trySpawnDaemon` — used by + * the recycle retry path. Best-effort: caps at `spawnWaitMs` and + * returns regardless so the retry attempt can run. + */ + private async waitForDaemonReady(): Promise { + const deadline = Date.now() + this.spawnWaitMs; + while (Date.now() < deadline) { + if (existsSync(this.socketPath)) return; + await new Promise((r) => setTimeout(r, 50)); + } + } + + /** + * Send a `hello` on first successful connect per EmbedClient instance. + * If the daemon answers with a path that doesn't match our configured + * daemonEntry — typical after a marketplace upgrade replaced the bundle + * — SIGTERM the daemon + clear sock/pid so the next call spawns from the + * current bundle. + * + * `helloVerified` is set ONLY after we've seen a compatible response, + * so a transient probe failure or a recycle-triggering mismatch leaves + * the flag false; the next reconnect re-runs verification against + * whatever daemon is then live (typically the fresh spawn). + */ + private async verifyDaemonOnce(sock: Socket): Promise { + if (this.helloVerified) return false; + if (!this.daemonEntry) { + // No expectation to verify against (e.g. canonical-shared-deps mode, + // or pi's fallback). Mark verified so we don't re-enter on every + // connect for the same EmbedClient. + this.helloVerified = true; + return false; + } + const id = String(++this.nextId); + const req: HelloRequest = { op: "hello", id }; + let resp: DaemonResponse; + try { + resp = await this.sendAndWait(sock, req); + } catch (e: unknown) { + // Daemon doesn't understand `hello` (older protocol) or connection + // hiccup. Don't kill on a transient — let embed proceed and surface + // any real problem there. Leave `helloVerified` false so the next + // reconnect attempts verification again (the current probe was + // inconclusive, not "definitely compatible"). + log(`hello probe failed (inconclusive, will retry next connect): ${e instanceof Error ? e.message : String(e)}`); + return false; + } + const hello = resp as HelloResponse; + // Recycle triggers — in order of severity: + // + // 1. No `daemonPath` in the response: the daemon predates this protocol + // (i.e. `{ error: "unknown op" }` from an older bundle). It's an + // incompatible older binary that needs to be replaced. + // + // 2. `daemonPath` is set but the file no longer exists on disk: the + // bundle that spawned it was GC'd (typical after Claude Code prunes + // old marketplace versions). The daemon is orphaned and a fresh + // spawn would use the current bundle. + // + // Note we DO NOT recycle on plain path mismatch when both paths exist + // — that's the multi-agent case (e.g. claude-code spawned the daemon, + // codex now wants to use it). All bundled daemons at the same + // protocolVersion are functionally identical, so any of them serves + // every agent fine. Recycling here would cause endless thrash. + if (_recycledStuckDaemon) { + // Another EmbedClient already triggered a recycle in this process; + // skip the check (but don't mark verified — the next reconnect + // against the freshly spawned daemon will run hello again, which + // is a single round-trip and harmless). + return false; + } + if (!hello.daemonPath) { + _recycledStuckDaemon = true; + log(`daemon does not implement hello (older protocol); recycling`); + this.recycleDaemon(hello.pid); + return true; + } + if (hello.daemonPath !== this.daemonEntry && !existsSync(hello.daemonPath)) { + _recycledStuckDaemon = true; + log(`daemon path no longer on disk — running=${hello.daemonPath} (gone) expected=${this.daemonEntry}; recycling`); + this.recycleDaemon(hello.pid); + return true; + } + // Compatible — same path, or different path but functionally identical + // (multi-agent sharing of one warm daemon). Only NOW do we mark the + // EmbedClient as verified. + this.helloVerified = true; + return false; + } + + /** + * On a transformers-missing error from the daemon, SIGTERM the stuck + * daemon (the bundle daemon that can't find its deps) and clear + * sock/pid so the next call spawns fresh. Also enqueue a one-time + * notification telling the user to run `hivemind embeddings install` + * — but only when the user has opted in. Suppressed when + * embeddingsStatus() === "user-disabled" so we don't nag users who + * explicitly chose to turn embeddings off. + */ + private handleTransformersMissing(detail: string): void { + if (!_recycledStuckDaemon) { + _recycledStuckDaemon = true; + this.recycleDaemon(null); + } + if (_signalledMissingDeps) return; + _signalledMissingDeps = true; + let status: string; + try { status = embeddingsStatus(); } catch { status = "enabled"; } + if (status === "user-disabled") return; // user said no, don't nag + // Fire-and-forget. `enqueueNotification` is now async (it may yield + // the event loop on lock contention); we don't await it so we never + // block the capture hot path on a notification write. Errors land in + // the .catch instead of being swallowed silently by the outer caller. + enqueueNotification({ + id: "embed-deps-missing", + severity: "warn", + title: "Hivemind embeddings disabled — deps missing", + body: `Semantic memory search is off because @huggingface/transformers is not installed where the daemon can find it. Run \`hivemind embeddings install\` to enable.`, + dedupKey: { reason: "transformers-missing", detail: detail.slice(0, 200) }, + }).catch((e: unknown) => { + log(`enqueue embed-deps-missing failed: ${e instanceof Error ? e.message : String(e)}`); + }); + } + + /** + * Best-effort SIGTERM + sock/pid cleanup. Tolerant of every missing-file + * combination and dead-PID cases. + * + * Identity check: gate the SIGTERM on the daemon's socket file still + * existing. We know the daemon was alive moments ago (we either just + * got a hello response or the caller saw a transformers-missing error + * the daemon emitted), but if the socket file is gone by the time we + * try to kill, the daemon process is also gone and the PID we + * captured may already have been recycled by the OS to an unrelated + * user process. Mirrors the gate added to `killEmbedDaemon` in the + * CLI — same failure mode, rarer trigger. + */ + private recycleDaemon(reportedPid: number | null): void { + let pid: number | null = reportedPid; + if (pid === null) { + try { + pid = Number.parseInt(readFileSync(this.pidPath, "utf-8").trim(), 10); + } catch { /* no pidfile */ } + } + if (Number.isFinite(pid) && pid !== null && pid > 0 && existsSync(this.socketPath)) { + try { process.kill(pid, "SIGTERM"); } catch { /* already dead */ } + } else if (pid !== null) { + log(`recycle: socket gone, skipping SIGTERM on possibly-stale pid ${pid}`); + } + try { unlinkSync(this.socketPath); } catch { /* not present */ } + try { unlinkSync(this.pidPath); } catch { /* not present */ } + } + /** * Wait up to spawnWaitMs for the daemon to accept connections, spawning if * necessary. Meant for SessionStart / long-running batches — not the hot path. @@ -221,7 +434,7 @@ export class EmbedClient { throw new Error("daemon did not become ready within spawnWaitMs"); } - private sendAndWait(sock: Socket, req: EmbedRequest): Promise { + private sendAndWait(sock: Socket, req: EmbedRequest | HelloRequest): Promise { return new Promise((resolve, reject) => { let buf = ""; const to = setTimeout(() => { @@ -256,6 +469,34 @@ function sleep(ms: number): Promise { return new Promise(r => setTimeout(r, ms)); } +/** + * Detect daemon-side errors that indicate `@huggingface/transformers` is + * not resolvable from the daemon's bundle location. Matches: + * - The actionable wrapper we throw from `defaultImportTransformers` + * (contains the literal `hivemind embeddings install`), or + * - A Node module-resolution error that specifically names + * `@huggingface/transformers`. + * + * Bare `MODULE_NOT_FOUND` (without the package name) used to fall here + * too, but that overshoots — it also caught onnxruntime-node / sharp + * / etc. missing-dep failures, recycled the daemon for problems + * `hivemind embeddings install` can't fix, and surfaced the wrong user + * guidance. Any daemon-side import failure of an unrelated dependency + * is a packaging bug we should hear about separately, not a request to + * reinstall transformers. + */ +export function isTransformersMissingError(err: string): boolean { + if (/hivemind embeddings install/i.test(err)) return true; + return /@huggingface\/transformers/i.test(err); +} + +// ── Test helpers ──────────────────────────────────────────────────────────── + +export function _resetClientStateForTesting(): void { + _signalledMissingDeps = false; + _recycledStuckDaemon = false; +} + let singleton: EmbedClient | null = null; export function getEmbedClient(): EmbedClient { if (!singleton) singleton = new EmbedClient(); diff --git a/src/embeddings/daemon.ts b/src/embeddings/daemon.ts index b41508ab..200ea0d0 100644 --- a/src/embeddings/daemon.ts +++ b/src/embeddings/daemon.ts @@ -9,11 +9,13 @@ import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "nod import { NomicEmbedder } from "./nomic.js"; import { DEFAULT_IDLE_TIMEOUT_MS, + PROTOCOL_VERSION, pidPathFor, socketPathFor, type DaemonRequest, type DaemonResponse, type EmbedRequest, + type HelloRequest, type PingRequest, } from "./protocol.js"; import { log as _log } from "../utils/debug.js"; @@ -31,6 +33,8 @@ export interface DaemonOptions { dims?: number; dtype?: string; repo?: string; + /** Path of the script invoked to start this daemon. Defaults to argv[1]. */ + daemonPath?: string; } export class EmbedDaemon { @@ -40,6 +44,7 @@ export class EmbedDaemon { private pidPath: string; private idleTimeoutMs: number; private idleTimer: NodeJS.Timeout | null = null; + private daemonPath: string; constructor(opts: DaemonOptions = {}) { const uid = getUid(); @@ -48,6 +53,7 @@ export class EmbedDaemon { this.pidPath = pidPathFor(uid, dir); this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims }); + this.daemonPath = opts.daemonPath ?? process.argv[1] ?? ""; } async start(): Promise { @@ -141,6 +147,15 @@ export class EmbedDaemon { } private async dispatch(req: DaemonRequest): Promise { + if (req.op === "hello") { + const h = req as HelloRequest; + return { + id: h.id, + daemonPath: this.daemonPath, + pid: process.pid, + protocolVersion: PROTOCOL_VERSION, + }; + } if (req.op === "ping") { const p = req as PingRequest; return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims }; diff --git a/src/embeddings/disable.ts b/src/embeddings/disable.ts index 7a67b512..a9674197 100644 --- a/src/embeddings/disable.ts +++ b/src/embeddings/disable.ts @@ -3,57 +3,59 @@ import { homedir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; +import { getEmbeddingsEnabled } from "../user-config.js"; + /** * Master opt-out for the embedding feature. * * Embeddings are off when EITHER: * - * 1. `HIVEMIND_EMBEDDINGS=false` is set — explicit opt-out for air-gapped / - * no-network installs, CI / benchmarks that want pure-lexical retrieval, - * and users who don't want the ~110 MB nomic download. + * 1. The user has opted out via `~/.deeplake/config.json` → + * `embeddings.enabled: false`. Set by `hivemind embeddings disable` or + * `hivemind embeddings uninstall`, or by the one-shot migration that + * seeds the config from `HIVEMIND_EMBEDDINGS` on first run. * - * 2. `@huggingface/transformers` is not resolvable from this bundle — the - * plugin ships without it (it has native deps that can't be bundled into - * the daemon). A fresh marketplace install lacks it; the README documents - * the optional `npm install @huggingface/transformers` step. When absent, - * we degrade silently to lexical-only mode rather than spawning a daemon - * that will crash on `import("@huggingface/transformers")` and emit - * confusing logs. + * 2. `@huggingface/transformers` is not resolvable — the plugin ships + * without it (native deps can't be bundled). A fresh marketplace install + * lacks it until the user runs `hivemind embeddings install`. When + * absent, we degrade silently to lexical-only mode rather than spawning + * a daemon that will crash on import. * - * In either case: SessionStart skips the warmup, capture / wiki-worker write - * rows with NULL in the embedding column, and `Grep` falls back to BM25 / - * ILIKE matching on text columns. Existing rows' embeddings remain readable. + * In either case: SessionStart skips the warmup, capture / wiki-worker + * write rows with NULL in the embedding column, and `Grep` falls back to + * BM25 / ILIKE matching on text columns. Existing rows' embeddings remain + * readable. * - * Read-once: cached for the lifetime of the (short-lived) hook process so a - * live `export HIVEMIND_EMBEDDINGS=...` takes effect on the next session. + * Read-once: the status is cached for the lifetime of the (short-lived) + * hook process. `hivemind embeddings enable|disable` takes effect on the + * next session, after the daemon is recycled. */ -export type EmbeddingsStatus = "enabled" | "env-disabled" | "no-transformers"; +export type EmbeddingsStatus = "enabled" | "user-disabled" | "no-transformers"; let cachedStatus: EmbeddingsStatus | null = null; function defaultResolveTransformers(): void { - // Resolve from this module's location — the same node_modules walk Node - // would do for the spawned daemon, since the daemon lives in the same - // bundle dir tree (true for CC/codex/cursor/hermes which symlink their - // plugin's node_modules to the shared deps). + // Try the canonical shared-deps location first — this is the location + // `hivemind embeddings install` populates, and the location the daemon + // resolves from in production. Probing here matches what will actually + // be loaded at runtime, eliminating the previous probe/use asymmetry + // (probe said enabled, daemon then failed with MODULE_NOT_FOUND). + const sharedDir = join(homedir(), ".hivemind", "embed-deps"); try { - createRequire(import.meta.url).resolve("@huggingface/transformers"); + createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); return; } catch { /* fall through */ } - // Fall back to the canonical shared deps location. Pi (and any future - // agent that doesn't ship a per-agent bundle adjacent to a node_modules) - // lands here: the shared deps at ~/.hivemind/embed-deps/node_modules - // are populated by `hivemind embeddings install`, and the daemon spawn - // resolves transformers via that exact dir. - const sharedDir = join(homedir(), ".hivemind", "embed-deps"); - createRequire(pathToFileURL(`${sharedDir}/`).href).resolve("@huggingface/transformers"); + // Bundle-relative walk for the dev tree or any future install layout + // that colocates `node_modules` next to the running file. + createRequire(import.meta.url).resolve("@huggingface/transformers"); } let _resolve: () => void = defaultResolveTransformers; +let _readEnabled: () => boolean = getEmbeddingsEnabled; function detectStatus(): EmbeddingsStatus { - if (process.env.HIVEMIND_EMBEDDINGS === "false") return "env-disabled"; + if (!_readEnabled()) return "user-disabled"; try { _resolve(); return "enabled"; @@ -73,16 +75,22 @@ export function embeddingsDisabled(): boolean { } // ── Test helpers ──────────────────────────────────────────────────────────── -// Exposed so unit tests can simulate "transformers not installed" without -// actually uninstalling the package. Underscore-prefixed and intentionally -// not re-exported from any public entry point — runtime never calls these. +// Exposed so unit tests can simulate "transformers not installed" or +// "user opted out" without touching real env or disk. Underscore-prefixed +// and intentionally not re-exported from any public entry point. export function _setResolveForTesting(fn: () => void): void { _resolve = fn; cachedStatus = null; } +export function _setEnabledReaderForTesting(fn: () => boolean): void { + _readEnabled = fn; + cachedStatus = null; +} + export function _resetForTesting(): void { _resolve = defaultResolveTransformers; + _readEnabled = getEmbeddingsEnabled; cachedStatus = null; } diff --git a/src/embeddings/nomic.ts b/src/embeddings/nomic.ts index d9dd77db..9dcc7b20 100644 --- a/src/embeddings/nomic.ts +++ b/src/embeddings/nomic.ts @@ -2,6 +2,11 @@ // process — hooks never import this. Kept isolated so the heavyweight transformer // dependency is not pulled into every bundled hook. +import { createRequire } from "node:module"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + import { DEFAULT_DIMS, DEFAULT_DTYPE, @@ -13,12 +18,80 @@ import { type Embedder = (input: string | string[], opts: Record) => Promise<{ data: Float32Array | number[] }>; +type TransformersModule = typeof import("@huggingface/transformers"); +type TransformersImporter = () => Promise; + export interface NomicOptions { repo?: string; dtype?: string; dims?: number; } +// ── transformers resolution ───────────────────────────────────────────────── +// The daemon may have been spawned from any plugin bundle path (marketplace +// versioned caches, dev tree, etc.). Bundle-relative `node_modules` resolution +// is unreliable across marketplace upgrades, so we explicitly look in the +// canonical shared-deps location that `hivemind embeddings install` populates, +// and only fall back to the bare specifier (dev tree / colocated install). + +export async function _importFromCanonicalSharedDeps( + sharedDir: string = join(homedir(), ".hivemind", "embed-deps"), +): Promise { + const base = pathToFileURL(`${sharedDir}/`).href; + // `createRequire(base).resolve(...)` honors the package's `"require"` + // conditional export, which for @huggingface/transformers v3 points at + // the CJS bundle (`./dist/transformers.node.cjs`). The dynamic + // `import()` of a CJS file wraps it as `{ default: }`, so + // top-level `env` / `pipeline` are not directly accessible. Normalize + // both shapes (ESM .mjs would put names at the top level; CJS .cjs + // hides them under `.default`). + const absMain = createRequire(base).resolve("@huggingface/transformers"); + const mod = await import(pathToFileURL(absMain).href); + return _normalizeTransformersModule(mod); +} + +export async function _importFromBareSpecifier(): Promise { + const mod = await import("@huggingface/transformers"); + return _normalizeTransformersModule(mod); +} + +export function _normalizeTransformersModule(mod: unknown): TransformersModule { + const m = mod as { default?: TransformersModule } & TransformersModule; + if (m.default && typeof m.default === "object" && "pipeline" in m.default) { + return m.default; + } + return m; +} + +export async function defaultImportTransformers( + canonical: () => Promise = _importFromCanonicalSharedDeps, + bare: () => Promise = _importFromBareSpecifier, +): Promise { + let canonicalErr: unknown; + try { + return await canonical(); + } catch (err) { + canonicalErr = err; + } + try { + return await bare(); + } catch (bareErr) { + const detail = bareErr instanceof Error ? bareErr.message : String(bareErr); + const canonicalDetail = canonicalErr instanceof Error ? canonicalErr.message : String(canonicalErr); + throw new Error( + `@huggingface/transformers is not installed anywhere reachable. ` + + `Run \`hivemind embeddings install\` to install it. ` + + `(canonical: ${canonicalDetail}; bare: ${detail})`, + ); + } +} + +// `defaultImportTransformers` has all-defaulted params, so calling it bare +// (`defaultImportTransformers()`) is fine — assign the function reference +// directly instead of wrapping in an arrow that v8 counts as a separate +// uncovered function. +let _importTransformers: TransformersImporter = defaultImportTransformers; + export class NomicEmbedder { private pipeline: Embedder | null = null; private loading: Promise | null = null; @@ -36,7 +109,7 @@ export class NomicEmbedder { if (this.pipeline) return; if (this.loading) return this.loading; this.loading = (async () => { - const mod = await import("@huggingface/transformers"); + const mod = await _importTransformers(); mod.env.allowLocalModels = false; mod.env.useFSCache = true; this.pipeline = (await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype as "fp32" | "q8" })) as unknown as Embedder; @@ -88,3 +161,16 @@ export class NomicEmbedder { return head; } } + +// ── Test helpers ──────────────────────────────────────────────────────────── +// Production never calls these. They let unit tests bypass the +// canonical-shared-deps resolver (which would otherwise hit the real +// ~/.hivemind/embed-deps/ on dev machines and ignore vi.mock). + +export function _setTransformersImporterForTesting(fn: TransformersImporter): void { + _importTransformers = fn; +} + +export function _resetTransformersImporterForTesting(): void { + _importTransformers = defaultImportTransformers; +} diff --git a/src/embeddings/protocol.ts b/src/embeddings/protocol.ts index b5518bd7..210208a9 100644 --- a/src/embeddings/protocol.ts +++ b/src/embeddings/protocol.ts @@ -29,8 +29,32 @@ export interface PingResponse { error?: string; } -export type DaemonRequest = EmbedRequest | PingRequest; -export type DaemonResponse = EmbedResponse | PingResponse; +// Wire-level handshake. Client sends a `hello` immediately after connecting +// the first time per process; daemon answers with its own `daemonPath` (the +// script that was actually spawned) so the client can verify that the +// running daemon is the same binary it would have spawned itself. On +// mismatch — typically after a marketplace plugin upgrade replaced the +// bundle but the old daemon process kept its socket — the client SIGTERMs +// and re-spawns from the new path. +export interface HelloRequest { + op: "hello"; + id: string; +} + +export interface HelloResponse { + id: string; + daemonPath: string; + pid: number; + protocolVersion: number; + error?: string; +} + +export type DaemonRequest = EmbedRequest | PingRequest | HelloRequest; +export type DaemonResponse = EmbedResponse | PingResponse | HelloResponse; + +// Increment when the wire protocol changes in a non-backward-compatible +// way. Used by the client's handshake mismatch check. +export const PROTOCOL_VERSION = 1; export const DEFAULT_SOCKET_DIR = "/tmp"; export const DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5"; diff --git a/src/embeddings/self-heal.ts b/src/embeddings/self-heal.ts new file mode 100644 index 00000000..c1cee59e --- /dev/null +++ b/src/embeddings/self-heal.ts @@ -0,0 +1,128 @@ +// Self-heal the per-plugin-version node_modules symlink that +// `hivemind embeddings install` creates. +// +// Why: `install` symlinks `/node_modules` to +// `~/.hivemind/embed-deps/node_modules` so Node's standard module +// resolution finds @huggingface/transformers from anywhere inside +// `/bundle/…`. But Claude Code's marketplace auto-upgrades drop +// new versioned cache dirs (`cache/hivemind/hivemind/0.7.27/`, +// `0.7.28/`, …) WITHOUT the symlink. Without intervention the user +// would have to re-run `hivemind embeddings install` after every +// marketplace upgrade — and most users won't, so embeddings would +// silently degrade. +// +// This helper runs from the capture hook on every session. The first +// session under a new plugin version creates the symlink; every other +// invocation is a cheap no-op. + +import { existsSync, lstatSync, mkdirSync, readlinkSync, renameSync, rmSync, symlinkSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, dirname, join } from "node:path"; + +export type SelfHealResult = + | { kind: "linked"; target: string; link: string } + | { kind: "already-linked"; target: string; link: string } + | { kind: "shared-deps-missing"; target: string } + | { kind: "plugin-owns-node-modules"; link: string } + | { kind: "linked-elsewhere"; link: string; existingTarget: string } + | { kind: "stale-link-removed"; link: string; danglingTarget: string } + | { kind: "not-bundle-layout"; bundleDir: string } + | { kind: "error"; detail: string }; + +export interface SelfHealOptions { + /** Absolute path to the agent's `bundle/` dir (passed by the capture hook). */ + bundleDir: string; + /** Override the target node_modules location (tests only). */ + sharedNodeModules?: string; +} + +/** + * Ensure `/node_modules` is a symlink to + * `~/.hivemind/embed-deps/node_modules`. Atomic, idempotent, conservative: + * never clobbers an existing real `node_modules` dir, never overrides a + * symlink that points elsewhere, and removes a dangling symlink (target + * no longer exists) so the next call can re-create it. + */ +export function ensurePluginNodeModulesLink(opts: SelfHealOptions): SelfHealResult { + // Guard against running from a non-bundle layout — e.g. tests that + // import the capture hook from `src/hooks/capture.ts` shouldn't accidentally + // symlink `src/node_modules` to the user's real shared deps. Every + // shipped agent bundle puts the capture hook at `/bundle/capture.js`, + // so the bundleDir's basename is always "bundle" in production. + if (basename(opts.bundleDir) !== "bundle") { + return { kind: "not-bundle-layout", bundleDir: opts.bundleDir }; + } + + const target = opts.sharedNodeModules ?? join(homedir(), ".hivemind", "embed-deps", "node_modules"); + const pluginDir = dirname(opts.bundleDir); + const link = join(pluginDir, "node_modules"); + + // No shared deps installed yet — leave the plugin dir alone. The capture + // hook's notification path covers user-facing surface for this case. + if (!existsSync(target)) { + return { kind: "shared-deps-missing", target }; + } + + // Check what currently exists at the link path. + let linkStat; + try { + linkStat = lstatSync(link); + } catch { + // Nothing there — go create. + return createSymlinkAtomic(target, link); + } + + if (linkStat.isSymbolicLink()) { + let existingTarget: string; + try { + existingTarget = readlinkSync(link); + } catch (e: unknown) { + return { kind: "error", detail: `readlink failed: ${e instanceof Error ? e.message : String(e)}` }; + } + if (existingTarget === target) { + return { kind: "already-linked", target, link }; + } + // Symlink to somewhere else — check whether the existing target + // resolves to a real directory. If it doesn't, the link is dangling + // and safe to remove + immediately re-create. Recreating in the same + // call (rather than returning "stale-link-removed" and waiting for + // the next session-start) means the CURRENT hook run lands with a + // healed link, not the one after it. + try { + statSync(link); // follows symlink — throws on dangling + // Real directory at a different target → don't override the user's choice. + return { kind: "linked-elsewhere", link, existingTarget }; + } catch { + try { rmSync(link); } catch { /* best-effort */ } + // Fall through to atomic re-create. If that fails we return its + // error rather than the stale-link-removed marker, since the link + // is now genuinely absent. + const recreated = createSymlinkAtomic(target, link); + // Keep the diagnostic that a stale link was repaired so callers + // can log the recovery — overload the existing variant with the + // dangling target the link used to point at. + if (recreated.kind === "linked") { + return { kind: "stale-link-removed", link, danglingTarget: existingTarget }; + } + return recreated; + } + } + + // Real directory or file at the link path — don't clobber. + return { kind: "plugin-owns-node-modules", link }; +} + +function createSymlinkAtomic(target: string, link: string): SelfHealResult { + try { + const parent = dirname(link); + if (!existsSync(parent)) mkdirSync(parent, { recursive: true }); + const tmp = `${link}.tmp.${process.pid}`; + // If a stale tmp exists from a crashed prior run, remove it first. + try { rmSync(tmp, { force: true }); } catch { /* best-effort */ } + symlinkSync(target, tmp); + renameSync(tmp, link); + return { kind: "linked", target, link }; + } catch (e: unknown) { + return { kind: "error", detail: e instanceof Error ? e.message : String(e) }; + } +} diff --git a/src/hooks/capture.ts b/src/hooks/capture.ts index b7d2bc62..3f72721a 100644 --- a/src/hooks/capture.ts +++ b/src/hooks/capture.ts @@ -25,6 +25,7 @@ import { tryStopCounterTrigger } from "../skillify/triggers.js"; import { EmbedClient } from "../embeddings/client.js"; import { embeddingSqlLiteral } from "../embeddings/sql.js"; import { embeddingsDisabled } from "../embeddings/disable.js"; +import { ensurePluginNodeModulesLink } from "../embeddings/self-heal.js"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { getInstalledVersion } from "../utils/version-check.js"; @@ -37,6 +38,15 @@ function resolveEmbedDaemonPath(): string { const __bundleDir = dirname(fileURLToPath(import.meta.url)); const PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".claude-plugin") ?? ""; +// Self-heal the shared-deps symlink for this plugin version. Marketplace +// auto-upgrades drop new versioned cache dirs without the symlink that +// `hivemind embeddings install` originally created; this restores it on +// first capture after each upgrade. No-op when the symlink already exists, +// shared deps are not installed, or the user has disabled embeddings. +if (!embeddingsDisabled()) { + try { ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); } catch { /* best-effort */ } +} + interface HookInput { session_id: string; transcript_path?: string; diff --git a/src/hooks/codex/capture.ts b/src/hooks/codex/capture.ts index 4adeaf46..d8b15327 100644 --- a/src/hooks/codex/capture.ts +++ b/src/hooks/codex/capture.ts @@ -21,6 +21,7 @@ import { buildSessionPath } from "../../utils/session-path.js"; import { EmbedClient } from "../../embeddings/client.js"; import { embeddingSqlLiteral } from "../../embeddings/sql.js"; import { embeddingsDisabled } from "../../embeddings/disable.js"; +import { ensurePluginNodeModulesLink } from "../../embeddings/self-heal.js"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { @@ -41,6 +42,14 @@ function resolveEmbedDaemonPath(): string { const __bundleDir = dirname(fileURLToPath(import.meta.url)); const PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".codex-plugin") ?? ""; +// Self-heal the shared-deps symlink for this plugin version. Marketplace +// auto-upgrades drop new versioned cache dirs without the symlink that +// `hivemind embeddings install` originally created; this restores it on +// first capture after each upgrade. +if (!embeddingsDisabled()) { + try { ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); } catch { /* best-effort */ } +} + interface CodexHookInput { session_id: string; transcript_path?: string | null; diff --git a/src/hooks/cursor/capture.ts b/src/hooks/cursor/capture.ts index 7955a8b7..d959fd4a 100644 --- a/src/hooks/cursor/capture.ts +++ b/src/hooks/cursor/capture.ts @@ -22,6 +22,7 @@ import { buildSessionPath } from "../../utils/session-path.js"; import { EmbedClient } from "../../embeddings/client.js"; import { embeddingSqlLiteral } from "../../embeddings/sql.js"; import { embeddingsDisabled } from "../../embeddings/disable.js"; +import { ensurePluginNodeModulesLink } from "../../embeddings/self-heal.js"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { @@ -44,6 +45,14 @@ function resolveEmbedDaemonPath(): string { const __bundleDir = dirname(fileURLToPath(import.meta.url)); const PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".claude-plugin") ?? ""; +// Self-heal the shared-deps symlink for this plugin version. Marketplace +// auto-upgrades drop new versioned cache dirs without the symlink that +// `hivemind embeddings install` originally created; this restores it on +// first capture after each upgrade. +if (!embeddingsDisabled()) { + try { ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); } catch { /* best-effort */ } +} + interface CursorCaptureInput { conversation_id?: string; hook_event_name?: string; diff --git a/src/hooks/cursor/session-start.ts b/src/hooks/cursor/session-start.ts index b517fef1..3601e3f1 100644 --- a/src/hooks/cursor/session-start.ts +++ b/src/hooks/cursor/session-start.ts @@ -57,7 +57,14 @@ Organization management — each argument is SEPARATE (do NOT quote subcommands - hivemind remove — remove member SKILLS (skillify) — mine + share reusable skills across the org: -${renderSkillifyCommands()}`; +${renderSkillifyCommands()} + +Embeddings (semantic memory search) — opt-in, persisted in ~/.deeplake/config.json: +- hivemind embeddings install — download deps (~600MB), symlink agents, set enabled:true +- hivemind embeddings enable — flip enabled:true (run install first if deps missing) +- hivemind embeddings disable — flip enabled:false + SIGTERM daemon (deps stay on disk) +- hivemind embeddings uninstall [--prune] — remove agent symlinks + disable; --prune wipes deps too +- hivemind embeddings status — show config + deps + per-agent link state`; interface CursorSessionStartInput { session_id?: string; diff --git a/src/hooks/hermes/capture.ts b/src/hooks/hermes/capture.ts index ff6269ed..a24e2afb 100644 --- a/src/hooks/hermes/capture.ts +++ b/src/hooks/hermes/capture.ts @@ -23,6 +23,7 @@ import { buildSessionPath } from "../../utils/session-path.js"; import { EmbedClient } from "../../embeddings/client.js"; import { embeddingSqlLiteral } from "../../embeddings/sql.js"; import { embeddingsDisabled } from "../../embeddings/disable.js"; +import { ensurePluginNodeModulesLink } from "../../embeddings/self-heal.js"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { @@ -45,6 +46,14 @@ function resolveEmbedDaemonPath(): string { const __bundleDir = dirname(fileURLToPath(import.meta.url)); const PLUGIN_VERSION = getInstalledVersion(__bundleDir, ".claude-plugin") ?? ""; +// Self-heal the shared-deps symlink for this plugin version. Marketplace +// auto-upgrades drop new versioned cache dirs without the symlink that +// `hivemind embeddings install` originally created; this restores it on +// first capture after each upgrade. +if (!embeddingsDisabled()) { + try { ensurePluginNodeModulesLink({ bundleDir: __bundleDir }); } catch { /* best-effort */ } +} + interface HermesCaptureInput { hook_event_name?: string; session_id?: string; diff --git a/src/hooks/hermes/session-start.ts b/src/hooks/hermes/session-start.ts index 0685ac25..2380f954 100644 --- a/src/hooks/hermes/session-start.ts +++ b/src/hooks/hermes/session-start.ts @@ -49,7 +49,14 @@ Organization management — each argument is SEPARATE (do NOT quote subcommands - hivemind remove — remove member SKILLS (skillify) — mine + share reusable skills across the org: -${renderSkillifyCommands()}`; +${renderSkillifyCommands()} + +Embeddings (semantic memory search) — opt-in, persisted in ~/.deeplake/config.json: +- hivemind embeddings install — download deps (~600MB), symlink agents, set enabled:true +- hivemind embeddings enable — flip enabled:true (run install first if deps missing) +- hivemind embeddings disable — flip enabled:false + SIGTERM daemon (deps stay on disk) +- hivemind embeddings uninstall [--prune] — remove agent symlinks + disable; --prune wipes deps too +- hivemind embeddings status — show config + deps + per-agent link state`; interface HermesSessionStartInput { hook_event_name?: string; diff --git a/src/hooks/session-start-setup.ts b/src/hooks/session-start-setup.ts index ed27db24..fa2ef0f5 100644 --- a/src/hooks/session-start-setup.ts +++ b/src/hooks/session-start-setup.ts @@ -80,8 +80,8 @@ async function main(): Promise { if (embeddingsDisabled()) { const status = embeddingsStatus(); const reason = status === "no-transformers" - ? "@huggingface/transformers not installed (see README to enable embeddings)" - : "HIVEMIND_EMBEDDINGS=false"; + ? "@huggingface/transformers not installed (run `hivemind embeddings install` to enable)" + : "embeddings disabled in ~/.deeplake/config.json (run `hivemind embeddings enable` to opt in)"; log(`embed daemon warmup skipped: ${reason}`); } else if (process.env.HIVEMIND_EMBED_WARMUP !== "false") { try { diff --git a/src/hooks/session-start.ts b/src/hooks/session-start.ts index 6ad852e0..e956b058 100644 --- a/src/hooks/session-start.ts +++ b/src/hooks/session-start.ts @@ -64,6 +64,13 @@ Organization management — each argument is SEPARATE (do NOT quote subcommands Skill management (mine + share reusable Claude skills across the org): ${renderSkillifyCommands()} +Embeddings (semantic memory search) — opt-in, persisted in ~/.deeplake/config.json: +- hivemind embeddings install — download deps (~600MB), symlink agents, set enabled:true +- hivemind embeddings enable — flip enabled:true (run install first if deps missing) +- hivemind embeddings disable — flip enabled:false + SIGTERM daemon (deps stay on disk) +- hivemind embeddings uninstall [--prune] — remove agent symlinks + disable; --prune wipes deps too +- hivemind embeddings status — show config + deps + per-agent link state + IMPORTANT: Only use bash commands (cat, ls, grep, echo, jq, head, tail, etc.) to interact with ~/.deeplake/memory/. Do NOT use python, python3, node, curl, or other interpreters — they are not available in the memory filesystem. Avoid bash brace expansions like \`{1..10}\` (not fully supported); spell out paths explicitly. Bash output is capped at 10MB total — avoid \`for f in *.json; do cat $f\` style loops on the whole sessions dir. LIMITS: Do NOT spawn subagents to read deeplake memory. If a file returns empty after 2 attempts, skip it and move on. Report what you found rather than exhaustively retrying. diff --git a/src/notifications/queue.ts b/src/notifications/queue.ts index cf634c10..d24a37be 100644 --- a/src/notifications/queue.ts +++ b/src/notifications/queue.ts @@ -10,18 +10,45 @@ * hook consumes at next session). The file is the cross-process boundary. */ -import { readFileSync, writeFileSync, renameSync, mkdirSync } from "node:fs"; +import { readFileSync, writeFileSync, renameSync, mkdirSync, openSync, closeSync, unlinkSync, statSync } from "node:fs"; import { join, resolve } from "node:path"; import { homedir } from "node:os"; +import { setTimeout as sleep } from "node:timers/promises"; import type { Notification, NotificationsQueue } from "./types.js"; import { log as _log } from "../utils/debug.js"; const log = (msg: string) => _log("notifications-queue", msg); +// Cross-process lock parameters for enqueueNotification's +// read-modify-write. Lock file lives next to the queue. Stale-lock +// reclaim threshold is well above any plausible enqueue duration +// (a few ms) but below any session-start timeout. Tests override these +// via `_setLockTimingForTesting` so the give-up / reclaim branches don't +// have to wait 6 s of real time per test. +let LOCK_RETRY_MAX = 50; +let LOCK_RETRY_BASE_MS = 5; +let LOCK_STALE_MS = 5000; + +export function _setLockTimingForTesting(opts: { retryMax?: number; retryBaseMs?: number; staleMs?: number }): void { + if (opts.retryMax !== undefined) LOCK_RETRY_MAX = opts.retryMax; + if (opts.retryBaseMs !== undefined) LOCK_RETRY_BASE_MS = opts.retryBaseMs; + if (opts.staleMs !== undefined) LOCK_STALE_MS = opts.staleMs; +} + +export function _resetLockTimingForTesting(): void { + LOCK_RETRY_MAX = 50; + LOCK_RETRY_BASE_MS = 5; + LOCK_STALE_MS = 5000; +} + export function queuePath(): string { return join(homedir(), ".deeplake", "notifications-queue.json"); } +function lockPath(): string { + return `${queuePath()}.lock`; +} + export function readQueue(): NotificationsQueue { try { const raw = readFileSync(queuePath(), "utf-8"); @@ -36,10 +63,22 @@ export function readQueue(): NotificationsQueue { } } +/** + * Defense-in-depth: refuse to write the queue if its resolved path + * escapes `$HOME`. Extracted so tests can exercise the guard directly + * without monkey-patching `homedir()` (vitest's ESM mode can't spy on + * `os.homedir`, and we don't want to mock the whole module). + */ +export function _isQueuePathInsideHome(path: string, home: string): boolean { + const r = resolve(path); + const h = resolve(home); + return r.startsWith(h + "/") || r === h; +} + export function writeQueue(q: NotificationsQueue): void { const path = queuePath(); const home = resolve(homedir()); - if (!resolve(path).startsWith(home + "/") && resolve(path) !== home) { + if (!_isQueuePathInsideHome(path, home)) { throw new Error(`notifications-queue write blocked: ${path} is outside ${home}`); } mkdirSync(join(home, ".deeplake"), { recursive: true, mode: 0o700 }); @@ -48,9 +87,92 @@ export function writeQueue(q: NotificationsQueue): void { renameSync(tmp, path); } -/** Append a notification to the persistent queue. Cross-process safe. */ -export function enqueueNotification(n: Notification): void { - const q = readQueue(); - q.queue.push(n); - writeQueue(q); +/** + * Acquire an exclusive advisory lock on the queue, run `fn`, then release. + * Uses `O_EXCL` on a `.lock` file — the only operation guaranteed atomic + * across processes on POSIX. Retries with backoff on EEXIST; if the lock + * has been held longer than LOCK_STALE_MS we assume the holder died and + * reclaim it. Always best-effort: a lock failure logs but does NOT block + * the caller (the only legitimate caller is `enqueueNotification`, and + * the contract there is "best-effort, never throw into the hook hot path"). + */ +async function withQueueLock(fn: () => T): Promise { + const path = lockPath(); + mkdirSync(join(homedir(), ".deeplake"), { recursive: true, mode: 0o700 }); + let fd: number | null = null; + for (let attempt = 0; attempt < LOCK_RETRY_MAX; attempt++) { + try { + fd = openSync(path, "wx", 0o600); + break; + } catch (e: unknown) { + const code = (e as NodeJS.ErrnoException).code; + if (code !== "EEXIST") throw e; + // Stale-lock reclaim: if the file is older than LOCK_STALE_MS, + // assume the previous holder died and try to remove it. Then loop + // back to retry the open. + try { + const age = Date.now() - statSync(path).mtimeMs; + if (age > LOCK_STALE_MS) { + unlinkSync(path); + continue; + } + } catch { /* stat/unlink may race with another reclaim — ignore */ } + // Standard contention: yield the event loop instead of spinning + // CPU. The earlier `while (Date.now() < end) {}` busy-wait could + // hold the loop for up to ~6 s at production defaults, freezing + // every other timer/IO callback in the hook process — including + // the in-flight embed daemon response. `await sleep(delay)` yields + // cleanly with the same backoff curve. + const delay = LOCK_RETRY_BASE_MS * (attempt + 1); + await sleep(delay); + } + } + if (fd === null) { + log(`lock acquisition gave up after ${LOCK_RETRY_MAX} attempts — proceeding unlocked (last-writer-wins)`); + return fn(); + } + try { + return fn(); + } finally { + try { closeSync(fd); } catch { /* best-effort */ } + try { unlinkSync(path); } catch { /* best-effort */ } + } +} + +function sameDedupKey(a: Notification, b: Notification): boolean { + if (a.id !== b.id) return false; + // JSON.stringify is canonical enough here — dedupKey values come from + // a small set of producers we control (transformers-missing detail, + // welcome-shown timestamps, summarization counts). Field-order + // determinism comes from the producers writing object literals in a + // stable shape, which we already rely on for state.ts dedup. + return JSON.stringify(a.dedupKey) === JSON.stringify(b.dedupKey); +} + +/** + * Append a notification to the persistent queue. Cross-process safe via + * an advisory `.lock` file: concurrent producers serialize on the lock so + * read-modify-write can't lose entries. Without the lock, two hooks that + * race here would both read the same starting state, push their own + * entry, and the second `rename(2)` would clobber the first writer's + * addition. + * + * Idempotent under (id, dedupKey): if an equivalent notification is + * already queued (i.e. a previous hook enqueued the same warning but the + * SessionStart drain hasn't run yet), the second call is a no-op. Without + * this, every hook process that hits an `embed-deps-missing` would pile + * another copy onto the queue between drains — the in-process + * `_signalledMissingDeps` flag in client.ts only dedups inside one + * process. The drain layer already dedups against the *shown* state in + * state.ts; this guard prevents redundant queue growth between drains. + */ +export async function enqueueNotification(n: Notification): Promise { + await withQueueLock(() => { + const q = readQueue(); + if (q.queue.some(existing => sameDedupKey(existing, n))) { + return; + } + q.queue.push(n); + writeQueue(q); + }); } diff --git a/src/shell/deeplake-fs.ts b/src/shell/deeplake-fs.ts index 6e681249..a69f5175 100644 --- a/src/shell/deeplake-fs.ts +++ b/src/shell/deeplake-fs.ts @@ -52,7 +52,13 @@ function normalizeSessionMessage(path: string, message: unknown): string { } function resolveEmbedDaemonPath(): string { - return join(dirname(fileURLToPath(import.meta.url)), "embeddings", "embed-daemon.js"); + // This module is bundled to `/bundle/shell/deeplake-shell.js`, + // while the embed daemon lives one level up at + // `/bundle/embeddings/embed-daemon.js`. The earlier resolver + // forgot the `..` and pointed at the non-existent + // `bundle/shell/embeddings/embed-daemon.js`, which silently broke the + // pre-tool-use shell embed path on every agent. + return join(dirname(fileURLToPath(import.meta.url)), "..", "embeddings", "embed-daemon.js"); } function joinSessionMessages(path: string, messages: unknown[]): string { diff --git a/src/user-config.ts b/src/user-config.ts new file mode 100644 index 00000000..b0528b8f --- /dev/null +++ b/src/user-config.ts @@ -0,0 +1,139 @@ +// Persistent user preferences for the plugin, stored at +// `~/.deeplake/config.json`. Separate from `~/.deeplake/credentials.json` +// (auth) — this file holds opt-in/out flags and other settings that survive +// across sessions, agents, and machines. +// +// Currently the only setting is `embeddings.enabled`, which gates whether +// capture / wiki / grep paths invoke the embed daemon. The previous +// `HIVEMIND_EMBEDDINGS=false` env var is read EXACTLY ONCE — during the +// first run of the new code on a machine that has no `embeddings.enabled` +// key yet — to seed the config, then never consulted again. + +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; + +export interface UserConfig { + embeddings?: { + enabled?: boolean; + }; +} + +let _configPath: () => string = () => + process.env.HIVEMIND_CONFIG_PATH ?? join(homedir(), ".deeplake", "config.json"); + +// In-memory cache so the migration's env-var read and resulting write happen +// at most once per process. The file on disk is the source of truth; the +// cache only avoids re-parsing JSON on every call. +let _cache: UserConfig | null = null; +let _migrated = false; + +export function readUserConfig(): UserConfig { + if (_cache !== null) return _cache; + const path = _configPath(); + if (!existsSync(path)) { + _cache = {}; + return _cache; + } + try { + const raw = readFileSync(path, "utf-8"); + const parsed = JSON.parse(raw) as unknown; + _cache = isPlainObject(parsed) ? (parsed as UserConfig) : {}; + } catch { + // Corrupt or unreadable — treat as empty, but DON'T overwrite (the user + // may want to fix it by hand). A subsequent write will overwrite. + _cache = {}; + } + return _cache; +} + +export function writeUserConfig(patch: Partial): UserConfig { + const current = readUserConfig(); + const merged = deepMerge(current, patch); + const path = _configPath(); + const dir = dirname(path); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + const tmp = `${path}.tmp.${process.pid}`; + writeFileSync(tmp, JSON.stringify(merged, null, 2) + "\n", "utf-8"); + renameSync(tmp, path); + _cache = merged; + return merged; +} + +// Reads the embeddings-enabled flag, performing the one-shot env-var +// migration if no value has ever been persisted. Returns the final boolean. +// +// Migration rule (per design): +// HIVEMIND_EMBEDDINGS=false OR unset → enabled: false +// HIVEMIND_EMBEDDINGS=true (or any other truthy) → enabled: true +// +// Subsequent calls read straight from config; the env var is never touched +// again. `hivemind embeddings install/enable/disable/uninstall` mutate the +// config via writeUserConfig(). +export function getEmbeddingsEnabled(): boolean { + const cfg = readUserConfig(); + if (cfg.embeddings && typeof cfg.embeddings.enabled === "boolean") { + return cfg.embeddings.enabled; + } + if (_migrated) { + // Migration ran this process but couldn't persist (read-only fs etc.). + // Fall back to the env var directly to avoid spinning the migration on + // every call. Cached for the lifetime of the process. + return migrationValueFromEnv(); + } + _migrated = true; + const enabled = migrationValueFromEnv(); + try { + writeUserConfig({ embeddings: { enabled } }); + } catch { + // Persist failed (perms, full disk, etc.) — keep the in-memory cache so + // the rest of the session sees a stable value. + _cache = { ...(cfg ?? {}), embeddings: { ...(cfg?.embeddings ?? {}), enabled } }; + } + return enabled; +} + +function migrationValueFromEnv(): boolean { + const raw = process.env.HIVEMIND_EMBEDDINGS; + if (raw === undefined) return false; + if (raw === "false") return false; + // Anything else (including "true", "1", etc.) → enabled. + return true; +} + +export function setEmbeddingsEnabled(enabled: boolean): void { + writeUserConfig({ embeddings: { enabled } }); +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function deepMerge(base: UserConfig, patch: Partial): UserConfig { + const out: UserConfig = { ...base }; + for (const key of Object.keys(patch) as Array) { + const patchVal = patch[key]; + const baseVal = base[key]; + if (isPlainObject(patchVal) && isPlainObject(baseVal)) { + (out as any)[key] = { ...(baseVal as object), ...(patchVal as object) }; + } else if (patchVal !== undefined) { + (out as any)[key] = patchVal; + } + } + return out; +} + +// ── Test helpers ──────────────────────────────────────────────────────────── + +export function _setConfigPathForTesting(fn: () => string): void { + _configPath = fn; + _cache = null; + _migrated = false; +} + +export function _resetUserConfigForTesting(): void { + _configPath = () => + process.env.HIVEMIND_CONFIG_PATH ?? join(homedir(), ".deeplake", "config.json"); + _cache = null; + _migrated = false; +} diff --git a/tests/claude-code/embeddings-bundle-scan.test.ts b/tests/claude-code/embeddings-bundle-scan.test.ts new file mode 100644 index 00000000..b578e783 --- /dev/null +++ b/tests/claude-code/embeddings-bundle-scan.test.ts @@ -0,0 +1,168 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +/** + * Bundle-level guards that the embed-daemon fix actually lands in every + * shipped agent bundle. Per the project testing philosophy: source tests + * prove the helpers are correct, bundle tests prove the build didn't drop + * the helpers, re-inline an old pattern, or otherwise regress on the + * shipped artifact. + * + * A 30-second reviewer guardrail: scan the shipped JS for the literal + * strings that prove each fix shipped to each agent. + */ + +const repoRoot = process.cwd(); + +interface AgentBundle { + agent: "claude-code" | "codex" | "cursor" | "hermes"; + embedDaemon: string; + captureHook: string; +} + +const AGENTS: AgentBundle[] = [ + { + agent: "claude-code", + embedDaemon: join(repoRoot, "claude-code", "bundle", "embeddings", "embed-daemon.js"), + captureHook: join(repoRoot, "claude-code", "bundle", "capture.js"), + }, + { + agent: "codex", + embedDaemon: join(repoRoot, "codex", "bundle", "embeddings", "embed-daemon.js"), + captureHook: join(repoRoot, "codex", "bundle", "capture.js"), + }, + { + agent: "cursor", + embedDaemon: join(repoRoot, "cursor", "bundle", "embeddings", "embed-daemon.js"), + captureHook: join(repoRoot, "cursor", "bundle", "capture.js"), + }, + { + agent: "hermes", + embedDaemon: join(repoRoot, "hermes", "bundle", "embeddings", "embed-daemon.js"), + captureHook: join(repoRoot, "hermes", "bundle", "capture.js"), + }, +]; + +describe("shipped embed-daemon.js — explicit transformers resolver", () => { + for (const a of AGENTS) { + describe(a.agent, () => { + it(`embed-daemon.js exists at the shipped path`, () => { + expect(existsSync(a.embedDaemon), `missing: ${a.embedDaemon}`).toBe(true); + }); + + it(`embed-daemon.js loads transformers via the canonical shared-deps location`, () => { + const src = readFileSync(a.embedDaemon, "utf-8"); + // Positive: canonical shared-deps path (".hivemind" + "embed-deps" + // adjacent string literals survive esbuild's join() reformatting). + expect(src).toMatch(/\.hivemind/); + expect(src).toMatch(/embed-deps/); + // Positive: createRequire-rooted resolve survived bundling. + expect(src).toMatch(/createRequire/); + }); + + it(`embed-daemon.js throws an actionable error pointing at "hivemind embeddings install"`, () => { + const src = readFileSync(a.embedDaemon, "utf-8"); + // The wrapper error message must survive the bundle so the + // client-side log line tells the user what to do. + expect(src).toContain("hivemind embeddings install"); + }); + }); + } +}); + +describe("shipped capture.js — self-heal + visible-failure notification", () => { + for (const a of AGENTS) { + describe(a.agent, () => { + it(`capture.js exists`, () => { + expect(existsSync(a.captureHook), `missing: ${a.captureHook}`).toBe(true); + }); + + it(`capture.js invokes the self-heal helper`, () => { + const src = readFileSync(a.captureHook, "utf-8"); + expect(src).toContain("ensurePluginNodeModulesLink"); + }); + + it(`capture.js carries the embed-deps-missing notification dedupKey`, () => { + const src = readFileSync(a.captureHook, "utf-8"); + // The notification ID is what the SessionStart drain renders. + expect(src).toContain("embed-deps-missing"); + }); + + it(`capture.js still suppresses notifications when user-disabled (no nag for explicit opt-out)`, () => { + const src = readFileSync(a.captureHook, "utf-8"); + // The guard must survive in the shipped artifact. + expect(src).toMatch(/user-disabled/); + }); + }); + } +}); + +describe("shipped shell/deeplake-shell.js — embed daemon path resolves to an existing file", () => { + // Regression guard for CodeRabbit #6/#7/#11: the in-bundle resolver + // computed `dirname(import.meta.url) + "embeddings/embed-daemon.js"`, + // which when run from `/bundle/shell/` pointed at the missing + // path `/bundle/shell/embeddings/embed-daemon.js`. The fix + // adds `..` so it correctly lands at `/bundle/embeddings/`. + // We verify both literally (the `..` survived bundling) AND + // structurally (the actual bundled daemon file exists where the + // bundled shell would look for it). + const SHELL_BUNDLES: Array<[string, string, string]> = [ + ["claude-code", + join(repoRoot, "claude-code", "bundle", "shell", "deeplake-shell.js"), + join(repoRoot, "claude-code", "bundle", "embeddings", "embed-daemon.js")], + ["codex", + join(repoRoot, "codex", "bundle", "shell", "deeplake-shell.js"), + join(repoRoot, "codex", "bundle", "embeddings", "embed-daemon.js")], + ["cursor", + join(repoRoot, "cursor", "bundle", "shell", "deeplake-shell.js"), + join(repoRoot, "cursor", "bundle", "embeddings", "embed-daemon.js")], + ]; + + it.each(SHELL_BUNDLES)("%s shell bundle exists", (_label, shellPath) => { + expect(existsSync(shellPath), `missing: ${shellPath}`).toBe(true); + }); + + it.each(SHELL_BUNDLES)( + "%s daemon sibling exists at the parent-of-shell path", + (_label, _shellPath, daemonPath) => { + expect(existsSync(daemonPath), `missing: ${daemonPath}`).toBe(true); + }, + ); + + it.each(SHELL_BUNDLES)( + "%s shell bundle resolves daemonEntry via parent-of-shell (`..` survived bundling)", + (_label, shellPath) => { + const src = readFileSync(shellPath, "utf-8"); + // The resolver builds a path like: + // join(dirname(import.meta.url), "..", "embeddings", "embed-daemon.js") + // After esbuild minification the literals "..", "embeddings", + // "embed-daemon.js" stay intact. Without the `..` we'd see + // `"embeddings", "embed-daemon.js"` adjacent. Match the corrected + // shape: a `..` immediately followed by `embeddings`. + expect(src).toMatch(/"\.\.",\s*"embeddings",\s*"embed-daemon\.js"/); + }, + ); +}); + +describe("shipped bundle/cli.js — full embeddings subcommand surface", () => { + const cliPath = join(repoRoot, "bundle", "cli.js"); + + it("bundle/cli.js exists", () => { + expect(existsSync(cliPath), `missing: ${cliPath}`).toBe(true); + }); + + it("dispatcher recognises every embeddings subcommand", () => { + const src = readFileSync(cliPath, "utf-8"); + expect(src).toContain('"install"'); + expect(src).toContain('"enable"'); + expect(src).toContain('"disable"'); + expect(src).toContain('"uninstall"'); + expect(src).toContain('"status"'); + }); + + it("CLI references ~/.deeplake/config.json so the model knows where state lives", () => { + const src = readFileSync(cliPath, "utf-8"); + expect(src).toContain("~/.deeplake/config.json"); + }); +}); diff --git a/tests/claude-code/embeddings-client.test.ts b/tests/claude-code/embeddings-client.test.ts index 71097bcc..68057fe0 100644 --- a/tests/claude-code/embeddings-client.test.ts +++ b/tests/claude-code/embeddings-client.test.ts @@ -1,14 +1,24 @@ // Unit tests for the embedding client — avoid loading the model by spinning up // a tiny fake daemon that speaks the protocol. -import { describe, it, expect, afterEach } from "vitest"; +import { describe, it, expect, afterEach, beforeEach, vi } from "vitest"; import { createServer, type Server, type Socket } from "node:net"; -import { mkdtempSync, rmSync, existsSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, existsSync, writeFileSync, unlinkSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { execSync } from "node:child_process"; -import { EmbedClient, getEmbedClient } from "../../src/embeddings/client.js"; + +const enqueueNotificationMock = vi.fn(); +vi.mock("../../src/notifications/queue.js", async () => { + const actual = await vi.importActual( + "../../src/notifications/queue.js", + ); + return { ...actual, enqueueNotification: (...a: unknown[]) => enqueueNotificationMock(...a) }; +}); + +import { EmbedClient, getEmbedClient, isTransformersMissingError, _resetClientStateForTesting } from "../../src/embeddings/client.js"; import type { DaemonRequest, DaemonResponse } from "../../src/embeddings/protocol.js"; +import { _setEnabledReaderForTesting, _resetForTesting as _resetDisableForTesting } from "../../src/embeddings/disable.js"; let servers: Server[] = []; let tmpDirs: string[] = []; @@ -58,7 +68,12 @@ describe("EmbedClient", () => { if (req.op === "embed") return { id: req.id, embedding: [0.1, 0.2, 0.3] }; return { id: req.id, ready: true }; }); - const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + // daemonEntry: "" → falsy, so verifyDaemonOnce early-returns without + // probing. Tests that care about embed semantics (not handshake) opt + // out of verification this way; without it the dev-machine fallback + // to SHARED_DAEMON_PATH would resolve a real path and the handshake + // mismatch (hello returns no daemonPath) would trip the recycle path. + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false, daemonEntry: "" }); const vec = await client.embed("hello", "document"); expect(vec).toEqual([0.1, 0.2, 0.3]); }); @@ -354,3 +369,426 @@ describe("EmbedClient", () => { try { execSync(`pkill -f ${daemonScript}`); } catch { /* already exited */ } }); }); + +describe("isTransformersMissingError", () => { + it("matches Node errors that specifically name @huggingface/transformers", () => { + expect(isTransformersMissingError("Cannot find module '@huggingface/transformers'")).toBe(true); + expect(isTransformersMissingError( + "Error [ERR_MODULE_NOT_FOUND]: Cannot find package '@huggingface/transformers' imported from /a/b", + )).toBe(true); + expect(isTransformersMissingError( + "MODULE_NOT_FOUND when resolving @huggingface/transformers from /tmp", + )).toBe(true); + }); + + it("matches the actionable wrapper thrown by defaultImportTransformers", () => { + expect(isTransformersMissingError( + "@huggingface/transformers is not installed anywhere reachable. Run `hivemind embeddings install`...", + )).toBe(true); + }); + + it("does NOT match bare MODULE_NOT_FOUND for unrelated dependencies (regression for #10/#14)", () => { + // The old matcher classified any MODULE_NOT_FOUND as a transformers + // issue, so an onnxruntime-node / sharp / etc. missing-dep failure + // would falsely trigger the recycle + "run hivemind embeddings + // install" guidance — a command that can't fix non-transformers + // problems. The matcher must require @huggingface/transformers OR + // the actionable wrapper string to land. + expect(isTransformersMissingError("MODULE_NOT_FOUND while loading onnxruntime-node")).toBe(false); + expect(isTransformersMissingError("Cannot find module 'sharp'")).toBe(false); + expect(isTransformersMissingError( + "Error [ERR_MODULE_NOT_FOUND]: Cannot find package 'onnxruntime-node' imported from /a/b", + )).toBe(false); + }); + + it("does not match unrelated daemon errors", () => { + expect(isTransformersMissingError("model load timeout")).toBe(false); + expect(isTransformersMissingError("unknown op")).toBe(false); + expect(isTransformersMissingError("")).toBe(false); + }); +}); + +describe("EmbedClient — transformers-missing handling", () => { + beforeEach(() => { + enqueueNotificationMock.mockReset(); + _resetClientStateForTesting(); + _resetDisableForTesting(); + }); + + afterEach(() => { + _resetClientStateForTesting(); + _resetDisableForTesting(); + }); + + it("enqueues an embed-deps-missing notification when daemon reports the transformers wrapper error", async () => { + _setEnabledReaderForTesting(() => true); + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") return { id: req.id, daemonPath: "/somewhere", pid: 1, protocolVersion: 1 }; + return { id: req.id, error: "@huggingface/transformers is not installed anywhere reachable. Run `hivemind embeddings install`" }; + }); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false, daemonEntry: "" }); + const vec = await client.embed("hello"); + expect(vec).toBeNull(); + expect(enqueueNotificationMock).toHaveBeenCalledTimes(1); + const arg = enqueueNotificationMock.mock.calls[0][0]; + expect(arg.id).toBe("embed-deps-missing"); + expect(arg.severity).toBe("warn"); + expect(arg.body).toMatch(/hivemind embeddings install/); + expect(arg.dedupKey.reason).toBe("transformers-missing"); + }); + + it("does NOT enqueue when the user has disabled embeddings (no nag for explicit opt-out)", async () => { + _setEnabledReaderForTesting(() => false); + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") return { id: req.id, daemonPath: "/somewhere", pid: 1, protocolVersion: 1 }; + return { id: req.id, error: "MODULE_NOT_FOUND @huggingface/transformers" }; + }); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + await client.embed("hello"); + expect(enqueueNotificationMock).not.toHaveBeenCalled(); + }); + + it("deduplicates within a single process: second failing call does not double-enqueue", async () => { + _setEnabledReaderForTesting(() => true); + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") return { id: req.id, daemonPath: "/somewhere", pid: 1, protocolVersion: 1 }; + return { id: req.id, error: "Cannot find package '@huggingface/transformers'" }; + }); + const c1 = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false, daemonEntry: "" }); + const c2 = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false, daemonEntry: "" }); + await c1.embed("a"); + await c2.embed("b"); + expect(enqueueNotificationMock).toHaveBeenCalledTimes(1); + }); + + it("does not enqueue on a generic daemon error unrelated to transformers", async () => { + _setEnabledReaderForTesting(() => true); + const dir = makeTmpDir(); + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") return { id: req.id, daemonPath: "/somewhere", pid: 1, protocolVersion: 1 }; + return { id: req.id, error: "model load timeout" }; + }); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + await client.embed("hello"); + expect(enqueueNotificationMock).not.toHaveBeenCalled(); + }); +}); + +describe("EmbedClient — hello handshake / stuck daemon recycle", () => { + beforeEach(() => { + enqueueNotificationMock.mockReset(); + _resetClientStateForTesting(); + }); + + afterEach(() => { + _resetClientStateForTesting(); + }); + + it("does NOT recycle the daemon when hello returns the expected daemonPath", async () => { + const dir = makeTmpDir(); + const expectedPath = "/expected/daemon.js"; + let lastReq: DaemonRequest | null = null; + await startFakeDaemon(dir, (req) => { + lastReq = req; + if (req.op === "hello") { + return { id: req.id, daemonPath: expectedPath, pid: 99999, protocolVersion: 1 }; + } + if (req.op === "embed") return { id: req.id, embedding: [0.1, 0.2] }; + return { id: req.id, error: "unknown" }; + }); + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: false, + daemonEntry: expectedPath, + }); + const vec = await client.embed("hi"); + expect(vec).toEqual([0.1, 0.2]); + expect(lastReq).not.toBeNull(); + // pidfile / sockfile should be untouched (we created the sock via the fake daemon) + const uid = String(process.getuid?.() ?? "test"); + expect(existsSync(join(dir, `hivemind-embed-${uid}.sock`))).toBe(true); + }); + + it("recycles when the daemon returns 'unknown op' on hello (older protocol)", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + const pidPath = join(dir, `hivemind-embed-${uid}.pid`); + writeFileSync(pidPath, "1"); // init pid — kill will fail silently + + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") { + // Mimic a pre-handshake daemon that doesn't recognize the op. + return { id: req.id, error: "unknown op" }; + } + if (req.op === "embed") return { id: req.id, embedding: [0.5] }; + return { id: req.id, error: "unknown" }; + }); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: false, + daemonEntry: "/expected/new/bundle/daemon.js", + }); + await client.embed("hi"); + // Recycle should have unlinked sock + pidfile so the next call respawns. + expect(existsSync(sockPath)).toBe(false); + expect(existsSync(pidPath)).toBe(false); + }); + + it("recycles when the running daemon's path no longer exists on disk (GC'd marketplace bundle)", async () => { + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + const pidPath = join(dir, `hivemind-embed-${uid}.pid`); + writeFileSync(pidPath, "1"); + + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") { + // Stale path — bundle was GC'd by Claude Code's plugin-cache cleanup. + return { id: req.id, daemonPath: "/non/existent/old/bundle/embed-daemon.js", pid: 1, protocolVersion: 1 }; + } + if (req.op === "embed") return { id: req.id, embedding: [0.5] }; + return { id: req.id, error: "unknown" }; + }); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: false, + daemonEntry: "/new/bundle/embed-daemon.js", + }); + await client.embed("hi"); + expect(existsSync(pidPath)).toBe(false); + expect(existsSync(sockPath)).toBe(false); + }); + + it("does NOT recycle when paths differ but the running daemon's bundle still exists (multi-agent share)", async () => { + // Simulates: claude-code spawned the daemon; now codex connects. + // Both bundle files are present on disk → daemons are functionally + // identical → codex must NOT kill claude-code's daemon. Recycling + // here would cause endless thrash between the agents. + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + + // Two real daemon-binary paths on disk (just empty files; we only + // need existsSync(...) to return true). + const claudePath = join(dir, "claude-code-daemon.js"); + const codexPath = join(dir, "codex-daemon.js"); + writeFileSync(claudePath, ""); + writeFileSync(codexPath, ""); + + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") { + return { id: req.id, daemonPath: claudePath, pid: 99999, protocolVersion: 1 }; + } + if (req.op === "embed") return { id: req.id, embedding: [0.5] }; + return { id: req.id, error: "unknown" }; + }); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: false, + daemonEntry: codexPath, + }); + const vec = await client.embed("hi"); + expect(vec).toEqual([0.5]); // happily reused claude-code's daemon + expect(existsSync(sockPath)).toBe(true); // socket NOT recycled + }); + + it("only verifies hello once per EmbedClient instance (subsequent calls skip)", async () => { + const dir = makeTmpDir(); + let helloCount = 0; + let embedCount = 0; + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") { + helloCount += 1; + return { id: req.id, daemonPath: "/match", pid: 1, protocolVersion: 1 }; + } + if (req.op === "embed") { + embedCount += 1; + return { id: req.id, embedding: [0.1] }; + } + return { id: req.id, error: "unknown" }; + }); + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: false, + daemonEntry: "/match", + }); + await client.embed("a"); + await client.embed("b"); + await client.embed("c"); + expect(helloCount).toBe(1); + expect(embedCount).toBe(3); + }); + + it("recycled probe + autoSpawn=true triggers spawn attempt and retries embed via waitForDaemonReady", async () => { + // Drives the retry path: verifyDaemonOnce returns "recycled", the + // outer wrapper calls trySpawnDaemon() + waitForDaemonReady(), then + // calls embedAttempt() a second time. With no real daemon spawn + // available (no daemonEntry on disk), the retry's connectOnce() will + // fail and the wrapper returns null. The point of this test is to + // exercise the waitForDaemonReady() poll-deadline branch and the + // outer retry composition, not to assert a successful round-trip. + const dir = makeTmpDir(); + let helloCount = 0; + let embedCount = 0; + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") { helloCount += 1; return { id: req.id, ready: true } as any; } + embedCount += 1; + return { id: req.id, embedding: [0.1] }; + }); + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 200, + // autoSpawn ON exercises the new retry path… + autoSpawn: true, + // …but daemonEntry points at a non-existent file, so the + // trySpawnDaemon call inside the retry no-ops (existsSync(daemonEntry) + // check inside trySpawnDaemon short-circuits) and waitForDaemonReady + // runs out its deadline without seeing a new sock file. + daemonEntry: "/nonexistent-bundle-path/embed-daemon.js", + // Keep the spawn-wait short so the test doesn't sit on the deadline. + spawnWaitMs: 100, + }); + const v = await client.embed("retry-with-autospawn"); + expect(v).toBeNull(); + // Probe ran once on the stale socket (no daemonPath → recycle). + // Verify embed wasn't sent on the dead connection either time. + expect(embedCount).toBe(0); + expect(helloCount).toBe(1); + }); + + it("recycled probe + autoSpawn=false returns null cleanly (no hang on dead socket)", async () => { + // Regression for CodeRabbit #9: previously `embed()` proceeded with + // its embed request on the SAME socket after `verifyDaemonOnce()` + // had SIGTERMed the daemon — the request silently dropped onto a + // dead connection. The fix splits `embed()` into an attempt that + // returns the sentinel "recycled" when the verify step killed the + // daemon, so the outer call can spawn fresh + retry or (with + // autoSpawn off) bail to null instead of stalling. + const dir = makeTmpDir(); + let embedAttempts = 0; + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") { + // No daemonPath → triggers "older protocol" recycle branch. + return { id: req.id, ready: true } as any; + } + embedAttempts += 1; + // If the bug returned, the test would see this count tick to 1 on + // the now-dead socket — we want it to stay 0. + return { id: req.id, embedding: [0.9, 0.9, 0.9] }; + }); + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: false, + daemonEntry: "/expected-but-not-this", + }); + const v = await client.embed("recycle-me"); + expect(v).toBeNull(); + // The whole point: we did NOT send an embed on the recycled socket. + expect(embedAttempts).toBe(0); + }); + + it("does NOT mark helloVerified after a probe failure — next reconnect retries verification", async () => { + // Regression for CodeRabbit #5: previously the client set + // `helloVerified = true` *before* awaiting the probe response, so a + // genuinely transient probe failure (socket dies before responding) + // on the first connect permanently disabled verification for every + // later embed call on the same EmbedClient. + // + // Simulate a transient failure by destroying the socket on the FIRST + // hello (no response written). That triggers the catch branch in + // verifyDaemonOnce (vs. an error-shaped JSON response, which routes + // through the recycle path instead). + const dir = makeTmpDir(); + const uid = String(process.getuid?.() ?? "test"); + const sockPath = join(dir, `hivemind-embed-${uid}.sock`); + let helloAttempts = 0; + let embedAttempts = 0; + const srv = createServer((sock: Socket) => { + let buf = ""; + sock.setEncoding("utf-8"); + sock.on("data", (chunk: string) => { + buf += chunk; + let nl: number; + while ((nl = buf.indexOf("\n")) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + if (!line) continue; + const req = JSON.parse(line) as DaemonRequest; + if (req.op === "hello") { + helloAttempts += 1; + if (helloAttempts === 1) { + // Drop the connection without responding — sendAndWait + // resolves with an error from the socket close event. + sock.destroy(); + return; + } + sock.write(JSON.stringify({ id: req.id, daemonPath: "/match", pid: 42, protocolVersion: 1 }) + "\n"); + } else if (req.op === "embed") { + embedAttempts += 1; + sock.write(JSON.stringify({ id: req.id, embedding: [0.5, 0.6] }) + "\n"); + } + } + }); + sock.on("error", () => { /* */ }); + }); + servers.push(srv); + await new Promise((resolve) => srv.listen(sockPath, resolve)); + + const client = new EmbedClient({ + socketDir: dir, + timeoutMs: 500, + autoSpawn: false, + daemonEntry: "/match", + }); + await client.embed("first"); + await client.embed("second"); + await client.embed("third"); + // Fix: probe is retried on the second connect because the first + // attempt was inconclusive (catch branch). After the second connect, + // the response is compatible so the flag is set and the third call + // skips the probe. + // + // embedAttempts is 2 (not 3) because the first connect's socket gets + // destroyed by the server during the failed probe, so the first + // embed() returns null without ever reaching the daemon's embed + // handler. The CORE invariant under test is that helloAttempts === 2 + // — proving the second connect did re-run verification. + expect(helloAttempts).toBe(2); + expect(embedAttempts).toBe(2); + }); + + it("does not send hello when daemonEntry is empty (nothing to compare against)", async () => { + // Force the resolver to land on a falsy daemonEntry by setting the env + // override to empty — env wins over the SHARED_DAEMON_PATH fallback, + // and "" is falsy, so verifyDaemonOnce returns early. + const prev = process.env.HIVEMIND_EMBED_DAEMON; + process.env.HIVEMIND_EMBED_DAEMON = ""; + try { + const dir = makeTmpDir(); + let helloCount = 0; + await startFakeDaemon(dir, (req) => { + if (req.op === "hello") { helloCount += 1; return { id: req.id, daemonPath: "/x", pid: 1, protocolVersion: 1 }; } + return { id: req.id, embedding: [0.1] }; + }); + const client = new EmbedClient({ socketDir: dir, timeoutMs: 500, autoSpawn: false }); + await client.embed("hi"); + expect(helloCount).toBe(0); + } finally { + if (prev === undefined) delete process.env.HIVEMIND_EMBED_DAEMON; + else process.env.HIVEMIND_EMBED_DAEMON = prev; + } + }); +}); diff --git a/tests/claude-code/embeddings-disable.test.ts b/tests/claude-code/embeddings-disable.test.ts index fb17d317..fe3d18a3 100644 --- a/tests/claude-code/embeddings-disable.test.ts +++ b/tests/claude-code/embeddings-disable.test.ts @@ -3,69 +3,48 @@ import { embeddingsDisabled, embeddingsStatus, _setResolveForTesting, + _setEnabledReaderForTesting, _resetForTesting, } from "../../src/embeddings/disable.js"; -const originalEnv = process.env.HIVEMIND_EMBEDDINGS; - -function restoreEnv(): void { - if (originalEnv === undefined) delete process.env.HIVEMIND_EMBEDDINGS; - else process.env.HIVEMIND_EMBEDDINGS = originalEnv; -} - -describe("embeddingsStatus / embeddingsDisabled — env branch", () => { - beforeEach(() => { - delete process.env.HIVEMIND_EMBEDDINGS; - _resetForTesting(); - }); +beforeEach(() => { + _resetForTesting(); + // Default: user has embeddings enabled. Individual tests flip this. + _setEnabledReaderForTesting(() => true); +}); - afterEach(() => { - restoreEnv(); - _resetForTesting(); - }); +afterEach(() => { + _resetForTesting(); +}); - it("is 'enabled' when env is unset and the package resolves", () => { +describe("embeddingsStatus / embeddingsDisabled — user-config branch", () => { + it("is 'enabled' when config says enabled and the package resolves", () => { + _setEnabledReaderForTesting(() => true); _setResolveForTesting(() => { /* no throw → installed */ }); expect(embeddingsStatus()).toBe("enabled"); expect(embeddingsDisabled()).toBe(false); }); - it("is 'env-disabled' when HIVEMIND_EMBEDDINGS is exactly 'false'", () => { - process.env.HIVEMIND_EMBEDDINGS = "false"; + it("is 'user-disabled' when config says embeddings.enabled === false", () => { + _setEnabledReaderForTesting(() => false); // Resolver should never be consulted — set it to throw so this fails - // loudly if the env-check is ever removed. + // loudly if the gate is ever removed. _setResolveForTesting(() => { throw new Error("must not be called"); }); - expect(embeddingsStatus()).toBe("env-disabled"); + expect(embeddingsStatus()).toBe("user-disabled"); expect(embeddingsDisabled()).toBe(true); }); - it("env-disabled wins over a missing package (single, definitive signal)", () => { - process.env.HIVEMIND_EMBEDDINGS = "false"; + it("user-disabled wins over missing transformers (single, definitive signal)", () => { + _setEnabledReaderForTesting(() => false); _setResolveForTesting(() => { throw new Error("MODULE_NOT_FOUND"); }); - expect(embeddingsStatus()).toBe("env-disabled"); + expect(embeddingsStatus()).toBe("user-disabled"); expect(embeddingsDisabled()).toBe(true); }); - - it("stays 'enabled' for any non-'false' truthy env value (avoid surprise kills)", () => { - for (const value of ["0", "no", "true", "", "FALSE", "False"]) { - process.env.HIVEMIND_EMBEDDINGS = value; - _resetForTesting(); - _setResolveForTesting(() => { /* installed */ }); - expect(embeddingsStatus()).toBe("enabled"); - expect(embeddingsDisabled()).toBe(false); - } - }); }); describe("embeddingsStatus / embeddingsDisabled — transformers-presence branch", () => { beforeEach(() => { - delete process.env.HIVEMIND_EMBEDDINGS; - _resetForTesting(); - }); - - afterEach(() => { - restoreEnv(); - _resetForTesting(); + _setEnabledReaderForTesting(() => true); }); it("is 'enabled' when @huggingface/transformers resolves cleanly", () => { @@ -118,17 +97,20 @@ describe("embeddingsStatus / embeddingsDisabled — transformers-presence branch _setResolveForTesting(() => { throw new Error("simulated missing"); }); expect(embeddingsStatus()).toBe("no-transformers"); _resetForTesting(); + _setEnabledReaderForTesting(() => true); // Real resolver runs against this test process, which has the package // installed via the worktree's node_modules → comes back 'enabled'. expect(embeddingsStatus()).toBe("enabled"); }); - it("real default resolver finds @huggingface/transformers in this repo", () => { - // Smoke check: in the dev / CI environment the package IS installed, - // so the actual createRequire-based resolver succeeds. Guards against - // a regression in the resolution path itself (wrong base URL, wrong - // package name spelling, build-time vs runtime path drift, etc.). + it("real default resolver finds @huggingface/transformers via the shared-deps probe", () => { + // Smoke check: in the dev / CI environment the package IS installed + // (either at ~/.hivemind/embed-deps/ or in the worktree's node_modules + // via the bundle walk fallback). Guards against a regression in the + // resolver chain (wrong base URL, wrong package name, build-time vs + // runtime path drift, etc.). _resetForTesting(); + _setEnabledReaderForTesting(() => true); expect(embeddingsStatus()).toBe("enabled"); }); }); diff --git a/tests/claude-code/embeddings-nomic.test.ts b/tests/claude-code/embeddings-nomic.test.ts index aa4300cc..8a16cda1 100644 --- a/tests/claude-code/embeddings-nomic.test.ts +++ b/tests/claude-code/embeddings-nomic.test.ts @@ -1,9 +1,23 @@ -import { describe, it, expect, vi } from "vitest"; -import { NomicEmbedder } from "../../src/embeddings/nomic.js"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + NomicEmbedder, + defaultImportTransformers, + _setTransformersImporterForTesting, + _resetTransformersImporterForTesting, + _normalizeTransformersModule, + _importFromBareSpecifier, + _importFromCanonicalSharedDeps, +} from "../../src/embeddings/nomic.js"; // Mock the heavy transformers import so these tests don't pull in -// onnxruntime-node or download any model weights. `load()` uses -// `await import("@huggingface/transformers")` — vi.mock intercepts. +// onnxruntime-node or download any model weights. `load()` resolves +// transformers via an injected importer (default goes through the canonical +// shared-deps walk + bare fallback); we inject one that returns this mock so +// the test env on developer machines doesn't accidentally load the real +// installed copy at ~/.hivemind/embed-deps/. vi.mock("@huggingface/transformers", () => { const embed = vi.fn((input: string | string[], _opts: Record) => { const texts = Array.isArray(input) ? input : [input]; @@ -15,11 +29,25 @@ vi.mock("@huggingface/transformers", () => { return Promise.resolve({ data: out }); }); return { + // Explicit `default: undefined` so that `_normalizeTransformersModule`'s + // `m.default && ...` probe doesn't trip the vitest auto-mock proxy, + // which throws on access of any export not declared in this factory. + default: undefined, env: { allowLocalModels: false, useFSCache: false }, pipeline: vi.fn(async () => embed), }; }); +beforeEach(() => { + // Route the embedder's loader through the vi.mock-intercepted bare specifier + // instead of the real canonical-shared-deps resolver. + _setTransformersImporterForTesting(() => import("@huggingface/transformers") as any); +}); + +afterEach(() => { + _resetTransformersImporterForTesting(); +}); + describe("NomicEmbedder", () => { it("loads lazily and reuses the pipeline across calls", async () => { const e = new NomicEmbedder({ dims: 4 }); @@ -87,7 +115,6 @@ describe("NomicEmbedder", () => { // Reach through the private helper via a custom mock that returns zeros. const mod: any = await import("@huggingface/transformers"); const origPipeline = mod.pipeline; - const zeroPipe = vi.fn(async () => [0, 0, 0, 0]); const wrapped = vi.fn(() => Promise.resolve(() => Promise.resolve({ data: [0, 0, 0, 0] }))); (mod as any).pipeline = wrapped; try { @@ -147,3 +174,148 @@ describe("NomicEmbedder", () => { expect(lastCall).toEqual(["search_query: hi"]); }); }); + +describe("defaultImportTransformers resolution", () => { + // These tests bypass the beforeEach DI hook above and call + // defaultImportTransformers() directly with stub resolvers, exercising the + // canonical → bare fallback chain and the actionable error path. + + it("uses the canonical shared-deps resolver first when reachable", async () => { + const canonical = vi.fn().mockResolvedValue({ marker: "canonical" }); + const bare = vi.fn().mockResolvedValue({ marker: "bare" }); + const mod = await defaultImportTransformers(canonical as any, bare as any); + expect((mod as any).marker).toBe("canonical"); + expect(canonical).toHaveBeenCalledTimes(1); + expect(bare).not.toHaveBeenCalled(); + }); + + it("falls back to the bare specifier when canonical throws", async () => { + const canonical = vi.fn().mockRejectedValue(new Error("ENOENT shared-deps")); + const bare = vi.fn().mockResolvedValue({ marker: "bare" }); + const mod = await defaultImportTransformers(canonical as any, bare as any); + expect((mod as any).marker).toBe("bare"); + expect(canonical).toHaveBeenCalledTimes(1); + expect(bare).toHaveBeenCalledTimes(1); + }); + + it("throws an actionable error referencing `hivemind embeddings install` when both fail", async () => { + const canonical = vi.fn().mockRejectedValue(new Error("ENOENT shared-deps")); + const bare = vi.fn().mockRejectedValue(new Error("Cannot find package '@huggingface/transformers'")); + await expect(defaultImportTransformers(canonical as any, bare as any)).rejects.toThrow( + /hivemind embeddings install/, + ); + }); + + it("preserves both underlying error messages in the thrown error for diagnostics", async () => { + const canonical = vi.fn().mockRejectedValue(new Error("canonical-error-marker")); + const bare = vi.fn().mockRejectedValue(new Error("bare-error-marker")); + await expect(defaultImportTransformers(canonical as any, bare as any)).rejects.toThrow( + /canonical-error-marker.*bare-error-marker/, + ); + }); + + it("wraps non-Error rejections in the combined error message", async () => { + // The catch branches normalize string/object rejections via the `instanceof Error` + // check; this asserts that the String(err) fallback path is exercised. + const canonical = vi.fn().mockRejectedValue("plain-string-canonical"); + const bare = vi.fn().mockRejectedValue({ toString: () => "plain-object-bare" }); + await expect(defaultImportTransformers(canonical as any, bare as any)).rejects.toThrow( + /plain-string-canonical.*plain-object-bare/, + ); + }); +}); + +describe("_normalizeTransformersModule (CJS-default-unwrap helper)", () => { + // The CJS bundle of @huggingface/transformers v3 lives at + // `dist/transformers.node.cjs`; `await import()` wraps the CJS + // exports under `.default`. The ESM .mjs build exposes names at top level. + // The normalizer must accept both shapes and return one with top-level + // `pipeline` / `env`. + + it("unwraps the .default key when CJS-style module has `default.pipeline`", () => { + const inner = { pipeline: () => "x", env: { allowLocalModels: false }, marker: "inner" }; + const wrapped = { default: inner }; + const out = _normalizeTransformersModule(wrapped) as any; + expect(out.marker).toBe("inner"); + expect(out.pipeline).toBe(inner.pipeline); + expect(out.env).toBe(inner.env); + }); + + it("returns the module as-is when ESM-style exposes `pipeline` at the top level", () => { + const top = { pipeline: () => "y", env: { allowLocalModels: false }, marker: "top" }; + const out = _normalizeTransformersModule(top) as any; + expect(out.marker).toBe("top"); + }); + + it("returns the module as-is when `.default` exists but doesn't carry `pipeline`", () => { + // ESM modules without a default export still get a `.default` namespace key + // pointing at the module record itself when bundled by some tools — make + // sure we don't accidentally unwrap into something that lacks `pipeline`. + const mod = { pipeline: () => "z", default: { someOtherKey: 1 }, marker: "top" }; + const out = _normalizeTransformersModule(mod) as any; + expect(out.marker).toBe("top"); + expect(out.pipeline).toBe(mod.pipeline); + }); + + it("returns the module as-is when `.default` is falsy (null/undefined)", () => { + const mod = { pipeline: () => "w", default: null, marker: "top" }; + const out = _normalizeTransformersModule(mod) as any; + expect(out.marker).toBe("top"); + }); +}); + +describe("_importFromBareSpecifier", () => { + // The bare-specifier importer relies on whatever the Node resolver picks + // up; in this test file `vi.mock("@huggingface/transformers")` (at the + // top) intercepts that resolution, so the importer should return the + // mocked module after normalization. + it("returns the mocked transformers module after normalization", async () => { + const mod = await _importFromBareSpecifier(); + expect(mod).toBeDefined(); + expect(typeof (mod as any).pipeline).toBe("function"); + expect((mod as any).env).toMatchObject({ allowLocalModels: false }); + }); +}); + +describe("_importFromCanonicalSharedDeps", () => { + // Build a real on-disk fixture that looks like a hivemind-installed + // shared-deps directory, then point the importer at it. Avoids any + // mocking gymnastics around `createRequire` / dynamic `import()`. + + let sharedDir: string; + + beforeEach(() => { + sharedDir = mkdtempSync(join(tmpdir(), "nomic-shared-deps-")); + const pkgDir = join(sharedDir, "node_modules", "@huggingface", "transformers"); + mkdirSync(pkgDir, { recursive: true }); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ name: "@huggingface/transformers", main: "index.cjs" }), + ); + // Minimal CJS shim exposing the same surface the daemon touches. + writeFileSync( + join(pkgDir, "index.cjs"), + "module.exports = { pipeline: function () { return 'fixture'; }, env: { allowLocalModels: false } };", + ); + }); + + afterEach(() => { + rmSync(sharedDir, { recursive: true, force: true }); + }); + + it("resolves transformers from the canonical shared-deps dir and normalizes the result", async () => { + const mod = await _importFromCanonicalSharedDeps(sharedDir); + expect(typeof (mod as any).pipeline).toBe("function"); + expect((mod as any).pipeline()).toBe("fixture"); + expect((mod as any).env.allowLocalModels).toBe(false); + }); + + it("propagates the underlying require error when transformers is missing under the base", async () => { + const emptyDir = mkdtempSync(join(tmpdir(), "nomic-empty-")); + try { + await expect(_importFromCanonicalSharedDeps(emptyDir)).rejects.toThrow(/transformers/); + } finally { + rmSync(emptyDir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/claude-code/embeddings-self-heal.test.ts b/tests/claude-code/embeddings-self-heal.test.ts new file mode 100644 index 00000000..3bfe3913 --- /dev/null +++ b/tests/claude-code/embeddings-self-heal.test.ts @@ -0,0 +1,141 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readlinkSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ensurePluginNodeModulesLink } from "../../src/embeddings/self-heal.js"; + +let root: string; +let pluginDir: string; +let bundleDir: string; +let sharedNodeModules: string; +let link: string; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "hvm-self-heal-")); + pluginDir = join(root, "plugin-v1"); + bundleDir = join(pluginDir, "bundle"); + sharedNodeModules = join(root, ".hivemind", "embed-deps", "node_modules"); + link = join(pluginDir, "node_modules"); + mkdirSync(bundleDir, { recursive: true }); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("ensurePluginNodeModulesLink", () => { + it("creates the symlink when shared deps exist and plugin has no node_modules", () => { + mkdirSync(sharedNodeModules, { recursive: true }); + const r = ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + expect(r.kind).toBe("linked"); + expect(lstatSync(link).isSymbolicLink()).toBe(true); + expect(readlinkSync(link)).toBe(sharedNodeModules); + }); + + it("returns shared-deps-missing (no-op) when the target node_modules does not exist", () => { + // Don't create sharedNodeModules. + const r = ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + expect(r.kind).toBe("shared-deps-missing"); + expect(existsSync(link)).toBe(false); + }); + + it("is idempotent: re-call when link already points at target returns already-linked", () => { + mkdirSync(sharedNodeModules, { recursive: true }); + ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + const r = ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + expect(r.kind).toBe("already-linked"); + expect(readlinkSync(link)).toBe(sharedNodeModules); + }); + + it("does NOT clobber an existing real node_modules directory", () => { + mkdirSync(sharedNodeModules, { recursive: true }); + mkdirSync(link, { recursive: true }); + writeFileSync(join(link, "marker.txt"), "do not delete"); + const r = ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + expect(r.kind).toBe("plugin-owns-node-modules"); + // Real dir still there with its marker. + expect(existsSync(join(link, "marker.txt"))).toBe(true); + expect(lstatSync(link).isSymbolicLink()).toBe(false); + }); + + it("does NOT clobber a symlink that points somewhere else (real target)", () => { + mkdirSync(sharedNodeModules, { recursive: true }); + const elsewhere = join(root, "elsewhere-nm"); + mkdirSync(elsewhere); + symlinkSync(elsewhere, link); + const r = ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + expect(r.kind).toBe("linked-elsewhere"); + if (r.kind === "linked-elsewhere") { + expect(r.existingTarget).toBe(elsewhere); + } + // Pre-existing symlink preserved. + expect(readlinkSync(link)).toBe(elsewhere); + }); + + it("repairs a DANGLING symlink in the SAME call (no two-pass recovery)", () => { + // Regression for CodeRabbit #3/#13: previously this branch removed + // the stale link and returned, leaving the current hook run without + // a working `node_modules` link until a second invocation. Now the + // helper removes the dangling link AND immediately re-creates it + // pointing at the correct shared target, so a single call is enough. + mkdirSync(sharedNodeModules, { recursive: true }); + const danglingTarget = join(root, "gone"); + mkdirSync(danglingTarget); + symlinkSync(danglingTarget, link); + // Now delete the target — link is dangling. + rmSync(danglingTarget, { recursive: true, force: true }); + + const r = ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + // Diagnostic preserved: the result kind still reports we removed a + // stale link (with the original dangling target) so callers can log + // the recovery. But the link is now alive and points at shared. + expect(r.kind).toBe("stale-link-removed"); + expect(existsSync(link)).toBe(true); + expect(readlinkSync(link)).toBe(sharedNodeModules); + }); + + it("computes pluginDir as dirname(bundleDir) (mirrors the agent layout)", () => { + // If the helper miscomputed pluginDir (e.g. used bundleDir directly), + // the symlink would land at /node_modules. Assert it lands at + // /node_modules. + mkdirSync(sharedNodeModules, { recursive: true }); + ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + expect(existsSync(join(pluginDir, "node_modules"))).toBe(true); + expect(existsSync(join(bundleDir, "node_modules"))).toBe(false); + }); + + it("creates the parent directory if missing (defensive against unusual install layouts)", () => { + mkdirSync(sharedNodeModules, { recursive: true }); + // Remove pluginDir entirely (no bundle/, nothing). + rmSync(pluginDir, { recursive: true, force: true }); + mkdirSync(bundleDir, { recursive: true }); // recreate bundle (parent comes back) + rmSync(pluginDir, { recursive: true, force: true }); + // Now pluginDir is gone again; the helper should mkdir it before symlinking. + const r = ensurePluginNodeModulesLink({ bundleDir, sharedNodeModules }); + expect(r.kind).toBe("linked"); + expect(readlinkSync(link)).toBe(sharedNodeModules); + }); + + it("refuses to act when bundleDir basename is not 'bundle' (test-tree / source-tree safety)", () => { + mkdirSync(sharedNodeModules, { recursive: true }); + // Mimic the source-tree path where capture.ts lives in `src/hooks/`, + // not in a `bundle/` dir. Without this gate, importing the capture + // module from tests would silently symlink src/node_modules to the + // user's real shared deps. + const wrongDir = join(root, "wrong-layout", "hooks"); + mkdirSync(wrongDir, { recursive: true }); + const r = ensurePluginNodeModulesLink({ bundleDir: wrongDir, sharedNodeModules }); + expect(r.kind).toBe("not-bundle-layout"); + // No symlink created in the bogus parent. + expect(existsSync(join(root, "wrong-layout", "node_modules"))).toBe(false); + }); +}); diff --git a/tests/claude-code/notifications-coverage.test.ts b/tests/claude-code/notifications-coverage.test.ts index 17d3cdf6..20413d82 100644 --- a/tests/claude-code/notifications-coverage.test.ts +++ b/tests/claude-code/notifications-coverage.test.ts @@ -317,7 +317,7 @@ describe("drainSessionStart — queue drained even when nothing fresh", () => { }); const n: Notification = { id: "x", title: "T", body: "B", dedupKey: { v: 1 } }; - enqueueNotification(n); + await enqueueNotification(n); // First drain: fires, marks as shown await drainSessionStart({ agent: "claude-code", creds: null }); @@ -325,7 +325,7 @@ describe("drainSessionStart — queue drained even when nothing fresh", () => { expect(readQueue().queue.length).toBe(0); // Re-enqueue same notification with same dedupKey → fresh.length === 0 - enqueueNotification(n); + await enqueueNotification(n); expect(readQueue().queue.length).toBe(1); writes.length = 0; @@ -348,7 +348,7 @@ describe("drainSessionStart — queue drained even when nothing fresh", () => { vi.spyOn(stateModule, "tryClaim").mockReturnValue(false); const n: Notification = { id: "y", title: "T2", body: "B2", dedupKey: { v: 99 } }; - enqueueNotification(n); + await enqueueNotification(n); await drainSessionStart({ agent: "claude-code", creds: null }); expect(writes.length).toBe(0); expect(readQueue().queue.length).toBe(0); // drained anyway diff --git a/tests/claude-code/notifications-queue-lock.test.ts b/tests/claude-code/notifications-queue-lock.test.ts new file mode 100644 index 00000000..a2d1f9af --- /dev/null +++ b/tests/claude-code/notifications-queue-lock.test.ts @@ -0,0 +1,192 @@ +/** + * Branch coverage for src/notifications/queue.ts — focused on the new + * `withQueueLock` paths that the cross-process safety fix introduced. + * + * Tests overlap with notifications.test.ts on the happy path (subprocess + * pool); this file isolates the synthetic branches (stale-lock reclaim, + * give-up after MAX retries, write-outside-home guard, malformed JSON, + * unknown-error rethrow) so vitest can hit them deterministically + * without needing the 6 s real-time wait the production constants + * imply. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + closeSync, + existsSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + enqueueNotification, + queuePath, + readQueue, + writeQueue, + _isQueuePathInsideHome, + _setLockTimingForTesting, + _resetLockTimingForTesting, +} from "../../src/notifications/queue.js"; + +let tmpHome = ""; +let origHome: string | undefined; + +beforeEach(() => { + tmpHome = mkdtempSync(join(tmpdir(), "queue-lock-test-")); + origHome = process.env.HOME; + process.env.HOME = tmpHome; + // Short retries + short stale window so the synthetic branches resolve + // in milliseconds, not the production 6 s. + _setLockTimingForTesting({ retryMax: 5, retryBaseMs: 1, staleMs: 50 }); +}); + +afterEach(() => { + _resetLockTimingForTesting(); + if (origHome === undefined) delete process.env.HOME; else process.env.HOME = origHome; + rmSync(tmpHome, { recursive: true, force: true }); +}); + +describe("withQueueLock — stale-lock reclaim", () => { + it("reclaims a lock file older than LOCK_STALE_MS and proceeds with the enqueue", async () => { + mkdirSync(join(tmpHome, ".deeplake"), { recursive: true }); + const lockFile = `${queuePath()}.lock`; + // Create the lock file and age it past the (test-shrunk) stale window. + const fd = openSync(lockFile, "wx", 0o600); + closeSync(fd); + const ancient = (Date.now() - 5000) / 1000; + utimesSync(lockFile, ancient, ancient); + + await enqueueNotification({ + id: "test-stale-reclaim", + title: "T", body: "B", + dedupKey: { tag: "stale" }, + }); + expect(readQueue().queue.length).toBe(1); + expect(readQueue().queue[0].id).toBe("test-stale-reclaim"); + // The reclaim-then-release sequence leaves no lock behind. + expect(existsSync(lockFile)).toBe(false); + }); +}); + +describe("withQueueLock — give up after MAX retries (degrades to unlocked)", () => { + it("when the lock can't be acquired, still runs fn and persists the enqueue", async () => { + mkdirSync(join(tmpHome, ".deeplake"), { recursive: true }); + const lockFile = `${queuePath()}.lock`; + // Fresh, recently-mtime'd lock that the reclaim branch won't touch. + const fd = openSync(lockFile, "wx", 0o600); + closeSync(fd); + // mtime is "now" → not stale → every attempt hits EEXIST → exhausts retries. + + await enqueueNotification({ + id: "test-giveup", + title: "T", body: "B", + dedupKey: { tag: "giveup" }, + }); + // The unlocked fallback still wrote the queue. + expect(readQueue().queue.length).toBe(1); + expect(readQueue().queue[0].id).toBe("test-giveup"); + // The lock file we held is still there (we didn't own it, so we + // didn't unlink it on release). + expect(existsSync(lockFile)).toBe(true); + }); +}); + +describe("readQueue — malformed JSON branch", () => { + it("returns empty queue when the on-disk file is not valid JSON", () => { + mkdirSync(join(tmpHome, ".deeplake"), { recursive: true }); + writeFileSync(queuePath(), "not-json-at-all", "utf-8"); + expect(readQueue()).toEqual({ queue: [] }); + }); + + it("returns empty queue when the JSON shape is wrong (missing `queue` array)", () => { + mkdirSync(join(tmpHome, ".deeplake"), { recursive: true }); + writeFileSync(queuePath(), JSON.stringify({ wrong: "shape" }), "utf-8"); + expect(readQueue()).toEqual({ queue: [] }); + }); +}); + +describe("enqueueNotification — sameDedupKey branches", () => { + it("skips append when an equivalent (id, dedupKey) is already queued (same-process dedup)", async () => { + const n = { + id: "embed-deps-missing", + title: "T", + body: "B", + dedupKey: { reason: "transformers-missing", detail: "exact" }, + }; + await enqueueNotification(n); + await enqueueNotification(n); + await enqueueNotification(n); + expect(readQueue().queue.length).toBe(1); + }); + + it("appends a second entry when id differs but dedupKey matches (id discriminates)", async () => { + // Hits the `a.id !== b.id` early-return inside sameDedupKey. + await enqueueNotification({ + id: "id-A", title: "T", body: "B", + dedupKey: { v: 1 }, + }); + await enqueueNotification({ + id: "id-B", title: "T", body: "B", + dedupKey: { v: 1 }, + }); + expect(readQueue().queue.length).toBe(2); + expect(readQueue().queue.map(n => n.id).sort()).toEqual(["id-A", "id-B"]); + }); + + it("appends a second entry when id matches but dedupKey differs (key discriminates)", async () => { + // Hits the JSON.stringify comparison returning `false`. + await enqueueNotification({ id: "shared", title: "T", body: "B", dedupKey: { v: 1 } }); + await enqueueNotification({ id: "shared", title: "T", body: "B", dedupKey: { v: 2 } }); + expect(readQueue().queue.length).toBe(2); + }); +}); + +describe("_isQueuePathInsideHome — outside-HOME guard", () => { + // Defense-in-depth invariant: the guard inside writeQueue refuses to + // touch the filesystem if the resolved queue path would escape $HOME. + // The actual `writeQueue` call can only hit this branch via a homedir() + // race (ESM doesn't let us spy on os.homedir reliably), so we test the + // extracted predicate directly. + + it("returns true when the path is a direct child of home", () => { + expect(_isQueuePathInsideHome("/home/u/.deeplake/notifications-queue.json", "/home/u")).toBe(true); + }); + + it("returns true when the path equals home itself", () => { + expect(_isQueuePathInsideHome("/home/u", "/home/u")).toBe(true); + }); + + it("returns true when home has a trailing slash (resolved normalizes)", () => { + expect(_isQueuePathInsideHome("/home/u/.deeplake/notifications-queue.json", "/home/u/")).toBe(true); + }); + + it("returns FALSE when the path is in a sibling directory of home", () => { + expect(_isQueuePathInsideHome("/etc/.deeplake/notifications-queue.json", "/home/u")).toBe(false); + }); + + it("returns FALSE on a prefix-match attack (path starts with home substring but differs)", () => { + // The naive `startsWith(home)` would let `/home/userspace/...` slip + // through when home is `/home/user`. Adding the explicit `home + "/"` + // separator (which the helper does internally) blocks it. + expect(_isQueuePathInsideHome("/home/userspace/.deeplake/notifications-queue.json", "/home/user")).toBe(false); + }); + + it("returns FALSE for a relative path that resolves outside home", () => { + // resolve("../../etc/passwd") relative to cwd lands somewhere far + // from a tmp home, so the guard rejects. + const outside = "/etc/.deeplake/notifications-queue.json"; + const home = mkdtempSync(join(tmpdir(), "queue-outside-guard-")); + try { + expect(_isQueuePathInsideHome(outside, home)).toBe(false); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/claude-code/notifications.test.ts b/tests/claude-code/notifications.test.ts index 769fc71c..d21fa660 100644 --- a/tests/claude-code/notifications.test.ts +++ b/tests/claude-code/notifications.test.ts @@ -284,6 +284,80 @@ describe("drainSessionStart with welcome rule registered", () => { // queue (push-based notifications) // --------------------------------------------------------------------------- +describe("enqueueNotification cross-process safety", () => { + // Regression for CodeRabbit #4: previously `enqueueNotification` did + // read-modify-write on the queue JSON without any cross-process lock, + // so two concurrent producers would race and the later `rename(2)` + // would clobber the earlier one's append. Spawn N subprocesses that + // each enqueue one notification and assert the final queue length + // equals N — without the lock, the count would be < N. + const modPath = new URL("../../src/notifications/queue.ts", import.meta.url).pathname; + + it("cross-process producers with identical (id, dedupKey) collapse to one queue entry", async () => { + // Regression for CodeRabbit #8/#12: previously the dedup gate + // (`_signalledMissingDeps`) lived in-process, so every fresh hook + // process would re-enqueue the same `embed-deps-missing` warning + // until the next drain. Two subprocesses with identical + // (id, dedupKey) must now produce exactly one entry in the queue. + const code = + `import("${modPath}").then(async m => { ` + + ` await m.enqueueNotification({ ` + + ` id: "embed-deps-missing", ` + + ` title: "T", body: "B", ` + + ` dedupKey: { reason: "transformers-missing", detail: "same" } ` + + ` }); ` + + ` process.stdout.write("ok"); ` + + `});`; + for (let i = 0; i < 3; i++) { + const r = spawnSync("npx", ["tsx", "-e", code], { + env: { ...process.env, HOME: TEMP_HOME }, + encoding: "utf-8", + timeout: 30_000, + }); + expect(r.status, `producer ${i} stderr=${(r.stderr || "").slice(0, 300)}`).toBe(0); + } + const q = readQueue().queue; + expect(q.length).toBe(1); + expect(q[0].id).toBe("embed-deps-missing"); + }, 60_000); + + it("N parallel producers each append exactly once (no lost writes)", async () => { + const N = 12; + // Each subprocess imports the queue module and enqueues a uniquely- + // identified notification. They all share the same $HOME (tmp dir + // from outer beforeEach) so they target the same queue file. + const code = + `import("${modPath}").then(async m => { ` + + ` const idx = process.env.PRODUCER_IDX; ` + + ` await m.enqueueNotification({ id: "test-cross-proc", title: "T" + idx, body: "B" + idx, dedupKey: { idx } }); ` + + ` process.stdout.write("ok"); ` + + `});`; + + const runs = Array.from({ length: N }, (_, i) => + new Promise((resolve, reject) => { + const r = spawnSync("npx", ["tsx", "-e", code], { + env: { ...process.env, HOME: TEMP_HOME, PRODUCER_IDX: String(i) }, + encoding: "utf-8", + timeout: 30_000, + }); + if (r.status !== 0) { + reject(new Error(`producer ${i} exit=${r.status} stderr=${(r.stderr || "").slice(0, 300)}`)); + } else { + resolve(); + } + }), + ); + await Promise.all(runs); + + const finalQueue = readQueue().queue; + expect(finalQueue.length).toBe(N); + // Every producer index 0..N-1 must appear exactly once. + const idxs = finalQueue.map(n => (n.dedupKey as { idx: string }).idx).sort(); + const expected = Array.from({ length: N }, (_, i) => String(i)).sort(); + expect(idxs).toEqual(expected); + }, 60_000); +}); + describe("enqueueNotification + drainSessionStart", () => { let writes: string[] = []; @@ -300,7 +374,7 @@ describe("enqueueNotification + drainSessionStart", () => { }); it("delivers a queued notification on the next drain and clears the queue", async () => { - enqueueNotification({ + await enqueueNotification({ id: "summarization-due", title: "Time for a summary refresh", body: "You've captured 50 sessions since the last summary update.", @@ -324,21 +398,21 @@ describe("enqueueNotification + drainSessionStart", () => { body: "B", dedupKey: { v: 1 }, }; - enqueueNotification(n); + await enqueueNotification(n); await drainSessionStart({ agent: "claude-code", creds: null }); writes.length = 0; - enqueueNotification(n); + await enqueueNotification(n); await drainSessionStart({ agent: "claude-code", creds: null }); expect(writes.length).toBe(0); }); it("re-delivers a queue item with the same id but different dedupKey", async () => { - enqueueNotification({ id: "foo", title: "T", body: "B1", dedupKey: { v: 1 } }); + await enqueueNotification({ id: "foo", title: "T", body: "B1", dedupKey: { v: 1 } }); await drainSessionStart({ agent: "claude-code", creds: null }); writes.length = 0; - enqueueNotification({ id: "foo", title: "T", body: "B2", dedupKey: { v: 2 } }); + await enqueueNotification({ id: "foo", title: "T", body: "B2", dedupKey: { v: 2 } }); await drainSessionStart({ agent: "claude-code", creds: null }); expect(writes.length).toBe(1); expect(JSON.parse(writes[0]).hookSpecificOutput.additionalContext).toContain("B2"); diff --git a/tests/claude-code/session-start-setup-hook.test.ts b/tests/claude-code/session-start-setup-hook.test.ts index 44183722..15235afd 100644 --- a/tests/claude-code/session-start-setup-hook.test.ts +++ b/tests/claude-code/session-start-setup-hook.test.ts @@ -48,21 +48,38 @@ vi.mock("../../src/embeddings/client.js", () => ({ // (it's installed once into ~/.hivemind/embed-deps via `hivemind embeddings // install`). Without this mock the warmup branch is never reached and every // assertion below would land on the "skipped: no-transformers" log line. We -// still respect HIVEMIND_EMBEDDINGS=false so the master-flag branch test below -// behaves like production. +// still honor the EMBEDDINGS_DISABLED_FOR_TEST env so the master-flag branch +// test below behaves like the production user-disabled path. vi.mock("../../src/embeddings/disable.js", () => ({ - embeddingsDisabled: () => process.env.HIVEMIND_EMBEDDINGS === "false", + embeddingsDisabled: () => process.env.EMBEDDINGS_DISABLED_FOR_TEST === "1", embeddingsStatus: () => - process.env.HIVEMIND_EMBEDDINGS === "false" ? "disabled-by-env" : "enabled", + process.env.EMBEDDINGS_DISABLED_FOR_TEST === "1" ? "user-disabled" : "enabled", })); // We also need to control global.fetch for the GitHub version lookup. const originalFetch = global.fetch; const fetchMock = vi.fn(); +// Env keys touched by tests in this file. Recorded so afterEach() can +// restore them — without this, a test that sets e.g. +// EMBEDDINGS_DISABLED_FOR_TEST=1 would leak the disabled state into +// every later test in the same vitest worker (next runHook() call without +// that key wouldn't clear it, since runHook() only updates the keys +// passed in). That's exactly the order-dependence CodeRabbit flagged. +const TOUCHED_ENV_KEYS = [ + "HIVEMIND_WIKI_WORKER", + "HIVEMIND_EMBED_WARMUP", + "EMBEDDINGS_DISABLED_FOR_TEST", +] as const; +const _origEnv: Record = {}; + async function runHook(env: Record = {}): Promise { + for (const k of TOUCHED_ENV_KEYS) { + if (!(k in _origEnv)) _origEnv[k] = process.env[k]; + } delete process.env.HIVEMIND_WIKI_WORKER; for (const [k, v] of Object.entries(env)) { + if (!(k in _origEnv)) _origEnv[k] = process.env[k]; if (v === undefined) delete process.env[k]; else process.env[k] = v; } @@ -100,6 +117,14 @@ beforeEach(() => { afterEach(() => { vi.restoreAllMocks(); global.fetch = originalFetch; + // Restore env keys the tests may have mutated via runHook(), so later + // tests in this file (and other test files in the same worker) start + // from a clean process.env. + for (const [k, v] of Object.entries(_origEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + for (const k of Object.keys(_origEnv)) delete _origEnv[k]; }); describe("session-start-setup hook — guards", () => { @@ -227,11 +252,11 @@ describe("session-start-setup hook — embed daemon warmup", () => { ); }); - it("skips warmup when the master HIVEMIND_EMBEDDINGS=false flag is set", async () => { - await runHook({ HIVEMIND_EMBEDDINGS: "false" }); + it("skips warmup when the user has disabled embeddings in config", async () => { + await runHook({ EMBEDDINGS_DISABLED_FOR_TEST: "1" }); expect(embedWarmupMock).not.toHaveBeenCalled(); expect(debugLogMock).toHaveBeenCalledWith( - "embed daemon warmup skipped: HIVEMIND_EMBEDDINGS=false", + "embed daemon warmup skipped: embeddings disabled in ~/.deeplake/config.json (run `hivemind embeddings enable` to opt in)", ); }); }); diff --git a/tests/claude-code/user-config.test.ts b/tests/claude-code/user-config.test.ts new file mode 100644 index 00000000..5931852e --- /dev/null +++ b/tests/claude-code/user-config.test.ts @@ -0,0 +1,172 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + readUserConfig, + writeUserConfig, + getEmbeddingsEnabled, + setEmbeddingsEnabled, + _setConfigPathForTesting, + _resetUserConfigForTesting, +} from "../../src/user-config.js"; + +let dir: string; +let configPath: string; + +const originalEnv = process.env.HIVEMIND_EMBEDDINGS; + +function restoreEnv(): void { + if (originalEnv === undefined) delete process.env.HIVEMIND_EMBEDDINGS; + else process.env.HIVEMIND_EMBEDDINGS = originalEnv; +} + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "hivemind-user-config-")); + configPath = join(dir, "config.json"); + _setConfigPathForTesting(() => configPath); + delete process.env.HIVEMIND_EMBEDDINGS; +}); + +afterEach(() => { + _resetUserConfigForTesting(); + rmSync(dir, { recursive: true, force: true }); + restoreEnv(); +}); + +describe("readUserConfig", () => { + it("returns {} when the config file does not exist", () => { + expect(readUserConfig()).toEqual({}); + }); + + it("parses an existing valid config", () => { + writeFileSync(configPath, JSON.stringify({ embeddings: { enabled: true } }), "utf-8"); + _resetUserConfigForTesting(); + _setConfigPathForTesting(() => configPath); + expect(readUserConfig()).toEqual({ embeddings: { enabled: true } }); + }); + + it("returns {} on corrupt JSON without throwing (don't crash the hook)", () => { + writeFileSync(configPath, "{ not json", "utf-8"); + _resetUserConfigForTesting(); + _setConfigPathForTesting(() => configPath); + expect(readUserConfig()).toEqual({}); + }); + + it("returns {} when the root JSON value is not an object (e.g. an array)", () => { + writeFileSync(configPath, JSON.stringify([1, 2, 3]), "utf-8"); + _resetUserConfigForTesting(); + _setConfigPathForTesting(() => configPath); + expect(readUserConfig()).toEqual({}); + }); + + it("caches the parsed config across calls (single file read per process)", () => { + writeFileSync(configPath, JSON.stringify({ embeddings: { enabled: false } }), "utf-8"); + _resetUserConfigForTesting(); + _setConfigPathForTesting(() => configPath); + const first = readUserConfig(); + // Mutate file under the cache — readUserConfig should NOT re-read. + writeFileSync(configPath, JSON.stringify({ embeddings: { enabled: true } }), "utf-8"); + const second = readUserConfig(); + expect(second).toEqual(first); + }); +}); + +describe("writeUserConfig", () => { + it("creates the file with the patched contents when none existed", () => { + writeUserConfig({ embeddings: { enabled: true } }); + expect(existsSync(configPath)).toBe(true); + const written = JSON.parse(readFileSync(configPath, "utf-8")); + expect(written).toEqual({ embeddings: { enabled: true } }); + }); + + it("deep-merges into existing keys without clobbering siblings", () => { + writeFileSync( + configPath, + JSON.stringify({ embeddings: { enabled: true, other: "keep" }, unrelated: { x: 1 } }), + "utf-8", + ); + _resetUserConfigForTesting(); + _setConfigPathForTesting(() => configPath); + writeUserConfig({ embeddings: { enabled: false } }); + const written = JSON.parse(readFileSync(configPath, "utf-8")); + expect(written).toEqual({ + embeddings: { enabled: false, other: "keep" }, + unrelated: { x: 1 }, + }); + }); + + it("writes atomically: no .tmp file remains after a successful write", () => { + writeUserConfig({ embeddings: { enabled: true } }); + const dirEntries = require("node:fs").readdirSync(dir); + expect(dirEntries.filter((f: string) => f.endsWith(".tmp") || f.includes(".tmp."))).toEqual([]); + }); + + it("creates the parent directory if missing", () => { + rmSync(dir, { recursive: true, force: true }); + writeUserConfig({ embeddings: { enabled: true } }); + expect(existsSync(configPath)).toBe(true); + }); + + it("setEmbeddingsEnabled is a one-line wrapper that writes the right shape", () => { + setEmbeddingsEnabled(false); + const written = JSON.parse(readFileSync(configPath, "utf-8")); + expect(written).toEqual({ embeddings: { enabled: false } }); + }); +}); + +describe("getEmbeddingsEnabled — migration from HIVEMIND_EMBEDDINGS", () => { + it("writes enabled:false and returns false when env is unset on first run", () => { + delete process.env.HIVEMIND_EMBEDDINGS; + expect(getEmbeddingsEnabled()).toBe(false); + const written = JSON.parse(readFileSync(configPath, "utf-8")); + expect(written).toEqual({ embeddings: { enabled: false } }); + }); + + it("writes enabled:false and returns false when env is 'false' on first run", () => { + process.env.HIVEMIND_EMBEDDINGS = "false"; + expect(getEmbeddingsEnabled()).toBe(false); + const written = JSON.parse(readFileSync(configPath, "utf-8")); + expect(written).toEqual({ embeddings: { enabled: false } }); + }); + + it("writes enabled:true and returns true when env is 'true' on first run", () => { + process.env.HIVEMIND_EMBEDDINGS = "true"; + expect(getEmbeddingsEnabled()).toBe(true); + const written = JSON.parse(readFileSync(configPath, "utf-8")); + expect(written).toEqual({ embeddings: { enabled: true } }); + }); + + it("writes enabled:true on any non-'false' truthy env (lenient migration)", () => { + process.env.HIVEMIND_EMBEDDINGS = "1"; + expect(getEmbeddingsEnabled()).toBe(true); + }); + + it("does NOT re-read the env var once a value is persisted", () => { + process.env.HIVEMIND_EMBEDDINGS = "false"; + expect(getEmbeddingsEnabled()).toBe(false); // migration runs + // Flip the env: should be ignored on subsequent reads. + process.env.HIVEMIND_EMBEDDINGS = "true"; + _resetUserConfigForTesting(); + _setConfigPathForTesting(() => configPath); + expect(getEmbeddingsEnabled()).toBe(false); // reads from persisted config + }); + + it("returns the persisted value when config already has embeddings.enabled set", () => { + writeFileSync(configPath, JSON.stringify({ embeddings: { enabled: true } }), "utf-8"); + _resetUserConfigForTesting(); + _setConfigPathForTesting(() => configPath); + // Env says false; persisted value should win. + process.env.HIVEMIND_EMBEDDINGS = "false"; + expect(getEmbeddingsEnabled()).toBe(true); + }); + + it("setEmbeddingsEnabled overrides a prior migration value (last write wins)", () => { + delete process.env.HIVEMIND_EMBEDDINGS; + expect(getEmbeddingsEnabled()).toBe(false); // migration → false + setEmbeddingsEnabled(true); + expect(getEmbeddingsEnabled()).toBe(true); + const written = JSON.parse(readFileSync(configPath, "utf-8")); + expect(written).toEqual({ embeddings: { enabled: true } }); + }); +}); diff --git a/tests/claude-code/wiki-worker-plugin-version.test.ts b/tests/claude-code/wiki-worker-plugin-version.test.ts index b76d0fc0..256dec0b 100644 --- a/tests/claude-code/wiki-worker-plugin-version.test.ts +++ b/tests/claude-code/wiki-worker-plugin-version.test.ts @@ -326,11 +326,13 @@ describe("wiki-worker resume + embeddings-disabled branches — per agent", () = expect(uploadSummaryMock.mock.calls[0][1].pluginVersion).toBe("9.9.9"); }); - it(`${v.agent}: HIVEMIND_EMBEDDINGS=false skips the embed daemon`, async () => { + it(`${v.agent}: user-disabled embeddings skip the embed daemon`, async () => { // Hit the embeddingsDisabled() branch — uploadSummary should still // be called, but with embedding === null (skipped daemon hop). - const prev = process.env.HIVEMIND_EMBEDDINGS; - process.env.HIVEMIND_EMBEDDINGS = "false"; + const tmpConfig = join(rootDir, "user-config.json"); + writeFileSync(tmpConfig, JSON.stringify({ embeddings: { enabled: false } }), "utf-8"); + const prev = process.env.HIVEMIND_CONFIG_PATH; + process.env.HIVEMIND_CONFIG_PATH = tmpConfig; try { await runVariant(v, "9.9.9"); expect(uploadSummaryMock).toHaveBeenCalledOnce(); @@ -338,8 +340,8 @@ describe("wiki-worker resume + embeddings-disabled branches — per agent", () = expect(params.embedding).toBeNull(); expect(params.pluginVersion).toBe("9.9.9"); } finally { - if (prev === undefined) delete process.env.HIVEMIND_EMBEDDINGS; - else process.env.HIVEMIND_EMBEDDINGS = prev; + if (prev === undefined) delete process.env.HIVEMIND_CONFIG_PATH; + else process.env.HIVEMIND_CONFIG_PATH = prev; } }); } diff --git a/tests/cli/cli-embeddings.test.ts b/tests/cli/cli-embeddings.test.ts index 1f0aaa4f..afa66c06 100644 --- a/tests/cli/cli-embeddings.test.ts +++ b/tests/cli/cli-embeddings.test.ts @@ -1,8 +1,22 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, mkdirSync, writeFileSync, symlinkSync, lstatSync, readlinkSync, rmSync, existsSync } from "node:fs"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, symlinkSync, lstatSync, readlinkSync, rmSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { findHivemindInstalls, isSharedDepsInstalled, linkStateFor, SHARED_DAEMON_PATH, SHARED_NODE_MODULES, TRANSFORMERS_PKG } from "../../src/cli/embeddings.js"; +import { + disableEmbeddings, + enableEmbeddings, + findHivemindInstalls, + installEmbeddings, + isSharedDepsInstalled, + killEmbedDaemon, + linkStateFor, + SHARED_DAEMON_PATH, + SHARED_NODE_MODULES, + TRANSFORMERS_PKG, + uninstallEmbeddings, + _linkAgentForTesting, +} from "../../src/cli/embeddings.js"; +import { _resetUserConfigForTesting, _setConfigPathForTesting, getEmbeddingsEnabled } from "../../src/user-config.js"; /** * Tests for the shared-deps embeddings installer's pure helpers. The @@ -141,3 +155,175 @@ describe("linkStateFor", () => { expect(state.kind).toBe("owns-own-node-modules"); }); }); + +// ── lightweight enable / disable: config-only, no fs install ────────────── + +describe("enableEmbeddings / disableEmbeddings — config flag mutation", () => { + let cfgPath: string; + + beforeEach(() => { + cfgPath = join(tmpHome, "config.json"); + _setConfigPathForTesting(() => cfgPath); + }); + + afterEach(() => { + _resetUserConfigForTesting(); + }); + + it("enableEmbeddings writes embeddings.enabled:true to ~/.deeplake/config.json", () => { + enableEmbeddings(); + expect(existsSync(cfgPath)).toBe(true); + expect(JSON.parse(readFileSync(cfgPath, "utf-8"))).toEqual({ embeddings: { enabled: true } }); + expect(getEmbeddingsEnabled()).toBe(true); + }); + + it("disableEmbeddings writes embeddings.enabled:false to ~/.deeplake/config.json", () => { + enableEmbeddings(); + disableEmbeddings(); + expect(JSON.parse(readFileSync(cfgPath, "utf-8"))).toEqual({ embeddings: { enabled: false } }); + expect(getEmbeddingsEnabled()).toBe(false); + }); + + it("disableEmbeddings is idempotent (no error when no daemon and no config)", () => { + expect(() => disableEmbeddings()).not.toThrow(); + expect(getEmbeddingsEnabled()).toBe(false); + }); + + it("enableEmbeddings overrides a prior disableEmbeddings (last write wins)", () => { + disableEmbeddings(); + enableEmbeddings(); + expect(getEmbeddingsEnabled()).toBe(true); + }); +}); + +// ── killEmbedDaemon: tolerant of every combination of missing files ─────── + +describe("killEmbedDaemon", () => { + it("returns silently when there is no pidfile or socket (fresh machine)", () => { + // SOCKET_DIR defaults to /tmp/.hivemind-embed-/ in production — we + // can't redirect that without monkey-patching. But the function only ever + // reads + best-effort-deletes, so calling it when nothing exists is a + // no-op by design. + expect(() => killEmbedDaemon()).not.toThrow(); + }); +}); + +// ── uninstall: writes config:false even when shared deps absent ─────────── + +describe("killEmbedDaemon — verifies socket before SIGTERM (#2)", () => { + // Regression for CodeRabbit #2: previously killEmbedDaemon read the PID + // from the pidfile and blindly SIGTERMed it. If the daemon had crashed + // and the OS recycled that PID to an unrelated user process, + // `hivemind embeddings disable` would silently kill that process. The + // fix gates the SIGTERM on `_isDaemonAliveOnSocket` — if the UDS path + // doesn't accept a connect within a short timeout, the daemon is dead + // and the PID in the file is stale, so we only clean up sock+pid. + it("skips SIGTERM when the socket is dead (stale pidfile path)", async () => { + const { killEmbedDaemon: kill, _isDaemonAliveOnSocket } = await import( + "../../src/cli/embeddings.js" + ); + + // Test isolation: use a per-test tmp dir for the sock/pid files + // instead of the real /tmp/hivemind-embed-.* paths. Without + // this, the test would clobber any real daemon's socket/pidfile + // for the same uid on the dev machine or CI worker. + const { pidPathFor, socketPathFor } = await import("../../src/embeddings/protocol.js"); + const sockDir = mkdtempSync(join(tmpdir(), "kill-test-")); + const uid = String(process.getuid?.() ?? 0); + const pidPath = pidPathFor(uid, sockDir); + const sockPath = socketPathFor(uid, sockDir); + + try { + // Write the *current process's* pid into the file. If the broken + // code ran, our test runner would receive SIGTERM and die. With + // the fix, the socket-alive probe sees no socket bound and + // killEmbedDaemon should skip the SIGTERM step entirely. + writeFileSync(pidPath, String(process.pid)); + // sockPath doesn't exist (we never wrote it), so the probe sees no + // socket binding. + + // Probe asserts the socket isn't alive. + expect(_isDaemonAliveOnSocket(sockPath, 100)).toBe(false); + + // The call must NOT crash the test runner (i.e. we must NOT + // receive SIGTERM). Passing the per-test sockDir keeps the call + // bound to our tmp paths. + kill(sockDir); + + // Sock+pid file cleanup still runs against the tmp paths. + expect(existsSync(pidPath)).toBe(false); + expect(existsSync(sockPath)).toBe(false); + } finally { + rmSync(sockDir, { recursive: true, force: true }); + } + }, 30_000); +}); + +describe("linkAgent — preserves real node_modules directory (#1)", () => { + // Regression for CodeRabbit #1: previously `linkAgent` went straight + // through `symlinkForce` → `unlinkSync` on whatever existed at + // `/node_modules`. If the path was a real directory (a + // marketplace plugin shipping its own deps, or a dev `npm install`), + // `unlinkSync` threw EISDIR and aborted `hivemind embeddings install` + // partway through, leaving some agents linked and others not. + it("skips linking when a real node_modules directory already exists at the link path", () => { + const pluginDir = join(tmpHome, ".fake-agent", "hivemind"); + mkDir(join(pluginDir, "bundle")); + // Existing real `node_modules/` dir with content (simulates a + // plugin that already shipped deps). + const realNm = join(pluginDir, "node_modules"); + mkDir(realNm); + writeFileSync(join(realNm, "marker.txt"), "preserved"); + + // Must NOT throw. + expect(() => + _linkAgentForTesting({ id: "fake-agent", pluginDir }) + ).not.toThrow(); + + // Real dir is intact, marker file untouched. + expect(existsSync(realNm)).toBe(true); + expect(lstatSync(realNm).isDirectory()).toBe(true); + expect(lstatSync(realNm).isSymbolicLink()).toBe(false); + expect(readFileSync(join(realNm, "marker.txt"), "utf-8")).toBe("preserved"); + }); + + it("still replaces a stale symlink at the link path (normal install path unaffected)", () => { + const pluginDir = join(tmpHome, ".fake-agent2", "hivemind"); + mkDir(join(pluginDir, "bundle")); + // Simulate a shared-deps target so symlinkForce has somewhere to point. + const fakeShared = join(tmpHome, ".hivemind", "embed-deps", "node_modules"); + mkDir(fakeShared); + // Pre-existing symlink to a stale location. + const stale = join(tmpHome, "stale"); + mkDir(stale); + symlinkSync(stale, join(pluginDir, "node_modules")); + + // Without HOME override the real SHARED_NODE_MODULES is used, so we + // can only assert "no throw" + "still a symlink after". The exact + // target depends on the runtime HOME, but the call must succeed. + expect(() => + _linkAgentForTesting({ id: "fake-agent2", pluginDir }) + ).not.toThrow(); + expect(lstatSync(join(pluginDir, "node_modules")).isSymbolicLink()).toBe(true); + }); +}); + +describe("uninstallEmbeddings — config flag side effect", () => { + let cfgPath: string; + + beforeEach(() => { + cfgPath = join(tmpHome, "config.json"); + _setConfigPathForTesting(() => cfgPath); + }); + + afterEach(() => { + _resetUserConfigForTesting(); + }); + + it("flips embeddings.enabled:false even when there are no agent installs and no shared deps", () => { + // No installs detected, no shared deps dir — uninstall still flips the flag. + enableEmbeddings(); + uninstallEmbeddings(); + expect(getEmbeddingsEnabled()).toBe(false); + }); +}); diff --git a/tests/cli/cli-index.test.ts b/tests/cli/cli-index.test.ts index 1acc28ec..b6971d90 100644 --- a/tests/cli/cli-index.test.ts +++ b/tests/cli/cli-index.test.ts @@ -83,12 +83,16 @@ vi.mock("../../src/cli/version.js", () => ({ vi.mock("../../src/cli/update.js", () => ({ runUpdate: (...a: unknown[]) => runUpdateMock(...a), })); +const installEmbeddingsMock = vi.fn(); const enableEmbeddingsMock = vi.fn(); const disableEmbeddingsMock = vi.fn(); +const uninstallEmbeddingsMock = vi.fn(); const statusEmbeddingsMock = vi.fn(); vi.mock("../../src/cli/embeddings.js", () => ({ + installEmbeddings: (...a: unknown[]) => installEmbeddingsMock(...a), enableEmbeddings: (...a: unknown[]) => enableEmbeddingsMock(...a), disableEmbeddings: (...a: unknown[]) => disableEmbeddingsMock(...a), + uninstallEmbeddings: (...a: unknown[]) => uninstallEmbeddingsMock(...a), statusEmbeddings: (...a: unknown[]) => statusEmbeddingsMock(...a), })); @@ -104,8 +108,10 @@ beforeEach(() => { allPlatformIdsMock.mockReset().mockReturnValue(["claude", "codex", "claw", "cursor", "hermes", "pi"]); getVersionMock.mockReset().mockReturnValue("1.2.3"); runUpdateMock.mockReset().mockResolvedValue(0); + installEmbeddingsMock.mockReset(); enableEmbeddingsMock.mockReset(); disableEmbeddingsMock.mockReset(); + uninstallEmbeddingsMock.mockReset(); statusEmbeddingsMock.mockReset(); stdoutMock.mockReset(); stderrMock.mockReset(); @@ -336,29 +342,40 @@ describe("hivemind update", () => { }); describe("hivemind embeddings", () => { - it.each([["install"], ["enable"]] as const)( - "'embeddings %s' calls enableEmbeddings exactly once", - async (sub) => { - await runCli(["embeddings", sub]); - expect(enableEmbeddingsMock).toHaveBeenCalledTimes(1); - expect(disableEmbeddingsMock).not.toHaveBeenCalled(); - expect(statusEmbeddingsMock).not.toHaveBeenCalled(); - }, - ); + it("'embeddings install' calls installEmbeddings (heavy: deps + symlinks + enabled:true)", async () => { + await runCli(["embeddings", "install"]); + expect(installEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(enableEmbeddingsMock).not.toHaveBeenCalled(); + expect(disableEmbeddingsMock).not.toHaveBeenCalled(); + expect(uninstallEmbeddingsMock).not.toHaveBeenCalled(); + }); - it.each([["uninstall"], ["disable"]] as const)( - "'embeddings %s' calls disableEmbeddings({ prune: false }) by default", - async (sub) => { - await runCli(["embeddings", sub]); - expect(disableEmbeddingsMock).toHaveBeenCalledTimes(1); - expect(disableEmbeddingsMock.mock.calls[0][0]).toEqual({ prune: false }); - }, - ); + it("'embeddings enable' calls enableEmbeddings (light: flip config flag only)", async () => { + await runCli(["embeddings", "enable"]); + expect(enableEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(installEmbeddingsMock).not.toHaveBeenCalled(); + expect(disableEmbeddingsMock).not.toHaveBeenCalled(); + expect(uninstallEmbeddingsMock).not.toHaveBeenCalled(); + }); + + it("'embeddings disable' calls disableEmbeddings (light: flip flag + kill daemon)", async () => { + await runCli(["embeddings", "disable"]); + expect(disableEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(disableEmbeddingsMock.mock.calls[0]).toEqual([]); + expect(uninstallEmbeddingsMock).not.toHaveBeenCalled(); + }); + + it("'embeddings uninstall' calls uninstallEmbeddings({ prune: false }) by default", async () => { + await runCli(["embeddings", "uninstall"]); + expect(uninstallEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(uninstallEmbeddingsMock.mock.calls[0][0]).toEqual({ prune: false }); + expect(disableEmbeddingsMock).not.toHaveBeenCalled(); + }); it("'embeddings uninstall --prune' passes prune: true", async () => { await runCli(["embeddings", "uninstall", "--prune"]); - expect(disableEmbeddingsMock).toHaveBeenCalledTimes(1); - expect(disableEmbeddingsMock.mock.calls[0][0]).toEqual({ prune: true }); + expect(uninstallEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(uninstallEmbeddingsMock.mock.calls[0][0]).toEqual({ prune: true }); }); it("'embeddings status' calls statusEmbeddings once", async () => { @@ -366,26 +383,36 @@ describe("hivemind embeddings", () => { expect(statusEmbeddingsMock).toHaveBeenCalledTimes(1); }); - it("unknown 'embeddings' subcommand exits 1 with usage warning", async () => { + it("unknown 'embeddings' subcommand exits 1 with usage warning that lists all 5 subcommands", async () => { await runCli(["embeddings", "bogus"]); expect(exitSpy).toHaveBeenCalledWith(1); - expect(stderrText()).toContain("Usage: hivemind embeddings"); + const text = stderrText(); + expect(text).toContain("Usage: hivemind embeddings"); + expect(text).toContain("install"); + expect(text).toContain("enable"); + expect(text).toContain("disable"); + expect(text).toContain("uninstall"); + expect(text).toContain("status"); + expect(installEmbeddingsMock).not.toHaveBeenCalled(); expect(enableEmbeddingsMock).not.toHaveBeenCalled(); expect(disableEmbeddingsMock).not.toHaveBeenCalled(); + expect(uninstallEmbeddingsMock).not.toHaveBeenCalled(); expect(statusEmbeddingsMock).not.toHaveBeenCalled(); }); - it("'install --with-embeddings' enables embeddings after the install loop", async () => { + it("'install --with-embeddings' runs installEmbeddings (heavy path) after the install loop", async () => { detectPlatformsMock.mockReturnValue([{ id: "claude", markerDir: "/x/.claude" }]); await runCli(["install", "--with-embeddings"]); expect(installs.installClaude).toHaveBeenCalledTimes(1); - expect(enableEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(installEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(enableEmbeddingsMock).not.toHaveBeenCalled(); }); - it("' install --with-embeddings' enables embeddings after the per-agent install", async () => { + it("' install --with-embeddings' runs installEmbeddings (heavy path) after the per-agent install", async () => { await runCli(["cursor", "install", "--with-embeddings"]); expect(installs.installCursor).toHaveBeenCalledTimes(1); - expect(enableEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(installEmbeddingsMock).toHaveBeenCalledTimes(1); + expect(enableEmbeddingsMock).not.toHaveBeenCalled(); }); }); diff --git a/tests/cursor/cursor-capture-hook.test.ts b/tests/cursor/cursor-capture-hook.test.ts index e280aab8..1d291e89 100644 --- a/tests/cursor/cursor-capture-hook.test.ts +++ b/tests/cursor/cursor-capture-hook.test.ts @@ -269,13 +269,20 @@ describe("cursor capture hook — message_embedding column", () => { expect(sql).toContain("'::jsonb, NULL,"); }); - it("HIVEMIND_EMBEDDINGS=false short-circuits to NULL without invoking EmbedClient", async () => { + it("user-disabled embeddings short-circuit to NULL without invoking EmbedClient", async () => { stdinMock.mockResolvedValue({ conversation_id: "sid-emb-3", hook_event_name: "beforeSubmitPrompt", prompt: "disabled", }); - await runHook({ HIVEMIND_EMBEDDINGS: "false" }); + // Point user-config at a throwaway path that says enabled:false. + const { writeFileSync, mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const dir = mkdtempSync(join(tmpdir(), "cursor-cap-disabled-")); + const cfgPath = join(dir, "config.json"); + writeFileSync(cfgPath, JSON.stringify({ embeddings: { enabled: false } }), "utf-8"); + await runHook({ HIVEMIND_CONFIG_PATH: cfgPath }); const sql = queryMock.mock.calls[0][0] as string; expect(sql).toContain("'::jsonb, NULL,"); expect(sql).toMatch(/, message_embedding,/); diff --git a/tests/hermes/hermes-capture-hook.test.ts b/tests/hermes/hermes-capture-hook.test.ts index 522b1adc..864df5e5 100644 --- a/tests/hermes/hermes-capture-hook.test.ts +++ b/tests/hermes/hermes-capture-hook.test.ts @@ -41,9 +41,26 @@ const validConfig = { sessionsTableName: "sessions", }; +// Env keys + tmp dirs that tests in this file mutate via runHook(). The +// afterEach hook reads from these to restore the process env and clean +// up any tmp dirs the user-disabled-embeddings test creates — without +// it, the next test in the same vitest worker would inherit a stray +// HIVEMIND_CONFIG_PATH pointing at a (deleted) tmp file, which silently +// alters how the embeddings module resolves its on-disk config. +const TOUCHED_ENV_KEYS = [ + "HIVEMIND_CAPTURE", + "HIVEMIND_CONFIG_PATH", +] as const; +const _origEnv: Record = {}; +const _tmpDirsToClean: string[] = []; + async function runHook(env: Record = {}): Promise { + for (const k of TOUCHED_ENV_KEYS) { + if (!(k in _origEnv)) _origEnv[k] = process.env[k]; + } delete process.env.HIVEMIND_CAPTURE; for (const [k, v] of Object.entries(env)) { + if (!(k in _origEnv)) _origEnv[k] = process.env[k]; if (v === undefined) delete process.env[k]; else process.env[k] = v; } @@ -62,7 +79,21 @@ beforeEach(() => { buildSessionPathMock.mockReset().mockReturnValue("/sessions/alice/foo.jsonl"); }); -afterEach(() => { vi.restoreAllMocks(); }); +afterEach(async () => { + vi.restoreAllMocks(); + // Restore env keys touched by runHook() so later tests start clean. + for (const [k, v] of Object.entries(_origEnv)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + for (const k of Object.keys(_origEnv)) delete _origEnv[k]; + // Clean any tmp dirs the user-disabled test created. + if (_tmpDirsToClean.length > 0) { + const { rmSync } = await import("node:fs"); + for (const d of _tmpDirsToClean) try { rmSync(d, { recursive: true, force: true }); } catch { /* */ } + _tmpDirsToClean.length = 0; + } +}); describe("hermes capture hook — guards", () => { it("HIVEMIND_CAPTURE=false → no stdin read", async () => { @@ -265,14 +296,21 @@ describe("hermes capture hook — message_embedding column", () => { expect(sql).toContain("'::jsonb, NULL,"); }); - it("HIVEMIND_EMBEDDINGS=false short-circuits to NULL without invoking EmbedClient", async () => { + it("user-disabled embeddings short-circuit to NULL without invoking EmbedClient", async () => { stdinMock.mockResolvedValue({ hook_event_name: "pre_llm_call", session_id: "sid-emb-3", cwd: "/work/proj", extra: { prompt: "disabled" }, }); - await runHook({ HIVEMIND_EMBEDDINGS: "false" }); + const { writeFileSync, mkdtempSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const dir = mkdtempSync(join(tmpdir(), "hermes-cap-disabled-")); + _tmpDirsToClean.push(dir); + const cfgPath = join(dir, "config.json"); + writeFileSync(cfgPath, JSON.stringify({ embeddings: { enabled: false } }), "utf-8"); + await runHook({ HIVEMIND_CONFIG_PATH: cfgPath }); const sql = queryMock.mock.calls[0][0] as string; expect(sql).toContain("'::jsonb, NULL,"); expect(sql).toMatch(/, message_embedding,/); diff --git a/tests/test-setup.ts b/tests/test-setup.ts new file mode 100644 index 00000000..6c1fc4ab --- /dev/null +++ b/tests/test-setup.ts @@ -0,0 +1,27 @@ +// Global vitest setup. Runs once before any test file. +// +// Why: as of the embeddings-config refactor, `~/.deeplake/config.json` is the +// source of truth for `embeddings.enabled`. The migration helper in +// src/user-config.ts writes to that file on first read if no key is present. +// Without isolation, every test run would mutate the developer's real config. +// This setup pins `HIVEMIND_CONFIG_PATH` to a per-process tmp dir so all +// reads / writes land in throwaway state. + +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll } from "vitest"; + +const tmpDir = mkdtempSync(join(tmpdir(), "hivemind-test-config-")); +process.env.HIVEMIND_CONFIG_PATH = join(tmpDir, "config.json"); + +// Default to embeddings-enabled in the test env so existing tests that +// expect the embed code path to run aren't surprised by the new +// opt-in-required default. Tests that exercise the disabled path set their +// own values via _setEnabledReaderForTesting or by writing the config file +// directly. +process.env.HIVEMIND_EMBEDDINGS = "true"; + +afterAll(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 86a7d396..05c9c567 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -27,6 +27,7 @@ export default defineConfig({ "tests/openclaw/**/*.test.ts", "tests/pi/**/*.test.ts", ], + setupFiles: ["./tests/test-setup.ts"], environment: "node", coverage: { provider: "v8",