Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
510 changes: 342 additions & 168 deletions bundle/cli.js

Large diffs are not rendered by default.

440 changes: 374 additions & 66 deletions claude-code/bundle/capture.js

Large diffs are not rendered by default.

60 changes: 56 additions & 4 deletions claude-code/bundle/embeddings/embed-daemon.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,14 @@
import { createServer } from "node:net";
import { unlinkSync, writeFileSync, existsSync, mkdirSync, chmodSync } from "node:fs";

// dist/src/embeddings/nomic.js
import { createRequire } from "node:module";
import { homedir } from "node:os";
import { join } from "node:path";
import { pathToFileURL } from "node:url";

// dist/src/embeddings/protocol.js
var PROTOCOL_VERSION = 1;
var DEFAULT_SOCKET_DIR = "/tmp";
var DEFAULT_MODEL_REPO = "nomic-ai/nomic-embed-text-v1.5";
var DEFAULT_DTYPE = "q8";
Expand All @@ -20,6 +27,40 @@ function pidPathFor(uid, dir = DEFAULT_SOCKET_DIR) {
}

// dist/src/embeddings/nomic.js
async function importFromCanonicalSharedDeps() {
const sharedDir = join(homedir(), ".hivemind", "embed-deps");
const base = pathToFileURL(`${sharedDir}/`).href;
const absMain = createRequire(base).resolve("@huggingface/transformers");
const mod = await import(pathToFileURL(absMain).href);
return normalizeTransformersModule(mod);
}
async function importFromBareSpecifier() {
const mod = await import("@huggingface/transformers");
return normalizeTransformersModule(mod);
}
function normalizeTransformersModule(mod) {
const m = mod;
if (m.default && typeof m.default === "object" && "pipeline" in m.default) {
return m.default;
}
return m;
}
async function defaultImportTransformers(canonical = importFromCanonicalSharedDeps, bare = importFromBareSpecifier) {
let canonicalErr;
try {
return await canonical();
} catch (err) {
canonicalErr = err;
}
try {
return await bare();
} catch (bareErr) {
const detail = bareErr instanceof Error ? bareErr.message : String(bareErr);
const canonicalDetail = canonicalErr instanceof Error ? canonicalErr.message : String(canonicalErr);
throw new Error(`@huggingface/transformers is not installed anywhere reachable. Run \`hivemind embeddings install\` to install it. (canonical: ${canonicalDetail}; bare: ${detail})`);
}
}
var _importTransformers = () => defaultImportTransformers();
var NomicEmbedder = class {
pipeline = null;
loading = null;
Expand All @@ -37,7 +78,7 @@ var NomicEmbedder = class {
if (this.loading)
return this.loading;
this.loading = (async () => {
const mod = await import("@huggingface/transformers");
const mod = await _importTransformers();
mod.env.allowLocalModels = false;
mod.env.useFSCache = true;
this.pipeline = await mod.pipeline("feature-extraction", this.repo, { dtype: this.dtype });
Expand Down Expand Up @@ -94,10 +135,10 @@ var NomicEmbedder = class {

// dist/src/utils/debug.js
import { appendFileSync } from "node:fs";
import { join } from "node:path";
import { homedir } from "node:os";
import { join as join2 } from "node:path";
import { homedir as homedir2 } from "node:os";
var DEBUG = process.env.HIVEMIND_DEBUG === "1";
var LOG = join(homedir(), ".deeplake", "hook-debug.log");
var LOG = join2(homedir2(), ".deeplake", "hook-debug.log");
function log(tag, msg) {
if (!DEBUG)
return;
Expand All @@ -118,13 +159,15 @@ var EmbedDaemon = class {
pidPath;
idleTimeoutMs;
idleTimer = null;
daemonPath;
constructor(opts = {}) {
const uid = getUid();
const dir = opts.socketDir ?? "/tmp";
this.socketPath = socketPathFor(uid, dir);
this.pidPath = pidPathFor(uid, dir);
this.idleTimeoutMs = opts.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
this.embedder = new NomicEmbedder({ repo: opts.repo, dtype: opts.dtype, dims: opts.dims });
this.daemonPath = opts.daemonPath ?? process.argv[1] ?? "";
}
async start() {
mkdirSync(this.socketPath.replace(/\/[^/]+$/, ""), { recursive: true });
Expand Down Expand Up @@ -216,6 +259,15 @@ var EmbedDaemon = class {
}
}
async dispatch(req) {
if (req.op === "hello") {
const h = req;
return {
id: h.id,
daemonPath: this.daemonPath,
pid: process.pid,
protocolVersion: PROTOCOL_VERSION
};
}
if (req.op === "ping") {
const p = req;
return { id: p.id, ready: true, model: this.embedder.repo, dims: this.embedder.dims };
Expand Down
Loading
Loading