Merge pull request #2 from supermemoryai/12-16-setup_project_structure_with_tree-sitter_and_effect

setup project structure with tree-sitter and effect

Author: nexxeln

.gitignore

@@ -18,3 +18,5 @@ yarn-error.log*
 .env.test.local
 .env.production.local
 .turbo
+todo.md
+plan.md

bun.lock

@@ -4,6 +4,16 @@
   "workspaces": {
     "": {
       "name": "astchunk",
+      "dependencies": {
+        "effect": "^3.19.12",
+        "tree-sitter-go": "^0.25.0",
+        "tree-sitter-java": "^0.23.5",
+        "tree-sitter-javascript": "^0.25.0",
+        "tree-sitter-python": "^0.25.0",
+        "tree-sitter-rust": "^0.24.0",
+        "tree-sitter-typescript": "^0.23.2",
+        "web-tree-sitter": "^0.26.3",
+      },
       "devDependencies": {
         "@biomejs/biome": "^2.3.8",
         "@types/bun": "^1.3.4",
@@ -158,6 +168,8 @@
 
     "@oxc-transform/binding-win32-x64-msvc": ["@oxc-transform/[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-6QN3DEaEw3eWioWEFRgNsTvYq8czYSnpkjB2za+/WdLN0g5FzOl2ZEfNiPrBWIPnSmjUmDWtWVWcSjwY7fX5/Q=="],
 
+    "@standard-schema/spec": ["@standard-schema/[email protected]", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
+
     "@tybys/wasm-util": ["@tybys/[email protected]", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="],
 
     "@types/bun": ["@types/[email protected]", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="],
@@ -196,10 +208,14 @@
 
     "dotenv": ["[email protected]", "", {}, "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w=="],
 
+    "effect": ["[email protected]", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-7F9RGTrCTC3D7nh9Zw+3VlJWwZgo5k33KA+476BAaD0rKIXKZsY/jQ+ipyhR/Avo239Fi6GqAVFs1mqM1IJ7yg=="],
+
     "escalade": ["[email protected]", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="],
 
     "exsolve": ["[email protected]", "", {}, "sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA=="],
 
+    "fast-check": ["[email protected]", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="],
+
     "fdir": ["[email protected]", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="],
 
     "giget": ["[email protected]", "", { "dependencies": { "citty": "^0.1.6", "consola": "^3.4.0", "defu": "^6.1.4", "node-fetch-native": "^1.6.6", "nypm": "^0.6.0", "pathe": "^2.0.3" }, "bin": { "giget": "dist/cli.mjs" } }, "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA=="],
@@ -232,8 +248,12 @@
 
     "lightningcss-win32-x64-msvc": ["[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw=="],
 
+    "node-addon-api": ["[email protected]", "", {}, "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A=="],
+
     "node-fetch-native": ["[email protected]", "", {}, "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q=="],
 
+    "node-gyp-build": ["[email protected]", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="],
+
     "nypm": ["[email protected]", "", { "dependencies": { "citty": "^0.1.6", "consola": "^3.4.2", "pathe": "^2.0.3", "pkg-types": "^2.3.0", "tinyexec": "^1.0.1" }, "bin": { "nypm": "dist/cli.mjs" } }, "sha512-7eM+hpOtrKrBDCh7Ypu2lJ9Z7PNZBdi/8AT3AX8xoCj43BBVHD0hPSTEvMtkMpfs8FCqBGhxB+uToIQimA111g=="],
 
     "ohash": ["[email protected]", "", {}, "sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ=="],
@@ -256,6 +276,8 @@
 
     "pkg-types": ["[email protected]", "", { "dependencies": { "confbox": "^0.2.2", "exsolve": "^1.0.7", "pathe": "^2.0.3" } }, "sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig=="],
 
+    "pure-rand": ["[email protected]", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="],
+
     "rc9": ["[email protected]", "", { "dependencies": { "defu": "^6.1.4", "destr": "^2.0.3" } }, "sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg=="],
 
     "readdirp": ["[email protected]", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="],
@@ -272,6 +294,18 @@
 
     "tree-kill": ["[email protected]", "", { "bin": { "tree-kill": "cli.js" } }, "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A=="],
 
+    "tree-sitter-go": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.3.1", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-APBc/Dq3xz/e35Xpkhb1blu5UgW+2E3RyGWawZSCNcbGwa7jhSQPS8KsUupuzBla8PCo8+lz9W/JDJjmfRa2tw=="],
+
+    "tree-sitter-java": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.2" }, "peerDependencies": { "tree-sitter": "^0.21.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-Yju7oQ0Xx7GcUT01mUglPP+bYfvqjNCGdxqigTnew9nLGoII42PNVP3bHrYeMxswiCRM0yubWmN5qk+zsg0zMA=="],
+
+    "tree-sitter-javascript": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.3.1", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-1fCbmzAskZkxcZzN41sFZ2br2iqTYP3tKls1b/HKGNPQUVOpsUxpmGxdN/wMqAk3jYZnYBR1dd/y/0avMeU7dw=="],
+
+    "tree-sitter-python": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.5.0", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-eCmJx6zQa35GxaCtQD+wXHOhYqBxEL+bp71W/s3fcDMu06MrtzkVXR437dRrCrbrDbyLuUDJpAgycs7ncngLXw=="],
+
+    "tree-sitter-rust": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.4" }, "peerDependencies": { "tree-sitter": "^0.22.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-NWemUDf629Tfc90Y0Z55zuwPCAHkLxWnMf2RznYu4iBkkrQl2o/CHGB7Cr52TyN5F1DAx8FmUnDtCy9iUkXZEQ=="],
+
+    "tree-sitter-typescript": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.2", "tree-sitter-javascript": "^0.23.1" }, "peerDependencies": { "tree-sitter": "^0.21.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA=="],
+
     "ts-import-resolver": ["[email protected]", "", { "peerDependencies": { "typescript": ">=4.5.0" }, "optionalPeers": ["typescript"] }, "sha512-282pgr6j6aOvP3P2I6XugDxdBobkpdMmdbWjRjGl5gjPI1p0+oTNGDh1t924t75kRlyIkF65DiwhSIUysmyHQA=="],
 
     "tslib": ["[email protected]", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
@@ -280,8 +314,12 @@
 
     "undici-types": ["[email protected]", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
 
+    "web-tree-sitter": ["[email protected]", "", {}, "sha512-JIVgIKFS1w6lejxSntCtsS/QsE/ecTS00en809cMxMPxaor6MvUnQ+ovG8uTTTvQCFosSh4MeDdI5bSGw5SoBw=="],
+
     "yaml": ["[email protected]", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="],
 
     "zlye": ["[email protected]", "", { "dependencies": { "picocolors": "^1.1.1" }, "peerDependencies": { "typescript": ">=4.5.0" }, "optionalPeers": ["typescript"] }, "sha512-fwpeC841X3ElOLYRMKXbwX29pitNrsm6nRNvEhDMrRXDl3BhR2i03Bkr0GNrpyYgZJuEzUsBylXAYzgGPXXOCQ=="],
+
+    "tree-sitter-typescript/tree-sitter-javascript": ["[email protected]", "", { "dependencies": { "node-addon-api": "^8.2.2", "node-gyp-build": "^4.8.2" }, "peerDependencies": { "tree-sitter": "^0.21.1" }, "optionalPeers": ["tree-sitter"] }, "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA=="],
   }
 }

package.json

@@ -51,5 +51,15 @@
 		"./package.json": "./package.json"
 	},
 	"module": "./dist/index.js",
-	"types": "./dist/index.d.ts"
+	"types": "./dist/index.d.ts",
+	"dependencies": {
+		"effect": "^3.19.12",
+		"tree-sitter-go": "^0.25.0",
+		"tree-sitter-java": "^0.23.5",
+		"tree-sitter-javascript": "^0.25.0",
+		"tree-sitter-python": "^0.25.0",
+		"tree-sitter-rust": "^0.24.0",
+		"tree-sitter-typescript": "^0.23.2",
+		"web-tree-sitter": "^0.26.3"
+	}
 }

src/chunk.ts

@@ -0,0 +1,160 @@
+import { Effect } from 'effect'
+import { chunk as chunkInternal } from './chunking'
+import { extractEntities } from './extract'
+import { parseCode } from './parser'
+import { detectLanguage } from './parser/languages'
+import { buildScopeTree } from './scope'
+import type { Chunk, ChunkOptions, Language } from './types'
+
+/**
+ * Error thrown when chunking fails
+ */
+export class ChunkingError extends Error {
+	readonly _tag = 'ChunkingError'
+	override readonly cause?: unknown
+
+	constructor(message: string, cause?: unknown) {
+		super(message)
+		this.name = 'ChunkingError'
+		this.cause = cause
+	}
+}
+
+/**
+ * Error thrown when language detection fails
+ */
+export class UnsupportedLanguageError extends Error {
+	readonly _tag = 'UnsupportedLanguageError'
+	readonly filepath: string
+
+	constructor(filepath: string) {
+		super(`Unsupported file type: ${filepath}`)
+		this.name = 'UnsupportedLanguageError'
+		this.filepath = filepath
+	}
+}
+
+/**
+ * Internal Effect-based implementation of the chunking pipeline
+ *
+ * Orchestrates: parse -> extract -> scope -> chunk -> context
+ */
+const chunkEffect = (
+	filepath: string,
+	code: string,
+	options: ChunkOptions = {},
+): Effect.Effect<Chunk[], ChunkingError | UnsupportedLanguageError> => {
+	return Effect.gen(function* () {
+		// Step 1: Detect language (or use override)
+		const language: Language | null =
+			options.language ?? detectLanguage(filepath)
+
+		if (!language) {
+			return yield* Effect.fail(new UnsupportedLanguageError(filepath))
+		}
+
+		// Step 2: Parse the code
+		const parseResult = yield* Effect.tryPromise({
+			try: () => parseCode(code, language),
+			catch: (error: unknown) =>
+				new ChunkingError('Failed to parse code', error),
+		})
+
+		// Step 3: Extract entities from AST
+		const entities = yield* Effect.mapError(
+			extractEntities(parseResult.tree.rootNode, language, code),
+			(error: unknown) =>
+				new ChunkingError('Failed to extract entities', error),
+		)
+
+		// Step 4: Build scope tree
+		const scopeTree = yield* Effect.mapError(
+			buildScopeTree(entities),
+			(error: unknown) =>
+				new ChunkingError('Failed to build scope tree', error),
+		)
+
+		// Step 5: Chunk the code
+		const chunks = yield* Effect.mapError(
+			chunkInternal(
+				parseResult.tree.rootNode,
+				code,
+				scopeTree,
+				language,
+				options,
+			),
+			(error: unknown) => new ChunkingError('Failed to chunk code', error),
+		)
+
+		// If there was a parse error (but recoverable), attach it to chunk contexts
+		if (parseResult.error) {
+			const errorInfo = parseResult.error
+			return chunks.map((c: Chunk) => ({
+				...c,
+				context: {
+					...c.context,
+					parseError: errorInfo,
+				},
+			}))
+		}
+
+		return chunks
+	})
+}
+
+/**
+ * Chunk source code into pieces with semantic context
+ *
+ * This is the main entry point for the astchunk library. It takes source code
+ * and returns an array of chunks, each with contextual information about the
+ * code's structure.
+ *
+ * @param filepath - The file path (used for language detection)
+ * @param code - The source code to chunk
+ * @param options - Optional chunking configuration
+ * @returns Array of chunks with context
+ * @throws ChunkingError if chunking fails
+ * @throws UnsupportedLanguageError if the file type is not supported
+ *
+ * @example
+ * ```ts
+ * import { chunk } from 'astchunk'
+ *
+ * const chunks = await chunk('src/utils.ts', sourceCode)
+ * for (const chunk of chunks) {
+ *   console.log(chunk.text, chunk.context)
+ * }
+ * ```
+ */
+export async function chunk(
+	filepath: string,
+	code: string,
+	options?: ChunkOptions,
+): Promise<Chunk[]> {
+	return Effect.runPromise(chunkEffect(filepath, code, options))
+}
+
+/**
+ * Chunk source code synchronously (blocking)
+ *
+ * **WARNING: Not yet implemented.** This function will throw an error.
+ * The chunking pipeline requires async WASM loading which cannot run synchronously.
+ * Use the async `chunk()` function instead.
+ *
+ * @param _filepath - The file path (unused)
+ * @param _code - The source code (unused)
+ * @param _options - Optional chunking configuration (unused)
+ * @throws Error Always throws - sync chunking is not supported
+ *
+ * @deprecated Use `chunk()` instead. This will be implemented in a future version
+ * if there's demand for sync operation with pre-initialized parsers.
+ */
+export function chunkSync(
+	_filepath: string,
+	_code: string,
+	_options?: ChunkOptions,
+): Chunk[] {
+	throw new Error(
+		'chunkSync is not supported. The chunking pipeline requires async WASM loading. Use chunk() instead.',
+	)
+}

src/chunker.ts

@@ -0,0 +1,97 @@
+import { chunk } from './chunk'
+import type { Chunk, Chunker, ChunkOptions } from './types'
+
+/**
+ * Default options for the chunker
+ */
+const DEFAULT_OPTIONS: ChunkOptions = {
+	maxChunkSize: 4096,
+	contextMode: 'full',
+	siblingDetail: 'signatures',
+	filterImports: false,
+}
+
+/**
+ * Implementation of the Chunker interface
+ *
+ * Provides a stateful wrapper around the chunk function that:
+ * - Stores default options
+ * - Tracks the filepath for language detection
+ */
+class ChunkerImpl implements Chunker {
+	private readonly filepath: string
+	private readonly defaultOptions: ChunkOptions
+
+	constructor(filepath: string, options: ChunkOptions = {}) {
+		this.filepath = filepath
+		this.defaultOptions = { ...DEFAULT_OPTIONS, ...options }
+	}
+
+	/**
+	 * Chunk source code into pieces with context
+	 *
+	 * @param source - The source code to chunk
+	 * @param options - Optional overrides for chunking options
+	 * @returns Promise resolving to array of chunks
+	 */
+	async chunk(source: string, options?: ChunkOptions): Promise<Chunk[]> {
+		const mergedOptions = { ...this.defaultOptions, ...options }
+		return chunk(this.filepath, source, mergedOptions)
+	}
+
+	/**
+	 * Stream chunks as they are generated
+	 *
+	 * @param source - The source code to chunk
+	 * @param options - Optional overrides for chunking options
+	 * @returns Async iterable of chunks
+	 *
+	 * TODO: Implement true streaming - for now, this just iterates the array
+	 */
+	async *stream(source: string, options?: ChunkOptions): AsyncIterable<Chunk> {
+		const mergedOptions = { ...this.defaultOptions, ...options }
+		const chunks = await chunk(this.filepath, source, mergedOptions)
+
+		for (const c of chunks) {
+			yield c
+		}
+	}
+}
+
+/**
+ * Create a new Chunker instance for a specific file
+ *
+ * The Chunker provides a convenient interface for chunking source code
+ * with pre-configured options. It's particularly useful when you need to
+ * chunk multiple versions of the same file or want to stream chunks.
+ *
+ * @param filepath - The file path (used for language detection)
+ * @param options - Default options for all chunking operations
+ * @returns A Chunker instance
+ *
+ * @example
+ * ```ts
+ * import { createChunker } from 'astchunk'
+ *
+ * const chunker = createChunker('src/utils.ts', { maxChunkSize: 2048 })
+ *
+ * // Chunk synchronously
+ * const chunks = await chunker.chunk(sourceCode)
+ *
+ * // Or stream chunks
+ * for await (const chunk of chunker.stream(sourceCode)) {
+ *   process.stdout.write(chunk.text)
+ * }
+ * ```
+ */
+export function createChunker(
+	filepath: string,
+	options?: ChunkOptions,
+): Chunker {
+	return new ChunkerImpl(filepath, options)
+}
+
+/**
+ * Re-export the Chunker type for convenience
+ */
+export type { Chunker } from './types'