feat: support more types in wasmify and wasi logging

closes #159

Author: hugomrdias

packages/homestar/package.json

@@ -64,13 +64,17 @@
   "dependencies": {
     "@bytecodealliance/componentize-js": "^0.7.0",
     "@fission-codes/channel": "workspace:^",
+    "@fission-codes/homestar-wit": "workspace:^",
     "@ipld/dag-cbor": "^9.2.0",
     "@ipld/dag-json": "^10.2.0",
     "@multiformats/sha3": "^3.0.2",
     "@ts-ast-parser/core": "^0.7.0",
+    "del": "^7.1.0",
     "emittery": "^1.0.3",
     "esbuild": "^0.20.1",
+    "esbuild-plugin-replace-regex": "^0.0.2",
     "get-tsconfig": "^4.7.2",
+    "just-kebab-case": "^4.2.0",
     "multiformats": "^13.1.0",
     "object-path": "^0.11.8",
     "zod": "^3.22.4"

packages/homestar/src/types.ts

@@ -59,3 +59,10 @@ export type HomestarService = Service<
       result: Opaque<number[], Schemas.EventNotification>
     }
 >
+
+export interface WitOptions {
+  filePath?: string
+  source?: string
+  worldName: string
+  wasiImports?: Set<string>
+}

packages/homestar/src/wasmify/index.js

@@ -1,134 +1,104 @@
 import { writeFile } from 'node:fs/promises'
 import path from 'path'
-import { parseFromFiles } from '@ts-ast-parser/core'
+import { createRequire } from 'node:module'
+import { base32 } from 'iso-base/rfc4648'
 import * as esbuild from 'esbuild'
-import { getTsconfig } from 'get-tsconfig'
+import { deleteAsync } from 'del'
 
 // @ts-ignore
-import { componentize } from '@bytecodealliance/componentize-js'
-
-/**
- * @param {import('@ts-ast-parser/core').Type} type
- * @returns {string}
- */
-function primitiveType(type) {
-  if (type.text === 'string') {
-    return 'string'
-  }
-
-  if (type.text === 'boolean') {
-    return 'bool'
-  }
-
-  if (type.text === 'number') {
-    return 's64'
-  }
-
-  if (type.text === 'Uint8Array') {
-    return 'list<u8>'
-  }
+import replacePlugin from 'esbuild-plugin-replace-regex'
 
-  if (type.kind === 'Array' && type.elementType) {
-    return `list<${primitiveType(type.elementType)}>`
-  }
+// @ts-ignore
+import { componentize } from '@bytecodealliance/componentize-js'
+import { wit } from './wit.js'
 
-  throw new Error(`Unknown type: ${JSON.stringify(type)}`)
-}
+const require = createRequire(import.meta.url)
 
 /**
- *
- * Generate a WIT file from a TypeScript file
+ * Bundle the js code with WASI support
  *
  * @param {string} filePath - Path to a TypeScript file
  */
-async function wit(filePath) {
-  const cfg = getTsconfig(filePath)
-  if (!cfg) {
-    throw new Error('No tsconfig found')
-  }
-
-  const { project, errors } = await parseFromFiles([filePath], {
-    tsConfigFilePath: cfg.path,
-  })
-
-  if (errors.length > 0) {
-    console.error(errors)
-    // Handle the errors
-
-    // process.exit(1)
-  }
-
-  const result = project?.getModules().map((m) => m.serialize()) ?? []
-  if (result.length > 0) {
-    // console.log(
-    //   '🚀 ~ file: cli.js:23 ~ reflectedModules:',
-    //   JSON.stringify(result, null, 2)
-    // )
-    const { sourcePath, declarations } = result[0]
-    const world = path.basename(sourcePath).replace('.ts', '')
-    const exports = declarations.map((d) => {
-      if (d.kind === 'Function') {
-        /** @type {string[]} */
-        const params = d.signatures[0].parameters
-          ? d.signatures[0].parameters.map(
-              (p) => `${p.name}: ${primitiveType(p.type)}`
-            )
-          : []
-        const name = d.name
-        const returnType = primitiveType(d.signatures[0].return.type)
-
-        return `  export ${name}: func(${params.join(', ')}) -> ${returnType};`
-      }
-
-      return ''
-    })
-
-    const wit = `
-package local:${world};
-
-world ${world} {
-${exports.join('\n')}
-}
-    `
-    // console.log('🚀 ~ WIT World\n\n', wit)
-    return wit
-  } else {
-    throw new Error('No modules found')
+async function bundle(filePath) {
+  const wasiImports = new Set()
+  /** @type {import('esbuild').Plugin} */
+  const wasiPlugin = {
+    name: 'wasi imports',
+    setup(build) {
+      // @ts-ignore
+      build.onResolve({ filter: /^wasi:.*/ }, (args) => {
+        wasiImports.add(`import ${args.path};`)
+      })
+    },
   }
-}
 
-/**
- * @param {string} filePath - Path to a TypeScript file
- */
-async function bundle(filePath) {
   const result = await esbuild.build({
     entryPoints: [filePath],
     bundle: true,
     format: 'esm',
     platform: 'browser',
     write: false,
+    plugins: [
+      replacePlugin({
+        filter: /homestar-wit\/src\/logging.js$/,
+        patterns: [
+          ['let wasiLog', '// let wasiLog'],
+          [
+            "// import { log as wasiLog } from 'wasi:logging/logging'",
+            "import { log as wasiLog } from 'wasi:logging/logging'",
+          ],
+        ],
+      }),
+      wasiPlugin,
+    ],
+    external: ['wasi:*'],
   })
-  // console.log('🚀 ~ bundle ~ result:', result.outputFiles[0].hash)
-  return result.outputFiles[0].text
+  return {
+    src: result.outputFiles[0].text,
+    hash: result.outputFiles[0].hash,
+    wasiImports,
+  }
 }
 
 /**
+ * Generate wasm component from a TypeScript file
+ *
  * @param {string} filePath - Path to a TypeScript file
  * @param {string} outDir - Path to a directory to write the Wasm component file
  */
 export async function build(filePath, outDir = process.cwd()) {
-  const outName = path
-    .basename(filePath)
-    .replace(path.extname(filePath), '.wasm')
-  const outPath = path.join(outDir, outName)
-
-  const { component } = await componentize(
-    await bundle(filePath),
-    await wit(filePath)
-  )
-  await writeFile(outPath, component)
-
-  return {
-    outPath,
+  const pkgPath = require.resolve('@fission-codes/homestar-wit')
+  const witPath = path.join(pkgPath, '..', '..', 'wit')
+  let witHash
+
+  // Clean up any old WIT files in the wit directory
+  // componentize process.exit(1) if it fails so we can't clean up after it
+  await deleteAsync([`${witPath}/*.wit`], { force: true })
+
+  try {
+    const { hash, src, wasiImports } = await bundle(filePath)
+    witHash = base32.encode(hash).toLowerCase()
+
+    // TODO: check the wit hash and only componentize if it has changed
+    const witSource = await wit({ filePath, worldName: witHash, wasiImports })
+    const witFile = path.join(witPath, `${witHash}.wit`)
+    await writeFile(witFile, witSource, 'utf8')
+    const { component } = await componentize(src, {
+      witPath,
+      worldName: witHash,
+      // debug: true,
+      sourceName: filePath,
+    })
+    const outPath = path.join(outDir, `${witHash}.wasm`)
+    await writeFile(outPath, component)
+
+    return {
+      outPath,
+      witHash,
+    }
+  } finally {
+    if (witHash) {
+      await deleteAsync([path.join(witPath, `${witHash}.wit`)], { force: true })
+    }
   }
 }