feat: tsdefs improvements & cli, expose & build schema-schema, fix type exports

Author: rvagg

bin/cli.js

@@ -4,6 +4,7 @@ import { validate } from './validate.js'
 import { toJSON } from './to-json.js'
 import { toSchema } from './to-schema.js'
 import { toJS } from './to-js.js'
+import { toTSDefs } from './to-tsdefs.js'
 import { jsonToSchema } from './json-to-schema.js'
 import _yargs from 'yargs'
 import { hideBin } from 'yargs/helpers'
@@ -13,6 +14,14 @@ const toOpts = {
     alias: 't',
     type: 'boolean',
     describe: 'print with tabs instead of spaces'
+  },
+  'include-comments': {
+    type: 'boolean',
+    describe: 'include comments in the output'
+  },
+  'include-annotations': {
+    type: 'boolean',
+    describe: 'include annotations in the output'
   }
 }
 
@@ -35,6 +44,12 @@ const yargs = _yargs(hideBin(process.argv))
       cjs: { boolean: true }
     }
   )
+  .command('to-tsdefs',
+    'Accepts .ipldsch and .md files, if none are passed will read from stdin, prints a TypeScript module implementing schema interfaces for the <root type>',
+    {
+      cjs: { boolean: true }
+    }
+  )
   .command('json-to-schema',
     'Accepts .json files, if none are passed will read from stdin, prints the canonical IPLD Schema form of the schema represented by the JSON',
     // @ts-ignore
@@ -70,6 +85,9 @@ switch (yargs.argv._[0]) {
   case 'to-js':
     runCommand(toJS)
     break
+  case 'to-tsdefs':
+    runCommand(toTSDefs)
+    break
   case 'json-to-schema':
     runCommand(jsonToSchema)
     break

bin/to-js.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 
-import { readFile } from 'fs/promises'
 import parser from '../lib/parser.cjs'
 import { transformError } from '../lib/util.js'
+import { pkgDecjsor } from './util.js'
 import { collectInput } from './collect-input.js'
 import { Builder, safeReference } from '../lib/typed.js'
 
@@ -20,6 +20,7 @@ export async function toJS (files, options) {
   let schema
   for (const { filename, contents } of input) {
     try {
+      /** @type {any} */
       const parsed = parser.parse(contents)
       if (!schema) {
         schema = parsed
@@ -55,8 +56,3 @@ ${options.cjs === true ? `module.exports${safeReference(type)}` : `export const
 }`)
   }
 }
-
-async function pkgDecjsor () {
-  const p = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'))
-  return `${p.name}@v${p.version}`
-}

bin/to-json.js

@@ -12,21 +12,28 @@ let indent = '  '
 
 /**
  * @param {string[]} files
- * @param {{tabs?:boolean}} options
+ * @param {{tabs?:boolean, 'include-comments'?:boolean, 'include-annotations'?:boolean}} options
  * @returns
  */
 export async function toJSON (files, options) {
   if (options.tabs) {
     indent = '\t'
   }
+  const parseOptions = {}
+  if (options['include-comments']) {
+    parseOptions.includeComments = true
+  }
+  if (options['include-annotations']) {
+    parseOptions.includeAnnotations = true
+  }
 
   const input = await collectInput(files)
 
   /** @type {Schema|null} */
   let schema = null
   for (const { filename, contents } of input) {
     try {
-      const parsed = parser.parse(contents)
+      const parsed = /** @type {Schema} */(parser.parse(contents, parseOptions))
       if (schema == null) {
         schema = parsed
       } else {

bin/to-tsdefs.js

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+
+import parser from '../lib/parser.cjs'
+import { transformError } from '../lib/util.js'
+import { pkgDecjsor } from './util.js'
+import { collectInput } from './collect-input.js'
+import { generateTypeScript } from '../lib/gen.js'
+
+/**
+ * @typedef {import('../schema-schema').Schema} Schema
+ */
+
+/**
+ * @param {string[]} files
+ * @param {{cjs:boolean}} _options
+ * @returns
+ */
+export async function toTSDefs (files, _options) {
+  const input = await collectInput(files)
+  let schema
+  for (const { filename, contents } of input) {
+    try {
+      /** @type {any} */
+      const parsed = parser.parse(contents, { includeComments: true, includeAnnotations: true })
+      if (!schema) {
+        schema = parsed
+      } else {
+        for (const [type, defn] of Object.entries(parsed.types)) {
+          if (schema.types[type]) {
+            console.error(`Error: duplicate type "${type}" found in schema(s)`)
+            return process.exit(1)
+          }
+          schema.types[type] = defn
+        }
+      }
+    } catch (err) {
+      // @ts-ignore
+      console.error(`Error parsing ${filename}: ${transformError(err).message}`)
+      process.exit(1)
+    }
+  }
+
+  const schemaContent = input.map(({ contents }) => contents).join('\n').replace(/^/mg, ' * ').replace(/\s+$/mg, '')
+  console.log(`/** Auto-generated with ${await pkgDecjsor()} at ${new Date().toDateString()} from IPLD Schema:\n *\n${schemaContent}\n */\n`)
+  console.log(generateTypeScript(schema))
+}

bin/util.js

@@ -0,0 +1,6 @@
+import { readFile } from 'fs/promises'
+
+export async function pkgDecjsor () {
+  const p = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'))
+  return `${p.name}@v${p.version}`
+}

ipld-schema.pegjs

@@ -100,10 +100,10 @@ TypeDef =
   }
   const typ = Object.keys(definition)[0]
   const comments = processComments(precomments)
-  if (comments) {
+  if (options.includeComments && comments) {
     definition[typ].comments = extend(definition[typ].comments || {}, { type: comments })
   }
-  if (annotations && annotations.length) {
+  if (options.includeComments && annotations && annotations.length) {
     definition[typ].annotations = extend(definition[typ].annotations || {}, { type: annotations })
   }
   return { [name]: definition }

lib/from-dsl.js

@@ -7,11 +7,12 @@ import { transformError } from './util.js'
 
 /**
  * @param {string} input
+ * @param {Record<string, any>} [options]
  * @returns {Schema}
  */
-export function fromDSL (input) {
+export function fromDSL (input, options = {}) {
   try {
-    return /** @type {Schema} */(parser.parse(input))
+    return /** @type {Schema} */(parser.parse(input, options))
   } catch (err) {
     throw transformError(err)
   }

lib/gen/typescript.js

@@ -9,14 +9,24 @@
  */
 export function generateTypeScript (schema) {
   /** @type {[string, string][]} */
-  let imports = []
+  const imports = []
 
   let typesrc = ''
   for (const [typeName, typeDefn] of Object.entries(schema.types)) {
+    if (Object.keys(typeDefn).length !== 1) {
+      throw new Error('Unexpected type definition: ' + JSON.stringify(typeDefn))
+    }
+    const typeKind = Object.keys(typeDefn)[0]
     if ('struct' in typeDefn) {
       typesrc += `export type ${typeName} = {\n`
 
+      /** @type {string[]} */
+      const fieldValidators = []
+      let requiredFieldCount = 0
       for (let [fieldName, fieldDefn] of Object.entries(typeDefn.struct.fields)) {
+        if (!fieldDefn.optional && !fieldDefn.optional) {
+          requiredFieldCount++
+        }
         /** @type { { [k in string]: string }[]} */
         let annotations = []
         if (typeof typeDefn.struct.annotations === 'object' && typeof typeDefn.struct.annotations.type === 'object') {
@@ -60,17 +70,97 @@ export function generateTypeScript (schema) {
         }
         fieldType = fixTypeScriptType(imports, fieldType, slice)
         fieldName = fixTypeScriptName(annotations, fieldName)
-        typesrc += `  ${fieldName}: ${fieldType}${linecomment}\n`
+        typesrc += `  ${fieldName}${fieldDefn.optional ? '?' : ''}: ${fieldType}${linecomment}\n`
+        const inCheck = !fieldDefn.optional && !fieldDefn.optional ? `'${fieldName}' in value &&` : `!('${fieldName}' in value) ||`
+        if (fieldType.endsWith('[]')) {
+          let elementType = getTypeScriptType([], fieldType.slice(0, -2))
+          elementType = fixTypeScriptType(imports, elementType, false)
+          fieldValidators.push(`      (${inCheck} (${fieldDefn.nullable ? `value.${fieldName} === null || ` : ''}(Array.isArray(value.${fieldName}) && value.${fieldName}.every(${elementType}.is${elementType}))))`)
+        } else {
+          fieldValidators.push(`      (${inCheck} (${fieldDefn.nullable ? `value.${fieldName} === null || ` : ''}(${fieldType}.is${fieldType}(value.${fieldName}))))`)
+        }
       }
 
       typesrc += '}\n\n'
+
+      const kind = fixTypeScriptType(imports, '@ipld/schema/schema-schema.js#KindMap', false)
+      typesrc += `export namespace ${typeName} {\n`
+      typesrc += `  export function is${typeName}(value: any): value is ${typeName} {\n`
+      typesrc += `    if (!${kind}.is${kind}(value)) {\n`
+      typesrc += '      return false\n'
+      typesrc += '    }\n'
+      typesrc += '    const keyCount = Object.keys(value).length\n'
+      typesrc += '    return '
+      if (requiredFieldCount === Object.keys(typeDefn.struct.fields).length) {
+        typesrc += `keyCount === ${requiredFieldCount} &&\n`
+      } else {
+        // TODO: this isn't really a complete check, we probably should check for extra fields
+        typesrc += `keyCount >= ${requiredFieldCount} && keyCount <= ${Object.keys(typeDefn.struct.fields).length} &&\n`
+      }
+      typesrc += fieldValidators.join(' &&\n')
+      typesrc += '\n  }\n'
+      typesrc += '}\n\n'
+    } else if ('list' in typeDefn) {
+      if (typeof typeDefn.list.valueType !== 'string') {
+        throw new Error('Unhandled list value type: ' + JSON.stringify(typeDefn))
+      }
+      let valueType = getTypeScriptType([], typeDefn.list.valueType)
+      valueType = fixTypeScriptType(imports, valueType, false)
+      typesrc += `export type ${typeName} = ${valueType}[]\n\n`
+      typesrc += `export namespace ${typeName} {\n`
+      typesrc += `  export function is${typeName}(value: any): value is ${typeName} {\n`
+      typesrc += `    return Array.isArray(value) && value.every(${valueType}.is${valueType})\n`
+      typesrc += '  }\n'
+      typesrc += '}\n\n'
+    } else if ('copy' in typeDefn) {
+      const { fromType } = typeDefn.copy
+      typesrc += `export type ${typeName} = ${fromType}\n\n`
+      typesrc += `export namespace ${typeName} {\n`
+      typesrc += `  export function is${typeName}(value: any): value is ${typeName} {\n`
+      typesrc += `    return ${fromType}.is${fromType}(value)\n`
+      typesrc += '  }\n'
+      typesrc += '}\n\n'
+    } else if (['bool', 'string', 'bytes', 'int', 'float', 'link', 'null'].includes(typeKind)) {
+      const kind = fixTypeScriptType(imports, `@ipld/schema/schema-schema.js#Kind${typeKind.charAt(0).toUpperCase()}${typeKind.slice(1)}`, false)
+      typesrc += `export type ${typeName} = ${kind}\n\n`
+      typesrc += `export namespace ${typeName} {\n`
+      typesrc += `  export function is${typeName}(value: any): value is ${typeName} {\n`
+      typesrc += `    return ${kind}.is${kind}(value)\n`
+      typesrc += '  }\n'
+      typesrc += '}\n\n'
+    } else if ('union' in typeDefn) {
+      if (!('kinded' in typeDefn.union.representation)) {
+        throw new Error('Unhandled union representation: ' + Object.keys(typeDefn.union.representation)[0])
+      }
+      if (typeDefn.union.members.some((member) => typeof member !== 'string')) {
+        throw new Error('Unhandled union member type(s): ' + JSON.stringify(typeDefn.union.members))
+      }
+      const kinds = typeDefn.union.members.map((member) => {
+        return fixTypeScriptType(imports, getTypeScriptType([], String(member)), false)
+      })
+      typesrc += `export type ${typeName} = ${kinds.join(' | ')}\n\n`
+      typesrc += `export namespace ${typeName} {\n`
+      typesrc += `  export function is${typeName}(value: any): value is ${typeName} {\n`
+      typesrc += `    return ${kinds.map((kind) => `${kind}.is${kind}(value)`).join(' || ')}\n`
+      typesrc += '  }\n'
+      typesrc += '}\n\n'
+    } else {
+      throw new Error('Unimplemented type kind: ' + typeKind)
     }
   }
 
   let ts = ''
-  imports = fixTypeScriptImports(imports)
-  for (const imp of imports) {
-    ts += `import { ${imp[1]} } from '${imp[0]}'\n`
+  const fixedImports = fixTypeScriptImports(imports)
+  for (const imp of fixedImports) {
+    if (imp[1].length === 1) {
+      ts += `import { ${imp[1]} } from '${imp[0]}'\n`
+    } else {
+      ts += 'import {\n'
+      for (const imported of imp[1]) {
+        ts += `  ${imported},\n`
+      }
+      ts += `} from '${imp[0]}'\n`
+    }
   }
   if (imports.length > 0) {
     ts += '\n'
@@ -93,8 +183,7 @@ function fixTypeScriptName (annotations, fieldName) {
       }
     }
   }
-  // snakeCase with lower-case first letter
-  return fieldName.charAt(0).toLowerCase() + fieldName.slice(1)
+  return fieldName
 }
 
 /**
@@ -112,26 +201,21 @@ function getTypeScriptType (annotations, ipldType) {
     }
   }
   switch (ipldType) {
-    case 'Int':
-      return 'number'
-    case 'Float':
-      return 'number'
     case 'Bool':
-      return 'boolean'
     case 'String':
-      return 'string'
     case 'Bytes':
-      return 'Uint8Array'
-    case 'Link':
-      return 'multiformats/cid#CID'
+    case 'Int':
+    case 'Float':
+    case 'Null':
     case 'Map':
-      return 'object'
     case 'List':
-      return 'any[]'
+    case 'Link':
     case 'Union':
-      return 'any' // TODO:
+    case 'Struct':
+    case 'Enum':
+      return `@ipld/schema/schema-schema.js#Kind${ipldType}`
     case 'Any':
-      return 'any' // TODO:
+      return 'any' // TODO: something here?
   }
 
   return ipldType
@@ -160,9 +244,25 @@ function fixTypeScriptType (imports, tstype, slice) {
 
 /**
  * @param {[string, string][]} imports
- * @returns {[string, string][]}
+ * @returns {[string, string[]][]}
  */
 function fixTypeScriptImports (imports) {
-  // TODO: implement for user imports
-  return imports
+  /** @type {Record<string, string[]>} */
+  const groupedImports = {}
+  for (const [source, imported] of imports) {
+    if (!groupedImports[source]) {
+      groupedImports[source] = []
+    }
+    if (!groupedImports[source].includes(imported)) {
+      groupedImports[source].push(imported)
+    }
+  }
+
+  /** @type {[string, string[]][]} */
+  const result = []
+  for (const source in groupedImports) {
+    result.push([source, groupedImports[source].sort()])
+  }
+
+  return result
 }