diff --git a/src/CLIOptions.types.ts b/src/CLIOptions.types.ts new file mode 100644 index 000000000..201986ffb --- /dev/null +++ b/src/CLIOptions.types.ts @@ -0,0 +1,32 @@ +import type { LanguageName, RendererOptions } from "quicktype-core"; + +export interface CLIOptions { + // We use this to access the inference flags + // biome-ignore lint/suspicious/noExplicitAny: + [option: string]: any; + additionalSchema: string[]; + allPropertiesOptional: boolean; + alphabetizeProperties: boolean; + buildMarkovChain?: string; + debug?: string; + graphqlIntrospect?: string; + graphqlSchema?: string; + help: boolean; + httpHeader?: string[]; + httpMethod?: string; + lang: Lang; + + noRender: boolean; + out?: string; + quiet: boolean; + + rendererOptions: RendererOptions; + + src: string[]; + srcLang: string; + srcUrls?: string; + telemetry?: string; + topLevel: string; + + version: boolean; +} diff --git a/src/GraphQLIntrospection.ts b/src/GraphQLIntrospection.ts index db2093a0a..aa59ab9f0 100644 --- a/src/GraphQLIntrospection.ts +++ b/src/GraphQLIntrospection.ts @@ -32,6 +32,7 @@ export async function introspectServer( headers[matches[1]] = matches[2]; } + // biome-ignore lint/suspicious/noImplicitAnyLet: let result; try { const response = await fetch(url, { diff --git a/src/cli.options.ts b/src/cli.options.ts new file mode 100644 index 000000000..407422200 --- /dev/null +++ b/src/cli.options.ts @@ -0,0 +1,126 @@ +import { exceptionToString } from "@glideapps/ts-necessities"; +import { + // biome-ignore lint/suspicious/noShadowRestrictedNames: + hasOwnProperty, +} from "collection-utils"; +import commandLineArgs from "command-line-args"; +import _ from "lodash"; + +import { + type OptionDefinition, + type RendererOptions, + type TargetLanguage, + assert, + defaultTargetLanguages, + getTargetLanguage, + isLanguageName, + messageError, +} from "quicktype-core"; + +import { inferCLIOptions } from "./inference"; +import { makeOptionDefinitions } from "./optionDefinitions"; +import type { CLIOptions } from "./CLIOptions.types"; + +// Parse the options in argv and split them into global options and renderer options, +// according to each option definition's `renderer` field. If `partial` is false this +// will throw if it encounters an unknown option. +function parseOptions( + definitions: OptionDefinition[], + argv: string[], + partial: boolean, +): Partial { + let opts: commandLineArgs.CommandLineOptions; + try { + opts = commandLineArgs( + definitions.map((def) => ({ + ...def, + type: def.optionType === "boolean" ? Boolean : String, + })), + { argv, partial }, + ); + } catch (e) { + assert(!partial, "Partial option parsing should not have failed"); + return messageError("DriverCLIOptionParsingFailed", { + message: exceptionToString(e), + }); + } + + for (const k of Object.keys(opts)) { + if (opts[k] === null) { + return messageError("DriverCLIOptionParsingFailed", { + message: `Missing value for command line option "${k}"`, + }); + } + } + + const options: { + [key: string]: unknown; + rendererOptions: RendererOptions; + } = { rendererOptions: {} }; + for (const optionDefinition of definitions) { + if (!hasOwnProperty(opts, optionDefinition.name)) { + continue; + } + + const optionValue = opts[optionDefinition.name] as string; + if (optionDefinition.kind !== "cli") { + ( + options.rendererOptions as Record< + typeof optionDefinition.name, + unknown + > + )[optionDefinition.name] = optionValue; + } else { + const k = _.lowerFirst( + optionDefinition.name.split("-").map(_.upperFirst).join(""), + ); + options[k] = optionValue; + } + } + + return options; +} + +export function parseCLIOptions( + argv: string[], + inputTargetLanguage?: TargetLanguage, +): CLIOptions { + if (argv.length === 0) { + return inferCLIOptions({ help: true }, inputTargetLanguage); + } + + const targetLanguages = inputTargetLanguage + ? [inputTargetLanguage] + : defaultTargetLanguages; + const optionDefinitions = makeOptionDefinitions(targetLanguages); + + // We can only fully parse the options once we know which renderer is selected, + // because there are renderer-specific options. But we only know which renderer + // is selected after we've parsed the options. Hence, we parse the options + // twice. This is the first parse to get the renderer: + const incompleteOptions = inferCLIOptions( + parseOptions(optionDefinitions, argv, true), + inputTargetLanguage, + ); + + let targetLanguage = inputTargetLanguage as TargetLanguage; + if (inputTargetLanguage === undefined) { + const languageName = isLanguageName(incompleteOptions.lang) + ? incompleteOptions.lang + : "typescript"; + targetLanguage = getTargetLanguage(languageName); + } + + const rendererOptionDefinitions = + targetLanguage.cliOptionDefinitions.actual; + // Use the global options as well as the renderer options from now on: + const allOptionDefinitions = _.concat( + optionDefinitions, + rendererOptionDefinitions, + ); + // This is the parse that counts: + return inferCLIOptions( + parseOptions(allOptionDefinitions, argv, false), + targetLanguage, + ); +} diff --git a/src/index.ts b/src/index.ts index 856542c96..566851235 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,1177 +1,27 @@ #!/usr/bin/env node + import * as fs from "node:fs"; import * as path from "node:path"; -import { exceptionToString } from "@glideapps/ts-necessities"; import chalk from "chalk"; -import { - definedMap, - // biome-ignore lint/suspicious/noShadowRestrictedNames: - hasOwnProperty, - mapFromObject, - mapMap, - withDefault, -} from "collection-utils"; -import commandLineArgs from "command-line-args"; -import getUsage from "command-line-usage"; -import * as _ from "lodash"; -import type { Readable } from "readable-stream"; -import stringToStream from "string-to-stream"; -import _wordwrap from "wordwrap"; import { - FetchingJSONSchemaStore, - InputData, IssueAnnotationData, - JSONInput, - JSONSchemaInput, - type JSONSourceData, - type LanguageName, - type OptionDefinition, - type Options, - type RendererOptions, type SerializedRenderResult, - type TargetLanguage, - assert, - assertNever, - capitalize, - defaultTargetLanguages, - defined, - getStream, - getTargetLanguage, - inferenceFlagNames, - inferenceFlags, - isLanguageName, - languageNamed, - messageAssert, - messageError, - panic, - parseJSON, quicktypeMultiFile, - readFromFileOrURL, - readableFromFileOrURL, - sourcesFromPostmanCollection, - splitIntoWords, - trainMarkovChain, } from "quicktype-core"; -import { GraphQLInput } from "quicktype-graphql-input"; -import { schemaForTypeScriptSources } from "quicktype-typescript-input"; - -import { CompressedJSONFromStream } from "./CompressedJSONFromStream"; -import { introspectServer } from "./GraphQLIntrospection"; -import type { - GraphQLTypeSource, - JSONTypeSource, - SchemaTypeSource, - TypeSource, -} from "./TypeSource"; -import { urlsFromURLGrammar } from "./URLGrammar"; - -// eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires -const packageJSON = require("../package.json"); - -const wordWrap: (s: string) => string = _wordwrap(90); - -export interface CLIOptions { - // We use this to access the inference flags - // biome-ignore lint/suspicious/noExplicitAny: - [option: string]: any; - additionalSchema: string[]; - allPropertiesOptional: boolean; - alphabetizeProperties: boolean; - buildMarkovChain?: string; - debug?: string; - graphqlIntrospect?: string; - graphqlSchema?: string; - help: boolean; - httpHeader?: string[]; - httpMethod?: string; - lang: Lang; - - noRender: boolean; - out?: string; - quiet: boolean; - - rendererOptions: RendererOptions; - - src: string[]; - srcLang: string; - srcUrls?: string; - telemetry?: string; - topLevel: string; - - version: boolean; -} - -const defaultDefaultTargetLanguageName = "go"; - -async function sourceFromFileOrUrlArray( - name: string, - filesOrUrls: string[], - httpHeaders?: string[], -): Promise { - const samples = await Promise.all( - filesOrUrls.map( - async (file) => await readableFromFileOrURL(file, httpHeaders), - ), - ); - return { kind: "json", name, samples }; -} - -function typeNameFromFilename(filename: string): string { - const name = path.basename(filename); - return name.substring(0, name.lastIndexOf(".")); -} - -async function samplesFromDirectory( - dataDir: string, - httpHeaders?: string[], -): Promise { - async function readFilesOrURLsInDirectory( - d: string, - ): Promise { - const files = fs - .readdirSync(d) - .map((x) => path.join(d, x)) - .filter((x) => fs.lstatSync(x).isFile()); - // Each file is a (Name, JSON | URL) - const sourcesInDir: TypeSource[] = []; - const graphQLSources: GraphQLTypeSource[] = []; - let graphQLSchema: Readable | undefined = undefined; - let graphQLSchemaFileName: string | undefined = undefined; - for (let file of files) { - const name = typeNameFromFilename(file); - - let fileOrUrl = file; - file = file.toLowerCase(); - - // If file is a URL string, download it - if (file.endsWith(".url")) { - fileOrUrl = fs.readFileSync(file, "utf8").trim(); - } - - if (file.endsWith(".url") || file.endsWith(".json")) { - sourcesInDir.push({ - kind: "json", - name, - samples: [ - await readableFromFileOrURL(fileOrUrl, httpHeaders), - ], - }); - } else if (file.endsWith(".schema")) { - sourcesInDir.push({ - kind: "schema", - name, - uris: [fileOrUrl], - }); - } else if (file.endsWith(".gqlschema")) { - messageAssert( - graphQLSchema === undefined, - "DriverMoreThanOneGraphQLSchemaInDir", - { - dir: dataDir, - }, - ); - graphQLSchema = await readableFromFileOrURL( - fileOrUrl, - httpHeaders, - ); - graphQLSchemaFileName = fileOrUrl; - } else if (file.endsWith(".graphql")) { - graphQLSources.push({ - kind: "graphql", - name, - schema: undefined, - query: await getStream( - await readableFromFileOrURL(fileOrUrl, httpHeaders), - ), - }); - } - } - - if (graphQLSources.length > 0) { - if (graphQLSchema === undefined) { - return messageError("DriverNoGraphQLSchemaInDir", { - dir: dataDir, - }); - } - - const schema = parseJSON( - await getStream(graphQLSchema), - "GraphQL schema", - graphQLSchemaFileName, - ); - for (const source of graphQLSources) { - source.schema = schema; - sourcesInDir.push(source); - } - } - - return sourcesInDir; - } - - const contents = fs.readdirSync(dataDir).map((x) => path.join(dataDir, x)); - const directories = contents.filter((x) => fs.lstatSync(x).isDirectory()); - - let sources = await readFilesOrURLsInDirectory(dataDir); - - for (const dir of directories) { - let jsonSamples: Readable[] = []; - const schemaSources: SchemaTypeSource[] = []; - const graphQLSources: GraphQLTypeSource[] = []; - - for (const source of await readFilesOrURLsInDirectory(dir)) { - switch (source.kind) { - case "json": - jsonSamples = jsonSamples.concat(source.samples); - break; - case "schema": - schemaSources.push(source); - break; - case "graphql": - graphQLSources.push(source); - break; - default: - return assertNever(source); - } - } - - if ( - jsonSamples.length > 0 && - schemaSources.length + graphQLSources.length > 0 - ) { - return messageError("DriverCannotMixJSONWithOtherSamples", { - dir: dir, - }); - } - - // FIXME: rewrite this to be clearer - const oneUnlessEmpty = (xs: TypeSource[]): 0 | 1 => - Math.sign(xs.length) as 0 | 1; - if ( - oneUnlessEmpty(schemaSources) + oneUnlessEmpty(graphQLSources) > - 1 - ) { - return messageError("DriverCannotMixNonJSONInputs", { dir: dir }); - } - - if (jsonSamples.length > 0) { - sources.push({ - kind: "json", - name: path.basename(dir), - samples: jsonSamples, - }); - } - - sources = sources.concat(schemaSources); - sources = sources.concat(graphQLSources); - } - - return sources; -} - -function inferLang( - options: Partial, - defaultLanguage: LanguageName, -): string | LanguageName { - // Output file extension determines the language if language is undefined - if (options.out !== undefined) { - const extension = path.extname(options.out); - if (extension === "") { - return messageError("DriverNoLanguageOrExtension", {}); - } - - return extension.slice(1); - } - - return defaultLanguage; -} - -function inferTopLevel(options: Partial): string { - // Output file name determines the top-level if undefined - if (options.out !== undefined) { - const extension = path.extname(options.out); - const without = path.basename(options.out).replace(extension, ""); - return without; - } - - // Source determines the top-level if undefined - if (options.src !== undefined && options.src.length === 1) { - const src = options.src[0]; - const extension = path.extname(src); - const without = path.basename(src).replace(extension, ""); - return without; - } - - return "TopLevel"; -} - -function inferCLIOptions( - opts: Partial, - targetLanguage: TargetLanguage | undefined, -): CLIOptions { - let srcLang = opts.srcLang; - if ( - opts.graphqlSchema !== undefined || - opts.graphqlIntrospect !== undefined - ) { - messageAssert( - srcLang === undefined || srcLang === "graphql", - "DriverSourceLangMustBeGraphQL", - {}, - ); - srcLang = "graphql"; - } else if ( - opts.src !== undefined && - opts.src.length > 0 && - opts.src.every((file) => _.endsWith(file, ".ts")) - ) { - srcLang = "typescript"; - } else { - messageAssert(srcLang !== "graphql", "DriverGraphQLSchemaNeeded", {}); - srcLang = withDefault(srcLang, "json"); - } - - let language: TargetLanguage; - if (targetLanguage !== undefined) { - language = targetLanguage; - } else { - const languageName = - opts.lang ?? inferLang(opts, defaultDefaultTargetLanguageName); - - if (isLanguageName(languageName)) { - language = languageNamed(languageName); - } else { - return messageError("DriverUnknownOutputLanguage", { - lang: languageName, - }); - } - } - - const options: CLIOptions = { - src: opts.src ?? [], - srcUrls: opts.srcUrls, - srcLang: srcLang, - lang: language.name as LanguageName, - topLevel: opts.topLevel ?? inferTopLevel(opts), - noRender: !!opts.noRender, - alphabetizeProperties: !!opts.alphabetizeProperties, - allPropertiesOptional: !!opts.allPropertiesOptional, - rendererOptions: opts.rendererOptions ?? {}, - help: opts.help ?? false, - quiet: opts.quiet ?? false, - version: opts.version ?? false, - out: opts.out, - buildMarkovChain: opts.buildMarkovChain, - additionalSchema: opts.additionalSchema ?? [], - graphqlSchema: opts.graphqlSchema, - graphqlIntrospect: opts.graphqlIntrospect, - httpMethod: opts.httpMethod, - httpHeader: opts.httpHeader, - debug: opts.debug, - telemetry: opts.telemetry, - }; - for (const flagName of inferenceFlagNames) { - const cliName = negatedInferenceFlagName(flagName); - options[cliName] = !!opts[cliName]; - } - - return options; -} - -function makeLangTypeLabel(targetLanguages: readonly TargetLanguage[]): string { - assert( - targetLanguages.length > 0, - "Must have at least one target language", - ); - return targetLanguages - .map((r) => _.minBy(r.names, (s) => s.length)) - .join("|"); -} - -function negatedInferenceFlagName(name: string): string { - const prefix = "infer"; - if (name.startsWith(prefix)) { - name = name.slice(prefix.length); - } - - return "no" + capitalize(name); -} - -function dashedFromCamelCase(name: string): string { - return splitIntoWords(name) - .map((w) => w.word.toLowerCase()) - .join("-"); -} - -function makeOptionDefinitions( - targetLanguages: readonly TargetLanguage[], -): OptionDefinition[] { - const beforeLang: OptionDefinition[] = [ - { - name: "out", - alias: "o", - optionType: "string", - typeLabel: "FILE", - description: "The output file. Determines --lang and --top-level.", - kind: "cli", - }, - { - name: "top-level", - alias: "t", - optionType: "string", - typeLabel: "NAME", - description: "The name for the top level type.", - kind: "cli", - }, - ]; - const lang: OptionDefinition[] = - targetLanguages.length < 2 - ? [] - : [ - { - name: "lang", - alias: "l", - optionType: "string", - typeLabel: "LANG", - description: "The target language.", - kind: "cli", - }, - ]; - const afterLang: OptionDefinition[] = [ - { - name: "src-lang", - alias: "s", - optionType: "string", - defaultValue: undefined, - typeLabel: "SRC_LANG", - description: "The source language (default is json).", - kind: "cli", - }, - { - name: "src", - optionType: "string", - multiple: true, - typeLabel: "FILE|URL|DIRECTORY", - description: "The file, url, or data directory to type.", - kind: "cli", - defaultOption: true, - }, - { - name: "src-urls", - optionType: "string", - typeLabel: "FILE", - description: "Tracery grammar describing URLs to crawl.", - kind: "cli", - }, - ]; - const inference: OptionDefinition[] = Array.from( - mapMap(mapFromObject(inferenceFlags), (flag, name) => { - return { - name: dashedFromCamelCase(negatedInferenceFlagName(name)), - optionType: "boolean" as const, - description: flag.negationDescription + ".", - kind: "cli" as const, - }; - }).values(), - ); - const afterInference: OptionDefinition[] = [ - { - name: "graphql-schema", - optionType: "string", - typeLabel: "FILE", - description: "GraphQL introspection file.", - kind: "cli", - }, - { - name: "graphql-introspect", - optionType: "string", - typeLabel: "URL", - description: "Introspect GraphQL schema from a server.", - kind: "cli", - }, - { - name: "http-method", - optionType: "string", - typeLabel: "METHOD", - description: - "HTTP method to use for the GraphQL introspection query.", - kind: "cli", - }, - { - name: "http-header", - optionType: "string", - multiple: true, - typeLabel: "HEADER", - description: - "Header(s) to attach to all HTTP requests, including the GraphQL introspection query.", - kind: "cli", - }, - { - name: "additional-schema", - alias: "S", - optionType: "string", - multiple: true, - typeLabel: "FILE", - description: "Register the $id's of additional JSON Schema files.", - kind: "cli", - }, - { - name: "no-render", - optionType: "boolean", - description: "Don't render output.", - kind: "cli", - }, - { - name: "alphabetize-properties", - optionType: "boolean", - description: "Alphabetize order of class properties.", - kind: "cli", - }, - { - name: "all-properties-optional", - optionType: "boolean", - description: "Make all class properties optional.", - kind: "cli", - }, - { - name: "build-markov-chain", - optionType: "string", - typeLabel: "FILE", - description: "Markov chain corpus filename.", - kind: "cli", - }, - { - name: "quiet", - optionType: "boolean", - description: "Don't show issues in the generated code.", - kind: "cli", - }, - { - name: "debug", - optionType: "string", - typeLabel: "OPTIONS or all", - description: - "Comma separated debug options: print-graph, print-reconstitution, print-gather-names, print-transformations, print-schema-resolving, print-times, provenance", - kind: "cli", - }, - { - name: "telemetry", - optionType: "string", - typeLabel: "enable|disable", - description: "Enable anonymous telemetry to help improve quicktype", - kind: "cli", - }, - { - name: "help", - alias: "h", - optionType: "boolean", - description: "Get some help.", - kind: "cli", - }, - { - name: "version", - alias: "v", - optionType: "boolean", - description: "Display the version of quicktype", - kind: "cli", - }, - ]; - return beforeLang.concat(lang, afterLang, inference, afterInference); -} - -interface ColumnDefinition { - name: string; - padding?: { left: string; right: string }; - width?: number; -} - -interface TableOptions { - columns: ColumnDefinition[]; -} - -interface UsageSection { - content?: string | string[]; - header?: string; - hide?: string[]; - optionList?: OptionDefinition[]; - tableOptions?: TableOptions; -} - -const tableOptionsForOptions: TableOptions = { - columns: [ - { - name: "option", - width: 60, - }, - { - name: "description", - width: 60, - }, - ], -}; - -function makeSectionsBeforeRenderers( - targetLanguages: readonly TargetLanguage[], -): UsageSection[] { - const langDisplayNames = targetLanguages - .map((r) => r.displayName) - .join(", "); - - return [ - { - header: "Synopsis", - content: [ - `$ quicktype [${chalk.bold("--lang")} LANG] [${chalk.bold("--src-lang")} SRC_LANG] [${chalk.bold( - "--out", - )} FILE] FILE|URL ...`, - "", - ` LANG ... ${makeLangTypeLabel(targetLanguages)}`, - "", - "SRC_LANG ... json|schema|graphql|postman|typescript", - ], - }, - { - header: "Description", - content: `Given JSON sample data, quicktype outputs code for working with that data in ${langDisplayNames}.`, - }, - { - header: "Options", - optionList: makeOptionDefinitions(targetLanguages), - hide: ["no-render", "build-markov-chain"], - tableOptions: tableOptionsForOptions, - }, - ]; -} - -const sectionsAfterRenderers: UsageSection[] = [ - { - header: "Examples", - content: [ - chalk.dim("Generate C# to parse a Bitcoin API"), - "$ quicktype -o LatestBlock.cs https://blockchain.info/latestblock", - "", - chalk.dim( - "Generate Go code from a directory of samples containing:", - ), - chalk.dim( - ` - Foo.json - + Bar - - bar-sample-1.json - - bar-sample-2.json - - Baz.url`, - ), - "$ quicktype -l go samples", - "", - chalk.dim("Generate JSON Schema, then TypeScript"), - "$ quicktype -o schema.json https://blockchain.info/latestblock", - "$ quicktype -o bitcoin.ts --src-lang schema schema.json", - ], - }, - { - content: `Learn more at ${chalk.bold("quicktype.io")}`, - }, -]; - -export function parseCLIOptions( - argv: string[], - targetLanguage?: TargetLanguage, -): CLIOptions { - if (argv.length === 0) { - return inferCLIOptions({ help: true }, targetLanguage); - } - - const targetLanguages = - targetLanguage === undefined - ? defaultTargetLanguages - : [targetLanguage]; - const optionDefinitions = makeOptionDefinitions(targetLanguages); - - // We can only fully parse the options once we know which renderer is selected, - // because there are renderer-specific options. But we only know which renderer - // is selected after we've parsed the options. Hence, we parse the options - // twice. This is the first parse to get the renderer: - const incompleteOptions = inferCLIOptions( - parseOptions(optionDefinitions, argv, true), - targetLanguage, - ); - if (targetLanguage === undefined) { - const languageName = isLanguageName(incompleteOptions.lang) - ? incompleteOptions.lang - : "typescript"; - targetLanguage = getTargetLanguage(languageName); - } - - const rendererOptionDefinitions = - targetLanguage.cliOptionDefinitions.actual; - // Use the global options as well as the renderer options from now on: - const allOptionDefinitions = _.concat( - optionDefinitions, - rendererOptionDefinitions, - ); - // This is the parse that counts: - return inferCLIOptions( - parseOptions(allOptionDefinitions, argv, false), - targetLanguage, - ); -} - -// Parse the options in argv and split them into global options and renderer options, -// according to each option definition's `renderer` field. If `partial` is false this -// will throw if it encounters an unknown option. -function parseOptions( - definitions: OptionDefinition[], - argv: string[], - partial: boolean, -): Partial { - let opts: commandLineArgs.CommandLineOptions; - try { - opts = commandLineArgs( - definitions.map((def) => ({ - ...def, - type: def.optionType === "boolean" ? Boolean : String, - })), - { argv, partial }, - ); - } catch (e) { - assert(!partial, "Partial option parsing should not have failed"); - return messageError("DriverCLIOptionParsingFailed", { - message: exceptionToString(e), - }); - } - - for (const k of Object.keys(opts)) { - if (opts[k] === null) { - return messageError("DriverCLIOptionParsingFailed", { - message: `Missing value for command line option "${k}"`, - }); - } - } - - const options: { - [key: string]: unknown; - rendererOptions: RendererOptions; - } = { rendererOptions: {} }; - for (const optionDefinition of definitions) { - if (!hasOwnProperty(opts, optionDefinition.name)) { - continue; - } - - const optionValue = opts[optionDefinition.name] as string; - if (optionDefinition.kind !== "cli") { - ( - options.rendererOptions as Record< - typeof optionDefinition.name, - unknown - > - )[optionDefinition.name] = optionValue; - } else { - const k = _.lowerFirst( - optionDefinition.name.split("-").map(_.upperFirst).join(""), - ); - options[k] = optionValue; - } - } - - return options; -} - -function usage(targetLanguages: readonly TargetLanguage[]): void { - const rendererSections: UsageSection[] = []; - - for (const language of targetLanguages) { - const definitions = language.cliOptionDefinitions.display; - if (definitions.length === 0) continue; - - rendererSections.push({ - header: `Options for ${language.displayName}`, - optionList: definitions, - tableOptions: tableOptionsForOptions, - }); - } - const sections = _.concat( - makeSectionsBeforeRenderers(targetLanguages), - rendererSections, - sectionsAfterRenderers, - ); - - console.log(getUsage(sections)); -} - -// Returns an array of [name, sourceURIs] pairs. -async function getSourceURIs( - options: CLIOptions, -): Promise> { - if (options.srcUrls !== undefined) { - const json = parseJSON( - await readFromFileOrURL(options.srcUrls, options.httpHeader), - "URL grammar", - options.srcUrls, - ); - const jsonMap = urlsFromURLGrammar(json); - const topLevels = Object.getOwnPropertyNames(jsonMap); - return topLevels.map( - (name) => [name, jsonMap[name]] as [string, string[]], - ); - } - if (options.src.length === 0) { - return [[options.topLevel, ["-"]]]; - } - - return []; -} - -async function typeSourcesForURIs( - name: string, - uris: string[], - options: CLIOptions, -): Promise { - switch (options.srcLang) { - case "json": - return [ - await sourceFromFileOrUrlArray(name, uris, options.httpHeader), - ]; - case "schema": - return uris.map( - (uri) => - ({ kind: "schema", name, uris: [uri] }) as SchemaTypeSource, - ); - default: - return panic( - `typeSourceForURIs must not be called for source language ${options.srcLang}`, - ); - } -} - -async function getSources(options: CLIOptions): Promise { - const sourceURIs = await getSourceURIs(options); - const sourceArrays = await Promise.all( - sourceURIs.map( - async ([name, uris]) => - await typeSourcesForURIs(name, uris, options), - ), - ); - let sources: TypeSource[] = ([] as TypeSource[]).concat(...sourceArrays); - - const exists = options.src.filter(fs.existsSync); - const directories = exists.filter((x) => fs.lstatSync(x).isDirectory()); - - for (const dataDir of directories) { - sources = sources.concat( - await samplesFromDirectory(dataDir, options.httpHeader), - ); - } - - // Every src that's not a directory is assumed to be a file or URL - const filesOrUrls = options.src.filter((x) => !_.includes(directories, x)); - if (!_.isEmpty(filesOrUrls)) { - sources.push( - ...(await typeSourcesForURIs( - options.topLevel, - filesOrUrls, - options, - )), - ); - } - - return sources; -} - -function makeTypeScriptSource(fileNames: string[]): SchemaTypeSource { - return Object.assign( - { kind: "schema" }, - schemaForTypeScriptSources(fileNames), - ) as SchemaTypeSource; -} - -export function jsonInputForTargetLanguage( - targetLanguage: string | TargetLanguage, - languages?: TargetLanguage[], - handleJSONRefs = false, -): JSONInput { - if (typeof targetLanguage === "string") { - const languageName = isLanguageName(targetLanguage) - ? targetLanguage - : "typescript"; - targetLanguage = defined(languageNamed(languageName, languages)); - } - - const compressedJSON = new CompressedJSONFromStream( - targetLanguage.dateTimeRecognizer, - handleJSONRefs, - ); - return new JSONInput(compressedJSON); -} - -async function makeInputData( - sources: TypeSource[], - targetLanguage: TargetLanguage, - additionalSchemaAddresses: readonly string[], - handleJSONRefs: boolean, - httpHeaders?: string[], -): Promise { - const inputData = new InputData(); - - for (const source of sources) { - switch (source.kind) { - case "graphql": - await inputData.addSource( - "graphql", - source, - () => new GraphQLInput(), - ); - break; - case "json": - await inputData.addSource("json", source, () => - jsonInputForTargetLanguage( - targetLanguage, - undefined, - handleJSONRefs, - ), - ); - break; - case "schema": - await inputData.addSource( - "schema", - source, - () => - new JSONSchemaInput( - new FetchingJSONSchemaStore(httpHeaders), - [], - additionalSchemaAddresses, - ), - ); - break; - default: - return assertNever(source); - } - } - - return inputData; -} - -function stringSourceDataToStreamSourceData( - src: JSONSourceData, -): JSONSourceData { - return { - name: src.name, - description: src.description, - samples: src.samples.map( - (sample) => stringToStream(sample) as Readable, - ), - }; -} - -export async function makeQuicktypeOptions( - options: CLIOptions, - targetLanguages?: TargetLanguage[], -): Promise | undefined> { - if (options.help) { - usage(targetLanguages ?? defaultTargetLanguages); - return undefined; - } - - if (options.version) { - console.log(`quicktype version ${packageJSON.version}`); - console.log("Visit quicktype.io for more info."); - return undefined; - } - - if (options.buildMarkovChain !== undefined) { - const contents = fs.readFileSync(options.buildMarkovChain).toString(); - const lines = contents.split("\n"); - const mc = trainMarkovChain(lines, 3); - console.log(JSON.stringify(mc)); - return undefined; - } - - let sources: TypeSource[] = []; - let leadingComments: string[] | undefined = undefined; - let fixedTopLevels = false; - switch (options.srcLang) { - case "graphql": - let schemaString: string | undefined = undefined; - let wroteSchemaToFile = false; - if (options.graphqlIntrospect !== undefined) { - schemaString = await introspectServer( - options.graphqlIntrospect, - withDefault(options.httpMethod, "POST"), - withDefault(options.httpHeader, []), - ); - if (options.graphqlSchema !== undefined) { - fs.writeFileSync(options.graphqlSchema, schemaString); - wroteSchemaToFile = true; - } - } - - const numSources = options.src.length; - if (numSources !== 1) { - if (wroteSchemaToFile) { - // We're done. - return undefined; - } - - if (numSources === 0) { - if (schemaString !== undefined) { - console.log(schemaString); - return undefined; - } - - return messageError("DriverNoGraphQLQueryGiven", {}); - } - } - - const gqlSources: GraphQLTypeSource[] = []; - for (const queryFile of options.src) { - let schemaFileName: string | undefined = undefined; - if (schemaString === undefined) { - schemaFileName = defined(options.graphqlSchema); - schemaString = fs.readFileSync(schemaFileName, "utf8"); - } - - const schema = parseJSON( - schemaString, - "GraphQL schema", - schemaFileName, - ); - const query = await getStream( - await readableFromFileOrURL(queryFile, options.httpHeader), - ); - const name = - numSources === 1 - ? options.topLevel - : typeNameFromFilename(queryFile); - gqlSources.push({ kind: "graphql", name, schema, query }); - } - - sources = gqlSources; - break; - case "json": - case "schema": - sources = await getSources(options); - break; - case "typescript": - sources = [makeTypeScriptSource(options.src)]; - break; - case "postman": - for (const collectionFile of options.src) { - const collectionJSON = fs.readFileSync(collectionFile, "utf8"); - const { sources: postmanSources, description } = - sourcesFromPostmanCollection( - collectionJSON, - collectionFile, - ); - for (const src of postmanSources) { - sources.push( - Object.assign( - { kind: "json" }, - stringSourceDataToStreamSourceData(src), - ) as JSONTypeSource, - ); - } - - if (postmanSources.length > 1) { - fixedTopLevels = true; - } - - if (description !== undefined) { - leadingComments = wordWrap(description).split("\n"); - } - } - - break; - default: - return messageError("DriverUnknownSourceLanguage", { - lang: options.srcLang, - }); - } - - const components = definedMap(options.debug, (d) => d.split(",")); - const debugAll = components?.includes("all"); - let debugPrintGraph = debugAll; - let checkProvenance = debugAll; - let debugPrintReconstitution = debugAll; - let debugPrintGatherNames = debugAll; - let debugPrintTransformations = debugAll; - let debugPrintSchemaResolving = debugAll; - let debugPrintTimes = debugAll; - if (components !== undefined) { - for (let component of components) { - component = component.trim(); - if (component === "print-graph") { - debugPrintGraph = true; - } else if (component === "print-reconstitution") { - debugPrintReconstitution = true; - } else if (component === "print-gather-names") { - debugPrintGatherNames = true; - } else if (component === "print-transformations") { - debugPrintTransformations = true; - } else if (component === "print-times") { - debugPrintTimes = true; - } else if (component === "print-schema-resolving") { - debugPrintSchemaResolving = true; - } else if (component === "provenance") { - checkProvenance = true; - } else if (component !== "all") { - return messageError("DriverUnknownDebugOption", { - option: component, - }); - } - } - } - - if (!isLanguageName(options.lang)) { - return messageError("DriverUnknownOutputLanguage", { - lang: options.lang, - }); - } - - const lang = languageNamed(options.lang, targetLanguages); - - const quicktypeOptions: Partial = { - lang, - alphabetizeProperties: options.alphabetizeProperties, - allPropertiesOptional: options.allPropertiesOptional, - fixedTopLevels, - noRender: options.noRender, - rendererOptions: options.rendererOptions, - leadingComments, - outputFilename: definedMap(options.out, path.basename), - debugPrintGraph, - checkProvenance, - debugPrintReconstitution, - debugPrintGatherNames, - debugPrintTransformations, - debugPrintSchemaResolving, - debugPrintTimes, - }; - for (const flagName of inferenceFlagNames) { - const cliName = negatedInferenceFlagName(flagName); - const v = options[cliName]; - if (typeof v === "boolean") { - quicktypeOptions[flagName] = !v; - } else { - quicktypeOptions[flagName] = true; - } - } - - quicktypeOptions.inputData = await makeInputData( - sources, - lang, - options.additionalSchema, - quicktypeOptions.ignoreJsonRefs !== true, - options.httpHeader, - ); - - return quicktypeOptions; -} +import { parseCLIOptions } from "./cli.options"; +import { inferCLIOptions } from "./inference"; +import { makeQuicktypeOptions } from "./quicktype.options"; +import type { CLIOptions } from "./CLIOptions.types"; +export type { CLIOptions }; export function writeOutput( cliOptions: CLIOptions, resultsByFilename: ReadonlyMap, ): void { - let onFirst = true; + let isFirstRun = true; for (const [filename, { lines, annotations }] of resultsByFilename) { const output = lines.join("\n"); @@ -1181,7 +31,7 @@ export function writeOutput( output, ); } else { - if (!onFirst) { + if (!isFirstRun) { process.stdout.write("\n"); } @@ -1196,10 +46,13 @@ export function writeOutput( continue; } - for (const sa of annotations) { - const annotation = sa.annotation; - if (!(annotation instanceof IssueAnnotationData)) continue; - const lineNumber = sa.span.start.line; + for (const sourceAnnotation of annotations) { + const annotation = sourceAnnotation.annotation; + if (!(annotation instanceof IssueAnnotationData)) { + continue; + } + + const lineNumber = sourceAnnotation.span.start.line; const humanLineNumber = lineNumber + 1; console.error( `\nIssue in line ${humanLineNumber}: ${annotation.message}`, @@ -1207,7 +60,7 @@ export function writeOutput( console.error(`${humanLineNumber}: ${lines[lineNumber]}`); } - onFirst = false; + isFirstRun = false; } } diff --git a/src/inference.ts b/src/inference.ts new file mode 100644 index 000000000..3b6de4d0e --- /dev/null +++ b/src/inference.ts @@ -0,0 +1,127 @@ +import * as path from "node:path"; + +import { withDefault } from "collection-utils"; + +import { + type LanguageName, + type TargetLanguage, + inferenceFlagNames, + isLanguageName, + languageNamed, + messageAssert, + messageError, +} from "quicktype-core"; + +import { negatedInferenceFlagName } from "./utils"; +import type { CLIOptions } from "./CLIOptions.types"; + +const defaultTargetLanguageName = "go"; + +function inferLang( + options: Partial, + defaultLanguage: LanguageName, +): string | LanguageName { + // Output file extension determines the language if language is undefined + if (options.out !== undefined) { + const extension = path.extname(options.out); + if (extension === "") { + return messageError("DriverNoLanguageOrExtension", {}); + } + + return extension.slice(1); + } + + return defaultLanguage; +} + +function inferTopLevel(options: Partial): string { + // Output file name determines the top-level if undefined + if (options.out !== undefined) { + const extension = path.extname(options.out); + const without = path.basename(options.out).replace(extension, ""); + return without; + } + + // Source determines the top-level if undefined + if (options.src !== undefined && options.src.length === 1) { + const src = options.src[0]; + const extension = path.extname(src); + const without = path.basename(src).replace(extension, ""); + return without; + } + + return "TopLevel"; +} + +export function inferCLIOptions( + opts: Partial, + targetLanguage: TargetLanguage | undefined, +): CLIOptions { + let srcLang = opts.srcLang; + if ( + opts.graphqlSchema !== undefined || + opts.graphqlIntrospect !== undefined + ) { + messageAssert( + srcLang === undefined || srcLang === "graphql", + "DriverSourceLangMustBeGraphQL", + {}, + ); + srcLang = "graphql"; + } else if ( + opts.src !== undefined && + opts.src.length > 0 && + opts.src.every((file) => file.endsWith(".ts")) + ) { + srcLang = "typescript"; + } else { + messageAssert(srcLang !== "graphql", "DriverGraphQLSchemaNeeded", {}); + srcLang = withDefault(srcLang, "json"); + } + + let language: TargetLanguage; + if (targetLanguage !== undefined) { + language = targetLanguage; + } else { + const languageName = + opts.lang ?? inferLang(opts, defaultTargetLanguageName); + + if (isLanguageName(languageName)) { + language = languageNamed(languageName); + } else { + return messageError("DriverUnknownOutputLanguage", { + lang: languageName, + }); + } + } + + const options: CLIOptions = { + src: opts.src ?? [], + srcUrls: opts.srcUrls, + srcLang: srcLang, + lang: language.name as LanguageName, + topLevel: opts.topLevel ?? inferTopLevel(opts), + noRender: !!opts.noRender, + alphabetizeProperties: !!opts.alphabetizeProperties, + allPropertiesOptional: !!opts.allPropertiesOptional, + rendererOptions: opts.rendererOptions ?? {}, + help: opts.help ?? false, + quiet: opts.quiet ?? false, + version: opts.version ?? false, + out: opts.out, + buildMarkovChain: opts.buildMarkovChain, + additionalSchema: opts.additionalSchema ?? [], + graphqlSchema: opts.graphqlSchema, + graphqlIntrospect: opts.graphqlIntrospect, + httpMethod: opts.httpMethod, + httpHeader: opts.httpHeader, + debug: opts.debug, + telemetry: opts.telemetry, + }; + for (const flagName of inferenceFlagNames) { + const cliName = negatedInferenceFlagName(flagName); + options[cliName] = !!opts[cliName]; + } + + return options; +} diff --git a/src/input.ts b/src/input.ts new file mode 100644 index 000000000..2570e1fd5 --- /dev/null +++ b/src/input.ts @@ -0,0 +1,86 @@ +import type { Readable } from "readable-stream"; + +import { + FetchingJSONSchemaStore, + InputData, + JSONInput, + JSONSchemaInput, + type TargetLanguage, + assertNever, + defined, + isLanguageName, + languageNamed, +} from "quicktype-core"; +import { GraphQLInput } from "quicktype-graphql-input"; + +import { CompressedJSONFromStream } from "./CompressedJSONFromStream"; +import type { TypeSource } from "./TypeSource"; + +export function jsonInputForTargetLanguage( + _targetLanguage: string | TargetLanguage, + languages?: TargetLanguage[], + handleJSONRefs = false, +): JSONInput { + let targetLanguage: TargetLanguage; + if (typeof _targetLanguage === "string") { + const languageName = isLanguageName(_targetLanguage) + ? _targetLanguage + : "typescript"; + targetLanguage = defined(languageNamed(languageName, languages)); + } else { + targetLanguage = _targetLanguage; + } + + const compressedJSON = new CompressedJSONFromStream( + targetLanguage.dateTimeRecognizer, + handleJSONRefs, + ); + return new JSONInput(compressedJSON); +} + +export async function makeInputData( + sources: TypeSource[], + targetLanguage: TargetLanguage, + additionalSchemaAddresses: readonly string[], + handleJSONRefs: boolean, + httpHeaders?: string[], +): Promise { + const inputData = new InputData(); + + for (const source of sources) { + switch (source.kind) { + case "graphql": + await inputData.addSource( + "graphql", + source, + () => new GraphQLInput(), + ); + break; + case "json": + await inputData.addSource("json", source, () => + jsonInputForTargetLanguage( + targetLanguage, + undefined, + handleJSONRefs, + ), + ); + break; + case "schema": + await inputData.addSource( + "schema", + source, + () => + new JSONSchemaInput( + new FetchingJSONSchemaStore(httpHeaders), + [], + additionalSchemaAddresses, + ), + ); + break; + default: + return assertNever(source); + } + } + + return inputData; +} diff --git a/src/optionDefinitions.ts b/src/optionDefinitions.ts new file mode 100644 index 000000000..f38189575 --- /dev/null +++ b/src/optionDefinitions.ts @@ -0,0 +1,186 @@ +import { mapFromObject, mapMap } from "collection-utils"; + +import { + type OptionDefinition, + type TargetLanguage, + inferenceFlags, +} from "quicktype-core"; + +import { dashedFromCamelCase, negatedInferenceFlagName } from "./utils"; + +export function makeOptionDefinitions( + targetLanguages: readonly TargetLanguage[], +): OptionDefinition[] { + const beforeLang: OptionDefinition[] = [ + { + name: "out", + alias: "o", + optionType: "string", + typeLabel: "FILE", + description: "The output file. Determines --lang and --top-level.", + kind: "cli", + defaultOption: true, + }, + { + name: "top-level", + alias: "t", + optionType: "string", + typeLabel: "NAME", + description: "The name for the top level type.", + kind: "cli", + }, + ]; + const lang: OptionDefinition[] = + targetLanguages.length < 2 + ? [] + : [ + { + name: "lang", + alias: "l", + optionType: "string", + typeLabel: "LANG", + description: "The target language.", + kind: "cli", + }, + ]; + const afterLang: OptionDefinition[] = [ + { + name: "src-lang", + alias: "s", + optionType: "string", + defaultValue: undefined, + typeLabel: "SRC_LANG", + description: "The source language (default is json).", + kind: "cli", + }, + { + name: "src", + optionType: "string", + multiple: true, + typeLabel: "FILE|URL|DIRECTORY", + description: "The file, url, or data directory to type.", + kind: "cli", + }, + { + name: "src-urls", + optionType: "string", + typeLabel: "FILE", + description: "Tracery grammar describing URLs to crawl.", + kind: "cli", + }, + ]; + const inference: OptionDefinition[] = Array.from( + mapMap(mapFromObject(inferenceFlags), (flag, name) => { + return { + name: dashedFromCamelCase(negatedInferenceFlagName(name)), + optionType: "boolean" as const, + // biome-ignore lint/style/useTemplate: + description: flag.negationDescription + ".", + kind: "cli" as const, + }; + }).values(), + ); + const afterInference: OptionDefinition[] = [ + { + name: "graphql-schema", + optionType: "string", + typeLabel: "FILE", + description: "GraphQL introspection file.", + kind: "cli", + }, + { + name: "graphql-introspect", + optionType: "string", + typeLabel: "URL", + description: "Introspect GraphQL schema from a server.", + kind: "cli", + }, + { + name: "http-method", + optionType: "string", + typeLabel: "METHOD", + description: + "HTTP method to use for the GraphQL introspection query.", + kind: "cli", + }, + { + name: "http-header", + optionType: "string", + multiple: true, + typeLabel: "HEADER", + description: + "Header(s) to attach to all HTTP requests, including the GraphQL introspection query.", + kind: "cli", + }, + { + name: "additional-schema", + alias: "S", + optionType: "string", + multiple: true, + typeLabel: "FILE", + description: "Register the $id's of additional JSON Schema files.", + kind: "cli", + }, + { + name: "no-render", + optionType: "boolean", + description: "Don't render output.", + kind: "cli", + }, + { + name: "alphabetize-properties", + optionType: "boolean", + description: "Alphabetize order of class properties.", + kind: "cli", + }, + { + name: "all-properties-optional", + optionType: "boolean", + description: "Make all class properties optional.", + kind: "cli", + }, + { + name: "build-markov-chain", + optionType: "string", + typeLabel: "FILE", + description: "Markov chain corpus filename.", + kind: "cli", + }, + { + name: "quiet", + optionType: "boolean", + description: "Don't show issues in the generated code.", + kind: "cli", + }, + { + name: "debug", + optionType: "string", + typeLabel: "OPTIONS or all", + description: + "Comma separated debug options: print-graph, print-reconstitution, print-gather-names, print-transformations, print-schema-resolving, print-times, provenance", + kind: "cli", + }, + { + name: "telemetry", + optionType: "string", + typeLabel: "enable|disable", + description: "Enable anonymous telemetry to help improve quicktype", + kind: "cli", + }, + { + name: "help", + alias: "h", + optionType: "boolean", + description: "Get some help.", + kind: "cli", + }, + { + name: "version", + alias: "v", + optionType: "boolean", + description: "Display the version of quicktype", + kind: "cli", + }, + ]; + return beforeLang.concat(lang, afterLang, inference, afterInference); +} diff --git a/src/quicktype.options.ts b/src/quicktype.options.ts new file mode 100644 index 000000000..d41d8ac38 --- /dev/null +++ b/src/quicktype.options.ts @@ -0,0 +1,246 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; + +import { definedMap, withDefault } from "collection-utils"; +import _wordwrap from "wordwrap"; + +import { + type Options, + type TargetLanguage, + defaultTargetLanguages, + defined, + getStream, + inferenceFlagNames, + isLanguageName, + languageNamed, + messageError, + parseJSON, + readableFromFileOrURL, + sourcesFromPostmanCollection, + trainMarkovChain, +} from "quicktype-core"; + +import { introspectServer } from "./GraphQLIntrospection"; +import type { + GraphQLTypeSource, + JSONTypeSource, + TypeSource, +} from "./TypeSource"; +import { makeInputData } from "./input"; +import { getSources, makeTypeScriptSource } from "./sources"; +import { usage } from "./usage"; +import { + negatedInferenceFlagName, + stringSourceDataToStreamSourceData, + typeNameFromFilename, +} from "./utils"; +import type { CLIOptions } from "./CLIOptions.types"; + +const packageJSON = require("../package.json"); + +const wordWrap: (s: string) => string = _wordwrap(90); + +export async function makeQuicktypeOptions( + options: CLIOptions, + targetLanguages?: TargetLanguage[], +): Promise | undefined> { + if (options.help) { + usage(targetLanguages ?? defaultTargetLanguages); + return; + } + + if (options.version) { + console.log(`quicktype version ${packageJSON.version}`); + console.log("Visit quicktype.io for more info."); + return; + } + + if (options.buildMarkovChain !== undefined) { + const contents = fs.readFileSync(options.buildMarkovChain).toString(); + const lines = contents.split("\n"); + const mc = trainMarkovChain(lines, 3); + console.log(JSON.stringify(mc)); + return; + } + + let sources: TypeSource[] = []; + let leadingComments: string[] | undefined = undefined; + let fixedTopLevels = false; + switch (options.srcLang) { + case "graphql": { + let schemaString: string | undefined = undefined; + let wroteSchemaToFile = false; + if (options.graphqlIntrospect !== undefined) { + schemaString = await introspectServer( + options.graphqlIntrospect, + withDefault(options.httpMethod, "POST"), + withDefault(options.httpHeader, []), + ); + if (options.graphqlSchema !== undefined) { + fs.writeFileSync(options.graphqlSchema, schemaString); + wroteSchemaToFile = true; + } + } + + const numSources = options.src.length; + if (numSources !== 1) { + if (wroteSchemaToFile) { + // We're done. + return; + } + + if (numSources === 0) { + if (schemaString !== undefined) { + console.log(schemaString); + return; + } + + return messageError("DriverNoGraphQLQueryGiven", {}); + } + } + + const gqlSources: GraphQLTypeSource[] = []; + for (const queryFile of options.src) { + let schemaFileName: string | undefined = undefined; + if (schemaString === undefined) { + schemaFileName = defined(options.graphqlSchema); + schemaString = fs.readFileSync(schemaFileName, "utf8"); + } + + const schema = parseJSON( + schemaString, + "GraphQL schema", + schemaFileName, + ); + const query = await getStream( + await readableFromFileOrURL(queryFile, options.httpHeader), + ); + const name = + numSources === 1 + ? options.topLevel + : typeNameFromFilename(queryFile); + gqlSources.push({ kind: "graphql", name, schema, query }); + } + + sources = gqlSources; + break; + } + case "json": + case "schema": + sources = await getSources(options); + break; + case "typescript": + sources = [makeTypeScriptSource(options.src)]; + break; + case "postman": + for (const collectionFile of options.src) { + const collectionJSON = fs.readFileSync(collectionFile, "utf8"); + const { sources: postmanSources, description } = + sourcesFromPostmanCollection( + collectionJSON, + collectionFile, + ); + for (const src of postmanSources) { + sources.push( + Object.assign( + { kind: "json" }, + stringSourceDataToStreamSourceData(src), + ) as JSONTypeSource, + ); + } + + if (postmanSources.length > 1) { + fixedTopLevels = true; + } + + if (description !== undefined) { + leadingComments = wordWrap(description).split("\n"); + } + } + + break; + default: + return messageError("DriverUnknownSourceLanguage", { + lang: options.srcLang, + }); + } + + const components = definedMap(options.debug, (d) => d.split(",")); + const debugAll = components?.includes("all"); + let debugPrintGraph = debugAll; + let checkProvenance = debugAll; + let debugPrintReconstitution = debugAll; + let debugPrintGatherNames = debugAll; + let debugPrintTransformations = debugAll; + let debugPrintSchemaResolving = debugAll; + let debugPrintTimes = debugAll; + if (components !== undefined) { + for (let component of components) { + component = component.trim(); + if (component === "print-graph") { + debugPrintGraph = true; + } else if (component === "print-reconstitution") { + debugPrintReconstitution = true; + } else if (component === "print-gather-names") { + debugPrintGatherNames = true; + } else if (component === "print-transformations") { + debugPrintTransformations = true; + } else if (component === "print-times") { + debugPrintTimes = true; + } else if (component === "print-schema-resolving") { + debugPrintSchemaResolving = true; + } else if (component === "provenance") { + checkProvenance = true; + } else if (component !== "all") { + return messageError("DriverUnknownDebugOption", { + option: component, + }); + } + } + } + + if (!isLanguageName(options.lang)) { + return messageError("DriverUnknownOutputLanguage", { + lang: options.lang, + }); + } + + const lang = languageNamed(options.lang, targetLanguages); + + const quicktypeOptions: Partial = { + lang, + alphabetizeProperties: options.alphabetizeProperties, + allPropertiesOptional: options.allPropertiesOptional, + fixedTopLevels, + noRender: options.noRender, + rendererOptions: options.rendererOptions, + leadingComments, + outputFilename: definedMap(options.out, path.basename), + debugPrintGraph, + checkProvenance, + debugPrintReconstitution, + debugPrintGatherNames, + debugPrintTransformations, + debugPrintSchemaResolving, + debugPrintTimes, + }; + for (const flagName of inferenceFlagNames) { + const cliName = negatedInferenceFlagName(flagName); + const v = options[cliName]; + if (typeof v === "boolean") { + quicktypeOptions[flagName] = !v; + } else { + quicktypeOptions[flagName] = true; + } + } + + quicktypeOptions.inputData = await makeInputData( + sources, + lang, + options.additionalSchema, + quicktypeOptions.ignoreJsonRefs !== true, + options.httpHeader, + ); + + return quicktypeOptions; +} diff --git a/src/sources.ts b/src/sources.ts new file mode 100644 index 000000000..f4adbfaf2 --- /dev/null +++ b/src/sources.ts @@ -0,0 +1,280 @@ +#!/usr/bin/env node + +import * as fs from "node:fs"; +import * as path from "node:path"; + +import * as _ from "lodash"; +import type { Readable } from "readable-stream"; +import _wordwrap from "wordwrap"; + +import { + assertNever, + getStream, + messageAssert, + messageError, + panic, + parseJSON, + readFromFileOrURL, + readableFromFileOrURL, +} from "quicktype-core"; +import { schemaForTypeScriptSources } from "quicktype-typescript-input"; + +import type { + GraphQLTypeSource, + JSONTypeSource, + SchemaTypeSource, + TypeSource, +} from "./TypeSource"; +import { urlsFromURLGrammar } from "./URLGrammar"; +import { typeNameFromFilename } from "./utils"; +import type { CLIOptions } from "./CLIOptions.types"; + +async function sourceFromFileOrUrlArray( + name: string, + filesOrUrls: string[], + httpHeaders?: string[], +): Promise { + const samples = await Promise.all( + filesOrUrls.map( + async (file) => await readableFromFileOrURL(file, httpHeaders), + ), + ); + return { kind: "json", name, samples }; +} + +async function samplesFromDirectory( + dataDir: string, + httpHeaders?: string[], +): Promise { + async function readFilesOrURLsInDirectory( + d: string, + ): Promise { + const files = fs + .readdirSync(d) + .map((x) => path.join(d, x)) + .filter((x) => fs.lstatSync(x).isFile()); + // Each file is a (Name, JSON | URL) + const sourcesInDir: TypeSource[] = []; + const graphQLSources: GraphQLTypeSource[] = []; + let graphQLSchema: Readable | undefined = undefined; + let graphQLSchemaFileName: string | undefined = undefined; + for (let file of files) { + const name = typeNameFromFilename(file); + + let fileOrUrl = file; + file = file.toLowerCase(); + + // If file is a URL string, download it + if (file.endsWith(".url")) { + fileOrUrl = fs.readFileSync(file, "utf8").trim(); + } + + if (file.endsWith(".url") || file.endsWith(".json")) { + sourcesInDir.push({ + kind: "json", + name, + samples: [ + await readableFromFileOrURL(fileOrUrl, httpHeaders), + ], + }); + } else if (file.endsWith(".schema")) { + sourcesInDir.push({ + kind: "schema", + name, + uris: [fileOrUrl], + }); + } else if (file.endsWith(".gqlschema")) { + messageAssert( + graphQLSchema === undefined, + "DriverMoreThanOneGraphQLSchemaInDir", + { + dir: dataDir, + }, + ); + graphQLSchema = await readableFromFileOrURL( + fileOrUrl, + httpHeaders, + ); + graphQLSchemaFileName = fileOrUrl; + } else if (file.endsWith(".graphql")) { + graphQLSources.push({ + kind: "graphql", + name, + schema: undefined, + query: await getStream( + await readableFromFileOrURL(fileOrUrl, httpHeaders), + ), + }); + } + } + + if (graphQLSources.length > 0) { + if (graphQLSchema === undefined) { + return messageError("DriverNoGraphQLSchemaInDir", { + dir: dataDir, + }); + } + + const schema = parseJSON( + await getStream(graphQLSchema), + "GraphQL schema", + graphQLSchemaFileName, + ); + for (const source of graphQLSources) { + source.schema = schema; + sourcesInDir.push(source); + } + } + + return sourcesInDir; + } + + const contents = fs.readdirSync(dataDir).map((x) => path.join(dataDir, x)); + const directories = contents.filter((x) => fs.lstatSync(x).isDirectory()); + + let sources = await readFilesOrURLsInDirectory(dataDir); + + for (const dir of directories) { + let jsonSamples: Readable[] = []; + const schemaSources: SchemaTypeSource[] = []; + const graphQLSources: GraphQLTypeSource[] = []; + + for (const source of await readFilesOrURLsInDirectory(dir)) { + switch (source.kind) { + case "json": + jsonSamples = jsonSamples.concat(source.samples); + break; + case "schema": + schemaSources.push(source); + break; + case "graphql": + graphQLSources.push(source); + break; + default: + return assertNever(source); + } + } + + if ( + jsonSamples.length > 0 && + schemaSources.length + graphQLSources.length > 0 + ) { + return messageError("DriverCannotMixJSONWithOtherSamples", { + dir: dir, + }); + } + + // FIXME: rewrite this to be clearer + const oneUnlessEmpty = (xs: TypeSource[]): 0 | 1 => + Math.sign(xs.length) as 0 | 1; + if ( + oneUnlessEmpty(schemaSources) + oneUnlessEmpty(graphQLSources) > + 1 + ) { + return messageError("DriverCannotMixNonJSONInputs", { dir: dir }); + } + + if (jsonSamples.length > 0) { + sources.push({ + kind: "json", + name: path.basename(dir), + samples: jsonSamples, + }); + } + + sources = sources.concat(schemaSources); + sources = sources.concat(graphQLSources); + } + + return sources; +} + +// Returns an array of [name, sourceURIs] pairs. +async function getSourceURIs( + options: CLIOptions, +): Promise> { + if (options.srcUrls !== undefined) { + const json = parseJSON( + await readFromFileOrURL(options.srcUrls, options.httpHeader), + "URL grammar", + options.srcUrls, + ); + const jsonMap = urlsFromURLGrammar(json); + const topLevels = Object.getOwnPropertyNames(jsonMap); + return topLevels.map( + (name) => [name, jsonMap[name]] as [string, string[]], + ); + } + if (options.src.length === 0) { + return [[options.topLevel, ["-"]]]; + } + + return []; +} + +async function typeSourcesForURIs( + name: string, + uris: string[], + options: CLIOptions, +): Promise { + switch (options.srcLang) { + case "json": + return [ + await sourceFromFileOrUrlArray(name, uris, options.httpHeader), + ]; + case "schema": + return uris.map( + (uri) => + ({ + kind: "schema", + name, + uris: [uri], + }) as SchemaTypeSource, + ); + default: + return panic( + `typeSourceForURIs must not be called for source language ${options.srcLang}`, + ); + } +} + +export async function getSources(options: CLIOptions): Promise { + const sourceURIs = await getSourceURIs(options); + const sourceArrays = await Promise.all( + sourceURIs.map( + async ([name, uris]) => + await typeSourcesForURIs(name, uris, options), + ), + ); + let sources: TypeSource[] = ([] as TypeSource[]).concat(...sourceArrays); + + const exists = options.src.filter(fs.existsSync); + const directories = exists.filter((x) => fs.lstatSync(x).isDirectory()); + + for (const dataDir of directories) { + sources = sources.concat( + await samplesFromDirectory(dataDir, options.httpHeader), + ); + } + + // Every src that's not a directory is assumed to be a file or URL + const filesOrUrls = options.src.filter((x) => !_.includes(directories, x)); + if (!_.isEmpty(filesOrUrls)) { + sources.push( + ...(await typeSourcesForURIs( + options.topLevel, + filesOrUrls, + options, + )), + ); + } + + return sources; +} + +export function makeTypeScriptSource(fileNames: string[]): SchemaTypeSource { + return Object.assign( + { kind: "schema" }, + schemaForTypeScriptSources(fileNames), + ) as SchemaTypeSource; +} diff --git a/src/usage.ts b/src/usage.ts new file mode 100644 index 000000000..b056e12b4 --- /dev/null +++ b/src/usage.ts @@ -0,0 +1,127 @@ +#!/usr/bin/env node + +import chalk from "chalk"; +import getUsage from "command-line-usage"; +import * as _ from "lodash"; +import _wordwrap from "wordwrap"; + +import type { OptionDefinition, TargetLanguage } from "quicktype-core"; + +import { makeOptionDefinitions } from "./optionDefinitions"; +import { makeLangTypeLabel } from "./utils"; + +interface ColumnDefinition { + name: string; + padding?: { left: string; right: string }; + width?: number; +} + +interface TableOptions { + columns: ColumnDefinition[]; +} + +interface UsageSection { + content?: string | string[]; + header?: string; + hide?: string[]; + optionList?: OptionDefinition[]; + tableOptions?: TableOptions; +} + +const tableOptionsForOptions: TableOptions = { + columns: [ + { + name: "option", + width: 60, + }, + { + name: "description", + width: 60, + }, + ], +}; + +function makeSectionsBeforeRenderers( + targetLanguages: readonly TargetLanguage[], +): UsageSection[] { + const langDisplayNames = targetLanguages + .map((r) => r.displayName) + .join(", "); + + return [ + { + header: "Synopsis", + content: [ + `$ quicktype [${chalk.bold("--lang")} LANG] [${chalk.bold("--src-lang")} SRC_LANG] [${chalk.bold( + "--out", + )} FILE] FILE|URL ...`, + "", + ` LANG ... ${makeLangTypeLabel(targetLanguages)}`, + "", + "SRC_LANG ... json|schema|graphql|postman|typescript", + ], + }, + { + header: "Description", + content: `Given JSON sample data, quicktype outputs code for working with that data in ${langDisplayNames}.`, + }, + { + header: "Options", + optionList: makeOptionDefinitions(targetLanguages), + hide: ["no-render", "build-markov-chain"], + tableOptions: tableOptionsForOptions, + }, + ]; +} + +const sectionsAfterRenderers: UsageSection[] = [ + { + header: "Examples", + content: [ + chalk.dim("Generate C# to parse a Bitcoin API"), + "$ quicktype -o LatestBlock.cs https://blockchain.info/latestblock", + "", + chalk.dim( + "Generate Go code from a directory of samples containing:", + ), + chalk.dim( + ` - Foo.json + + Bar + - bar-sample-1.json + - bar-sample-2.json + - Baz.url`, + ), + "$ quicktype -l go samples", + "", + chalk.dim("Generate JSON Schema, then TypeScript"), + "$ quicktype -o schema.json https://blockchain.info/latestblock", + "$ quicktype -o bitcoin.ts --src-lang schema schema.json", + ], + }, + { + content: `Learn more at ${chalk.bold("quicktype.io")}`, + }, +]; + +export function usage(targetLanguages: readonly TargetLanguage[]): void { + const rendererSections: UsageSection[] = []; + + for (const language of targetLanguages) { + const definitions = language.cliOptionDefinitions.display; + if (definitions.length === 0) continue; + + rendererSections.push({ + header: `Options for ${language.displayName}`, + optionList: definitions, + tableOptions: tableOptionsForOptions, + }); + } + + const sections = _.concat( + makeSectionsBeforeRenderers(targetLanguages), + rendererSections, + sectionsAfterRenderers, + ); + + console.log(getUsage(sections)); +} diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 000000000..39a116f54 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,58 @@ +import * as path from "node:path"; + +import * as _ from "lodash"; +import type { Readable } from "readable-stream"; +import stringToStream from "string-to-stream"; +import _wordwrap from "wordwrap"; + +import { + assert, + splitIntoWords, + type JSONSourceData, + type TargetLanguage, +} from "quicktype-core"; + +export function makeLangTypeLabel( + targetLanguages: readonly TargetLanguage[], +): string { + assert( + targetLanguages.length > 0, + "Must have at least one target language", + ); + return targetLanguages + .map((r) => _.minBy(r.names, (s) => s.length)) + .join("|"); +} + +export function negatedInferenceFlagName(inputName: string): string { + let name = inputName; + const prefix = "infer"; + if (name.startsWith(prefix)) { + name = name.slice(prefix.length); + } + + return `no${_.capitalize(name)}`; +} + +export function dashedFromCamelCase(name: string): string { + return splitIntoWords(name) + .map((w) => w.word.toLowerCase()) + .join("-"); +} + +export function typeNameFromFilename(filename: string): string { + const name = path.basename(filename); + return name.substring(0, name.lastIndexOf(".")); +} + +export function stringSourceDataToStreamSourceData( + src: JSONSourceData, +): JSONSourceData { + return { + name: src.name, + description: src.description, + samples: src.samples.map( + (sample) => stringToStream(sample) as Readable, + ), + }; +} diff --git a/test/buildkite.ts b/test/buildkite.ts index e0d2ac89d..5869329bf 100644 --- a/test/buildkite.ts +++ b/test/buildkite.ts @@ -11,10 +11,8 @@ function getChangedFiles(base: string, commit: string): string[] { return diff.trim().split("\n"); } -export function affectedFixtures( - changedFiles: string[] | undefined = undefined, -): Fixture[] { - if (changedFiles === undefined) { +export function affectedFixtures(_changedFiles?: string[]): Fixture[] { + if (_changedFiles === undefined) { const { GITHUB_BASE_REF: base, GITHUB_SHA: commit } = process.env; return commit === undefined ? allFixtures @@ -22,7 +20,9 @@ export function affectedFixtures( } // We can ignore changes in Markdown files - changedFiles = _.reject(changedFiles, (file) => _.endsWith(file, ".md")); + const changedFiles = _.reject(_changedFiles, (file) => + _.endsWith(file, ".md"), + ); // All fixtures are dirty if any changed file is not included as a sourceFile of some fixture. const fileDependencies = _.flatMap( diff --git a/test/fixtures.ts b/test/fixtures.ts index d4eb8f2fd..f31d484d9 100644 --- a/test/fixtures.ts +++ b/test/fixtures.ts @@ -103,7 +103,7 @@ function runEnvForLanguage( const newEnv = Object.assign({}, process.env); for (const option of Object.keys(additionalRendererOptions)) { - newEnv["QUICKTYPE_" + option.toUpperCase().replace("-", "_")] = ( + newEnv[`QUICKTYPE_${option.toUpperCase().replace("-", "_")}`] = ( additionalRendererOptions[ option as keyof typeof additionalRendererOptions ] as Option @@ -216,10 +216,6 @@ export abstract class Fixture { } abstract class LanguageFixture extends Fixture { - constructor(language: languages.Language) { - super(language); - } - async setup() { const setupCommand = this.language.setupCommand; if (!setupCommand || ONLY_OUTPUT) { diff --git a/test/languages.ts b/test/languages.ts index 9573f3e57..d53bbcbdd 100644 --- a/test/languages.ts +++ b/test/languages.ts @@ -1,6 +1,6 @@ import type { LanguageName } from "quicktype-core"; -import * as process from "process"; +import * as process from "node:process"; // @ts-ignore import type { RendererOptions } from "../dist/quicktype-core/Run"; @@ -672,7 +672,7 @@ export const ElmLanguage: Language = { export const SwiftLanguage: Language = { name: "swift", base: "test/fixtures/swift", - compileCommand: `swiftc -o quicktype main.swift quicktype.swift`, + compileCommand: "swiftc -o quicktype main.swift quicktype.swift", runCommand(sample: string) { return `./quicktype "${sample}"`; }, @@ -760,7 +760,7 @@ export const SwiftLanguage: Language = { export const ObjectiveCLanguage: Language = { name: "objective-c", base: "test/fixtures/objective-c", - compileCommand: `clang -Werror -framework Foundation *.m -o test`, + compileCommand: "clang -Werror -framework Foundation *.m -o test", runCommand(sample: string) { return `cp "${sample}" sample.json && ./test sample.json`; }, diff --git a/test/lib/deepEquals.ts b/test/lib/deepEquals.ts index eaeb1209a..8b7db9e15 100644 --- a/test/lib/deepEquals.ts +++ b/test/lib/deepEquals.ts @@ -3,6 +3,7 @@ import type { Moment } from "moment"; import type { ComparisonRelaxations } from "../utils"; function pathToString(path: string[]): string { + // biome-ignore lint/style/useTemplate: return "." + path.join("."); } @@ -13,9 +14,13 @@ declare namespace Math { function tryParseMoment(s: string): [Moment | undefined, boolean] { let m = moment(s); - if (m.isValid()) return [m, false]; + if (m.isValid()) { + return [m, false]; + } m = moment(s, "HH:mm:ss.SSZ"); - if (m.isValid()) return [m, true]; + if (m.isValid()) { + return [m, true]; + } return [undefined, false]; } diff --git a/test/lib/multicore.ts b/test/lib/multicore.ts index 117f28010..d4169709e 100644 --- a/test/lib/multicore.ts +++ b/test/lib/multicore.ts @@ -1,6 +1,5 @@ -import cluster from "cluster"; -import process from "process"; -import * as _ from "lodash"; +import cluster from "node:cluster"; +import process from "node:process"; const exit = require("exit"); @@ -18,8 +17,8 @@ function randomPick(arr: T[]): T { } function guys(n: number): string { - return _.range(n) - .map((_i) => randomPick(WORKERS)) + return Array.from({ length: n }) + .map(() => randomPick(WORKERS)) .join(" "); } @@ -63,13 +62,13 @@ export async function inParallel( await map(item, i); } } else { - _.range(workers).forEach((i) => + for (let i = 0; i < workers; i++) { cluster.fork({ worker: i, // https://github.com/TypeStrong/ts-node/issues/367 TS_NODE_PROJECT: "test/tsconfig.json", - }), - ); + }); + } } } else { // Setup a worker diff --git a/test/test.ts b/test/test.ts index f6015eafa..86d7ec05b 100755 --- a/test/test.ts +++ b/test/test.ts @@ -1,9 +1,8 @@ -import * as os from "os"; -import * as _ from "lodash"; +import * as os from "node:os"; import { inParallel } from "./lib/multicore"; import { execAsync, type Sample } from "./utils"; -import { type Fixture, allFixtures } from "./fixtures"; +import { allFixtures } from "./fixtures"; import { affectedFixtures, divideParallelJobs } from "./buildkite"; const exit = require("exit"); @@ -20,8 +19,8 @@ async function main(sources: string[]) { const fixturesFromCmdline = process.env.FIXTURE; if (fixturesFromCmdline) { const fixtureNames = fixturesFromCmdline.split(","); - fixtures = _.filter(fixtures, (fixture) => - _.some(fixtureNames, (name) => fixture.runForName(name)), + fixtures = fixtures.filter((fixture) => + fixtureNames.some((name) => fixture.runForName(name)), ); } @@ -34,24 +33,24 @@ async function main(sources: string[]) { // Get an array of all { sample, fixtureName } objects we'll run. // We can't just put the fixture in there because these WorkItems // will be sent in a message, removing all code. - const samples = _.map(fixtures, (fixture) => ({ + const samples = fixtures.map((fixture) => ({ fixtureName: fixture.name, samples: fixture.getSamples(sources), })); - const priority = _.flatMap(samples, (x) => - _.map(x.samples.priority, (s) => ({ - fixtureName: x.fixtureName, - sample: s, + const priority = samples.flatMap((sample) => + sample.samples.priority.map((prioritySample) => ({ + fixtureName: sample.fixtureName, + sample: prioritySample, })), ); - const others = _.flatMap(samples, (x) => - _.map(x.samples.others, (s) => ({ - fixtureName: x.fixtureName, - sample: s, + const others = samples.flatMap((sample) => + sample.samples.others.map((otherSample) => ({ + fixtureName: sample.fixtureName, + sample: otherSample, })), ); - const tests = divideParallelJobs(_.concat(priority, others)); + const tests = divideParallelJobs(priority.concat(others)); await inParallel({ queue: tests, @@ -63,17 +62,20 @@ async function main(sources: string[]) { ); for (const fixture of fixtures) { - await execAsync(`rm -rf test/runs`); - await execAsync(`mkdir -p test/runs`); + await execAsync("rm -rf test/runs"); + await execAsync("mkdir -p test/runs"); await fixture.setup(); } }, map: async ({ sample, fixtureName }: WorkItem, index) => { - const fixture = _.find(fixtures, { name: fixtureName }) as Fixture; + const fixture = fixtures.find( + (fixture) => fixture.name === fixtureName, + ); + try { - await fixture.runWithSample(sample, index, tests.length); + await fixture?.runWithSample(sample, index, tests.length); } catch (e) { console.trace(e); exit(1); diff --git a/test/utils.ts b/test/utils.ts index b532a2e8f..f2196f122 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -52,19 +52,16 @@ export function callAndExpectFailure(message: string, f: () => T): void { } export function exec( - s: string, - env: NodeJS.ProcessEnv | undefined, + str: string, + env: NodeJS.ProcessEnv = process.env, printFailure = true, ): { stdout: string; code: number } { - debug(s); - if (env === undefined) { - env = process.env; - } - const result = shell.exec(s, { silent: !DEBUG, env }); + debug(str); + const result = shell.exec(str, { silent: !DEBUG, env }); if (result.code !== 0) { const failureObj = { - command: s, + command: str, code: result.code, }; if (!printFailure) {