feat: allow parsing multiple input types (#1050)

* feat: allow parsing multiple input types

Parse now has a new method that allows the user to get all files and dependencies
by providing a list of directories, files or globs

* fix: performance test

* fix: improve validate file

* fix: pass directory

* feat: normalize test paths

Author: ferrandiaz

packages/cli/src/services/check-parser/__tests__/parse-files.spec.ts

@@ -0,0 +1,252 @@
+import { Parser } from '../parser'
+import * as path from 'path'
+import { pathToPosix } from '../../util'
+
+describe('project parser - getFilesAndDependencies()', () => {
+  it('should handle JS file with no dependencies', async () => {
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([path.join(__dirname, 'check-parser-fixtures', 'no-dependencies.js')])
+    expect(res.files).toHaveLength(1)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should handle JS file with dependencies', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'simple-example', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.js')])
+    const expectedFiles = [
+      'dep1.js',
+      'dep2.js',
+      'dep3.js',
+      'entrypoint.js',
+      'module-package/main.js',
+      'module-package/package.json',
+      'module/index.js',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('Should not repeat files if duplicated', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'simple-example', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.js'), toAbsolutePath('*.js')])
+    const expectedFiles = [
+      'dep1.js',
+      'dep2.js',
+      'dep3.js',
+      'entrypoint.js',
+      'module-package/main.js',
+      'module-package/package.json',
+      'module/index.js',
+      'unreachable.js',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should not fail on a non-existing directory', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'simple-example-that-does-not-exist', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('/')])
+    expect(res.files).toHaveLength(0)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should parse the cli in less than 400ms', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, '../../../', ...filepath)
+    const startTimestamp = Date.now().valueOf()
+    const res = await new Parser({}).getFilesAndDependencies([toAbsolutePath('/index.ts')])
+    const endTimestamp = Date.now().valueOf()
+    expect(res.files).not.toHaveLength(0)
+    expect(res.errors).toHaveLength(0)
+    const isCI = process.env.CI === 'true'
+    expect(endTimestamp - startTimestamp).toBeLessThan(isCI ? 2000 : 400)
+  })
+
+  it('should handle JS file with dependencies glob patterns', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'simple-example', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('*.js'), toAbsolutePath('*.json')])
+    const expectedFiles = [
+      'dep1.js',
+      'dep2.js',
+      'dep3.js',
+      'entrypoint.js',
+      'module-package/main.js',
+      'module-package/package.json',
+      'module/index.js',
+      'unreachable.js',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should parse typescript dependencies', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'typescript-example', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.ts')])
+    const expectedFiles = [
+      'dep1.ts',
+      'dep2.ts',
+      'dep3.ts',
+      'dep4.js',
+      'dep5.ts',
+      'dep6.ts',
+      'entrypoint.ts',
+      'module-package/main.js',
+      'module-package/package.json',
+      'module/index.ts',
+      'pages/external.first.page.js',
+      'pages/external.second.page.ts',
+      'type.ts',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should parse typescript dependencies using tsconfig', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'tsconfig-paths-sample-project', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('src', 'entrypoint.ts')])
+    const expectedFiles = [
+      'lib1/file1.ts',
+      'lib1/file2.ts',
+      'lib1/folder/file1.ts',
+      'lib1/folder/file2.ts',
+      'lib1/index.ts',
+      'lib1/package.json',
+      'lib1/tsconfig.json',
+      'lib2/index.ts',
+      'lib3/foo/bar.ts',
+      'src/entrypoint.ts',
+      'tsconfig.json',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should not include tsconfig if not needed', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'tsconfig-paths-unused', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('src', 'entrypoint.ts')])
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual([pathToPosix(toAbsolutePath('src', 'entrypoint.ts'))])
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should support importing ts extensions if allowed', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'tsconfig-allow-importing-ts-extensions', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('src', 'entrypoint.ts')])
+    const expectedFiles = [
+      'src/dep1.ts',
+      'src/dep2.ts',
+      'src/dep3.ts',
+      'src/entrypoint.ts',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+    expect(res.errors).toHaveLength(0)
+  })
+
+  it('should not import TS files from a JS file', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'no-import-ts-from-js', ...filepath)
+    const parser = new Parser({})
+    expect.assertions(1)
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.js')])
+    } catch (err) {
+      expect(err).toMatchObject({
+        missingFiles: [
+          pathToPosix(toAbsolutePath('dep1')),
+          pathToPosix(toAbsolutePath('dep1.ts')),
+          pathToPosix(toAbsolutePath('dep1.js')),
+        ],
+      })
+    }
+  })
+
+  it('should import JS files from a TS file', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'import-js-from-ts', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.ts')])
+    const expectedFiles = [
+      'dep1.js',
+      'dep2.js',
+      'dep3.ts',
+      'entrypoint.ts',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+  })
+
+  it('should handle ES Modules', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'esmodules-example', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.js')])
+    const expectedFiles = [
+      'dep1.js',
+      'dep2.js',
+      'dep3.js',
+      'dep5.js',
+      'dep6.js',
+      'entrypoint.js',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+  })
+
+  it('should handle Common JS and ES Modules', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'common-esm-example', ...filepath)
+    const parser = new Parser({})
+    const res = await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.mjs')])
+    const expectedFiles = [
+      'dep1.js',
+      'dep2.mjs',
+      'dep3.mjs',
+      'dep4.mjs',
+      'dep5.mjs',
+      'dep6.mjs',
+      'entrypoint.mjs',
+    ].map(file => pathToPosix(toAbsolutePath(file)))
+
+    expect(res.files.map(file => pathToPosix(file)).sort()).toEqual(expectedFiles)
+  })
+
+  it('should handle node: prefix for built-ins', async () => {
+    const toAbsolutePath = (...filepath: string[]) => path.join(__dirname, 'check-parser-fixtures', 'builtin-with-node-prefix', ...filepath)
+    const parser = new Parser({})
+    await parser.getFilesAndDependencies([toAbsolutePath('entrypoint.ts')])
+  })
+
+  /*
+   * There is an unhandled edge-case when require() is reassigned.
+   * Even though the check might execute fine, we throw an error for a missing dependency.
+   * We could address this by keeping track of assignments as we walk the AST.
+   */
+  it.skip('should ignore cases where require is reassigned', async () => {
+    const entrypoint = path.join(__dirname, 'check-parser-fixtures', 'reassign-require.js')
+    const parser = new Parser({})
+    await parser.getFilesAndDependencies([entrypoint])
+  })
+
+  // Checks run on Checkly are wrapped to support top level await.
+  // For consistency with checks created via the UI, the CLI should support this as well.
+  it('should allow top-level await', async () => {
+    const entrypoint = path.join(__dirname, 'check-parser-fixtures', 'top-level-await.js')
+    const parser = new Parser({})
+    await parser.getFilesAndDependencies([entrypoint])
+  })
+
+  it('should allow top-level await in TypeScript', async () => {
+    const entrypoint = path.join(__dirname, 'check-parser-fixtures', 'top-level-await.ts')
+    const parser = new Parser({})
+    await parser.getFilesAndDependencies([entrypoint])
+  })
+})

packages/cli/src/services/check-parser/parser.ts

@@ -1,12 +1,14 @@
 import * as path from 'path'
 import * as fs from 'fs'
+import * as fsAsync from 'fs/promises'
 import * as acorn from 'acorn'
 import * as walk from 'acorn-walk'
 import { Collector } from './collector'
 import { DependencyParseError } from './errors'
 import { PackageFilesResolver, Dependencies } from './package-files/resolver'
 // Only import types given this is an optional dependency
 import type { TSESTree, AST_NODE_TYPES } from '@typescript-eslint/typescript-estree'
+import { findFilesWithPattern, pathToPosix } from '../util'
 
 // Our custom configuration to handle walking errors
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -113,6 +115,91 @@ export class Parser {
     return false
   }
 
+  private async validateFileAsync (filePath: string): Promise<{ filePath: string, content: string }> {
+    const extension = path.extname(filePath)
+    if (extension !== '.js' && extension !== '.ts' && extension !== '.mjs') {
+      throw new Error(`Unsupported file extension for ${filePath}`)
+    }
+    try {
+      const content = await fsAsync.readFile(filePath, { encoding: 'utf-8' })
+      return { filePath, content }
+    } catch (err) {
+      throw new DependencyParseError(filePath, [filePath], [], [])
+    }
+  }
+
+  async getFilesAndDependencies (paths: string[]): Promise<{ files: string[], errors: string[] }> {
+    const files = new Set(await this.getFilesFromPaths(paths))
+    const errors = new Set<string>()
+    const missingFiles = new Set<string>()
+    const resultFileSet = new Set<string>()
+    for (const file of files) {
+      if (resultFileSet.has(file)) {
+        continue
+      }
+      if (file.endsWith('.json')) {
+        // Holds info about the main file and doesn't need to be parsed
+        resultFileSet.add(file)
+        continue
+      }
+      const item = await this.validateFileAsync(file)
+
+      const cache = this.cache.get(item.filePath)
+      const { module, error } = cache !== undefined
+        ? cache
+        : Parser.parseDependencies(item.filePath, item.content)
+
+      if (error) {
+        this.cache.set(item.filePath, { module, error })
+        errors.add(item.filePath)
+        continue
+      }
+      const resolvedDependencies = cache?.resolvedDependencies ??
+          this.resolver.resolveDependenciesForFilePath(item.filePath, module.dependencies)
+
+      for (const dep of resolvedDependencies.missing) {
+        missingFiles.add(pathToPosix(dep.filePath))
+      }
+
+      this.cache.set(item.filePath, { module, resolvedDependencies })
+
+      for (const dep of resolvedDependencies.local) {
+        if (resultFileSet.has(dep.sourceFile.meta.filePath)) {
+          continue
+        }
+        const filePath = dep.sourceFile.meta.filePath
+        files.add(filePath)
+      }
+      resultFileSet.add(pathToPosix(item.filePath))
+    }
+    if (missingFiles.size) {
+      throw new DependencyParseError(paths.join(', '), Array.from(missingFiles), [], [])
+    }
+    return { files: Array.from(resultFileSet), errors: Array.from(errors) }
+  }
+
+  private async getFilesFromPaths (paths: string[]): Promise<string[]> {
+    const files = paths.map(async (currPath) => {
+      const normalizedPath = pathToPosix(currPath)
+      try {
+        const stats = await fsAsync.lstat(normalizedPath)
+        if (stats.isDirectory()) {
+          return findFilesWithPattern(normalizedPath, '**/*.{js,ts,mjs}', [])
+        }
+        return [normalizedPath]
+      } catch (err) {
+        if (normalizedPath.includes('*') || normalizedPath.includes('?') || normalizedPath.includes('{')) {
+          return findFilesWithPattern(process.cwd(), normalizedPath, [])
+        } else {
+          return []
+        }
+      }
+    })
+
+    const filesArray = await Promise.all(files)
+    return filesArray.flat()
+  }
+
   parse (entrypoint: string) {
     const { content } = validateEntrypoint(entrypoint)
 

packages/cli/src/services/project-parser.ts

@@ -1,6 +1,5 @@
-import { glob } from 'glob'
 import * as path from 'path'
-import { loadJsFile, loadTsFile, pathToPosix } from './util'
+import { findFilesWithPattern, loadJsFile, loadTsFile, pathToPosix } from './util'
 import {
   Check, BrowserCheck, CheckGroup, Project, Session,
   PrivateLocation, PrivateLocationCheckAssignment, PrivateLocationGroupAssignment, MultiStepCheck,
@@ -223,18 +222,3 @@ async function loadAllPrivateLocationsSlugNames (
     })
   })
 }
-
-async function findFilesWithPattern (
-  directory: string,
-  pattern: string | string[],
-  ignorePattern: string[],
-): Promise<string[]> {
-  // The files are sorted to make sure that the processing order is deterministic.
-  const files = await glob(pattern, {
-    nodir: true,
-    cwd: directory,
-    ignore: ignorePattern,
-    absolute: true,
-  })
-  return files.sort()
-}