diff --git a/Cargo.lock b/Cargo.lock index b4e0da442..7a0b75f83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1803,6 +1803,25 @@ name = "grit_cloud_client" version = "0.1.0" source = "git+https://github.com/getgrit/grit_cloud_client.git#7221ad63b697fd2c14e14a1872de602a7b3c135a" +[[package]] +name = "gritql_napi" +version = "0.0.0" +dependencies = [ + "anyhow", + "grit-pattern-matcher", + "grit-util", + "marzano-core", + "marzano-gritmodule", + "marzano-language", + "marzano-util", + "marzano_messenger", + "napi", + "napi-build", + "napi-derive", + "serde_json", + "tokio", +] + [[package]] name = "h2" version = "0.3.24" @@ -2879,6 +2898,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "napi-build" +version = "2.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28acfa557c083f6e254a786e01ba253fc56f18ee000afcd4f79af735f73a6da" + [[package]] name = "napi-derive" version = "2.16.3" diff --git a/Cargo.toml b/Cargo.toml index f9abfebb2..a6e0acd07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/wasm-bindings", "crates/marzano_messenger", "crates/cli_bin", + "js/gritql", ] exclude = [ "resources", diff --git a/js/gritql/.github/workflows/CI.yml b/js/gritql/.github/workflows/CI.yml new file mode 100644 index 000000000..f975e361c --- /dev/null +++ b/js/gritql/.github/workflows/CI.yml @@ -0,0 +1,484 @@ +name: CI +env: + DEBUG: napi:* + APP_NAME: gritql + MACOSX_DEPLOYMENT_TARGET: '10.13' +permissions: + contents: write + id-token: write +'on': + push: + branches: + - main + tags-ignore: + - '**' + paths-ignore: + - '**/*.md' + - LICENSE + - '**/*.gitignore' + - .editorconfig + - docs/** + pull_request: null +jobs: + build: + strategy: + fail-fast: false + matrix: + settings: + - host: macos-latest + target: x86_64-apple-darwin + build: yarn build --target x86_64-apple-darwin + - host: windows-latest + build: yarn build --target x86_64-pc-windows-msvc + target: x86_64-pc-windows-msvc + - host: windows-latest + build: | + yarn build --target i686-pc-windows-msvc + yarn test + target: i686-pc-windows-msvc + - host: ubuntu-latest + target: x86_64-unknown-linux-gnu + docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian + build: yarn build --target x86_64-unknown-linux-gnu + - host: ubuntu-latest + target: x86_64-unknown-linux-musl + docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine + build: yarn build --target x86_64-unknown-linux-musl + - host: macos-latest + target: aarch64-apple-darwin + build: yarn build --target aarch64-apple-darwin + - host: ubuntu-latest + target: aarch64-unknown-linux-gnu + docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64 + build: yarn build --target aarch64-unknown-linux-gnu + - host: ubuntu-latest + target: armv7-unknown-linux-gnueabihf + setup: | + sudo apt-get update + sudo apt-get install gcc-arm-linux-gnueabihf -y + build: yarn build --target armv7-unknown-linux-gnueabihf + - host: ubuntu-latest + target: armv7-unknown-linux-musleabihf + build: yarn build --target armv7-unknown-linux-musleabihf + - host: ubuntu-latest + target: aarch64-linux-android + build: yarn build --target aarch64-linux-android + - host: ubuntu-latest + target: armv7-linux-androideabi + build: yarn build --target armv7-linux-androideabi + - host: ubuntu-latest + target: aarch64-unknown-linux-musl + docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine + build: |- + set -e && + rustup target add aarch64-unknown-linux-musl && + yarn build --target aarch64-unknown-linux-musl + - host: windows-latest + target: aarch64-pc-windows-msvc + build: yarn build --target aarch64-pc-windows-msvc + - host: ubuntu-latest + target: riscv64gc-unknown-linux-gnu + setup: | + sudo apt-get update + sudo apt-get install gcc-riscv64-linux-gnu -y + build: yarn build --target riscv64gc-unknown-linux-gnu + name: stable - ${{ matrix.settings.target }} - node@20 + runs-on: ${{ matrix.settings.host }} + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + if: ${{ !matrix.settings.docker }} + with: + node-version: 20 + cache: yarn + - name: Install + uses: dtolnay/rust-toolchain@stable + if: ${{ !matrix.settings.docker }} + with: + toolchain: stable + targets: ${{ matrix.settings.target }} + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + .cargo-cache + target/ + key: ${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }} + - uses: goto-bus-stop/setup-zig@v2 + if: ${{ matrix.settings.target == 'armv7-unknown-linux-gnueabihf' || matrix.settings.target == 'armv7-unknown-linux-musleabihf' }} + with: + version: 0.11.0 + - name: Setup toolchain + run: ${{ matrix.settings.setup }} + if: ${{ matrix.settings.setup }} + shell: bash + - name: Setup node x86 + if: matrix.settings.target == 'i686-pc-windows-msvc' + run: yarn config set supportedArchitectures.cpu "ia32" + shell: bash + - name: Install dependencies + run: yarn install + - name: Setup node x86 + uses: actions/setup-node@v4 + if: matrix.settings.target == 'i686-pc-windows-msvc' + with: + node-version: 20 + cache: yarn + architecture: x86 + - name: Build in docker + uses: addnab/docker-run-action@v3 + if: ${{ matrix.settings.docker }} + with: + image: ${{ matrix.settings.docker }} + options: '--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build' + run: ${{ matrix.settings.build }} + - name: Build + run: ${{ matrix.settings.build }} + if: ${{ !matrix.settings.docker }} + shell: bash + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bindings-${{ matrix.settings.target }} + path: ${{ env.APP_NAME }}.*.node + if-no-files-found: error + build-freebsd: + runs-on: macos-13 + name: Build FreeBSD + steps: + - uses: actions/checkout@v4 + - name: Build + id: build + uses: cross-platform-actions/action@v0.24.0 + env: + DEBUG: napi:* + RUSTUP_IO_THREADS: 1 + with: + operating_system: freebsd + version: '13.2' + memory: 8G + cpu_count: 3 + environment_variables: DEBUG RUSTUP_IO_THREADS + shell: bash + run: | + sudo pkg install -y -f curl node libnghttp2 npm + sudo npm install -g yarn --ignore-scripts + curl https://sh.rustup.rs -sSf --output rustup.sh + sh rustup.sh -y --profile minimal --default-toolchain stable + source "$HOME/.cargo/env" + echo "~~~~ rustc --version ~~~~" + rustc --version + echo "~~~~ node -v ~~~~" + node -v + echo "~~~~ yarn --version ~~~~" + yarn --version + pwd + ls -lah + whoami + env + freebsd-version + yarn install + yarn build + yarn test + rm -rf node_modules + rm -rf target + rm -rf .yarn/cache + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bindings-freebsd + path: ${{ env.APP_NAME }}.*.node + if-no-files-found: error + test-macOS-windows-binding: + name: Test bindings on ${{ matrix.settings.target }} - node@${{ matrix.node }} + needs: + - build + strategy: + fail-fast: false + matrix: + settings: + - host: macos-latest + target: x86_64-apple-darwin + - host: windows-latest + target: x86_64-pc-windows-msvc + node: + - '18' + - '20' + runs-on: ${{ matrix.settings.host }} + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: yarn + architecture: x64 + - name: Install dependencies + run: yarn install + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-${{ matrix.settings.target }} + path: . + - name: List packages + run: ls -R . + shell: bash + - name: Test bindings + run: yarn test + test-linux-x64-gnu-binding: + name: Test bindings on Linux-x64-gnu - node@${{ matrix.node }} + needs: + - build + strategy: + fail-fast: false + matrix: + node: + - '18' + - '20' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: yarn + - name: Install dependencies + run: yarn install + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-x86_64-unknown-linux-gnu + path: . + - name: List packages + run: ls -R . + shell: bash + - name: Test bindings + run: docker run --rm -v $(pwd):/build -w /build node:${{ matrix.node }}-slim yarn test + test-linux-x64-musl-binding: + name: Test bindings on x86_64-unknown-linux-musl - node@${{ matrix.node }} + needs: + - build + strategy: + fail-fast: false + matrix: + node: + - '18' + - '20' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: yarn + - name: Install dependencies + run: | + yarn config set supportedArchitectures.libc "musl" + yarn install + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-x86_64-unknown-linux-musl + path: . + - name: List packages + run: ls -R . + shell: bash + - name: Test bindings + run: docker run --rm -v $(pwd):/build -w /build node:${{ matrix.node }}-alpine yarn test + test-linux-aarch64-gnu-binding: + name: Test bindings on aarch64-unknown-linux-gnu - node@${{ matrix.node }} + needs: + - build + strategy: + fail-fast: false + matrix: + node: + - '18' + - '20' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-aarch64-unknown-linux-gnu + path: . + - name: List packages + run: ls -R . + shell: bash + - name: Install dependencies + run: | + yarn config set supportedArchitectures.cpu "arm64" + yarn config set supportedArchitectures.libc "glibc" + yarn install + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Setup and run tests + uses: addnab/docker-run-action@v3 + with: + image: node:${{ matrix.node }}-slim + options: '--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build' + run: | + set -e + yarn test + ls -la + test-linux-aarch64-musl-binding: + name: Test bindings on aarch64-unknown-linux-musl - node@${{ matrix.node }} + needs: + - build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-aarch64-unknown-linux-musl + path: . + - name: List packages + run: ls -R . + shell: bash + - name: Install dependencies + run: | + yarn config set supportedArchitectures.cpu "arm64" + yarn config set supportedArchitectures.libc "musl" + yarn install + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Setup and run tests + uses: addnab/docker-run-action@v3 + with: + image: node:lts-alpine + options: '--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build' + run: | + set -e + yarn test + test-linux-arm-gnueabihf-binding: + name: Test bindings on armv7-unknown-linux-gnueabihf - node@${{ matrix.node }} + needs: + - build + strategy: + fail-fast: false + matrix: + node: + - '18' + - '20' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: bindings-armv7-unknown-linux-gnueabihf + path: . + - name: List packages + run: ls -R . + shell: bash + - name: Install dependencies + run: | + yarn config set supportedArchitectures.cpu "arm" + yarn install + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm + - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Setup and run tests + uses: addnab/docker-run-action@v3 + with: + image: node:${{ matrix.node }}-bullseye-slim + options: '--platform linux/arm/v7 -v ${{ github.workspace }}:/build -w /build' + run: | + set -e + yarn test + ls -la + universal-macOS: + name: Build universal macOS binary + needs: + - build + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: yarn + - name: Install dependencies + run: yarn install + - name: Download macOS x64 artifact + uses: actions/download-artifact@v4 + with: + name: bindings-x86_64-apple-darwin + path: artifacts + - name: Download macOS arm64 artifact + uses: actions/download-artifact@v4 + with: + name: bindings-aarch64-apple-darwin + path: artifacts + - name: Combine binaries + run: yarn universal + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: bindings-universal-apple-darwin + path: ${{ env.APP_NAME }}.*.node + if-no-files-found: error + publish: + name: Publish + runs-on: ubuntu-latest + needs: + - build-freebsd + - test-macOS-windows-binding + - test-linux-x64-gnu-binding + - test-linux-x64-musl-binding + - test-linux-aarch64-gnu-binding + - test-linux-aarch64-musl-binding + - test-linux-arm-gnueabihf-binding + - universal-macOS + steps: + - uses: actions/checkout@v4 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: yarn + - name: Install dependencies + run: yarn install + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + - name: Move artifacts + run: yarn artifacts + - name: List packages + run: ls -R ./npm + shell: bash + - name: Publish + run: | + npm config set provenance true + if git log -1 --pretty=%B | grep "^[0-9]\+\.[0-9]\+\.[0-9]\+$"; + then + echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc + npm publish --access public + elif git log -1 --pretty=%B | grep "^[0-9]\+\.[0-9]\+\.[0-9]\+"; + then + echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc + npm publish --tag next --access public + else + echo "Not a release, skipping publish" + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/js/gritql/.gitignore b/js/gritql/.gitignore new file mode 100644 index 000000000..a2b5be15d --- /dev/null +++ b/js/gritql/.gitignore @@ -0,0 +1,197 @@ +# Created by https://www.toptal.com/developers/gitignore/api/node +# Edit at https://www.toptal.com/developers/gitignore?templates=node + +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test + +# parcel-bundler cache (https://parceljs.org/) +.cache + +# Next.js build output +.next + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# End of https://www.toptal.com/developers/gitignore/api/node + +# Created by https://www.toptal.com/developers/gitignore/api/macos +# Edit at https://www.toptal.com/developers/gitignore?templates=macos + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +# End of https://www.toptal.com/developers/gitignore/api/macos + +# Created by https://www.toptal.com/developers/gitignore/api/windows +# Edit at https://www.toptal.com/developers/gitignore?templates=windows + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/windows + +#Added by cargo + +/target +Cargo.lock + +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions + +*.node diff --git a/js/gritql/.npmignore b/js/gritql/.npmignore new file mode 100644 index 000000000..ec144db2a --- /dev/null +++ b/js/gritql/.npmignore @@ -0,0 +1,13 @@ +target +Cargo.lock +.cargo +.github +npm +.eslintrc +.prettierignore +rustfmt.toml +yarn.lock +*.node +.yarn +__test__ +renovate.json diff --git a/js/gritql/Cargo.toml b/js/gritql/Cargo.toml new file mode 100644 index 000000000..ec02cf5d4 --- /dev/null +++ b/js/gritql/Cargo.toml @@ -0,0 +1,36 @@ +[package] +edition = "2021" +name = "gritql_napi" +version = "0.0.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix +napi = { version = "2.16.4", default-features = false, features = [ + "napi8", + "async", +] } +napi-derive = "2.12.2" +anyhow = "1.0.70" +serde_json = "1.0.113" + + +marzano-gritmodule = { path = "../../crates/gritmodule" } +marzano-core = { path = "../../crates/core", features = [ + "non_wasm", +], default-features = false } +grit-util = { path = "../../crates/grit-util", features = [] } +grit-pattern-matcher = { path = "../../crates/grit-pattern-matcher", features = [ +] } +marzano-language = { path = "../../crates/language", features = ["finder"] } +marzano-util = { path = "../../crates/util", features = ["napi"] } + +marzano_messenger = { path = "../../crates/marzano_messenger", features = [] } + +[build-dependencies] +napi-build = "2.0.1" + +[dev-dependencies] +tokio = { version = "1.0", features = ["rt", "macros"] } diff --git a/js/gritql/__generated__/.gitkeep b/js/gritql/__generated__/.gitkeep new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/js/gritql/__generated__/.gitkeep @@ -0,0 +1 @@ +* diff --git a/js/gritql/__generated__/index.d.ts b/js/gritql/__generated__/index.d.ts new file mode 100644 index 000000000..3c6e06a45 --- /dev/null +++ b/js/gritql/__generated__/index.d.ts @@ -0,0 +1,109 @@ +/* tslint:disable */ +/* eslint-disable */ + +/* auto-generated by NAPI-RS */ + +export interface Position { + /** 1-based line number in the source file. */ + line: number + /** 1-based column number in the source file. */ + column: number +} +export interface RangeWithoutByte { + start: Position + end: Position +} +export interface RangePair { + before: RangeWithoutByte + after: RangeWithoutByte +} +export interface FileDiff { + oldPath?: string + newPath?: string + ranges: Array +} +export interface RichFile { + path: string + content: string +} +export enum SearchOutputOptions { + /** Emit absolute paths */ + AbsolutePaths = 'AbsolutePaths', + /** Emit paths relative to the provided search/root directory */ + RelativePaths = 'RelativePaths' +} +export interface SearchOptions { + targetPaths: Array + stepId: string + /** Skip emitting any results and applying changes */ + silent?: boolean + /** The format of the output paths */ + outputFormat?: SearchOutputOptions + /** Verify the query without applying any changes */ + verifyOnly?: boolean +} +export interface InputFile { + path: string + content: string +} +export interface InputOptions { + files: Array +} +export class JsResolvedBinding { + /** + * Retrieves the stringified representation of the binding. + * + * This method accesses the binding's text representation, using the current state's files + * and the specified language context. + */ + text(): string + /** Returns the range of the binding. */ + range(): RangeWithoutByte | null + /** Retrieve a variable's text value. */ + var(name: string): string | null + /** Retrieves the absolute file name of the file containing the current binding. */ + filename(): string + /** Inserts the provided text after the binding. */ + insertAfter(text: string): void +} +export class QueryBuilder { + /** Construct a new query, starting from a GritQL query */ + constructor(baseQuerySrc: string) + /** + * Add a callback to filter the results of the search + * + * @param filter A callback that that will be called with each match, it should return true to keep the match + */ + filter(filter: MatchCallback): void + /** + * Set a replacement rewrite to the query + * + * @param replacement The replacement rewrite to apply to the query + */ + setReplacement(replacement: string): void + /** + * Add an insertion to the query + * + * @param insertion The insertion to add to the query + */ + addInsertion(insertion: string): void + /** + * Pipe the query into another query + * When you pipe a query, it will stop directly emitting results itself. + * Instead, every *matching* file will be passed to the next query. + * @param query The query to pipe into + */ + pipe(secondQuery: QueryBuilder): void + /** + * Run the query on the provided files + * @param target_paths A list of paths to search for matches + * @returns A list of files that matched the query + */ + run(options: SearchOptions): Promise> + /** + * Apply the query to a single file and return the modified file (if any) + * @param file The file to apply the query to + * @returns The modified file (if any) + */ + applyToFile(file: InputFile): Promise +} diff --git a/js/gritql/__generated__/index.js b/js/gritql/__generated__/index.js new file mode 100644 index 000000000..074f85d82 --- /dev/null +++ b/js/gritql/__generated__/index.js @@ -0,0 +1,317 @@ +/* tslint:disable */ +/* eslint-disable */ +/* prettier-ignore */ + +/* auto-generated by NAPI-RS */ + +const { existsSync, readFileSync } = require('fs') +const { join } = require('path') + +const { platform, arch } = process + +let nativeBinding = null +let localFileExisted = false +let loadError = null + +function isMusl() { + // For Node 10 + if (!process.report || typeof process.report.getReport !== 'function') { + try { + const lddPath = require('child_process').execSync('which ldd').toString().trim() + return readFileSync(lddPath, 'utf8').includes('musl') + } catch (e) { + return true + } + } else { + const { glibcVersionRuntime } = process.report.getReport().header + return !glibcVersionRuntime + } +} + +switch (platform) { + case 'android': + switch (arch) { + case 'arm64': + localFileExisted = existsSync(join(__dirname, 'gritql.android-arm64.node')) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.android-arm64.node') + } else { + nativeBinding = require('@getgrit/gritql-android-arm64') + } + } catch (e) { + loadError = e + } + break + case 'arm': + localFileExisted = existsSync(join(__dirname, 'gritql.android-arm-eabi.node')) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.android-arm-eabi.node') + } else { + nativeBinding = require('@getgrit/gritql-android-arm-eabi') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on Android ${arch}`) + } + break + case 'win32': + switch (arch) { + case 'x64': + localFileExisted = existsSync( + join(__dirname, 'gritql.win32-x64-msvc.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.win32-x64-msvc.node') + } else { + nativeBinding = require('@getgrit/gritql-win32-x64-msvc') + } + } catch (e) { + loadError = e + } + break + case 'ia32': + localFileExisted = existsSync( + join(__dirname, 'gritql.win32-ia32-msvc.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.win32-ia32-msvc.node') + } else { + nativeBinding = require('@getgrit/gritql-win32-ia32-msvc') + } + } catch (e) { + loadError = e + } + break + case 'arm64': + localFileExisted = existsSync( + join(__dirname, 'gritql.win32-arm64-msvc.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.win32-arm64-msvc.node') + } else { + nativeBinding = require('@getgrit/gritql-win32-arm64-msvc') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on Windows: ${arch}`) + } + break + case 'darwin': + localFileExisted = existsSync(join(__dirname, 'gritql.darwin-universal.node')) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.darwin-universal.node') + } else { + nativeBinding = require('@getgrit/gritql-darwin-universal') + } + break + } catch {} + switch (arch) { + case 'x64': + localFileExisted = existsSync(join(__dirname, 'gritql.darwin-x64.node')) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.darwin-x64.node') + } else { + nativeBinding = require('@getgrit/gritql-darwin-x64') + } + } catch (e) { + loadError = e + } + break + case 'arm64': + localFileExisted = existsSync( + join(__dirname, 'gritql.darwin-arm64.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.darwin-arm64.node') + } else { + nativeBinding = require('@getgrit/gritql-darwin-arm64') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on macOS: ${arch}`) + } + break + case 'freebsd': + if (arch !== 'x64') { + throw new Error(`Unsupported architecture on FreeBSD: ${arch}`) + } + localFileExisted = existsSync(join(__dirname, 'gritql.freebsd-x64.node')) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.freebsd-x64.node') + } else { + nativeBinding = require('@getgrit/gritql-freebsd-x64') + } + } catch (e) { + loadError = e + } + break + case 'linux': + switch (arch) { + case 'x64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-x64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-x64-musl.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-x64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-x64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-x64-gnu.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-x64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 'arm64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-arm64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-arm64-musl.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-arm64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-arm64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-arm64-gnu.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-arm64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 'arm': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-arm-musleabihf.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-arm-musleabihf.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-arm-musleabihf') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-arm-gnueabihf.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-arm-gnueabihf.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-arm-gnueabihf') + } + } catch (e) { + loadError = e + } + } + break + case 'riscv64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-riscv64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-riscv64-musl.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-riscv64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-riscv64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-riscv64-gnu.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-riscv64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 's390x': + localFileExisted = existsSync( + join(__dirname, 'gritql.linux-s390x-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gritql.linux-s390x-gnu.node') + } else { + nativeBinding = require('@getgrit/gritql-linux-s390x-gnu') + } + } catch (e) { + loadError = e + } + break + default: + throw new Error(`Unsupported architecture on Linux: ${arch}`) + } + break + default: + throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`) +} + +if (!nativeBinding) { + if (loadError) { + throw loadError + } + throw new Error(`Failed to load native binding`) +} + +const { JsResolvedBinding, QueryBuilder, SearchOutputOptions } = nativeBinding + +module.exports.JsResolvedBinding = JsResolvedBinding +module.exports.QueryBuilder = QueryBuilder +module.exports.SearchOutputOptions = SearchOutputOptions diff --git a/js/gritql/__test__/apply.spec.mts b/js/gritql/__test__/apply.spec.mts new file mode 100644 index 000000000..5e70be9fe --- /dev/null +++ b/js/gritql/__test__/apply.spec.mts @@ -0,0 +1,74 @@ +import { expect, describe, it } from 'bun:test'; + +import { QueryBuilder } from '../__generated__/index.js'; + +describe('GritQL apply bindings', () => { + const file = { + path: 'test.js', + content: `console.log("hello") +console.log("world")`, + }; + + it('can apply a rewrite in a synthetic file', async () => { + const query = new QueryBuilder(`js"console.log($msg)" => js"console.error($msg)"`); + const modified = await query.applyToFile(file); + expect(modified).toBeDefined(); + expect(modified!.content).toBe(`console.error("hello") +console.error("world")`); + }); + + it('can apply a rewrite with a pattern', async () => { + const query = new QueryBuilder(`import_statement() as $candidate where { + $candidate <: within program($statements), + $statements <: some bubble($last_import) import_statement() as $this_import where $last_import = $this_import, + $candidate <: $last_import +}`); + query.filter((arg: any) => { + arg.insertAfter(`<<>>\n`); + return true; + }); + const modified = await query.applyToFile({ + path: 'test.js', + content: `import { foo } from 'bar'; +import { baz } from 'qux'; +import { quux } from 'corge'; + +function foo() { + console.log('foo'); +}`, + }); + console.log(modified); + expect(modified).toBeDefined(); + expect(modified!.content).toBe(`import { foo } from 'bar'; +import { baz } from 'qux'; +import { quux } from 'corge'; +<<>> + + +function foo() { + console.log('foo'); +}`); + }); + + it('can apply a rewrite via a callback', async () => { + const query = new QueryBuilder(`js"console.log($msg)"`); + query.filter((arg) => { + arg.insertAfter('\n^^^inserted^^^'); + return true; + }); + const modified = await query.applyToFile(file); + expect(modified).toBeDefined(); + expect(modified!.content).toBe( + `console.log("hello") +^^^inserted^^^ +console.log("world") +^^^inserted^^^`, + ); + }); + + it('returns null if no matches', async () => { + const query = new QueryBuilder(`js"console.loud($msg)" => js"console.error($msg)"`); + const modified = await query.applyToFile(file); + expect(modified).toBeNull(); + }); +}); diff --git a/js/gritql/__test__/napi.spec.mts b/js/gritql/__test__/napi.spec.mts new file mode 100644 index 000000000..4fec5190d --- /dev/null +++ b/js/gritql/__test__/napi.spec.mts @@ -0,0 +1,68 @@ +import { expect, describe, it } from 'bun:test'; +import { AsyncLocalStorage } from 'node:async_hooks'; + +import { QueryBuilder } from '../__generated__/index.js'; + +describe('Node API interfaces ', () => { + const file = { + path: 'test.js', + content: `console.log("hello") +console.log("world")`, + }; + + // Broken, see https://github.com/getgrit/rewriter/issues/10170 + it.skip('can run inside a storage context', async () => { + const testStorage = new AsyncLocalStorage(); + + const result = await testStorage.exit(async () => { + const query = new QueryBuilder(`js"console.log($msg)"`); + query.filter((log) => { + log.insertAfter('\n<>'); + return true; + }); + const modified = await query.applyToFile(file); + expect(modified).toBeDefined(); + expect(modified!.content).toBe(`console.log("hello") +<> +console.log("world") +<>`); + + return true; + }); + expect(result).toBeTrue(); + }); + + it('can do simple rewrites without callbacks', async () => { + const testStorage = new AsyncLocalStorage(); + + const result = await testStorage.exit(async () => { + const query = new QueryBuilder(`js"console.log($msg)"`); + query.setReplacement('console.log(REPLACED)'); + const modified = await query.applyToFile(file); + expect(modified).toBeDefined(); + expect(modified!.content).toBe(`console.log(REPLACED) +console.log(REPLACED)`); + + return true; + }); + expect(result).toBeTrue(); + }); + + it('can do simple inserts without callbacks', async () => { + const testStorage = new AsyncLocalStorage(); + + const result = await testStorage.exit(async () => { + const query = new QueryBuilder(`js"console.log($msg)"`); + query.addInsertion('\n<>'); + const modified = await query.applyToFile(file); + expect(modified).toBeDefined(); + expect(modified!.content).toBe(`console.log("hello") +<> +console.log("world") +<>`); + + return true; + }); + expect(result).toBeTrue(); + }); +}); diff --git a/js/gritql/build.rs b/js/gritql/build.rs new file mode 100644 index 000000000..9fc236788 --- /dev/null +++ b/js/gritql/build.rs @@ -0,0 +1,5 @@ +extern crate napi_build; + +fn main() { + napi_build::setup(); +} diff --git a/js/gritql/examples/.grit/.gitignore b/js/gritql/examples/.grit/.gitignore new file mode 100644 index 000000000..e4fdfb17c --- /dev/null +++ b/js/gritql/examples/.grit/.gitignore @@ -0,0 +1,2 @@ +.gritmodules* +*.log diff --git a/js/gritql/examples/.grit/grit.yaml b/js/gritql/examples/.grit/grit.yaml new file mode 100644 index 000000000..c3740aaf3 --- /dev/null +++ b/js/gritql/examples/.grit/grit.yaml @@ -0,0 +1,6 @@ +patterns: + - name: foo_bar + body: | + language js + + `foo` => `bar` diff --git a/js/gritql/examples/async.js b/js/gritql/examples/async.js new file mode 100644 index 000000000..86ca0f348 --- /dev/null +++ b/js/gritql/examples/async.js @@ -0,0 +1,26 @@ +const { AsyncLocalStorage } = require('node:async_hooks'); +const { QueryBuilder } = require('../__generated__/bridge.darwin-arm64.node'); + +const testStorage = new AsyncLocalStorage(); +// testStorage.enterWith(9); + +// testStorage.disable(); + +// testStorage.exit(() => { +const query = new QueryBuilder(`js"console.log($msg)"`); +query.filter((err, log) => { + console.log('filter was called', err, log); + log.insertAfter('\n<>'); + return true; +}); + +query + .applyToFile({ + path: 'test.js', + content: `console.log("hello")`, + }) + .then((result) => { + console.log(result); + process.exit(0); + }); +// }); diff --git a/js/gritql/examples/file.js b/js/gritql/examples/file.js new file mode 100644 index 000000000..d7a4e327c --- /dev/null +++ b/js/gritql/examples/file.js @@ -0,0 +1,3 @@ +function hello() { + return foo(); +} diff --git a/js/gritql/examples/file2.js b/js/gritql/examples/file2.js new file mode 100644 index 000000000..2732fd68a --- /dev/null +++ b/js/gritql/examples/file2.js @@ -0,0 +1,3 @@ +function helloTwo() { + return foo(); +} diff --git a/js/gritql/examples/pipe.ts b/js/gritql/examples/pipe.ts new file mode 100644 index 000000000..0836f932a --- /dev/null +++ b/js/gritql/examples/pipe.ts @@ -0,0 +1,26 @@ +import { QueryBuilder } from '../__generated__/index.js'; + +/// The first query is just filtering files +const query = new QueryBuilder(`file()`); + +// NOTE: THIS CAUSES THE PIPE TO BREAK +// query.filter((err, arg) => { +// console.log('file hit in layer 1', arg); +// return true; +// }); + +/// The second query actually looks for foo_bar +const query2 = new QueryBuilder(`js"function $_($_) { $_ }"`); + +query2.filter((node) => { + console.log('file hit in layer 2', node); + return true; +}); + +/// We pipe the first query into the second +query.pipe(query2); + +/// Then run the first query and it will run the second query as well +const files = await query.run({ targetPaths: [process.cwd()], stepId: 'test' }); +const total = files.length; +console.log(`Hello from search.ts!, we found ${total} files`); diff --git a/js/gritql/examples/search.ts b/js/gritql/examples/search.ts new file mode 100644 index 000000000..06a8be5d0 --- /dev/null +++ b/js/gritql/examples/search.ts @@ -0,0 +1,23 @@ +import { QueryBuilder } from '../__generated__/index.js'; + +const query = new QueryBuilder(`js"console.log($_)" as $match where { + $msg = $match, + $something = length($msg), + $something <: true +}`); + +query.filter((node) => { + console.log('got my first arg!', node.text()); + const length = arg.length; + const isOdd = length % 2 === 1; + return isOdd; +}); + +query.filter((node) => { + console.log('got my second arg!', node.text()); + return arg.length > 19; +}); + +const files = await query.run({ targetPaths: [process.cwd()], stepId: 'test' }); +const total = files.length; +console.log(`Hello from search.ts!, we found ${total} files`); diff --git a/js/gritql/npm/android-arm-eabi/README.md b/js/gritql/npm/android-arm-eabi/README.md new file mode 100644 index 000000000..8a323e13a --- /dev/null +++ b/js/gritql/npm/android-arm-eabi/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-android-arm-eabi` + +This is the **armv7-linux-androideabi** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/android-arm-eabi/package.json b/js/gritql/npm/android-arm-eabi/package.json new file mode 100644 index 000000000..6eebdbb33 --- /dev/null +++ b/js/gritql/npm/android-arm-eabi/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-android-arm-eabi", + "version": "0.0.0", + "os": [ + "android" + ], + "cpu": [ + "arm" + ], + "main": "gritql.android-arm-eabi.node", + "files": [ + "gritql.android-arm-eabi.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/android-arm64/README.md b/js/gritql/npm/android-arm64/README.md new file mode 100644 index 000000000..eaea2979f --- /dev/null +++ b/js/gritql/npm/android-arm64/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-android-arm64` + +This is the **aarch64-linux-android** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/android-arm64/package.json b/js/gritql/npm/android-arm64/package.json new file mode 100644 index 000000000..91fe8b5a8 --- /dev/null +++ b/js/gritql/npm/android-arm64/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-android-arm64", + "version": "0.0.0", + "os": [ + "android" + ], + "cpu": [ + "arm64" + ], + "main": "gritql.android-arm64.node", + "files": [ + "gritql.android-arm64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/darwin-arm64/README.md b/js/gritql/npm/darwin-arm64/README.md new file mode 100644 index 000000000..e120fde74 --- /dev/null +++ b/js/gritql/npm/darwin-arm64/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-darwin-arm64` + +This is the **aarch64-apple-darwin** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/darwin-arm64/package.json b/js/gritql/npm/darwin-arm64/package.json new file mode 100644 index 000000000..b3ad1edcb --- /dev/null +++ b/js/gritql/npm/darwin-arm64/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-darwin-arm64", + "version": "0.0.0", + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "main": "gritql.darwin-arm64.node", + "files": [ + "gritql.darwin-arm64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/darwin-universal/README.md b/js/gritql/npm/darwin-universal/README.md new file mode 100644 index 000000000..5b29ad913 --- /dev/null +++ b/js/gritql/npm/darwin-universal/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-darwin-universal` + +This is the **universal-apple-darwin** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/darwin-universal/package.json b/js/gritql/npm/darwin-universal/package.json new file mode 100644 index 000000000..49c89fdea --- /dev/null +++ b/js/gritql/npm/darwin-universal/package.json @@ -0,0 +1,15 @@ +{ + "name": "@getgrit/gritql-darwin-universal", + "version": "0.0.0", + "os": [ + "darwin" + ], + "main": "gritql.darwin-universal.node", + "files": [ + "gritql.darwin-universal.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/darwin-x64/README.md b/js/gritql/npm/darwin-x64/README.md new file mode 100644 index 000000000..647b580f2 --- /dev/null +++ b/js/gritql/npm/darwin-x64/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-darwin-x64` + +This is the **x86_64-apple-darwin** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/darwin-x64/package.json b/js/gritql/npm/darwin-x64/package.json new file mode 100644 index 000000000..848344e84 --- /dev/null +++ b/js/gritql/npm/darwin-x64/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-darwin-x64", + "version": "0.0.0", + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "main": "gritql.darwin-x64.node", + "files": [ + "gritql.darwin-x64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/freebsd-x64/README.md b/js/gritql/npm/freebsd-x64/README.md new file mode 100644 index 000000000..390bf4120 --- /dev/null +++ b/js/gritql/npm/freebsd-x64/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-freebsd-x64` + +This is the **x86_64-unknown-freebsd** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/freebsd-x64/package.json b/js/gritql/npm/freebsd-x64/package.json new file mode 100644 index 000000000..a3c97800f --- /dev/null +++ b/js/gritql/npm/freebsd-x64/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-freebsd-x64", + "version": "0.0.0", + "os": [ + "freebsd" + ], + "cpu": [ + "x64" + ], + "main": "gritql.freebsd-x64.node", + "files": [ + "gritql.freebsd-x64.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/linux-arm-gnueabihf/README.md b/js/gritql/npm/linux-arm-gnueabihf/README.md new file mode 100644 index 000000000..bb18b26c0 --- /dev/null +++ b/js/gritql/npm/linux-arm-gnueabihf/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-linux-arm-gnueabihf` + +This is the **armv7-unknown-linux-gnueabihf** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/linux-arm-gnueabihf/package.json b/js/gritql/npm/linux-arm-gnueabihf/package.json new file mode 100644 index 000000000..eea4435c9 --- /dev/null +++ b/js/gritql/npm/linux-arm-gnueabihf/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-linux-arm-gnueabihf", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "arm" + ], + "main": "gritql.linux-arm-gnueabihf.node", + "files": [ + "gritql.linux-arm-gnueabihf.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/linux-arm-musleabihf/README.md b/js/gritql/npm/linux-arm-musleabihf/README.md new file mode 100644 index 000000000..a63f66603 --- /dev/null +++ b/js/gritql/npm/linux-arm-musleabihf/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-linux-arm-musleabihf` + +This is the **armv7-unknown-linux-musleabihf** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/linux-arm-musleabihf/package.json b/js/gritql/npm/linux-arm-musleabihf/package.json new file mode 100644 index 000000000..dc60ebc96 --- /dev/null +++ b/js/gritql/npm/linux-arm-musleabihf/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-linux-arm-musleabihf", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "arm" + ], + "main": "gritql.linux-arm-musleabihf.node", + "files": [ + "gritql.linux-arm-musleabihf.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/linux-arm64-gnu/README.md b/js/gritql/npm/linux-arm64-gnu/README.md new file mode 100644 index 000000000..a5316db82 --- /dev/null +++ b/js/gritql/npm/linux-arm64-gnu/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-linux-arm64-gnu` + +This is the **aarch64-unknown-linux-gnu** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/linux-arm64-gnu/package.json b/js/gritql/npm/linux-arm64-gnu/package.json new file mode 100644 index 000000000..fb2431e0c --- /dev/null +++ b/js/gritql/npm/linux-arm64-gnu/package.json @@ -0,0 +1,21 @@ +{ + "name": "@getgrit/gritql-linux-arm64-gnu", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "main": "gritql.linux-arm64-gnu.node", + "files": [ + "gritql.linux-arm64-gnu.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "glibc" + ] +} \ No newline at end of file diff --git a/js/gritql/npm/linux-arm64-musl/README.md b/js/gritql/npm/linux-arm64-musl/README.md new file mode 100644 index 000000000..1bf8356f7 --- /dev/null +++ b/js/gritql/npm/linux-arm64-musl/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-linux-arm64-musl` + +This is the **aarch64-unknown-linux-musl** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/linux-arm64-musl/package.json b/js/gritql/npm/linux-arm64-musl/package.json new file mode 100644 index 000000000..776c2ea48 --- /dev/null +++ b/js/gritql/npm/linux-arm64-musl/package.json @@ -0,0 +1,21 @@ +{ + "name": "@getgrit/gritql-linux-arm64-musl", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "main": "gritql.linux-arm64-musl.node", + "files": [ + "gritql.linux-arm64-musl.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "musl" + ] +} \ No newline at end of file diff --git a/js/gritql/npm/linux-riscv64-gnu/README.md b/js/gritql/npm/linux-riscv64-gnu/README.md new file mode 100644 index 000000000..3673a6b0c --- /dev/null +++ b/js/gritql/npm/linux-riscv64-gnu/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-linux-riscv64-gnu` + +This is the **riscv64gc-unknown-linux-gnu** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/linux-riscv64-gnu/package.json b/js/gritql/npm/linux-riscv64-gnu/package.json new file mode 100644 index 000000000..41aebdf06 --- /dev/null +++ b/js/gritql/npm/linux-riscv64-gnu/package.json @@ -0,0 +1,21 @@ +{ + "name": "@getgrit/gritql-linux-riscv64-gnu", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "riscv64" + ], + "main": "gritql.linux-riscv64-gnu.node", + "files": [ + "gritql.linux-riscv64-gnu.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "glibc" + ] +} \ No newline at end of file diff --git a/js/gritql/npm/linux-x64-gnu/README.md b/js/gritql/npm/linux-x64-gnu/README.md new file mode 100644 index 000000000..a049181f0 --- /dev/null +++ b/js/gritql/npm/linux-x64-gnu/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-linux-x64-gnu` + +This is the **x86_64-unknown-linux-gnu** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/linux-x64-gnu/package.json b/js/gritql/npm/linux-x64-gnu/package.json new file mode 100644 index 000000000..6945d9cbd --- /dev/null +++ b/js/gritql/npm/linux-x64-gnu/package.json @@ -0,0 +1,21 @@ +{ + "name": "@getgrit/gritql-linux-x64-gnu", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "main": "gritql.linux-x64-gnu.node", + "files": [ + "gritql.linux-x64-gnu.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "glibc" + ] +} \ No newline at end of file diff --git a/js/gritql/npm/linux-x64-musl/README.md b/js/gritql/npm/linux-x64-musl/README.md new file mode 100644 index 000000000..6e1af89f3 --- /dev/null +++ b/js/gritql/npm/linux-x64-musl/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-linux-x64-musl` + +This is the **x86_64-unknown-linux-musl** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/linux-x64-musl/package.json b/js/gritql/npm/linux-x64-musl/package.json new file mode 100644 index 000000000..342103210 --- /dev/null +++ b/js/gritql/npm/linux-x64-musl/package.json @@ -0,0 +1,21 @@ +{ + "name": "@getgrit/gritql-linux-x64-musl", + "version": "0.0.0", + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "main": "gritql.linux-x64-musl.node", + "files": [ + "gritql.linux-x64-musl.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "libc": [ + "musl" + ] +} \ No newline at end of file diff --git a/js/gritql/npm/win32-arm64-msvc/README.md b/js/gritql/npm/win32-arm64-msvc/README.md new file mode 100644 index 000000000..cd8a72aa4 --- /dev/null +++ b/js/gritql/npm/win32-arm64-msvc/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-win32-arm64-msvc` + +This is the **aarch64-pc-windows-msvc** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/win32-arm64-msvc/package.json b/js/gritql/npm/win32-arm64-msvc/package.json new file mode 100644 index 000000000..3700a7ae4 --- /dev/null +++ b/js/gritql/npm/win32-arm64-msvc/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-win32-arm64-msvc", + "version": "0.0.0", + "os": [ + "win32" + ], + "cpu": [ + "arm64" + ], + "main": "gritql.win32-arm64-msvc.node", + "files": [ + "gritql.win32-arm64-msvc.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/win32-ia32-msvc/README.md b/js/gritql/npm/win32-ia32-msvc/README.md new file mode 100644 index 000000000..3dc81208d --- /dev/null +++ b/js/gritql/npm/win32-ia32-msvc/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-win32-ia32-msvc` + +This is the **i686-pc-windows-msvc** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/win32-ia32-msvc/package.json b/js/gritql/npm/win32-ia32-msvc/package.json new file mode 100644 index 000000000..23c6c2fc3 --- /dev/null +++ b/js/gritql/npm/win32-ia32-msvc/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-win32-ia32-msvc", + "version": "0.0.0", + "os": [ + "win32" + ], + "cpu": [ + "ia32" + ], + "main": "gritql.win32-ia32-msvc.node", + "files": [ + "gritql.win32-ia32-msvc.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/npm/win32-x64-msvc/README.md b/js/gritql/npm/win32-x64-msvc/README.md new file mode 100644 index 000000000..838d0c78c --- /dev/null +++ b/js/gritql/npm/win32-x64-msvc/README.md @@ -0,0 +1,3 @@ +# `@getgrit/gritql-win32-x64-msvc` + +This is the **x86_64-pc-windows-msvc** binary for `@getgrit/gritql` diff --git a/js/gritql/npm/win32-x64-msvc/package.json b/js/gritql/npm/win32-x64-msvc/package.json new file mode 100644 index 000000000..5af474b79 --- /dev/null +++ b/js/gritql/npm/win32-x64-msvc/package.json @@ -0,0 +1,18 @@ +{ + "name": "@getgrit/gritql-win32-x64-msvc", + "version": "0.0.0", + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "main": "gritql.win32-x64-msvc.node", + "files": [ + "gritql.win32-x64-msvc.node" + ], + "license": "MIT", + "engines": { + "node": ">= 10" + } +} \ No newline at end of file diff --git a/js/gritql/package.json b/js/gritql/package.json new file mode 100644 index 000000000..dfe16a456 --- /dev/null +++ b/js/gritql/package.json @@ -0,0 +1,24 @@ +{ + "name": "@getgrit/gritql", + "version": "0.0.1", + "main": "__generated__/index.js", + "types": "__generated__/index.d.ts", + "napi": { + "name": "gritql", + "triples": {} + }, + "license": "MIT", + "devDependencies": { + "@napi-rs/cli": "^2.18.4" + }, + "engines": { + "node": ">= 10" + }, + "scripts": { + "build": "RUSTFLAGS=\"-A warnings\" napi build --no-const-enum --platform --release --dts __generated__/index.d.ts --js __generated__/index.js && mv gritql.darwin-arm64.node npm/darwin-arm64/", + "test": "bun test", + "test:ci": "bun test --run", + "example:search": "bun run examples/search.ts", + "example:search:node": "node examples/search.js" + } +} diff --git a/js/gritql/src/binding.rs b/js/gritql/src/binding.rs new file mode 100644 index 000000000..db1d7684e --- /dev/null +++ b/js/gritql/src/binding.rs @@ -0,0 +1,98 @@ +use grit_pattern_matcher::context::ExecContext; +use grit_pattern_matcher::effects::insert_effect; +use grit_pattern_matcher::pattern::PatternOrResolved; +use grit_pattern_matcher::pattern::ResolvedPattern; +use grit_pattern_matcher::pattern::State; +use grit_pattern_matcher::pattern::Variable; + +use marzano_core::{ + marzano_context::MarzanoContext, marzano_resolved_pattern::MarzanoResolvedPattern, + problem::MarzanoQueryContext, +}; +use napi::{Env, Result}; + +#[napi] +pub struct JsResolvedBinding { + pub(crate) inner: &'static MarzanoResolvedPattern<'static>, + pub(crate) context: &'static MarzanoContext<'static>, + pub(crate) state: &'static mut State<'static, MarzanoQueryContext>, +} + +#[napi] +impl JsResolvedBinding { + /// Retrieves the stringified representation of the binding. + /// + /// This method accesses the binding's text representation, using the current state's files + /// and the specified language context. + #[napi] + pub fn text(&self, _env: Env) -> Result { + let stringified_result = self + .inner + .text(&self.state.files, self.context.language()) + .map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?; + let string = stringified_result.to_string(); + Ok(string) + } + + /// Returns the range of the binding. + #[napi] + pub fn range(&self, _env: Env) -> Result> { + let range = self.inner.position(self.context.language()); + Ok(range.map(|e| e.into())) + } + + /// Retrieve a variable's text value. + #[napi] + pub fn var(&self, _env: Env, name: String) -> Result> { + let var = self.state.find_var(&name); + let Some(var) = var else { + return Ok(None); + }; + let resolved = var + .get_pattern_or_resolved(self.state) + .map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?; + let Some(candidate) = resolved else { + return Ok(None); + }; + let PatternOrResolved::Resolved(resolved) = candidate else { + return Err(napi::Error::from_reason("No resolved pattern found")); + }; + let stringified_result = resolved + .text(&self.state.files, self.context.language()) + .map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?; + let string = stringified_result.to_string(); + Ok(Some(string)) + } + + /// Retrieves the absolute file name of the file containing the current binding. + #[napi] + pub fn filename(&self, _env: Env) -> Result { + let var = Variable::file_name(); + let resolved = var + .get_pattern_or_resolved(self.state) + .map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?; + let Some(candidate) = resolved else { + return Err(napi::Error::from_reason("No resolved pattern found")); + }; + let PatternOrResolved::Resolved(resolved) = candidate else { + return Err(napi::Error::from_reason("No resolved pattern found")); + }; + let stringified_result = resolved + .text(&self.state.files, self.context.language()) + .map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?; + let string = stringified_result.to_string(); + Ok(string) + } + + /// Inserts the provided text after the binding. + #[napi] + pub fn insert_after(&mut self, _env: Env, text: String) -> Result<()> { + let left = PatternOrResolved::Resolved::(self.inner); + let replacement = ResolvedPattern::from_string(text); + + insert_effect(&left, replacement, self.state, self.context) + .map_err(|e| napi::Error::from_reason(format!("{:?}", e)))?; + + Ok(()) + } +} diff --git a/js/gritql/src/lib.rs b/js/gritql/src/lib.rs new file mode 100644 index 000000000..39a5231ea --- /dev/null +++ b/js/gritql/src/lib.rs @@ -0,0 +1,9 @@ +#[macro_use] +extern crate napi_derive; + +use napi::bindgen_prelude::*; + +mod binding; +mod search; + +pub use search::QueryBuilder; diff --git a/js/gritql/src/search.rs b/js/gritql/src/search.rs new file mode 100644 index 000000000..ae937eb12 --- /dev/null +++ b/js/gritql/src/search.rs @@ -0,0 +1,522 @@ +use anyhow::bail; +use grit_pattern_matcher::pattern::{DynamicPattern, Pattern, StringConstant}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + sync::mpsc::{self, Receiver}, +}; +use tokio::task::JoinSet; + +use grit_pattern_matcher::pattern::State; + +use marzano_core::{ + api::{is_match, DoneFile, MatchResult}, + marzano_context::MarzanoContext, + marzano_resolved_pattern::MarzanoResolvedPattern, + pattern_compiler::{CompilationResult, CompiledPatternBuilder}, + problem::{MarzanoQueryContext, Problem}, +}; +use marzano_gritmodule::{ + config::{init_config_from_path, init_global_grit_modules}, + fetcher::KeepFetcherKind, + resolver::find_and_resolve_grit_dir, + utils::{extract_path, infer_pattern}, +}; +use napi::{ + bindgen_prelude::*, + threadsafe_function::{ErrorStrategy, ThreadsafeFunction}, + tokio::{self}, +}; + +use marzano_language::{ + grit_parser::MarzanoGritParser, + target_language::{expand_paths, PatternLanguage}, +}; +use marzano_messenger::emit::{FlushableMessenger, Messager}; +use marzano_messenger::testing::TestingMessenger; +use marzano_util::{ + cache::NullCache, + rich_path::RichFile, + runtime::{ExecutionContext, LanguageModelAPI}, +}; + +use crate::binding::JsResolvedBinding; + +type MatchCallback = ThreadsafeFunction; + +#[napi] +#[derive(Clone)] +pub struct QueryBuilder { + base_query_src: String, + filters: Vec, + piped_queries: Vec, + replacement: Option, + insertions: Vec, +} + +#[napi(string_enum)] +#[derive(Default)] +pub enum SearchOutputOptions { + /// Emit absolute paths + #[default] + AbsolutePaths, + /// Emit paths relative to the provided search/root directory + RelativePaths, +} + +#[napi(object)] +pub struct SearchOptions { + pub target_paths: Vec, + pub step_id: String, + /// Skip emitting any results and applying changes + pub silent: Option, + /// The format of the output paths + pub output_format: Option, + /// Verify the query without applying any changes + pub verify_only: Option, +} + +#[napi(object)] +pub struct InputFile { + pub path: String, + pub content: String, +} + +impl From for RichFile { + fn from(file: InputFile) -> Self { + RichFile::new(file.path, file.content) + } +} + +#[napi(object)] +pub struct InputOptions { + pub files: Vec, +} + +#[napi] +impl QueryBuilder { + /// Construct a new query, starting from a GritQL query + #[napi(constructor)] + pub fn new(base_query_src: String) -> Self { + Self { + base_query_src, + filters: Vec::new(), + piped_queries: Vec::new(), + replacement: None, + insertions: Vec::new(), + } + } + + /// Add a callback to filter the results of the search + /// + /// @param filter A callback that that will be called with each match, it should return true to keep the match + #[napi(js_name = "filter")] + pub fn add_filter(&mut self, filter: MatchCallback) { + self.filters.push(filter); + } + + /// Set a replacement rewrite to the query + /// + /// @param replacement The replacement rewrite to apply to the query + #[napi] + pub fn set_replacement(&mut self, replacement: String) { + self.replacement = Some(replacement); + } + + /// Add an insertion to the query + /// + /// @param insertion The insertion to add to the query + #[napi] + pub fn add_insertion(&mut self, insertion: String) { + self.insertions.push(insertion); + } + + /// Pipe the query into another query + /// When you pipe a query, it will stop directly emitting results itself. + /// Instead, every *matching* file will be passed to the next query. + /// @param query The query to pipe into + #[napi] + pub fn pipe(&mut self, second_query: &QueryBuilder) { + self.piped_queries.push(second_query.clone()); + } + + /// Run the query on the provided files + /// @param target_paths A list of paths to search for matches + /// @returns A list of files that matched the query + #[napi] + pub async fn run(&self, options: SearchOptions) -> Result> { + if options.target_paths.is_empty() { + return Ok(vec![]); + } + search_internal(self, options) + .await + .map_err(|e| napi::Error::from_reason(format!("Error: {:?}", e))) + .map(|paths| paths.into_iter().collect()) + } + + /// Apply the query to a single file and return the modified file (if any) + /// @param file The file to apply the query to + /// @returns The modified file (if any) + #[napi] + pub async fn apply_to_file(&self, file: InputFile) -> Result> { + let results = apply_internal(self, InputOptions { files: vec![file] }) + .await + .map_err(|e| napi::Error::from_reason(format!("Error: {:?}", e)))?; + let rewrite = results + .into_iter() + .find(|r| matches!(r, MatchResult::Rewrite(_))); + match rewrite { + Some(MatchResult::Rewrite(rewrite)) => Ok(Some(InputFile { + path: rewrite.rewritten.source_file, + content: rewrite.rewritten.content.unwrap_or_default(), + })), + _ => Ok(None), + } + } +} + +async fn prep_query( + query: QueryBuilder, + target_paths: Vec, +) -> anyhow::Result<(Problem, Vec)> { + let QueryBuilder { + base_query_src, + filters, + piped_queries, + replacement, + insertions, + } = query; + + let grit_files = if target_paths.is_empty() { + let init = init_global_grit_modules::(None).await?; + init.get_grit_files().await? + } else { + let path = PathBuf::from(target_paths.first().unwrap()); + init_config_from_path::(path.clone(), false).await?; + find_and_resolve_grit_dir(Some(path), None).await? + }; + + let (lang, _, pattern_body) = infer_pattern(&base_query_src, &grit_files); + + let pattern_language = match lang { + Some(l) => l, + None => PatternLanguage::default(), + }; + + let target_lang = pattern_language.try_into()?; + + let mut grit_parser = MarzanoGritParser::new()?; + + let libs = grit_files.get_language_directory_or_default(lang)?; + + let injected_builtins = None; + + let mut builder = CompiledPatternBuilder::start( + pattern_body, + &libs, + target_lang, + None, + &mut grit_parser, + injected_builtins, + )?; + + for filter in filters { + builder = builder.matches_callback(Box::new(move |binding, context, state, _logs| { + let runtime = context + .runtime + .handle + .as_ref() + .ok_or(anyhow::anyhow!("Async runtime required"))?; + + /* + THIS IS INHERENTLY UNSAFE + We cannot guarantee that the references live for the lifetime of the JavaScript object + This works for super basic cases around callbacks, but more advanced use cases will require + a more complex solution + */ + + let inner_context: &'static MarzanoContext = unsafe { std::mem::transmute(context) }; + let inner_state: &'static mut State = + unsafe { std::mem::transmute(state) }; + + let inner_binding: &'static MarzanoResolvedPattern = + unsafe { std::mem::transmute(binding) }; + + let js_binding = JsResolvedBinding { + inner: inner_binding, + context: inner_context, + state: inner_state, + }; + + let val = runtime.block_on(async { filter.call_async::(js_binding).await })?; + Ok(val) + })); + } + + for insertion in insertions { + let insertion = Pattern::StringConstant(StringConstant::new(insertion)); + builder = builder.wrap_with_accumulate(insertion); + } + + if let Some(replacement) = replacement { + let replacement = DynamicPattern::from_str_constant(&replacement)?; + builder = builder.wrap_with_rewrite(replacement); + } + + let CompilationResult { + problem, + compilation_warnings, + } = builder.compile(None, None, true)?; + + if !compilation_warnings.is_empty() { + println!("Warnings: {:?}", compilation_warnings); + } + + Ok((problem, piped_queries)) +} + +#[derive(Debug)] +struct PreppedQuery { + pattern: Problem, + children: Vec, +} + +/// Currently only 2 levels of queries are supported +async fn prep_queries( + root: &QueryBuilder, + target_paths: Vec, +) -> anyhow::Result { + let (root_pattern, root_child_queries) = prep_query(root.clone(), target_paths.clone()).await?; + let mut root_query = PreppedQuery { + pattern: root_pattern, + children: Vec::new(), + }; + + for child_query in root_child_queries { + let (pattern, child_queries) = prep_query(child_query, target_paths.clone()).await?; + let child_query = PreppedQuery { + pattern, + children: Vec::new(), + }; + + if !child_queries.is_empty() { + bail!("Nested query pipes are not currently supported"); + } + + root_query.children.push(child_query); + } + + Ok(root_query) +} + +enum EmbeddedMessenger { + Testing(TestingMessenger), +} + +impl EmbeddedMessenger { + fn emit(&mut self, message: &MatchResult) -> anyhow::Result<()> { + match self { + EmbeddedMessenger::Testing(messenger) => messenger.emit(message), + } + } + + async fn flush(&mut self) -> anyhow::Result<()> { + match self { + EmbeddedMessenger::Testing(messenger) => messenger.flush().await, + } + } + + fn emit_estimate(&mut self, total: usize) -> anyhow::Result<()> { + match self { + EmbeddedMessenger::Testing(messenger) => messenger.emit_estimate(total), + } + } +} + +fn get_messenger(root_address: &Path, step_id: String, silent: bool) -> EmbeddedMessenger { + if silent { + return EmbeddedMessenger::Testing(TestingMessenger::new()); + } + + EmbeddedMessenger::Testing(TestingMessenger::new()) +} + +async fn apply_internal( + query: &QueryBuilder, + options: InputOptions, +) -> anyhow::Result> { + let root_query = prep_queries(query, vec![]).await?; + let context = ExecutionContext::default(); + + if !root_query.children.is_empty() { + bail!("Nested query pipes are not currently supported for .apply()"); + } + + let files: Vec = options.files.into_iter().map(|f| f.into()).collect(); + let results = root_query.pattern.execute_files(files, &context); + + Ok(results) +} + +async fn search_internal( + query: &QueryBuilder, + options: SearchOptions, +) -> anyhow::Result> { + let SearchOptions { + target_paths, + step_id, + silent, + output_format, + verify_only, + } = options; + + let silent = silent.unwrap_or(false); + + let root_query = prep_queries(query, target_paths.clone()).await?; + + if verify_only.unwrap_or(false) { + return Ok(HashSet::new()); + } + + let root_path = PathBuf::from(target_paths.first().unwrap()); + + let mut final_handles = JoinSet::new(); + let mut stack: Vec<(PreppedQuery, Option>>)> = Vec::new(); + + // Push the root query onto the stack + stack.push((root_query, None)); + + while let Some((current_query, parent_rx)) = stack.pop() { + let (current_tx, current_rx) = mpsc::channel::>(); + + if !current_query.children.is_empty() { + let child_channels = current_query + .children + .iter() + .map(|_| mpsc::channel::>()) + .collect::>(); + + let (child_txs, child_rxs): (Vec<_>, Vec<_>) = child_channels.into_iter().unzip(); + + for (child_query, child_rx) in current_query.children.into_iter().zip(child_rxs) { + stack.push((child_query, Some(child_rx))); + } + + tokio::spawn(async move { + for matches in current_rx { + for tx in &child_txs { + match tx.send(matches.clone()) { + Ok(_) => {} + Err(e) => { + println!("Error sending to child: {:?}", e.to_string()); + } + } + } + } + }); + } else { + let path = root_path.clone(); + let step_id = step_id.clone(); + let root_path_str = root_path.to_string_lossy().to_string(); + final_handles.spawn(async move { + let mut paths = HashSet::new(); + + let mut messenger = get_messenger(&path, step_id, silent); + + for results in current_rx { + for match_result in results { + if is_match(&match_result) { + if let Some(path) = extract_path(&match_result) { + if matches!(output_format, Some(SearchOutputOptions::RelativePaths)) + { + paths.insert( + path.strip_prefix(&root_path_str) + .map(|p| format!(".{}", p)) + .unwrap_or(path.to_string()), + ); + } else { + paths.insert(path.to_string()); + } + } + } + messenger.emit(&match_result).unwrap(); + } + } + + (paths, messenger) + }); + } + + let context = ExecutionContext::default(); + + if let Some(parent_rx) = parent_rx { + current_query + .pattern + .execute_streaming_relay(parent_rx, &context, current_tx, &NullCache::new()) + .unwrap(); + } else { + let start_paths = target_paths.iter().map(PathBuf::from).collect::>(); + let file_walker = expand_paths( + &start_paths, + Some(&[current_query.pattern.language.to_module_language()]), + ) + .unwrap(); + + let mut language_paths = Vec::new(); + for file in file_walker { + let Ok(file) = file else { + continue; + }; + let Some(file_type) = file.file_type() else { + continue; + }; + if file_type.is_dir() { + continue; + } + if !¤t_query.pattern.language.match_extension( + file.path() + .extension() + .unwrap_or_default() + .to_str() + .unwrap_or_default(), + ) { + let done_file = MatchResult::DoneFile(DoneFile { + relative_file_path: file.path().to_string_lossy().to_string(), + has_results: Some(false), + file_hash: None, + from_cache: false, + }); + if let Err(e) = current_tx.send(vec![done_file]) { + println!("Error sending to parent: {:?}", e.to_string()); + } + continue; + } + let path = file.path(); + language_paths.push(path.to_path_buf()); + } + + let mut messenger = get_messenger(&root_path, step_id.clone(), silent); + messenger.emit_estimate(language_paths.len()).unwrap(); + // If we don't flush, we risk not sending the estimate + messenger.flush().await.unwrap(); + + current_query.pattern.execute_paths_streaming( + language_paths, + &context, + current_tx, + &NullCache::new(), + ); + } + } + + let mut all_paths = HashSet::new(); + + while let Some(res) = final_handles.join_next().await { + let (this_set, mut messenger) = res?; + all_paths.extend(this_set); + + messenger.flush().await?; + } + + Ok(all_paths) +}