Add support for concurrent file processing in ingestion module #5

	name: CI

	on:
	push:
	branches: [ main ]
	pull_request:
	branches: [ main ]

	jobs:
	test:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3

	- name: Set up Rust
	uses: actions-rs/toolchain@v1
	with:
	toolchain: stable
	profile: minimal
	components: rustfmt, clippy
	override: true

	- name: Cache Rust dependencies
	uses: actions/cache@v3
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

	- name: Check formatting
	run: cargo fmt -- --check

	- name: Run clippy
	run: cargo clippy -- -D warnings

	- name: Run tests
	run: cargo test

	- name: Run security audit
	run: \|
	cargo install cargo-audit
	cargo audit

	- name: Set up Node.js
	uses: actions/setup-node@v3
	with:
	node-version: '20'

	- name: Check JSON files
	run: \|
	npm install -g jsonlint prettier
	find . -name ".json" -not -path "./node_modules/" -not -path "./target/*" \| xargs -I{} jsonlint -q {}

	- name: Check Markdown files
	run: \|
	find . -name ".md" -not -path "./node_modules/" \| xargs -I{} prettier --check --parser markdown --print-width 80 --prose-wrap always {}

Provide feedback