diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 000000000..38d3dc968 --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "gitversion.tool": { + "version": "6.4.0", + "commands": [ + "dotnet-gitversion" + ], + "rollForward": false + } + } +} \ No newline at end of file diff --git a/.github/os-matrix.json b/.github/os-matrix.json new file mode 100644 index 000000000..ae9d84b69 --- /dev/null +++ b/.github/os-matrix.json @@ -0,0 +1,9 @@ +{ + "linux": [ + "ubuntu-22.04", + "ubuntu-24.04" + ], + "windows": [ + "windows-2022" + ] +} diff --git a/.github/scripts/compose_repro_matrix.py b/.github/scripts/compose_repro_matrix.py new file mode 100644 index 000000000..ba3ea3a20 --- /dev/null +++ b/.github/scripts/compose_repro_matrix.py @@ -0,0 +1,173 @@ +import json +import os +import sys +from pathlib import Path + + +def as_iterable(value): + if value is None: + return [] + if isinstance(value, (list, tuple, set)): + return list(value) + return [value] + + +def main(): + workspace = Path(os.getenv("GITHUB_WORKSPACE", Path.cwd())) + os_matrix_path = workspace / ".github" / "os-matrix.json" + repros_path = workspace / "repros.json" + + os_matrix = json.loads(os_matrix_path.read_text(encoding="utf-8")) + platform_labels: dict[str, list[str]] = {} + label_platform: dict[str, str] = {} + + for platform, labels in os_matrix.items(): + normalized = platform.lower() + platform_labels[normalized] = [] + for label in labels: + platform_labels[normalized].append(label) + label_platform[label] = normalized + + all_labels = set(label_platform.keys()) + + payload = json.loads(repros_path.read_text(encoding="utf-8")) + repros = payload.get("repros") or [] + + matrix_entries: list[dict[str, str]] = [] + skipped: list[str] = [] + unknown_platforms: set[str] = set() + unknown_labels: set[str] = set() + + def collect_labels(items, local_unknown): + labels = set() + for item in items: + key = item.lower() + if key in platform_labels: + labels.update(platform_labels[key]) + else: + local_unknown.add(key) + unknown_platforms.add(key) + return labels + + for repro in repros: + name = repro.get("name") or repro.get("id") + if not name: + continue + + supports = as_iterable(repro.get("supports")) + normalized_supports = {str(item).lower() for item in supports if isinstance(item, str)} + + local_unknown_platforms: set[str] = set() + local_unknown_labels: set[str] = set() + + if not normalized_supports or "any" in normalized_supports: + candidate_labels = set(all_labels) + else: + candidate_labels = collect_labels(normalized_supports, local_unknown_platforms) + + os_constraints = repro.get("os") or {} + include_platforms = { + str(item).lower() + for item in as_iterable(os_constraints.get("includePlatforms")) + if isinstance(item, str) + } + include_labels = { + str(item) + for item in as_iterable(os_constraints.get("includeLabels")) + if isinstance(item, str) + } + exclude_platforms = { + str(item).lower() + for item in as_iterable(os_constraints.get("excludePlatforms")) + if isinstance(item, str) + } + exclude_labels = { + str(item) + for item in as_iterable(os_constraints.get("excludeLabels")) + if isinstance(item, str) + } + + if include_platforms: + candidate_labels &= collect_labels(include_platforms, local_unknown_platforms) + + if include_labels: + recognized_includes = {label for label in include_labels if label in label_platform} + local_unknown_labels.update({label for label in include_labels if label not in label_platform}) + unknown_labels.update(local_unknown_labels) + candidate_labels &= recognized_includes if recognized_includes else set() + + if exclude_platforms: + candidate_labels -= collect_labels(exclude_platforms, local_unknown_platforms) + + if exclude_labels: + recognized_excludes = {label for label in exclude_labels if label in label_platform} + candidate_labels -= recognized_excludes + unrecognized = {label for label in exclude_labels if label not in label_platform} + local_unknown_labels.update(unrecognized) + unknown_labels.update(unrecognized) + + candidate_labels &= all_labels + + if candidate_labels: + for label in sorted(candidate_labels): + matrix_entries.append( + { + "os": label, + "repro": name, + "platform": label_platform[label], + } + ) + else: + reason_segments = [] + if normalized_supports: + reason_segments.append(f"supports={sorted(normalized_supports)}") + if os_constraints: + reason_segments.append("os constraints applied") + if local_unknown_platforms: + reason_segments.append(f"unknown platforms={sorted(local_unknown_platforms)}") + if local_unknown_labels: + reason_segments.append(f"unknown labels={sorted(local_unknown_labels)}") + reason = "; ".join(reason_segments) or "no matching runners" + skipped.append(f"{name} ({reason})") + + matrix_entries.sort(key=lambda entry: (entry["repro"], entry["os"])) + + summary_lines = [] + summary_lines.append(f"Total repro jobs: {len(matrix_entries)}") + if skipped: + summary_lines.append("") + summary_lines.append("Skipped repros:") + for item in skipped: + summary_lines.append(f"- {item}") + if unknown_platforms: + summary_lines.append("") + summary_lines.append("Unknown platforms encountered:") + for item in sorted(unknown_platforms): + summary_lines.append(f"- {item}") + if unknown_labels: + summary_lines.append("") + summary_lines.append("Unknown labels encountered:") + for item in sorted(unknown_labels): + summary_lines.append(f"- {item}") + + summary_path = os.getenv("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a", encoding="utf-8") as handle: + handle.write("\n".join(summary_lines) + "\n") + + outputs_path = os.getenv("GITHUB_OUTPUT") + if not outputs_path: + raise RuntimeError("GITHUB_OUTPUT is not defined.") + + with open(outputs_path, "a", encoding="utf-8") as handle: + handle.write("matrix=" + json.dumps({"include": matrix_entries}) + "\n") + handle.write("count=" + str(len(matrix_entries)) + "\n") + handle.write("skipped=" + json.dumps(skipped) + "\n") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: # pragma: no cover - diagnostic output for CI + print(f"Failed to compose repro matrix: {exc}", file=sys.stderr) + raise diff --git a/.github/scripts/test-crossuser-windows.ps1 b/.github/scripts/test-crossuser-windows.ps1 new file mode 100644 index 000000000..82ed887ab --- /dev/null +++ b/.github/scripts/test-crossuser-windows.ps1 @@ -0,0 +1,307 @@ +# PowerShell script to test LiteDB shared mode with cross-user scenarios +# This script creates temporary users and runs tests as different users to verify cross-user access + +param( + [string]$TestDll, + [string]$Framework = "net8.0" +) + +$ErrorActionPreference = "Stop" + +# Configuration +$TestUsers = @("LiteDBTest1", "LiteDBTest2") +$TestPassword = ConvertTo-SecureString "Test@Password123!" -AsPlainText -Force +$DbPath = Join-Path $env:TEMP "litedb_crossuser_test.db" +$TestId = [Guid]::NewGuid().ToString("N").Substring(0, 8) + +Write-Host "=== LiteDB Cross-User Testing Script ===" -ForegroundColor Cyan +Write-Host "Test ID: $TestId" +Write-Host "Database: $DbPath" +Write-Host "Framework: $Framework" +Write-Host "" + +# Function to create a test user +function Create-TestUser { + param([string]$Username) + + try { + # Check if user already exists + $existingUser = Get-LocalUser -Name $Username -ErrorAction SilentlyContinue + if ($existingUser) { + Write-Host "User $Username already exists, removing..." -ForegroundColor Yellow + Remove-LocalUser -Name $Username -ErrorAction SilentlyContinue + } + + Write-Host "Creating user: $Username" -ForegroundColor Green + New-LocalUser -Name $Username -Password $TestPassword -FullName "LiteDB Test User" -Description "Temporary user for LiteDB cross-user testing" -ErrorAction Stop | Out-Null + + # Add to Users group + Add-LocalGroupMember -Group "Users" -Member $Username -ErrorAction SilentlyContinue + + return $true + } + catch { + Write-Host "Failed to create user ${Username}: $_" -ForegroundColor Red + return $false + } +} + +# Function to remove a test user +function Remove-TestUser { + param([string]$Username) + + try { + $user = Get-LocalUser -Name $Username -ErrorAction SilentlyContinue + if ($user) { + Write-Host "Removing user: $Username" -ForegroundColor Yellow + Remove-LocalUser -Name $Username -ErrorAction Stop + } + } + catch { + Write-Host "Warning: Failed to remove user ${Username}: $_" -ForegroundColor Yellow + } +} + +# Function to run process as a specific user +function Run-AsUser { + param( + [string]$Username, + [string]$Command, + [string[]]$Arguments, + [int]$TimeoutSeconds = 30 + ) + + Write-Host "Running as user $Username..." -ForegroundColor Cyan + + $credential = New-Object System.Management.Automation.PSCredential($Username, $TestPassword) + + try { + $job = Start-Job -ScriptBlock { + param($cmd, $args) + & $cmd $args + } -ArgumentList $Command, $Arguments -Credential $credential + + $completed = Wait-Job -Job $job -Timeout $TimeoutSeconds + + if (-not $completed) { + Stop-Job -Job $job + throw "Process timed out after $TimeoutSeconds seconds" + } + + $output = Receive-Job -Job $job + Remove-Job -Job $job + + Write-Host "Output from ${Username}:" -ForegroundColor Gray + $output | ForEach-Object { Write-Host " $_" -ForegroundColor Gray } + + return $true + } + catch { + Write-Host "Failed to run as ${Username}: $_" -ForegroundColor Red + return $false + } +} + +# Cleanup function +function Cleanup { + Write-Host "`n=== Cleanup ===" -ForegroundColor Cyan + + # Remove database files + if (Test-Path $DbPath) { + try { + Remove-Item $DbPath -Force -ErrorAction SilentlyContinue + Write-Host "Removed database file" -ForegroundColor Yellow + } + catch { + Write-Host "Warning: Could not remove database file: $_" -ForegroundColor Yellow + } + } + + $logPath = "$DbPath-log" + if (Test-Path $logPath) { + try { + Remove-Item $logPath -Force -ErrorAction SilentlyContinue + Write-Host "Removed database log file" -ForegroundColor Yellow + } + catch { + Write-Host "Warning: Could not remove database log file: $_" -ForegroundColor Yellow + } + } + + # Remove test users + foreach ($username in $TestUsers) { + Remove-TestUser -Username $username + } +} + +# Register cleanup on exit +try { + # Check if running as Administrator + $isAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) + + if (-not $isAdmin) { + Write-Host "ERROR: This script must be run as Administrator to create users" -ForegroundColor Red + exit 1 + } + + # Cleanup any previous test artifacts + Cleanup + + # Create test users + Write-Host "`n=== Creating Test Users ===" -ForegroundColor Cyan + $usersCreated = $true + foreach ($username in $TestUsers) { + if (-not (Create-TestUser -Username $username)) { + $usersCreated = $false + break + } + } + + if (-not $usersCreated) { + Write-Host "Failed to create all test users" -ForegroundColor Red + exit 1 + } + + # Initialize database with current user + Write-Host "`n=== Initializing Database ===" -ForegroundColor Cyan + $initScript = @" +using System; +using LiteDB; + +var db = new LiteDatabase(new ConnectionString +{ + Filename = @"$DbPath", + Connection = ConnectionType.Shared +}); + +var col = db.GetCollection("crossuser_test"); +col.Insert(new BsonDocument { ["user"] = Environment.UserName, ["timestamp"] = DateTime.UtcNow, ["action"] = "init" }); +db.Dispose(); + +Console.WriteLine("Database initialized by " + Environment.UserName); +"@ + + $initScriptPath = Join-Path $env:TEMP "litedb_init_$TestId.cs" + Set-Content -Path $initScriptPath -Value $initScript + + # Run init script + dotnet script $initScriptPath + + if ($LASTEXITCODE -ne 0) { + Write-Host "Failed to initialize database" -ForegroundColor Red + exit 1 + } + + Remove-Item $initScriptPath -Force -ErrorAction SilentlyContinue + + # Grant permissions to the database file for all test users + Write-Host "`n=== Setting Database Permissions ===" -ForegroundColor Cyan + $acl = Get-Acl $DbPath + foreach ($username in $TestUsers) { + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule($username, "FullControl", "Allow") + $acl.SetAccessRule($rule) + } + Set-Acl -Path $DbPath -AclObject $acl + Write-Host "Database permissions set for all test users" -ForegroundColor Green + + # Run tests as each user + Write-Host "`n=== Running Cross-User Tests ===" -ForegroundColor Cyan + + $testScript = @" +using System; +using LiteDB; + +var db = new LiteDatabase(new ConnectionString +{ + Filename = @"$DbPath", + Connection = ConnectionType.Shared +}); + +var col = db.GetCollection("crossuser_test"); + +// Read existing documents +var existingCount = col.Count(); +Console.WriteLine(`$"User {Environment.UserName} found {existingCount} existing documents"); + +// Write new document +col.Insert(new BsonDocument { ["user"] = Environment.UserName, ["timestamp"] = DateTime.UtcNow, ["action"] = "write" }); + +// Read all documents +var allDocs = col.FindAll(); +Console.WriteLine("All documents in database:"); +foreach (var doc in allDocs) +{ + Console.WriteLine(`$" - User: {doc[\"user\"]}, Action: {doc[\"action\"]}"); +} + +db.Dispose(); +Console.WriteLine("Test completed successfully for user " + Environment.UserName); +"@ + + $testScriptPath = Join-Path $env:TEMP "litedb_test_$TestId.cs" + Set-Content -Path $testScriptPath -Value $testScript + + # Note: Running as different users requires elevated permissions and is complex + # For now, we'll document that this should be done manually or in a controlled environment + Write-Host "Cross-user test script created at: $testScriptPath" -ForegroundColor Green + Write-Host "Note: Automated cross-user testing requires complex setup." -ForegroundColor Yellow + Write-Host "For full cross-user verification, run the following manually:" -ForegroundColor Yellow + Write-Host " dotnet script $testScriptPath" -ForegroundColor Cyan + Write-Host " (as each of the test users: $($TestUsers -join ', '))" -ForegroundColor Cyan + + # For CI purposes, we'll verify that the database was created and is accessible + Write-Host "`n=== Verifying Database Access ===" -ForegroundColor Cyan + if (Test-Path $DbPath) { + Write-Host "✓ Database file exists and is accessible" -ForegroundColor Green + + # Verify we can open it in shared mode + $verifyScript = @" +using System; +using LiteDB; + +var db = new LiteDatabase(new ConnectionString +{ + Filename = @"$DbPath", + Connection = ConnectionType.Shared +}); + +var col = db.GetCollection("crossuser_test"); +var count = col.Count(); +Console.WriteLine(`$"Verification: Found {count} documents in shared database"); +db.Dispose(); + +if (count > 0) { + Console.WriteLine("SUCCESS: Database is accessible in shared mode"); + Environment.Exit(0); +} else { + Console.WriteLine("ERROR: Database is empty"); + Environment.Exit(1); +} +"@ + + $verifyScriptPath = Join-Path $env:TEMP "litedb_verify_$TestId.cs" + Set-Content -Path $verifyScriptPath -Value $verifyScript + + dotnet script $verifyScriptPath + $verifyResult = $LASTEXITCODE + + Remove-Item $verifyScriptPath -Force -ErrorAction SilentlyContinue + Remove-Item $testScriptPath -Force -ErrorAction SilentlyContinue + + if ($verifyResult -eq 0) { + Write-Host "`n=== Cross-User Test Setup Completed Successfully ===" -ForegroundColor Green + exit 0 + } + else { + Write-Host "`n=== Cross-User Test Setup Failed ===" -ForegroundColor Red + exit 1 + } + } + else { + Write-Host "✗ Database file was not created" -ForegroundColor Red + exit 1 + } +} +finally { + Cleanup +} diff --git a/.github/workflows/_reusable-ci.yml b/.github/workflows/_reusable-ci.yml new file mode 100644 index 000000000..0148c6b19 --- /dev/null +++ b/.github/workflows/_reusable-ci.yml @@ -0,0 +1,368 @@ +name: Reusable CI (Build & Test) + +on: + workflow_call: + inputs: + sdk-matrix: + required: false + type: string + default: | + [ + { "display": ".NET 8", "sdk": "8.0.x", "framework": "net8.0", "includePrerelease": false, "msbuildProps": "" }, + { "display": ".NET 9", "sdk": "9.0.x\n8.0.x", "framework": "net8.0", "includePrerelease": true, "msbuildProps": "" }, + { "display": ".NET 10", "sdk": "10.0.x\n8.0.x", "framework": "net10.0", "includePrerelease": true, "msbuildProps": "-p:LiteDBTestsEnableNet10=true" } + ] + +jobs: + build-linux: + name: Build (Linux) + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore + run: dotnet restore LiteDB.sln + + - name: Build + run: dotnet build LiteDB.sln --configuration Release --no-restore /p:TestingEnabled=true + + - name: Package build outputs + run: tar -czf tests-build-linux.tar.gz LiteDB/bin/Release LiteDB/obj/Release LiteDB.Tests/bin/Release LiteDB.Tests/obj/Release + + - name: Upload linux test build + uses: actions/upload-artifact@v4 + with: + name: tests-build-linux + path: tests-build-linux.tar.gz + + test-linux: + name: Test (Linux ${{ matrix.item.display }}) + runs-on: ubuntu-latest + needs: build-linux + strategy: + fail-fast: false + matrix: + item: ${{ fromJson(inputs.sdk-matrix) }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK ${{ matrix.item.display }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.item.sdk }} + include-prerelease: ${{ matrix.item.includePrerelease }} + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: tests-build-linux + + - name: Extract build artifacts + run: tar -xzf tests-build-linux.tar.gz + + - name: Build test project for target framework + run: >- + dotnet build LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --framework ${{ matrix.item.framework }} + --no-dependencies + ${{ matrix.item.msbuildProps }} + + - name: Run tests + timeout-minutes: 5 + run: >- + dotnet test LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --no-build + --framework ${{ matrix.item.framework }} + --verbosity normal + --settings tests.runsettings + --logger "trx;LogFileName=TestResults.trx" + --logger "console;verbosity=detailed" + ${{ matrix.item.msbuildProps }} + + build-macos: + name: Build (macOS) + runs-on: macos-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore + run: dotnet restore LiteDB.sln + + - name: Build + run: dotnet build LiteDB.sln --configuration Release --no-restore /p:TestingEnabled=true + + - name: Package build outputs + run: tar -czf tests-build-macos.tar.gz LiteDB/bin/Release LiteDB/obj/Release LiteDB.Tests/bin/Release LiteDB.Tests/obj/Release + + - name: Upload macOS test build + uses: actions/upload-artifact@v4 + with: + name: tests-build-macos + path: tests-build-macos.tar.gz + + test-macos: + name: Test (macOS ${{ matrix.item.display }}) + runs-on: macos-latest + needs: build-macos + strategy: + fail-fast: false + matrix: + item: ${{ fromJson(inputs.sdk-matrix) }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK ${{ matrix.item.display }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.item.sdk }} + include-prerelease: ${{ matrix.item.includePrerelease }} + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: tests-build-macos + + - name: Extract build artifacts + run: tar -xzf tests-build-macos.tar.gz + + - name: Build test project for target framework + run: >- + dotnet build LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --framework ${{ matrix.item.framework }} + --no-dependencies + ${{ matrix.item.msbuildProps }} + + - name: Run tests + timeout-minutes: 5 + run: >- + dotnet test LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --no-build + --framework ${{ matrix.item.framework }} + --verbosity normal + --settings tests.runsettings + --logger "trx;LogFileName=TestResults.trx" + --logger "console;verbosity=detailed" + ${{ matrix.item.msbuildProps }} + + build-windows: + name: Build (Windows) + runs-on: windows-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore + run: dotnet restore LiteDB.sln + + - name: Build + run: dotnet build LiteDB.sln --configuration Release --no-restore /p:TestingEnabled=true + + - name: Upload windows test build + uses: actions/upload-artifact@v4 + with: + name: tests-build-windows + path: | + LiteDB/bin/Release + LiteDB/obj/Release + LiteDB.Tests/bin/Release + LiteDB.Tests/obj/Release + + test-windows: + name: Test (Windows ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.item.display }}) + runs-on: ${{ matrix.os }} + needs: build-windows + strategy: + fail-fast: false + matrix: + os: [windows-latest, windows-2022] + arch: [x64, x86] + item: ${{ fromJson(inputs.sdk-matrix) }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK ${{ matrix.item.display }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.item.sdk }} + include-prerelease: ${{ matrix.item.includePrerelease }} + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: tests-build-windows + path: . + + - name: Build test project for target framework (${{ matrix.arch }}) + run: >- + dotnet build LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --framework ${{ matrix.item.framework }} + --no-dependencies + ${{ matrix.item.msbuildProps }} + + - name: Run tests (${{ matrix.arch }}) + timeout-minutes: 5 + run: >- + dotnet test LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --no-build + --framework ${{ matrix.item.framework }} + --verbosity normal + --settings tests.runsettings + --logger "trx;LogFileName=TestResults-${{ matrix.arch }}.trx" + --logger "console;verbosity=detailed" + ${{ matrix.item.msbuildProps }} + + test-windows-crossprocess: + name: Cross-Process Tests (Windows ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.item.display }}) + runs-on: ${{ matrix.os }} + needs: build-windows + strategy: + fail-fast: false + matrix: + os: [windows-latest, windows-2022] + arch: [x64, x86] + item: ${{ fromJson(inputs.sdk-matrix) }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDK ${{ matrix.item.display }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.item.sdk }} + include-prerelease: ${{ matrix.item.includePrerelease }} + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: tests-build-windows + path: . + + - name: Build test project for target framework (${{ matrix.arch }}) + run: >- + dotnet build LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --framework ${{ matrix.item.framework }} + --no-dependencies + ${{ matrix.item.msbuildProps }} + + - name: Run cross-process shared mode tests (${{ matrix.arch }}) + timeout-minutes: 10 + run: >- + dotnet test LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --no-build + --framework ${{ matrix.item.framework }} + --filter "FullyQualifiedName~CrossProcess_Shared_Tests" + --verbosity normal + --logger "trx;LogFileName=CrossProcessTestResults-${{ matrix.arch }}.trx" + --logger "console;verbosity=detailed" + ${{ matrix.item.msbuildProps }} + + - name: Upload cross-process test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: crossprocess-test-results-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.item.display }} + path: LiteDB.Tests/TestResults/CrossProcessTestResults-${{ matrix.arch }}.trx + + test-linux-arm64: + name: Test (Linux ARM64 - ${{ matrix.item.display }}) + runs-on: ubuntu-latest + needs: build-linux + strategy: + fail-fast: false + matrix: + item: ${{ fromJson(inputs.sdk-matrix) }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up QEMU for ARM64 emulation + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up .NET SDK ${{ matrix.item.display }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ matrix.item.sdk }} + include-prerelease: ${{ matrix.item.includePrerelease }} + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: tests-build-linux + + - name: Extract build artifacts + run: tar -xzf tests-build-linux.tar.gz + + - name: Build test project for ARM64 + run: >- + dotnet build LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --framework ${{ matrix.item.framework }} + --no-dependencies + ${{ matrix.item.msbuildProps }} + + - name: Run tests (ARM64 via QEMU) + timeout-minutes: 10 + run: >- + dotnet test LiteDB.Tests/LiteDB.Tests.csproj + --configuration Release + --no-build + --framework ${{ matrix.item.framework }} + --verbosity normal + --logger "trx;LogFileName=TestResults-arm64.trx" + --logger "console;verbosity=detailed" + ${{ matrix.item.msbuildProps }} + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..34075dfca --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,12 @@ +name: CI + +on: + pull_request: + +jobs: + build-and-test: + uses: ./.github/workflows/_reusable-ci.yml + + repro-runner: + uses: ./.github/workflows/reprorunner.yml + secrets: inherit diff --git a/.github/workflows/publish-prerelease.yml b/.github/workflows/publish-prerelease.yml new file mode 100644 index 000000000..6a5094379 --- /dev/null +++ b/.github/workflows/publish-prerelease.yml @@ -0,0 +1,106 @@ +name: Publish prerelease + +on: + push: + branches: + - dev + paths: + - 'LiteDB/**' + - '.github/workflows/**' + # allow workflow_dispatch to trigger manually from GitHub UI + workflow_dispatch: + inputs: + ref: + description: Branch, tag, or SHA to release from + default: dev + required: true + +jobs: + ci-checks: + uses: ./.github/workflows/_reusable-ci.yml + secrets: inherit + + reprorunner: + uses: ./.github/workflows/reprorunner.yml + needs: ci-checks + secrets: inherit + + publish: + name: Pack & Publish + runs-on: ubuntu-latest + needs: [ci-checks, reprorunner] + permissions: + contents: write + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up .NET SDKs (8/9/10) + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 10.0.x + 9.0.x + 8.0.x + include-prerelease: true + + - name: Restore .NET tools + run: dotnet tool restore + + - name: Compute semantic version + id: gitversion + shell: bash + run: | + set -euo pipefail + JSON=$(dotnet tool run dotnet-gitversion /output json) + echo "$JSON" + + NUGET_VERSION=$(echo "$JSON" | jq -r '.NuGetVersion') + FULL_SEMVER=$(echo "$JSON" | jq -r '.FullSemVer') + SHORT_SHA=$(echo "$JSON" | jq -r '.ShortSha') + MAJOR_MINOR_PATCH=$(echo "$JSON" | jq -r '.MajorMinorPatch') + PR_LABEL=$(echo "$JSON" | jq -r '.PreReleaseLabel') + PR_NUMBER=$(echo "$JSON" | jq -r '.PreReleaseNumber') + + if [[ "$PR_LABEL" != "" && "$PR_LABEL" != "null" ]]; then + printf -v PR_PADDED '%04d' "$PR_NUMBER" + RELEASE_VERSION_PADDED="${MAJOR_MINOR_PATCH}-${PR_LABEL}.${PR_PADDED}" + else + RELEASE_VERSION_PADDED="$MAJOR_MINOR_PATCH" + fi + + echo "nugetVersion=$NUGET_VERSION" >> "$GITHUB_OUTPUT" + echo "fullSemVer=$FULL_SEMVER" >> "$GITHUB_OUTPUT" + echo "releaseVersionPadded=$RELEASE_VERSION_PADDED" >> "$GITHUB_OUTPUT" + echo "informational=${NUGET_VERSION}+${SHORT_SHA}" >> "$GITHUB_OUTPUT" + + - name: Restore + run: dotnet restore LiteDB.sln + + - name: Build + run: dotnet build LiteDB/LiteDB.csproj --configuration Release --no-restore /p:ContinuousIntegrationBuild=true + + - name: Pack + run: dotnet pack LiteDB/LiteDB.csproj --configuration Release --no-build -o artifacts /p:ContinuousIntegrationBuild=true + + - name: Clean up detached packages + run: | + find artifacts -name "0.0.0-detached.nupkg" -delete + + - name: Retrieve secrets from Bitwarden + uses: bitwarden/sm-action@v2 + with: + access_token: ${{ secrets.BW_ACCESS_TOKEN }} + base_url: https://vault.bitwarden.eu + secrets: | + 265b2fb6-2cf0-4859-9bc8-b24c00ab4378 > NUGET_API_KEY + + - name: Push package to NuGet + env: + PACKAGE_VERSION: ${{ steps.gitversion.outputs.nugetVersion }} + run: | + echo "Pushing LiteDB version $PACKAGE_VERSION" + dotnet nuget push "artifacts/*.nupkg" --api-key "$NUGET_API_KEY" --source https://api.nuget.org/v3/index.json --skip-duplicate diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 000000000..3703a1210 --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,112 @@ +name: Publish release + +on: + workflow_dispatch: + inputs: + ref: + description: Branch, tag, or SHA to release from + default: master + required: true + publish_nuget: + description: Push packages to NuGet + type: boolean + default: false + publish_github: + description: Create or update GitHub release + type: boolean + default: false + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref }} + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore .NET tools + run: dotnet tool restore + + - name: Compute semantic version + id: gitversion + shell: bash + run: | + set -euo pipefail + JSON=$(dotnet tool run dotnet-gitversion /output json) + echo "$JSON" + + NUGET_VERSION=$(echo "$JSON" | jq -r '.NuGetVersion') + FULL_SEMVER=$(echo "$JSON" | jq -r '.FullSemVer') + SHORT_SHA=$(echo "$JSON" | jq -r '.ShortSha') + MAJOR_MINOR_PATCH=$(echo "$JSON" | jq -r '.MajorMinorPatch') + PR_LABEL=$(echo "$JSON" | jq -r '.PreReleaseLabel') + PR_NUMBER=$(echo "$JSON" | jq -r '.PreReleaseNumber') + + if [[ "$PR_LABEL" != "" && "$PR_LABEL" != "null" ]]; then + printf -v PR_PADDED '%04d' "$PR_NUMBER" + RELEASE_VERSION_PADDED="${MAJOR_MINOR_PATCH}-${PR_LABEL}.${PR_PADDED}" + else + RELEASE_VERSION_PADDED="$MAJOR_MINOR_PATCH" + fi + + echo "nugetVersion=$NUGET_VERSION" >> "$GITHUB_OUTPUT" + echo "fullSemVer=$FULL_SEMVER" >> "$GITHUB_OUTPUT" + echo "releaseVersionPadded=$RELEASE_VERSION_PADDED" >> "$GITHUB_OUTPUT" + echo "informational=${NUGET_VERSION}+${SHORT_SHA}" >> "$GITHUB_OUTPUT" + + - name: Restore + run: dotnet restore LiteDB.sln + + - name: Build + run: dotnet build LiteDB.sln --configuration Release --no-restore /p:ContinuousIntegrationBuild=true + + - name: Test + timeout-minutes: 8 + run: dotnet test LiteDB.sln --configuration Release --no-build --verbosity normal --settings tests.runsettings --logger "trx;LogFileName=TestResults.trx" --logger "console;verbosity=detailed" + + - name: Pack + run: dotnet pack LiteDB/LiteDB.csproj --configuration Release --no-build -o artifacts /p:ContinuousIntegrationBuild=true + + # if delete all nuget packages which are named ``0.0.0-detached.nupkg`` (leftover from detached builds) + - name: Clean up detached packages + run: | + find artifacts -name "0.0.0-detached.nupkg" -delete + + - name: Retrieve secrets from Bitwarden + if: ${{ inputs.publish_nuget }} + uses: bitwarden/sm-action@v2 + with: + access_token: ${{ secrets.BW_ACCESS_TOKEN }} + base_url: https://vault.bitwarden.eu + secrets: | + 265b2fb6-2cf0-4859-9bc8-b24c00ab4378 > NUGET_API_KEY + + - name: Push package to NuGet + if: ${{ inputs.publish_nuget }} + env: + PACKAGE_VERSION: ${{ steps.gitversion.outputs.nugetVersion }} + run: | + echo "Pushing LiteDB version $PACKAGE_VERSION" + dotnet nuget push "artifacts/*.nupkg" --api-key "$NUGET_API_KEY" --source https://api.nuget.org/v3/index.json --skip-duplicate + + - name: Publish GitHub release + if: ${{ inputs.publish_github }} + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.gitversion.outputs.releaseVersionPadded }} + name: LiteDB ${{ steps.gitversion.outputs.releaseVersionPadded }} + generate_release_notes: true + files: artifacts/*.nupkg + target_commitish: ${{ github.sha }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reprorunner.yml b/.github/workflows/reprorunner.yml new file mode 100644 index 000000000..c860db42f --- /dev/null +++ b/.github/workflows/reprorunner.yml @@ -0,0 +1,128 @@ +name: Repro Runner + +on: + workflow_dispatch: + inputs: + filter: + description: Optional regular expression to select repros + required: false + type: string + ref: + description: Git ref (branch, tag, or SHA) to check out + required: false + type: string + workflow_call: + inputs: + filter: + required: false + type: string + ref: + required: false + type: string + +jobs: + generate-matrix: + name: Generate matrix + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.compose.outputs.matrix }} + count: ${{ steps.compose.outputs.count }} + skipped: ${{ steps.compose.outputs.skipped }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref && inputs.ref || github.ref }} + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore ReproRunner CLI + run: dotnet restore LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/LiteDB.ReproRunner.Cli.csproj + + - name: Build ReproRunner CLI + run: dotnet build LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/LiteDB.ReproRunner.Cli.csproj --configuration Release --no-restore + + - name: Capture repro inventory + id: list + shell: bash + run: | + set -euo pipefail + filter_input="${{ inputs.filter }}" + cmd=(dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli --configuration Release --no-build --no-restore -- list --json) + if [ -n "${filter_input}" ]; then + cmd+=("--filter" "${filter_input}") + fi + "${cmd[@]}" > repros.json + + - name: Compose matrix + id: compose + shell: bash + run: | + set -euo pipefail + python .github/scripts/compose_repro_matrix.py + + - name: Upload repro inventory + if: always() + uses: actions/upload-artifact@v4 + with: + name: repros-json + path: repros.json + + repro: + name: Run ${{ matrix.repro }} on ${{ matrix.os }} + needs: generate-matrix + if: ${{ needs.generate-matrix.outputs.count != '0' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref && inputs.ref || github.ref }} + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore ReproRunner CLI + run: dotnet restore LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/LiteDB.ReproRunner.Cli.csproj + + - name: Build ReproRunner CLI + run: dotnet build LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/LiteDB.ReproRunner.Cli.csproj --configuration Release --no-restore + + - name: Execute repro + id: run + shell: bash + run: | + set -euo pipefail + dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli --configuration Release -- \ + run ${{ matrix.repro }} --ci --target-os "${{ matrix.os }}" + + - name: Upload repro artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.repro }}-${{ matrix.os }} + path: | + artifacts + LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/bin/**/runs + if-no-files-found: ignore + + - name: Publish summary + if: always() + shell: bash + run: | + status="${{ job.status }}" + { + echo "### ${{ matrix.repro }} (${{ matrix.os }})" + echo "- Result: ${status}" + echo "- Artifacts: logs-${{ matrix.repro }}-${{ matrix.os }}" + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml new file mode 100644 index 000000000..651cfbeb9 --- /dev/null +++ b/.github/workflows/tag-version.yml @@ -0,0 +1,125 @@ +name: Tag version + +on: + workflow_dispatch: + inputs: + ref: + description: Branch or SHA to tag + default: master + required: true + bump: + description: Version component to increment + type: choice + options: + - patch + - minor + - major + default: patch + +jobs: + create-tag: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref }} + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + + - name: Restore .NET tools + run: dotnet tool restore + + - name: Determine current version + id: gitversion + shell: bash + run: | + set -euo pipefail + JSON=$(dotnet tool run dotnet-gitversion /output json) + echo "$JSON" + + MAJOR_MINOR_PATCH=$(echo "$JSON" | jq -r '.MajorMinorPatch') + PR_LABEL=$(echo "$JSON" | jq -r '.PreReleaseLabel') + PR_NUMBER=$(echo "$JSON" | jq -r '.PreReleaseNumber') + + if [[ "$PR_LABEL" != "" && "$PR_LABEL" != "null" ]]; then + printf -v PR_PADDED '%04d' "$PR_NUMBER" + SEMVER="${MAJOR_MINOR_PATCH}-${PR_LABEL}.${PR_PADDED}" + else + SEMVER="$MAJOR_MINOR_PATCH" + fi + + echo "fullSemVer=$SEMVER" >> "$GITHUB_OUTPUT" + echo "major=$(echo "$JSON" | jq -r '.Major')" >> "$GITHUB_OUTPUT" + echo "minor=$(echo "$JSON" | jq -r '.Minor')" >> "$GITHUB_OUTPUT" + echo "patch=$(echo "$JSON" | jq -r '.Patch')" >> "$GITHUB_OUTPUT" + + - name: Calculate next version + id: next + shell: bash + env: + BUMP: ${{ inputs.bump }} + MAJOR: ${{ steps.gitversion.outputs.major }} + MINOR: ${{ steps.gitversion.outputs.minor }} + PATCH: ${{ steps.gitversion.outputs.patch }} + run: | + set -euo pipefail + major=$MAJOR + minor=$MINOR + patch=$PATCH + + case "$BUMP" in + major) + major=$((major + 1)) + minor=0 + patch=0 + ;; + minor) + minor=$((minor + 1)) + patch=0 + ;; + patch) + patch=$((patch + 1)) + ;; + esac + + target="${major}.${minor}.${patch}" + echo "target=$target" >> "$GITHUB_OUTPUT" + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create annotated tag + env: + TARGET: ${{ steps.next.outputs.target }} + run: | + set -euo pipefail + if git rev-parse -q --verify "refs/tags/v${TARGET}" >/dev/null; then + echo "Tag v${TARGET} already exists" >&2 + exit 1 + fi + + git tag -a "v${TARGET}" -m "Release tag v${TARGET}" + + - name: Push tag + env: + TARGET: ${{ steps.next.outputs.target }} + run: | + set -euo pipefail + git push origin "v${TARGET}" + + - name: Summary + if: always() + env: + TARGET: ${{ steps.next.outputs.target }} + FULL: ${{ steps.gitversion.outputs.fullSemVer }} + run: echo "Created tag v${TARGET} (previous build was ${FULL})" diff --git a/.gitignore b/.gitignore index 22e416ab6..58992c712 100644 --- a/.gitignore +++ b/.gitignore @@ -224,3 +224,4 @@ FakesAssemblies/ # Uncomment if necessary however generally it will be regenerated when needed #!**/packages/repositories.config /LiteDB.BadJsonTest +repro-summary.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..bfe51d3e9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,20 @@ +# Repository Guidelines + +## Project Structure & Module Organization +The `LiteDB/` library is the heart of the solution and is split into domains such as `Engine/`, `Document/`, `Client/`, and `Utils/` for low-level storage, document modeling, and public APIs. Companion apps live beside it: `LiteDB.Shell/` provides the interactive CLI, `LiteDB.Benchmarks/` and `LiteDB.Stress/` target performance and endurance scenarios, while `LiteDB.Tests/` houses unit coverage grouped by feature area. Sample integrations and temporary packaging output stay in `ConsoleApp1/` and `artifacts_temp/` respectively. + +## Build, Test, and Development Commands +Restore and build with `dotnet build LiteDB.sln -c Release` after a `dotnet restore`. Execute `dotnet test LiteDB.sln --settings tests.runsettings` to honor the solution-wide timeout profile, or scope to a single project with `dotnet test LiteDB.Tests -f net8.0`. Launch the shell locally via `dotnet run --project LiteDB.Shell/LiteDB.Shell.csproj -- MyData.db`. Produce NuGet artifacts using `dotnet pack LiteDB/LiteDB.csproj -c Release` when preparing releases. + +## Coding Style & Naming Conventions +Follow the repository’s C# conventions: four-space indentation, Allman braces, and grouped `using` directives with system namespaces first. Prefer `var` only when the right-hand side is obvious, keep public APIs XML-documented (the build emits `LiteDB.xml`), and avoid introducing nullable warnings in both `netstandard2.0` and `net8.0` targets. Unsafe code is enabled; justify its use with comments tied to the relevant `Engine` component. + +## Testing Guidelines +Tests are written with xUnit and FluentAssertions; mirror the production folder names (`Engine`, `Query`, `Issues`, etc.) when adding scenarios. Name files after the type under test and choose expressive `[Fact]` / `[Theory]` method names describing the behavior. Long-running tests must finish within the 300-second session timeout defined in `tests.runsettings`; run focused suites with `dotnet test LiteDB.Tests --filter FullyQualifiedName~Engine` to triage regressions quickly. + +## Commit & Pull Request Guidelines +Commits use concise, present-tense subject lines (e.g., `Add test run settings`) and may reference issues inline (`Fix #123`). Each PR should describe the problem, the approach, and include before/after notes or perf metrics when touching storage internals. Link to tracking issues, attach shell transcripts or benchmarks where relevant, and confirm `dotnet test` output so reviewers can spot regressions. + +## Versioning & Release Prep +Semantic versions are generated by MinVer; create annotated tags like `v6.0.0` on the main branch rather than editing project files manually. Before tagging, ensure Release builds are clean, pack outputs land in `artifacts_temp/`, and update any shell or benchmark usage notes affected by the change. Update this guide whenever you discover repository practices worth sharing. + diff --git a/ConsoleApp1/ConsoleApp1.csproj b/ConsoleApp1/ConsoleApp1.csproj index 3469f1a6c..e1c362a12 100644 --- a/ConsoleApp1/ConsoleApp1.csproj +++ b/ConsoleApp1/ConsoleApp1.csproj @@ -5,10 +5,16 @@ net8.0 enable enable + 5.0.20 + false - + + + + + diff --git a/ConsoleApp1/Program.cs b/ConsoleApp1/Program.cs index cfa7f4a6f..fec90ee1b 100644 --- a/ConsoleApp1/Program.cs +++ b/ConsoleApp1/Program.cs @@ -1,98 +1,66 @@ -using LiteDB; +using System.IO; +using System.Reflection; +using System.Threading; using LiteDB.Engine; -using System.Reflection.Emit; -using System.Reflection.PortableExecutable; +const string DatabaseName = "issue-2561-repro.db"; +var databasePath = Path.Combine(AppContext.BaseDirectory, DatabaseName); -var password = "46jLz5QWd5fI3m4LiL2r"; -var path = $"C:\\LiteDB\\Examples\\CrashDB_{DateTime.Now.Ticks}.db"; +if (File.Exists(databasePath)) +{ + File.Delete(databasePath); +} var settings = new EngineSettings { - AutoRebuild = true, - Filename = path, - Password = password + Filename = databasePath }; -var data = Enumerable.Range(1, 10_000).Select(i => new BsonDocument -{ - ["_id"] = i, - ["name"] = Faker.Fullname(), - ["age"] = Faker.Age(), - ["created"] = Faker.Birthday(), - ["lorem"] = Faker.Lorem(5, 25) -}).ToArray(); - -try -{ - using (var db = new LiteEngine(settings)) - { -#if DEBUG - db.SimulateDiskWriteFail = (page) => - { - var p = new BasePage(page); +Console.WriteLine($"LiteDB engine file: {databasePath}"); +Console.WriteLine("Creating an explicit transaction on the main thread..."); - if (p.PageID == 248) - { - page.Write((uint)123123123, 8192-4); - } - }; -#endif +using var engine = new LiteEngine(settings); +engine.BeginTrans(); - db.Pragma("USER_VERSION", 123); +var monitorField = typeof(LiteEngine).GetField("_monitor", BindingFlags.NonPublic | BindingFlags.Instance) ?? + throw new InvalidOperationException("Could not locate the transaction monitor field."); - db.EnsureIndex("col1", "idx_age", "$.age", false); +var monitor = monitorField.GetValue(engine) ?? + throw new InvalidOperationException("Failed to extract the transaction monitor instance."); - db.Insert("col1", data, BsonAutoId.Int32); - db.Insert("col2", data, BsonAutoId.Int32); +var getTransaction = monitor.GetType().GetMethod( + "GetTransaction", + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, + binder: null, + types: new[] { typeof(bool), typeof(bool), typeof(bool).MakeByRefType() }, + modifiers: null) ?? + throw new InvalidOperationException("Could not locate GetTransaction on the monitor."); - var col1 = db.Query("col1", Query.All()).ToList().Count; - var col2 = db.Query("col2", Query.All()).ToList().Count; +var getTransactionArgs = new object[] { false, false, null! }; +var transaction = getTransaction.Invoke(monitor, getTransactionArgs) ?? + throw new InvalidOperationException("LiteDB did not return the thread-bound transaction."); - Console.WriteLine("Inserted Col1: " + col1); - Console.WriteLine("Inserted Col2: " + col2); - } -} -catch (Exception ex) -{ - Console.WriteLine("ERROR: " + ex.Message); -} +Console.WriteLine("Main thread transaction captured. Launching a worker thread to mimic the finalizer..."); -Console.WriteLine("Recovering database..."); +var releaseTransaction = monitor.GetType().GetMethod( + "ReleaseTransaction", + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, + binder: null, + types: new[] { transaction.GetType() }, + modifiers: null) ?? + throw new InvalidOperationException("Could not locate ReleaseTransaction on the monitor."); -using (var db = new LiteEngine(settings)) +var worker = new Thread(() => { - var col1 = db.Query("col1", Query.All()).ToList().Count; - var col2 = db.Query("col2", Query.All()).ToList().Count; - - Console.WriteLine($"Col1: {col1}"); - Console.WriteLine($"Col2: {col2}"); - - var errors = new BsonArray(db.Query("_rebuild_errors", Query.All()).ToList()).ToString(); - - Console.WriteLine("Errors: " + errors); - -} - -/* -var errors = new List(); -var fr = new FileReaderV8(settings, errors); - -fr.Open(); -var pragmas = fr.GetPragmas(); -var cols = fr.GetCollections().ToArray(); -var indexes = fr.GetIndexes(cols[0]); - -var docs1 = fr.GetDocuments("col1").ToArray(); -var docs2 = fr.GetDocuments("col2").ToArray(); + Console.WriteLine($"Worker thread {Environment.CurrentManagedThreadId} releasing the transaction..."); + // This invocation throws LiteException("current thread must contains transaction parameter") + // because the worker thread never registered the transaction in its ThreadLocal slot. + releaseTransaction.Invoke(monitor, new[] { transaction }); +}); -Console.WriteLine("Recovered Col1: " + docs1.Length); -Console.WriteLine("Recovered Col2: " + docs2.Length); +worker.Start(); +worker.Join(); -Console.WriteLine("# Errors: "); -errors.ForEach(x => Console.WriteLine($"PageID: {x.PageID}/{x.Origin}/#{x.Position}[{x.Collection}]: " + x.Message)); -*/ +Console.WriteLine("If you see this message the repro did not trigger the crash."); -Console.WriteLine("\n\nEnd."); -Console.ReadKey(); \ No newline at end of file diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 000000000..b8ded0683 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,37 @@ + + + <_RootGitDirectory>$([System.IO.Path]::Combine('$(MSBuildThisFileDirectory)', '.git')) + <_RootGitHeadPath>$([System.IO.Path]::Combine('$(_RootGitDirectory)', 'HEAD')) + <_RootGitShallowPath>$([System.IO.Path]::Combine('$(_RootGitDirectory)', 'shallow')) + true + false + + + + true + false + + + + <_GitVersionPreReleaseNumberPadded Condition="'$(GitVersion_PreReleaseNumber)' != ''">$([System.String]::Format("{0:0000}", $(GitVersion_PreReleaseNumber))) + <_GitVersionCalculatedSemVer Condition="'$(GitVersion_PreReleaseLabel)' != ''">$(GitVersion_MajorMinorPatch)-$(GitVersion_PreReleaseLabel).$(_GitVersionPreReleaseNumberPadded) + <_GitVersionCalculatedSemVer Condition="'$(GitVersion_PreReleaseLabel)' == ''">$(GitVersion_MajorMinorPatch) + $(_GitVersionCalculatedSemVer) + $(_GitVersionCalculatedSemVer) + $(GitVersion_AssemblySemVer) + $(GitVersion_AssemblySemFileVer) + $(_GitVersionCalculatedSemVer)+$(GitVersion_ShortSha) + + + + 0.0.0-detached + 0.0.0-detached + 0.0.0.0 + 0.0.0.0 + 0.0.0-detached + + + + + + diff --git a/GitVersion.yml b/GitVersion.yml new file mode 100644 index 000000000..b532d865a --- /dev/null +++ b/GitVersion.yml @@ -0,0 +1,39 @@ +branches: + main: + regex: ^master$|^main$ + increment: Patch + label: '' + prevent-increment: + of-merged-branch: true + develop: + regex: ^dev(elop)?(ment)?$ + increment: Patch + label: prerelease + source-branches: + - main + prevent-increment: + of-merged-branch: true + feature: + regex: ^(feature|bugfix|chore|refactor)/.*$ + increment: Inherit + label: '' + source-branches: + - develop + - main + pull-request: + regex: ^(pull|pr)/.*$ + increment: Inherit + label: pr + source-branches: + - develop + - main + release: + regex: ^release/.*$ + increment: Patch + label: rc + source-branches: + - develop + - main +commit-message-incrementing: Enabled +commit-date-format: yyyy-MM-dd +tag-prefix: 'v' diff --git a/LiteDB.Benchmarks/Benchmarks/Queries/QueryWithVectorSimilarity.cs b/LiteDB.Benchmarks/Benchmarks/Queries/QueryWithVectorSimilarity.cs new file mode 100644 index 000000000..de44989c9 --- /dev/null +++ b/LiteDB.Benchmarks/Benchmarks/Queries/QueryWithVectorSimilarity.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Attributes; +using LiteDB; +using LiteDB.Benchmarks.Models; +using LiteDB.Benchmarks.Models.Generators; +using LiteDB.Vector; + +namespace LiteDB.Benchmarks.Benchmarks.Queries +{ + [BenchmarkCategory(Constants.Categories.QUERIES)] + public class QueryWithVectorSimilarity : BenchmarkBase + { + private ILiteCollection _fileMetaCollection; + private ILiteCollection _unindexedCollection; + private float[] _queryVector; + + [GlobalSetup] + public void GlobalSetup() + { + File.Delete(DatabasePath); + + DatabaseInstance = new LiteDatabase(ConnectionString()); + _fileMetaCollection = DatabaseInstance.GetCollection("withIndex"); + _unindexedCollection = DatabaseInstance.GetCollection("withoutIndex"); + + _fileMetaCollection.EnsureIndex(fileMeta => fileMeta.ShouldBeShown); + _unindexedCollection.EnsureIndex(fileMeta => fileMeta.ShouldBeShown); + _fileMetaCollection.EnsureIndex(fileMeta => fileMeta.Vectors, new VectorIndexOptions(128)); + + var rnd = new Random(); + var data = FileMetaGenerator.GenerateList(DatasetSize); + + _fileMetaCollection.Insert(data); // executed once per each N value + _unindexedCollection.Insert(data); + + _queryVector = Enumerable.Range(0, 128).Select(_ => (float)rnd.NextDouble()).ToArray(); + + DatabaseInstance.Checkpoint(); + } + + [Benchmark] + public List WhereNear_Filter() + { + return _unindexedCollection.Query() + .WhereNear(x => x.Vectors, _queryVector, maxDistance: 0.5) + .ToList(); + } + + [Benchmark] + public List WhereNear_Filter_Indexed() + { + return _fileMetaCollection.Query() + .WhereNear(x => x.Vectors, _queryVector, maxDistance: 0.5) + .ToList(); + } + + [Benchmark] + public List TopKNear_OrderLimit() + { + return _unindexedCollection.Query() + .TopKNear(x => x.Vectors, _queryVector, k: 10) + .ToList(); + } + + [Benchmark] + public List TopKNear_OrderLimit_Indexed() + { + return _fileMetaCollection.Query() + .TopKNear(x => x.Vectors, _queryVector, k: 10) + .ToList(); + } + } +} \ No newline at end of file diff --git a/LiteDB.Benchmarks/LiteDB.Benchmarks.csproj b/LiteDB.Benchmarks/LiteDB.Benchmarks.csproj index 80a1ebdde..1bb264165 100644 --- a/LiteDB.Benchmarks/LiteDB.Benchmarks.csproj +++ b/LiteDB.Benchmarks/LiteDB.Benchmarks.csproj @@ -1,10 +1,10 @@  - - Exe - net472;net6 - 8 - + + Exe + net8.0 + latest + diff --git a/LiteDB.Benchmarks/Models/FileMetaBase.cs b/LiteDB.Benchmarks/Models/FileMetaBase.cs index ca2f7e2c7..15b7f63aa 100644 --- a/LiteDB.Benchmarks/Models/FileMetaBase.cs +++ b/LiteDB.Benchmarks/Models/FileMetaBase.cs @@ -28,6 +28,8 @@ public class FileMetaBase public bool ShouldBeShown { get; set; } + public float[] Vectors { get; set; } + public virtual bool IsValid => ValidFrom == null || ValidFrom <= DateTimeOffset.UtcNow && ValidTo == null || ValidTo > DateTimeOffset.UtcNow; } } \ No newline at end of file diff --git a/LiteDB.Benchmarks/Models/Generators/FileMetaGenerator.cs b/LiteDB.Benchmarks/Models/Generators/FileMetaGenerator.cs index 6a551b502..903597882 100644 --- a/LiteDB.Benchmarks/Models/Generators/FileMetaGenerator.cs +++ b/LiteDB.Benchmarks/Models/Generators/FileMetaGenerator.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; namespace LiteDB.Benchmarks.Models.Generators { @@ -18,7 +19,8 @@ private static T Generate() Title = $"Document-{docGuid}", MimeType = "application/pdf", IsFavorite = _random.Next(10) >= 9, - ShouldBeShown = _random.Next(10) >= 7 + ShouldBeShown = _random.Next(10) >= 7, + Vectors = Enumerable.Range(0, 128).Select(_ => (float)_random.NextDouble()).ToArray() }; if (_random.Next(10) >= 5) diff --git a/LiteDB.Demo.Tools.VectorSearch/Commands/IngestCommand.cs b/LiteDB.Demo.Tools.VectorSearch/Commands/IngestCommand.cs new file mode 100644 index 000000000..788528100 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Commands/IngestCommand.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using LiteDB.Demo.Tools.VectorSearch.Configuration; +using LiteDB.Demo.Tools.VectorSearch.Embedding; +using LiteDB.Demo.Tools.VectorSearch.Models; +using LiteDB.Demo.Tools.VectorSearch.Services; +using LiteDB.Demo.Tools.VectorSearch.Utilities; +using Spectre.Console; +using Spectre.Console.Cli; +using ValidationResult = Spectre.Console.ValidationResult; + +namespace LiteDB.Demo.Tools.VectorSearch.Commands +{ + internal sealed class IngestCommand : AsyncCommand + { + public override async Task ExecuteAsync(CommandContext context, IngestCommandSettings settings) + { + if (!Directory.Exists(settings.SourceDirectory)) + { + throw new InvalidOperationException($"Source directory '{settings.SourceDirectory}' does not exist."); + } + + var embeddingOptions = settings.CreateEmbeddingOptions(); + + using var documentStore = new DocumentStore(settings.DatabasePath); + using var embeddingService = await GeminiEmbeddingService.CreateAsync(embeddingOptions, CancellationToken.None); + + var searchOption = settings.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; + var files = Directory.EnumerateFiles(settings.SourceDirectory, "*", searchOption) + .Where(TextUtilities.IsSupportedDocument) + .OrderBy(x => x) + .Select(Path.GetFullPath) + .ToArray(); + + if (files.Length == 0) + { + AnsiConsole.MarkupLine("[yellow]No supported text documents were found. Nothing to ingest.[/]"); + return 0; + } + + var skipUnchanged = !settings.Force; + var processed = 0; + var skipped = 0; + var errors = new List<(string Path, string Error)>(); + + await AnsiConsole.Progress() + .Columns(new ProgressColumn[] + { + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new ElapsedTimeColumn(), + new RemainingTimeColumn() + }) + .StartAsync(async ctx => + { + var task = ctx.AddTask("Embedding documents", maxValue: files.Length); + + foreach (var path in files) + { + try + { + var info = new FileInfo(path); + var rawContent = TextUtilities.ReadDocument(path); + var contentHash = TextUtilities.ComputeContentHash(rawContent); + + var existing = documentStore.FindByPath(path); + if (existing != null && skipUnchanged && string.Equals(existing.ContentHash, contentHash, StringComparison.Ordinal)) + { + skipped++; + continue; + } + + var chunkRecords = new List(); + var chunkIndex = 0; + var ensuredIndex = false; + + foreach (var chunk in TextUtilities.SplitIntoChunks(rawContent, settings.ChunkLength, settings.ChunkOverlap)) + { + var normalizedChunk = TextUtilities.NormalizeForEmbedding(chunk, embeddingOptions.MaxInputLength); + if (string.IsNullOrWhiteSpace(normalizedChunk)) + { + chunkIndex++; + continue; + } + + var embedding = await embeddingService.EmbedAsync(normalizedChunk, CancellationToken.None); + + if (!ensuredIndex) + { + documentStore.EnsureChunkVectorIndex(embedding.Length); + ensuredIndex = true; + } + + chunkRecords.Add(new IndexedDocumentChunk + { + Path = path, + ChunkIndex = chunkIndex, + Snippet = chunk.Trim(), + Embedding = embedding + }); + + chunkIndex++; + } + + var record = existing ?? new IndexedDocument(); + record.Path = path; + record.Title = Path.GetFileName(path); + record.Preview = TextUtilities.BuildPreview(rawContent, settings.PreviewLength); + record.Embedding = Array.Empty(); + record.LastModifiedUtc = info.LastWriteTimeUtc; + record.SizeBytes = info.Length; + record.ContentHash = contentHash; + record.IngestedUtc = DateTime.UtcNow; + + if (chunkRecords.Count == 0) + { + documentStore.Upsert(record); + documentStore.ReplaceDocumentChunks(path, Array.Empty()); + skipped++; + continue; + } + + documentStore.Upsert(record); + documentStore.ReplaceDocumentChunks(path, chunkRecords); + processed++; + } + catch (Exception ex) + { + errors.Add((path, ex.Message)); + } + finally + { + task.Increment(1); + } + } + }); + + if (settings.PruneMissing) + { + var indexedPaths = documentStore.GetTrackedPaths(); + var currentPaths = new HashSet(files, StringComparer.OrdinalIgnoreCase); + var missing = indexedPaths.Where(path => !currentPaths.Contains(path)).ToArray(); + + if (missing.Length > 0) + { + documentStore.RemoveMissingDocuments(missing); + AnsiConsole.MarkupLine($"[yellow]Removed {missing.Length} documents that no longer exist on disk.[/]"); + } + } + + var summary = new Table().Border(TableBorder.Rounded); + summary.AddColumn("Metric"); + summary.AddColumn("Value"); + summary.AddRow("Processed", processed.ToString()); + summary.AddRow("Skipped", skipped.ToString()); + summary.AddRow("Errors", errors.Count.ToString()); + + AnsiConsole.Write(summary); + + if (errors.Count > 0) + { + var errorTable = new Table().Border(TableBorder.Rounded); + errorTable.AddColumn("File"); + errorTable.AddColumn("Error"); + + foreach (var (path, message) in errors) + { + errorTable.AddRow(Markup.Escape(path), Markup.Escape(message)); + } + + AnsiConsole.Write(errorTable); + return 1; + } + + return 0; + } + } + + internal sealed class IngestCommandSettings : VectorSearchCommandSettings + { + [CommandOption("-s|--source ")] + public string SourceDirectory { get; set; } = string.Empty; + + [CommandOption("--preview-length ")] + public int PreviewLength { get; set; } = 240; + + [CommandOption("--no-recursive")] + public bool NoRecursive { get; set; } + + [CommandOption("--force")] + public bool Force { get; set; } + + [CommandOption("--prune-missing")] + public bool PruneMissing { get; set; } + + [CommandOption("--chunk-length ")] + public int ChunkLength { get; set; } = 600; + + [CommandOption("--chunk-overlap ")] + public int ChunkOverlap { get; set; } = 100; + + public bool Recursive => !NoRecursive; + + public override ValidationResult Validate() + { + var baseResult = base.Validate(); + if (!baseResult.Successful) + { + return baseResult; + } + + if (string.IsNullOrWhiteSpace(SourceDirectory)) + { + return ValidationResult.Error("A source directory is required."); + } + + if (PreviewLength <= 0) + { + return ValidationResult.Error("--preview-length must be greater than zero."); + } + + if (ChunkLength <= 0) + { + return ValidationResult.Error("--chunk-length must be greater than zero."); + } + + if (ChunkOverlap < 0) + { + return ValidationResult.Error("--chunk-overlap must be zero or greater."); + } + + if (ChunkOverlap >= ChunkLength) + { + return ValidationResult.Error("--chunk-overlap must be smaller than --chunk-length."); + } + + return ValidationResult.Success(); + } + } +} + + + + diff --git a/LiteDB.Demo.Tools.VectorSearch/Commands/SearchCommand.cs b/LiteDB.Demo.Tools.VectorSearch/Commands/SearchCommand.cs new file mode 100644 index 000000000..b5c778a86 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Commands/SearchCommand.cs @@ -0,0 +1,165 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using LiteDB.Demo.Tools.VectorSearch.Embedding; +using LiteDB.Demo.Tools.VectorSearch.Models; +using LiteDB.Demo.Tools.VectorSearch.Services; +using LiteDB.Demo.Tools.VectorSearch.Utilities; +using Spectre.Console; +using Spectre.Console.Cli; +using ValidationResult = Spectre.Console.ValidationResult; + +namespace LiteDB.Demo.Tools.VectorSearch.Commands +{ + internal sealed class SearchCommand : AsyncCommand + { + public override async Task ExecuteAsync(CommandContext context, SearchCommandSettings settings) + { + using var documentStore = new DocumentStore(settings.DatabasePath); + + var embeddingOptions = settings.CreateEmbeddingOptions(); + using var embeddingService = await GeminiEmbeddingService.CreateAsync(embeddingOptions, CancellationToken.None); + + var queryText = settings.Query; + if (string.IsNullOrWhiteSpace(queryText)) + { + queryText = AnsiConsole.Ask("Enter a search prompt:"); + } + + if (string.IsNullOrWhiteSpace(queryText)) + { + AnsiConsole.MarkupLine("[red]A non-empty query is required.[/]"); + return 1; + } + + var normalized = TextUtilities.NormalizeForEmbedding(queryText, embeddingOptions.MaxInputLength); + var queryEmbedding = await embeddingService.EmbedAsync(normalized, CancellationToken.None); + + var chunkResults = documentStore.TopNearestChunks(queryEmbedding, settings.Top) + .Select(chunk => new SearchHit(chunk, VectorMath.CosineSimilarity(chunk.Embedding, queryEmbedding))) + .ToList(); + + if (settings.MaxDistance.HasValue) + { + chunkResults = chunkResults + .Where(hit => VectorMath.CosineDistance(hit.Chunk.Embedding, queryEmbedding) <= settings.MaxDistance.Value) + .ToList(); + } + + if (chunkResults.Count == 0) + { + AnsiConsole.MarkupLine("[yellow]No matching documents were found.[/]"); + return 0; + } + + chunkResults.Sort((left, right) => right.Similarity.CompareTo(left.Similarity)); + + var table = new Table().Border(TableBorder.Rounded); + table.AddColumn("#"); + table.AddColumn("Score"); + table.AddColumn("Document"); + if (!settings.HidePath) + { + table.AddColumn("Path"); + } + table.AddColumn("Snippet"); + + var rank = 1; + var documentCache = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var hit in chunkResults) + { + var snippet = hit.Chunk.Snippet; + if (snippet.Length > settings.PreviewLength) + { + snippet = snippet[..settings.PreviewLength] + "\u2026"; + } + + if (!documentCache.TryGetValue(hit.Chunk.Path, out var parentDocument)) + { + parentDocument = documentStore.FindByPath(hit.Chunk.Path); + documentCache[hit.Chunk.Path] = parentDocument; + } + + var chunkNumber = hit.Chunk.ChunkIndex + 1; + var documentLabel = parentDocument != null + ? $"{parentDocument.Title} (Chunk {chunkNumber})" + : $"Chunk {chunkNumber}"; + + var rowData = new List + { + Markup.Escape(rank.ToString()), + Markup.Escape(hit.Similarity.ToString("F3")), + Markup.Escape(documentLabel) + }; + + if (!settings.HidePath) + { + var pathValue = parentDocument?.Path ?? hit.Chunk.Path; + rowData.Add(Markup.Escape(pathValue)); + } + + rowData.Add(Markup.Escape(snippet)); + + table.AddRow(rowData.ToArray()); + + rank++; + } + + AnsiConsole.Write(table); + return 0; + } + + private sealed record SearchHit(IndexedDocumentChunk Chunk, double Similarity); + } + + internal sealed class SearchCommandSettings : VectorSearchCommandSettings + { + [CommandOption("-q|--query ")] + public string? Query { get; set; } + + [CommandOption("--top ")] + public int Top { get; set; } = 5; + + [CommandOption("--max-distance ")] + public double? MaxDistance { get; set; } + + [CommandOption("--preview-length ")] + public int PreviewLength { get; set; } = 160; + + [CommandOption("--hide-path")] + public bool HidePath { get; set; } + + public override ValidationResult Validate() + { + var baseResult = base.Validate(); + if (!baseResult.Successful) + { + return baseResult; + } + + if (Top <= 0) + { + return ValidationResult.Error("--top must be greater than zero."); + } + + if (MaxDistance.HasValue && MaxDistance <= 0) + { + return ValidationResult.Error("--max-distance must be greater than zero when specified."); + } + + if (PreviewLength <= 0) + { + return ValidationResult.Error("--preview-length must be greater than zero."); + } + + return ValidationResult.Success(); + } + } +} + + + + diff --git a/LiteDB.Demo.Tools.VectorSearch/Commands/VectorSearchCommandSettings.cs b/LiteDB.Demo.Tools.VectorSearch/Commands/VectorSearchCommandSettings.cs new file mode 100644 index 000000000..3ef95ad43 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Commands/VectorSearchCommandSettings.cs @@ -0,0 +1,125 @@ +using System; +using Spectre.Console.Cli; +using LiteDB.Demo.Tools.VectorSearch.Configuration; +using ValidationResult = Spectre.Console.ValidationResult; + +namespace LiteDB.Demo.Tools.VectorSearch.Commands +{ + internal abstract class VectorSearchCommandSettings : CommandSettings + { + private const string DefaultModel = "gemini-embedding-001"; + private const string DefaultLocation = "us-central1"; + private const string ApiKeyEnvironmentVariable = "GOOGLE_VERTEX_API_KEY"; + private const string ApiKeyFallbackEnvironmentVariable = "GOOGLE_API_KEY"; + + [CommandOption("-d|--database ")] + public string DatabasePath { get; set; } = "vector-search.db"; + + [CommandOption("--project-id ")] + public string? ProjectId { get; set; } + + [CommandOption("--location ")] + public string? Location { get; set; } + + [CommandOption("--model ")] + public string? Model { get; set; } + + [CommandOption("--api-key ")] + public string? ApiKey { get; set; } + + [CommandOption("--max-input-length ")] + public int MaxInputLength { get; set; } = 7000; + + public GeminiEmbeddingOptions CreateEmbeddingOptions() + { + var model = ResolveModel(); + var apiKey = ResolveApiKey(); + + if (!string.IsNullOrWhiteSpace(apiKey)) + { + return GeminiEmbeddingOptions.ForApiKey(apiKey!, model, MaxInputLength); + } + + var projectId = ResolveProjectIdOrNull(); + if (string.IsNullOrWhiteSpace(projectId)) + { + throw new InvalidOperationException("Provide --api-key/GOOGLE_VERTEX_API_KEY or --project-id/GOOGLE_PROJECT_ID to configure Gemini embeddings."); + } + + var location = ResolveLocation(); + return GeminiEmbeddingOptions.ForServiceAccount(projectId!, location, model, MaxInputLength); + } + + public override ValidationResult Validate() + { + if (MaxInputLength <= 0) + { + return ValidationResult.Error("--max-input-length must be greater than zero."); + } + + if (string.IsNullOrWhiteSpace(DatabasePath)) + { + return ValidationResult.Error("A database path must be provided."); + } + + var hasApiKey = !string.IsNullOrWhiteSpace(ResolveApiKey()); + var hasProject = !string.IsNullOrWhiteSpace(ResolveProjectIdOrNull()); + + if (!hasApiKey && !hasProject) + { + return ValidationResult.Error("Authentication required. Supply --api-key (or GOOGLE_VERTEX_API_KEY/GOOGLE_API_KEY) or --project-id (or GOOGLE_PROJECT_ID)."); + } + + return ValidationResult.Success(); + } + + private string? ResolveProjectIdOrNull() + { + if (!string.IsNullOrWhiteSpace(ProjectId)) + { + return ProjectId; + } + + var fromEnv = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); + return string.IsNullOrWhiteSpace(fromEnv) ? null : fromEnv; + } + + private string ResolveLocation() + { + if (!string.IsNullOrWhiteSpace(Location)) + { + return Location; + } + + var fromEnv = Environment.GetEnvironmentVariable("GOOGLE_VERTEX_LOCATION"); + return string.IsNullOrWhiteSpace(fromEnv) ? DefaultLocation : fromEnv; + } + + private string ResolveModel() + { + if (!string.IsNullOrWhiteSpace(Model)) + { + return Model; + } + + var fromEnv = Environment.GetEnvironmentVariable("GOOGLE_VERTEX_EMBEDDING_MODEL"); + return string.IsNullOrWhiteSpace(fromEnv) ? DefaultModel : fromEnv; + } + + private string? ResolveApiKey() + { + if (!string.IsNullOrWhiteSpace(ApiKey)) + { + return ApiKey; + } + + var fromEnv = Environment.GetEnvironmentVariable(ApiKeyEnvironmentVariable); + if (string.IsNullOrWhiteSpace(fromEnv)) + { + fromEnv = Environment.GetEnvironmentVariable(ApiKeyFallbackEnvironmentVariable); + } + + return string.IsNullOrWhiteSpace(fromEnv) ? null : fromEnv; + } + } +} diff --git a/LiteDB.Demo.Tools.VectorSearch/Configuration/GeminiEmbeddingOptions.cs b/LiteDB.Demo.Tools.VectorSearch/Configuration/GeminiEmbeddingOptions.cs new file mode 100644 index 000000000..47a981045 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Configuration/GeminiEmbeddingOptions.cs @@ -0,0 +1,95 @@ +using System; + +namespace LiteDB.Demo.Tools.VectorSearch.Configuration +{ + internal sealed class GeminiEmbeddingOptions + { + private const string ApiModelPrefix = "models/"; + + private GeminiEmbeddingOptions(string? projectId, string? location, string model, int maxInputLength, string? apiKey) + { + Model = TrimModelPrefix(model); + + if (maxInputLength <= 0) + { + throw new ArgumentOutOfRangeException(nameof(maxInputLength)); + } + + ProjectId = projectId; + Location = location; + MaxInputLength = maxInputLength; + ApiKey = string.IsNullOrWhiteSpace(apiKey) ? null : apiKey; + } + + public static GeminiEmbeddingOptions ForServiceAccount(string projectId, string location, string model, int maxInputLength) + { + if (string.IsNullOrWhiteSpace(projectId)) + { + throw new ArgumentNullException(nameof(projectId)); + } + + if (string.IsNullOrWhiteSpace(location)) + { + throw new ArgumentNullException(nameof(location)); + } + + return new GeminiEmbeddingOptions(projectId, location, model, maxInputLength, apiKey: null); + } + + public static GeminiEmbeddingOptions ForApiKey(string apiKey, string model, int maxInputLength) + { + if (string.IsNullOrWhiteSpace(apiKey)) + { + throw new ArgumentNullException(nameof(apiKey)); + } + + return new GeminiEmbeddingOptions(projectId: null, location: null, model, maxInputLength, apiKey); + } + + public string? ProjectId { get; } + + public string? Location { get; } + + public string Model { get; } + + public int MaxInputLength { get; } + + public string? ApiKey { get; } + + public bool UseApiKey => !string.IsNullOrWhiteSpace(ApiKey); + + public string GetVertexEndpoint() + { + if (string.IsNullOrWhiteSpace(ProjectId) || string.IsNullOrWhiteSpace(Location)) + { + throw new InvalidOperationException("Vertex endpoint requires both project id and location."); + } + + return $"https://{Location}-aiplatform.googleapis.com/v1/projects/{ProjectId}/locations/{Location}/publishers/google/models/{Model}:predict"; + } + + public string GetApiEndpoint() + { + return $"https://generativelanguage.googleapis.com/v1beta/{GetApiModelIdentifier()}:embedContent"; //models/{GetApiModelIdentifier()}:embedContent"; + } + + public string GetApiModelIdentifier() + { + return Model.StartsWith(ApiModelPrefix, StringComparison.Ordinal) + ? Model + : $"{ApiModelPrefix}{Model}"; + } + + private static string TrimModelPrefix(string model) + { + if (string.IsNullOrWhiteSpace(model)) + { + throw new ArgumentNullException(nameof(model)); + } + + return model.StartsWith(ApiModelPrefix, StringComparison.OrdinalIgnoreCase) + ? model.Substring(ApiModelPrefix.Length) + : model; + } + } +} diff --git a/LiteDB.Demo.Tools.VectorSearch/Embedding/GeminiEmbeddingService.cs b/LiteDB.Demo.Tools.VectorSearch/Embedding/GeminiEmbeddingService.cs new file mode 100644 index 000000000..d475228b9 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Embedding/GeminiEmbeddingService.cs @@ -0,0 +1,190 @@ +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Google.Apis.Auth.OAuth2; +using LiteDB.Demo.Tools.VectorSearch.Configuration; + +namespace LiteDB.Demo.Tools.VectorSearch.Embedding +{ + internal sealed class GeminiEmbeddingService : IEmbeddingService, IDisposable + { + private static readonly JsonSerializerOptions SerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + private readonly HttpClient _httpClient; + private readonly GeminiEmbeddingOptions _options; + private readonly ITokenAccess? _tokenAccessor; + private bool _disposed; + + private GeminiEmbeddingService(HttpClient httpClient, GeminiEmbeddingOptions options, ITokenAccess? tokenAccessor) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _tokenAccessor = tokenAccessor; + } + + public static async Task CreateAsync(GeminiEmbeddingOptions options, CancellationToken cancellationToken) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + ITokenAccess? tokenAccessor = null; + + if (!options.UseApiKey) + { + var credential = await GoogleCredential.GetApplicationDefaultAsync(cancellationToken); + credential = credential.CreateScoped("https://www.googleapis.com/auth/cloud-platform"); + tokenAccessor = credential; + } + + var httpClient = new HttpClient(); + return new GeminiEmbeddingService(httpClient, options, tokenAccessor); + } + + public async Task EmbedAsync(string text, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(text)) + { + throw new ArgumentException("Text must be provided for embedding.", nameof(text)); + } + + EnsureNotDisposed(); + + var normalized = text.Length <= _options.MaxInputLength + ? text + : text[.._options.MaxInputLength]; + + var endpoint = _options.UseApiKey ? _options.GetApiEndpoint() : _options.GetVertexEndpoint(); + object payload = _options.UseApiKey + ? new + { + model = _options.GetApiModelIdentifier(), + content = new + { + parts = new[] + { + new + { + text = normalized + } + } + } + } + : new + { + instances = new[] + { + new + { + content = new + { + parts = new[] + { + new + { + text = normalized + } + } + } + } + } + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint); + var json = System.Text.Json.JsonSerializer.Serialize(payload, SerializerOptions); + var content = new StringContent(json, Encoding.UTF8); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + request.Content = content; + + if (_options.UseApiKey) + { + request.Headers.TryAddWithoutValidation("x-goog-api-key", _options.ApiKey); + } + else + { + if (_tokenAccessor == null) + { + throw new InvalidOperationException("Google credentials are required when no API key is provided."); + } + + var token = await _tokenAccessor.GetAccessTokenForRequestAsync(cancellationToken: cancellationToken); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + } + + using var response = await _httpClient.SendAsync(request, cancellationToken); + + if (!response.IsSuccessStatusCode) + { + var details = await response.Content.ReadAsStringAsync(cancellationToken); + throw new InvalidOperationException($"Embedding request failed ({response.StatusCode}). {details}"); + } + + using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + + if (TryReadValues(document.RootElement, out var values)) + { + return values; + } + + throw new InvalidOperationException("Embedding response did not contain any vector values."); + } + + private static bool TryReadValues(JsonElement root, out float[] values) + { + if (root.TryGetProperty("predictions", out var predictions) && predictions.GetArrayLength() > 0) + { + var embeddings = predictions[0].GetProperty("embeddings").GetProperty("values"); + values = ReadFloatArray(embeddings); + return true; + } + + if (root.TryGetProperty("embedding", out var embedding) && embedding.TryGetProperty("values", out var apiValues)) + { + values = ReadFloatArray(apiValues); + return true; + } + + values = Array.Empty(); + return false; + } + + private static float[] ReadFloatArray(JsonElement element) + { + var array = new float[element.GetArrayLength()]; + for (var i = 0; i < array.Length; i++) + { + array[i] = (float)element[i].GetDouble(); + } + + return array; + } + + private void EnsureNotDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(nameof(GeminiEmbeddingService)); + } + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _httpClient.Dispose(); + _disposed = true; + } + } +} diff --git a/LiteDB.Demo.Tools.VectorSearch/Embedding/IEmbeddingService.cs b/LiteDB.Demo.Tools.VectorSearch/Embedding/IEmbeddingService.cs new file mode 100644 index 000000000..2b0f4ff17 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Embedding/IEmbeddingService.cs @@ -0,0 +1,10 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace LiteDB.Demo.Tools.VectorSearch.Embedding +{ + internal interface IEmbeddingService + { + Task EmbedAsync(string text, CancellationToken cancellationToken); + } +} diff --git a/LiteDB.Demo.Tools.VectorSearch/LiteDB.Demo.Tools.VectorSearch.csproj b/LiteDB.Demo.Tools.VectorSearch/LiteDB.Demo.Tools.VectorSearch.csproj new file mode 100644 index 000000000..97e2ef33d --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/LiteDB.Demo.Tools.VectorSearch.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + diff --git a/LiteDB.Demo.Tools.VectorSearch/Models/IndexedDocument.cs b/LiteDB.Demo.Tools.VectorSearch/Models/IndexedDocument.cs new file mode 100644 index 000000000..deb4c5c67 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Models/IndexedDocument.cs @@ -0,0 +1,27 @@ +using System; +using LiteDB; + +namespace LiteDB.Demo.Tools.VectorSearch.Models +{ + public sealed class IndexedDocument + { + public ObjectId Id { get; set; } = ObjectId.Empty; + + public string Path { get; set; } = string.Empty; + + public string Title { get; set; } = string.Empty; + + public string Preview { get; set; } = string.Empty; + + public float[] Embedding { get; set; } = Array.Empty(); + + public DateTime LastModifiedUtc { get; set; } + + public long SizeBytes { get; set; } + + public string ContentHash { get; set; } = string.Empty; + + public DateTime IngestedUtc { get; set; } + } +} + diff --git a/LiteDB.Demo.Tools.VectorSearch/Models/IndexedDocumentChunk.cs b/LiteDB.Demo.Tools.VectorSearch/Models/IndexedDocumentChunk.cs new file mode 100644 index 000000000..a4b6e9ce9 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Models/IndexedDocumentChunk.cs @@ -0,0 +1,19 @@ +using System; +using LiteDB; + +namespace LiteDB.Demo.Tools.VectorSearch.Models +{ + public sealed class IndexedDocumentChunk + { + public ObjectId Id { get; set; } = ObjectId.Empty; + + public string Path { get; set; } = string.Empty; + + public int ChunkIndex { get; set; } + + public string Snippet { get; set; } = string.Empty; + + public float[] Embedding { get; set; } = Array.Empty(); + } +} + diff --git a/LiteDB.Demo.Tools.VectorSearch/Program.cs b/LiteDB.Demo.Tools.VectorSearch/Program.cs new file mode 100644 index 000000000..ec54e8c56 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Program.cs @@ -0,0 +1,40 @@ +using System.Diagnostics; +using System.Threading.Tasks; +using LiteDB.Demo.Tools.VectorSearch.Commands; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace LiteDB.Demo.Tools.VectorSearch +{ + public static class Program + { + public static async Task Main(string[] args) + { + var app = new CommandApp(); + + app.Configure(config => + { + config.SetApplicationName("litedb-vector-search"); + config.SetExceptionHandler(ex => + { + AnsiConsole.WriteException(ex, ExceptionFormats.ShortenEverything); + return -1; + }); + + config.AddCommand("ingest") + .WithDescription("Embed text documents from a folder into LiteDB for vector search."); + + config.AddCommand("search") + .WithDescription("Search previously embedded documents using vector similarity."); + + if (Debugger.IsAttached) + { + config.PropagateExceptions(); + } + }); + + return await app.RunAsync(args); + } + } +} + diff --git a/LiteDB.Demo.Tools.VectorSearch/Readme.md b/LiteDB.Demo.Tools.VectorSearch/Readme.md new file mode 100644 index 000000000..e21692d7a --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Readme.md @@ -0,0 +1,39 @@ +## Vector Search Demo CLI + +`LiteDB.Demo.Tools.VectorSearch` showcases the new vector index APIs with an end-to-end ingestion and query experience. It embeds text documents using Google Gemini embeddings and persists metadata plus the resulting vectors in LiteDB. + +### Requirements + +- Supply Gemini credentials using **one** of the following approaches: + - API key with `--api-key`, `GOOGLE_VERTEX_API_KEY`, or `GOOGLE_API_KEY` (Get from [AI Studio](https://aistudio.google.com/api-keys)) + - Service account credentials via `GOOGLE_APPLICATION_CREDENTIALS` (or other default `GoogleCredential` providers) together with project metadata. +- When targeting Vertex AI with a service account, the following settings apply (optionally via command options): + - `GOOGLE_PROJECT_ID` + - `GOOGLE_VERTEX_LOCATION` (defaults to `us-central1`) +- Model selection is controlled with `--model` or `GOOGLE_VERTEX_EMBEDDING_MODEL` and defaults to `gemini-embedding-001`. + +### Usage + +Restore and build the demo project: + +```bash +dotnet build LiteDB.Demo.Tools.VectorSearch.csproj -c Release +``` + +Index a folder of `.txt`/`.md` files (API key example): + +```bash +dotnet run --project LiteDB.Demo.Tools.VectorSearch.csproj -- ingest --source ./docs --database vector.db --api-key "$env:GOOGLE_VERTEX_API_KEY" +``` + +Run a semantic search over the ingested content (Vertex AI example): + +```bash +dotnet run --project LiteDB.Demo.Tools.VectorSearch.csproj -- search --database vector.db --query "Explain document storage guarantees" +``` + +Use `--help` on either command to list all supported options (preview length, pruning behaviour, auth mode, custom model identifiers, etc.). + +## License + +[MIT](http://opensource.org/licenses/MIT) \ No newline at end of file diff --git a/LiteDB.Demo.Tools.VectorSearch/Services/DocumentStore.cs b/LiteDB.Demo.Tools.VectorSearch/Services/DocumentStore.cs new file mode 100644 index 000000000..db9db0c0e --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Services/DocumentStore.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using LiteDB; +using LiteDB.Demo.Tools.VectorSearch.Models; +using LiteDB.Vector; + +namespace LiteDB.Demo.Tools.VectorSearch.Services +{ + internal sealed class DocumentStore : IDisposable + { + private const string DocumentCollectionName = "documents"; + private const string ChunkCollectionName = "chunks"; + + private readonly LiteDatabase _database; + private readonly ILiteCollection _documents; + private readonly ILiteCollection _chunks; + private ushort? _chunkVectorDimensions; + + public DocumentStore(string databasePath) + { + if (string.IsNullOrWhiteSpace(databasePath)) + { + throw new ArgumentException("Database path must be provided.", nameof(databasePath)); + } + + var fullPath = Path.GetFullPath(databasePath); + _database = new LiteDatabase(fullPath); + _documents = _database.GetCollection(DocumentCollectionName); + _documents.EnsureIndex(x => x.Path, true); + + _chunks = _database.GetCollection(ChunkCollectionName); + _chunks.EnsureIndex(x => x.Path); + _chunks.EnsureIndex(x => x.ChunkIndex); + } + + public IndexedDocument? FindByPath(string absolutePath) + { + if (string.IsNullOrWhiteSpace(absolutePath)) + { + return null; + } + + return _documents.FindOne(x => x.Path == absolutePath); + } + + public void EnsureChunkVectorIndex(int dimensions) + { + if (dimensions <= 0) + { + throw new ArgumentOutOfRangeException(nameof(dimensions), dimensions, "Vector dimensions must be positive."); + } + + var targetDimensions = (ushort)dimensions; + if (_chunkVectorDimensions == targetDimensions) + { + return; + } + + _chunks.EnsureIndex(x => x.Embedding, new VectorIndexOptions(targetDimensions, VectorDistanceMetric.Cosine)); + _chunkVectorDimensions = targetDimensions; + } + + public void Upsert(IndexedDocument document) + { + if (document == null) + { + throw new ArgumentNullException(nameof(document)); + } + + _documents.Upsert(document); + } + + public void ReplaceDocumentChunks(string documentPath, IEnumerable chunks) + { + if (string.IsNullOrWhiteSpace(documentPath)) + { + throw new ArgumentException("Document path must be provided.", nameof(documentPath)); + } + + _chunks.DeleteMany(chunk => chunk.Path == documentPath); + + if (chunks == null) + { + return; + } + + foreach (var chunk in chunks) + { + if (chunk == null) + { + continue; + } + + chunk.Path = documentPath; + + if (chunk.Id == ObjectId.Empty) + { + chunk.Id = ObjectId.NewObjectId(); + } + + _chunks.Insert(chunk); + } + } + + public IEnumerable TopNearestChunks(float[] embedding, int count) + { + if (embedding == null) + { + throw new ArgumentNullException(nameof(embedding)); + } + + if (count <= 0) + { + throw new ArgumentOutOfRangeException(nameof(count), count, "Quantity must be positive."); + } + + EnsureChunkVectorIndex(embedding.Length); + + return _chunks.Query() + .TopKNear(x => x.Embedding, embedding, count) + .ToEnumerable(); + } + + public IReadOnlyCollection GetTrackedPaths() + { + return _documents.FindAll() + .Select(doc => doc.Path) + .Where(path => !string.IsNullOrWhiteSpace(path)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + public void RemoveMissingDocuments(IEnumerable existingDocumentPaths) + { + if (existingDocumentPaths == null) + { + return; + } + + var keep = new HashSet(existingDocumentPaths, StringComparer.OrdinalIgnoreCase); + + foreach (var doc in _documents.FindAll().Where(doc => !keep.Contains(doc.Path))) + { + _documents.Delete(doc.Id); + _chunks.DeleteMany(chunk => chunk.Path == doc.Path); + } + } + + public void Dispose() + { + _database.Dispose(); + } + } +} diff --git a/LiteDB.Demo.Tools.VectorSearch/Utilities/TextUtilities.cs b/LiteDB.Demo.Tools.VectorSearch/Utilities/TextUtilities.cs new file mode 100644 index 000000000..1ead98ed9 --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Utilities/TextUtilities.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Text; + +namespace LiteDB.Demo.Tools.VectorSearch.Utilities +{ + internal static class TextUtilities + { + private static readonly HashSet _supportedExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".txt", + ".md", + ".markdown", + ".mdown" + }; + + private static readonly char[] _chunkBreakCharacters = { '\n', ' ', '\t' }; + + public static bool IsSupportedDocument(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return false; + } + + var extension = Path.GetExtension(path); + return !string.IsNullOrEmpty(extension) && _supportedExtensions.Contains(extension); + } + + public static string ReadDocument(string path) + { + return File.ReadAllText(path); + } + + public static string NormalizeForEmbedding(string content, int maxLength) + { + if (string.IsNullOrWhiteSpace(content)) + { + return string.Empty; + } + + if (maxLength <= 0) + { + return string.Empty; + } + + var normalized = content.Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .Trim(); + + if (normalized.Length <= maxLength) + { + return normalized; + } + + return normalized[..maxLength]; + } + + public static string BuildPreview(string content, int maxLength) + { + if (string.IsNullOrWhiteSpace(content) || maxLength <= 0) + { + return string.Empty; + } + + var collapsed = new StringBuilder(Math.Min(content.Length, maxLength)); + var previousWhitespace = false; + + foreach (var ch in content) + { + if (char.IsControl(ch) && ch != '\n' && ch != '\t') + { + continue; + } + + if (char.IsWhiteSpace(ch)) + { + if (!previousWhitespace) + { + collapsed.Append(' '); + } + + previousWhitespace = true; + } + else + { + previousWhitespace = false; + collapsed.Append(ch); + } + + if (collapsed.Length >= maxLength) + { + break; + } + } + + var preview = collapsed.ToString().Trim(); + return preview.Length <= maxLength ? preview : preview[..maxLength]; + } + + public static string ComputeContentHash(string content) + { + if (content == null) + { + return string.Empty; + } + + using var sha256 = SHA256.Create(); + var bytes = Encoding.UTF8.GetBytes(content); + var hash = sha256.ComputeHash(bytes); + + return Convert.ToHexString(hash); + } + + public static IEnumerable SplitIntoChunks(string content, int chunkLength, int chunkOverlap) + { + if (string.IsNullOrWhiteSpace(content)) + { + yield break; + } + + if (chunkLength <= 0) + { + yield break; + } + + if (chunkOverlap < 0 || chunkOverlap >= chunkLength) + { + throw new ArgumentOutOfRangeException(nameof(chunkOverlap), chunkOverlap, "Chunk overlap must be non-negative and smaller than the chunk length."); + } + + var normalized = content.Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n'); + + var step = chunkLength - chunkOverlap; + var position = 0; + + if (step <= 0) + { + yield break; + } + + while (position < normalized.Length) + { + var remaining = normalized.Length - position; + var take = Math.Min(chunkLength, remaining); + var window = normalized.Substring(position, take); + + if (take == chunkLength && position + take < normalized.Length) + { + var lastBreak = window.LastIndexOfAny(_chunkBreakCharacters); + if (lastBreak >= step) + { + window = window[..lastBreak]; + take = window.Length; + } + } + + var chunk = window.Trim(); + if (!string.IsNullOrWhiteSpace(chunk)) + { + yield return chunk; + } + + if (position + take >= normalized.Length) + { + yield break; + } + + position += step; + } + } + } +} diff --git a/LiteDB.Demo.Tools.VectorSearch/Utilities/VectorMath.cs b/LiteDB.Demo.Tools.VectorSearch/Utilities/VectorMath.cs new file mode 100644 index 000000000..b3609896f --- /dev/null +++ b/LiteDB.Demo.Tools.VectorSearch/Utilities/VectorMath.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace LiteDB.Demo.Tools.VectorSearch.Utilities +{ + internal static class VectorMath + { + public static double CosineSimilarity(IReadOnlyList left, IReadOnlyList right) + { + if (left == null || right == null) + { + return 0d; + } + + var length = Math.Min(left.Count, right.Count); + + if (length == 0) + { + return 0d; + } + + double dot = 0d; + double leftMagnitude = 0d; + double rightMagnitude = 0d; + + for (var i = 0; i < length; i++) + { + var l = left[i]; + var r = right[i]; + + dot += l * r; + leftMagnitude += l * l; + rightMagnitude += r * r; + } + + if (leftMagnitude <= double.Epsilon || rightMagnitude <= double.Epsilon) + { + return 0d; + } + + return dot / (Math.Sqrt(leftMagnitude) * Math.Sqrt(rightMagnitude)); + } + + public static double CosineDistance(IReadOnlyList left, IReadOnlyList right) + { + var similarity = CosineSimilarity(left, right); + return 1d - similarity; + } + } +} diff --git a/LiteDB.ReproRunner.Tests/CliApplicationTests.cs b/LiteDB.ReproRunner.Tests/CliApplicationTests.cs new file mode 100644 index 000000000..29563df31 --- /dev/null +++ b/LiteDB.ReproRunner.Tests/CliApplicationTests.cs @@ -0,0 +1,52 @@ +using LiteDB.ReproRunner.Cli.Commands; +using LiteDB.ReproRunner.Cli.Infrastructure; +using Spectre.Console; +using Spectre.Console.Testing; + +namespace LiteDB.ReproRunner.Tests; + +public sealed class CliApplicationTests +{ + [Fact] + public async Task Validate_ReturnsErrorCodeWhenManifestInvalid() + { + var tempRoot = Directory.CreateTempSubdirectory(); + try + { + var reproRoot = Path.Combine(tempRoot.FullName, "LiteDB.ReproRunner"); + var reproDirectory = Path.Combine(reproRoot, "Repros", "BadRepro"); + Directory.CreateDirectory(reproDirectory); + + var manifest = """ + { + "id": "Bad Repro", + "title": "", + "timeoutSeconds": 0, + "requiresParallel": false, + "defaultInstances": 0, + "state": "unknown" + } + """; + + await File.WriteAllTextAsync(Path.Combine(reproDirectory, "repro.json"), manifest); + await File.WriteAllTextAsync(Path.Combine(reproDirectory, "BadRepro.csproj"), ""); + + using var console = new TestConsole(); + var command = new ValidateCommand(console, new ReproRootLocator()); + var settings = new ValidateCommandSettings + { + All = true, + Root = reproRoot + }; + + var exitCode = command.Execute(null!, settings); + + Assert.Equal(2, exitCode); + Assert.Contains("INVALID", console.Output, StringComparison.OrdinalIgnoreCase); + } + finally + { + Directory.Delete(tempRoot.FullName, true); + } + } +} diff --git a/LiteDB.ReproRunner.Tests/LiteDB.ReproRunner.Tests.csproj b/LiteDB.ReproRunner.Tests/LiteDB.ReproRunner.Tests.csproj new file mode 100644 index 000000000..af7861839 --- /dev/null +++ b/LiteDB.ReproRunner.Tests/LiteDB.ReproRunner.Tests.csproj @@ -0,0 +1,29 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + + + + + + + + + diff --git a/LiteDB.ReproRunner.Tests/ManifestValidatorTests.cs b/LiteDB.ReproRunner.Tests/ManifestValidatorTests.cs new file mode 100644 index 000000000..91d03c09f --- /dev/null +++ b/LiteDB.ReproRunner.Tests/ManifestValidatorTests.cs @@ -0,0 +1,134 @@ +using System.Text.Json; +using LiteDB.ReproRunner.Cli.Manifests; + +namespace LiteDB.ReproRunner.Tests; + +public sealed class ManifestValidatorTests +{ + [Fact] + public void Validate_AllowsValidManifest() + { + const string json = """ + { + "id": "Issue_000_Demo", + "title": "Demo repro", + "issues": ["https://example.com/issue"], + "failingSince": "5.0.x", + "timeoutSeconds": 120, + "requiresParallel": false, + "defaultInstances": 1, + "sharedDatabaseKey": "demo", + "args": ["--flag"], + "tags": ["demo"], + "state": "red" + } + """; + + using var document = JsonDocument.Parse(json); + var validation = new ManifestValidationResult(); + var validator = new ManifestValidator(); + + var manifest = validator.Validate(document.RootElement, validation, out var rawId); + + Assert.NotNull(manifest); + Assert.True(validation.IsValid); + Assert.Equal("Issue_000_Demo", manifest!.Id); + Assert.Equal("Issue_000_Demo", rawId); + Assert.Equal(120, manifest.TimeoutSeconds); + Assert.Equal(ReproState.Red, manifest.State); + Assert.Null(manifest.ExpectedOutcomes.Package); + Assert.Null(manifest.ExpectedOutcomes.Latest); + } + + [Fact] + public void Validate_FailsWhenTimeoutOutOfRange() + { + const string json = """ + { + "id": "Issue_001", + "title": "Invalid timeout", + "timeoutSeconds": 0, + "requiresParallel": false, + "defaultInstances": 1, + "state": "green" + } + """; + + using var document = JsonDocument.Parse(json); + var validation = new ManifestValidationResult(); + var validator = new ManifestValidator(); + + var manifest = validator.Validate(document.RootElement, validation, out _); + + Assert.Null(manifest); + Assert.Contains(validation.Errors, error => error.Contains("$.timeoutSeconds", StringComparison.Ordinal)); + } + + [Fact] + public void Validate_ParsesExpectedOutcomes() + { + const string json = """ + { + "id": "Issue_002", + "title": "Expected outcomes", + "timeoutSeconds": 120, + "requiresParallel": false, + "defaultInstances": 1, + "state": "green", + "expectedOutcomes": { + "package": { + "kind": "hardFail", + "exitCode": -5, + "logContains": "NetworkException" + }, + "latest": { + "kind": "noRepro" + } + } + } + """; + + using var document = JsonDocument.Parse(json); + var validation = new ManifestValidationResult(); + var validator = new ManifestValidator(); + + var manifest = validator.Validate(document.RootElement, validation, out _); + + Assert.NotNull(manifest); + Assert.True(validation.IsValid); + Assert.Equal(ReproOutcomeKind.HardFail, manifest!.ExpectedOutcomes.Package!.Kind); + Assert.Equal(-5, manifest.ExpectedOutcomes.Package!.ExitCode); + Assert.Equal("NetworkException", manifest.ExpectedOutcomes.Package!.LogContains); + Assert.Equal(ReproOutcomeKind.NoRepro, manifest.ExpectedOutcomes.Latest!.Kind); + } + + [Fact] + public void Validate_FailsWhenLatestHardFailDeclared() + { + const string json = """ + { + "id": "Issue_003", + "title": "Invalid latest expectation", + "timeoutSeconds": 120, + "requiresParallel": false, + "defaultInstances": 1, + "state": "red", + "expectedOutcomes": { + "latest": { + "kind": "hardFail" + } + } + } + """; + + using var document = JsonDocument.Parse(json); + var validation = new ManifestValidationResult(); + var validator = new ManifestValidator(); + + var manifest = validator.Validate(document.RootElement, validation, out _); + + Assert.NotNull(manifest); + Assert.False(validation.IsValid); + Assert.Contains(validation.Errors, error => error.Contains("expectedOutcomes.latest.kind", StringComparison.Ordinal)); + } +} diff --git a/LiteDB.ReproRunner.Tests/ReproExecutorTests.cs b/LiteDB.ReproRunner.Tests/ReproExecutorTests.cs new file mode 100644 index 000000000..908942d5e --- /dev/null +++ b/LiteDB.ReproRunner.Tests/ReproExecutorTests.cs @@ -0,0 +1,101 @@ +using System; +using System.Text.Json; +using LiteDB.ReproRunner.Cli.Execution; +using LiteDB.ReproRunner.Shared; +using LiteDB.ReproRunner.Shared.Messaging; + +namespace LiteDB.ReproRunner.Tests; + +public sealed class ReproExecutorTests +{ + [Fact] + public void LogObserver_ReceivesEntries_WhenSuppressed() + { + var output = new StringWriter(); + var error = new StringWriter(); + var executor = new ReproExecutor(output, error) + { + SuppressConsoleLogOutput = true + }; + + var observed = new List(); + executor.LogObserver = entry => observed.Add(entry); + + var envelope = ReproHostMessageEnvelope.CreateLog("critical failure", ReproHostLogLevel.Error); + var json = JsonSerializer.Serialize(envelope, ReproJsonOptions.Default); + + var parsed = executor.TryProcessStructuredLine(json, 3); + + Assert.True(parsed); + Assert.Single(observed); + Assert.Equal(3, observed[0].InstanceIndex); + Assert.Equal("critical failure", observed[0].Message); + Assert.Equal(ReproHostLogLevel.Error, observed[0].Level); + Assert.Equal(string.Empty, output.ToString()); + Assert.Equal(string.Empty, error.ToString()); + } + + [Fact] + public void TryProcessStructuredLine_RejectsMessagesBeforeConfiguration() + { + var output = new StringWriter(); + var error = new StringWriter(); + var executor = new ReproExecutor(output, error); + executor.ConfigureExpectedConfiguration(false, "5.0.20", 1); + + var logEnvelope = ReproHostMessageEnvelope.CreateLog("hello", ReproHostLogLevel.Information); + var json = JsonSerializer.Serialize(logEnvelope, ReproJsonOptions.Default); + + var parsed = executor.TryProcessStructuredLine(json, 0); + + Assert.True(parsed); + Assert.Contains("configuration error", error.ToString()); + Assert.Contains("expected configuration handshake", error.ToString()); + Assert.Equal(string.Empty, output.ToString()); + } + + [Fact] + public void TryProcessStructuredLine_AllowsMessagesAfterValidConfiguration() + { + var output = new StringWriter(); + var error = new StringWriter(); + var executor = new ReproExecutor(output, error); + executor.ConfigureExpectedConfiguration(false, "5.0.20", 1); + + var configuration = ReproHostMessageEnvelope.CreateConfiguration(false, "5.0.20"); + var configurationJson = JsonSerializer.Serialize(configuration, ReproJsonOptions.Default); + var logEnvelope = ReproHostMessageEnvelope.CreateLog("ready", ReproHostLogLevel.Information); + var logJson = JsonSerializer.Serialize(logEnvelope, ReproJsonOptions.Default); + + var configParsed = executor.TryProcessStructuredLine(configurationJson, 0); + var logParsed = executor.TryProcessStructuredLine(logJson, 0); + + Assert.True(configParsed); + Assert.True(logParsed); + Assert.Contains("ready", output.ToString()); + Assert.Equal(string.Empty, error.ToString()); + } + + [Fact] + public void TryProcessStructuredLine_FlagsConfigurationMismatch() + { + var output = new StringWriter(); + var error = new StringWriter(); + var executor = new ReproExecutor(output, error); + executor.ConfigureExpectedConfiguration(false, "5.0.20", 1); + + var configuration = ReproHostMessageEnvelope.CreateConfiguration(true, "5.0.20"); + var configurationJson = JsonSerializer.Serialize(configuration, ReproJsonOptions.Default); + var parsed = executor.TryProcessStructuredLine(configurationJson, 0); + + Assert.True(parsed); + Assert.Contains("configuration error", error.ToString()); + Assert.Contains("UseProjectReference=True", error.ToString(), StringComparison.OrdinalIgnoreCase); + + var logEnvelope = ReproHostMessageEnvelope.CreateLog("ignored", ReproHostLogLevel.Information); + var logJson = JsonSerializer.Serialize(logEnvelope, ReproJsonOptions.Default); + executor.TryProcessStructuredLine(logJson, 0); + + Assert.Equal(string.Empty, output.ToString()); + } +} diff --git a/LiteDB.ReproRunner.Tests/ReproHostClientTests.cs b/LiteDB.ReproRunner.Tests/ReproHostClientTests.cs new file mode 100644 index 000000000..5bc987c3b --- /dev/null +++ b/LiteDB.ReproRunner.Tests/ReproHostClientTests.cs @@ -0,0 +1,100 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using LiteDB.ReproRunner.Cli.Execution; +using LiteDB.ReproRunner.Shared.Messaging; + +namespace LiteDB.ReproRunner.Tests; + +public sealed class ReproHostClientTests +{ + [Fact] + public async Task SendLogAsync_WritesStructuredEnvelope() + { + var writer = new StringWriter(); + var client = new ReproHostClient(writer, TextReader.Null, CreateOptions()); + + await client.SendLogAsync("hello world", ReproHostLogLevel.Warning); + + var output = writer.ToString().Trim(); + Assert.True(ReproHostMessageEnvelope.TryParse(output, out var envelope, out _)); + Assert.NotNull(envelope); + Assert.Equal(ReproHostMessageTypes.Log, envelope!.Type); + Assert.Equal(ReproHostLogLevel.Warning, envelope.Level); + Assert.Equal("hello world", envelope.Text); + } + + [Fact] + public async Task SendConfigurationAsync_WritesConfigurationEnvelope() + { + var writer = new StringWriter(); + var client = new ReproHostClient(writer, TextReader.Null, CreateOptions()); + + await client.SendConfigurationAsync(true, "5.0.20"); + + var output = writer.ToString().Trim(); + Assert.True(ReproHostMessageEnvelope.TryParse(output, out var envelope, out _)); + Assert.NotNull(envelope); + Assert.Equal(ReproHostMessageTypes.Configuration, envelope!.Type); + + var payload = envelope.DeserializePayload(); + Assert.NotNull(payload); + Assert.True(payload!.UseProjectReference); + Assert.Equal("5.0.20", payload.LiteDBPackageVersion); + } + + [Fact] + public async Task ReadAsync_ParsesHostReadyHandshake() + { + var envelope = ReproInputEnvelope.CreateHostReady("run-123", "/tmp/shared", 1, 2, "Issue_1234"); + var json = JsonSerializer.Serialize(envelope, CreateOptions()); + using var reader = new StringReader(json + Environment.NewLine); + var client = new ReproHostClient(TextWriter.Null, reader, CreateOptions()); + + var read = await client.ReadAsync(); + + Assert.NotNull(read); + Assert.Equal(ReproInputTypes.HostReady, read!.Type); + + var payload = read.DeserializePayload(); + Assert.NotNull(payload); + Assert.Equal("run-123", payload!.RunIdentifier); + Assert.Equal("/tmp/shared", payload.SharedDatabaseRoot); + Assert.Equal(1, payload.InstanceIndex); + Assert.Equal(2, payload.TotalInstances); + Assert.Equal("Issue_1234", payload.ManifestId); + } + + [Fact] + public void TryProcessStructuredLine_NotifiesObserver() + { + var output = new StringWriter(); + var error = new StringWriter(); + var executor = new ReproExecutor(output, error); + var observed = new List(); + executor.StructuredMessageObserver = (_, envelope) => observed.Add(envelope); + + var message = ReproHostMessageEnvelope.CreateResult(true, "completed"); + var json = JsonSerializer.Serialize(message, CreateOptions()); + + var parsed = executor.TryProcessStructuredLine(json, 0); + + Assert.True(parsed); + Assert.Single(observed); + Assert.Equal(ReproHostMessageTypes.Result, observed[0].Type); + Assert.True(observed[0].Success); + Assert.Equal("completed", observed[0].Text); + } + + private static JsonSerializerOptions CreateOptions() + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false + }; + + options.Converters.Add(new JsonStringEnumConverter()); + return options; + } +} diff --git a/LiteDB.ReproRunner.Tests/ReproOutcomeEvaluatorTests.cs b/LiteDB.ReproRunner.Tests/ReproOutcomeEvaluatorTests.cs new file mode 100644 index 000000000..83dfbaa7a --- /dev/null +++ b/LiteDB.ReproRunner.Tests/ReproOutcomeEvaluatorTests.cs @@ -0,0 +1,119 @@ +using System; +using LiteDB.ReproRunner.Cli.Execution; +using LiteDB.ReproRunner.Cli.Manifests; + +namespace LiteDB.ReproRunner.Tests; + +public sealed class ReproOutcomeEvaluatorTests +{ + private static readonly ReproOutcomeEvaluator Evaluator = new(); + + [Fact] + public void Evaluate_AllowsRedReproWhenLatestStillFails() + { + var manifest = CreateManifest(ReproState.Red); + var packageResult = CreateResult(useProjectReference: false, exitCode: 0); + var latestResult = CreateResult(useProjectReference: true, exitCode: 0); + + var evaluation = Evaluator.Evaluate(manifest, packageResult, latestResult); + + Assert.False(evaluation.ShouldFail); + Assert.True(evaluation.Package.Met); + Assert.True(evaluation.Latest.Met); + } + + [Fact] + public void Evaluate_FailsGreenWhenLatestStillReproduces() + { + var manifest = CreateManifest(ReproState.Green); + var packageResult = CreateResult(false, 0); + var latestResult = CreateResult(true, 0); + + var evaluation = Evaluator.Evaluate(manifest, packageResult, latestResult); + + Assert.True(evaluation.ShouldFail); + Assert.True(evaluation.Package.Met); + Assert.False(evaluation.Latest.Met); + } + + [Fact] + public void Evaluate_RespectsHardFailExpectationWhenLogMatches() + { + var expectation = new ReproVariantOutcomeExpectations( + new ReproOutcomeExpectation(ReproOutcomeKind.HardFail, -5, "NetworkException"), + null); + var manifest = CreateManifest(ReproState.Green, expectation); + var packageResult = CreateResult(false, -5, "NetworkException at socket"); + + var evaluation = Evaluator.Evaluate(manifest, packageResult, null); + + Assert.False(evaluation.Package.ShouldFail); + Assert.True(evaluation.Package.Met); + Assert.Equal(ReproOutcomeKind.HardFail, evaluation.Package.Expectation.Kind); + } + + [Fact] + public void Evaluate_FailsHardFailWhenLogMissing() + { + var expectation = new ReproVariantOutcomeExpectations( + new ReproOutcomeExpectation(ReproOutcomeKind.HardFail, -5, "NetworkException"), + null); + var manifest = CreateManifest(ReproState.Green, expectation); + var packageResult = CreateResult(false, -5, "No matching text"); + + var evaluation = Evaluator.Evaluate(manifest, packageResult, null); + + Assert.True(evaluation.Package.ShouldFail); + Assert.False(evaluation.Package.Met); + Assert.Contains("NetworkException", evaluation.Package.FailureReason); + } + + [Fact] + public void Evaluate_WarnsFlakyLatestMismatch() + { + var manifest = CreateManifest(ReproState.Flaky); + var packageResult = CreateResult(false, 0); + var latestResult = CreateResult(true, 1); + + var evaluation = Evaluator.Evaluate(manifest, packageResult, latestResult); + + Assert.False(evaluation.ShouldFail); + Assert.True(evaluation.ShouldWarn); + Assert.True(evaluation.Package.Met); + Assert.False(evaluation.Latest.Met); + } + + private static ReproManifest CreateManifest(ReproState state, ReproVariantOutcomeExpectations? expectations = null) + { + return new ReproManifest( + "Issue_Example", + "Example", + Array.Empty(), + null, + 120, + false, + 1, + null, + Array.Empty(), + Array.Empty(), + state, + expectations ?? ReproVariantOutcomeExpectations.Empty); + } + + private static ReproExecutionResult CreateResult(bool useProjectReference, int exitCode, string? output = null) + { + var captured = output is null + ? Array.Empty() + : new[] + { + new ReproExecutionCapturedLine(ReproExecutionStream.StandardOutput, output) + }; + + return new ReproExecutionResult( + useProjectReference, + exitCode == 0, + exitCode, + TimeSpan.FromSeconds(1), + captured); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ListCommand.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ListCommand.cs new file mode 100644 index 000000000..bd1d70178 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ListCommand.cs @@ -0,0 +1,147 @@ +using System.IO; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using LiteDB.ReproRunner.Cli.Infrastructure; +using LiteDB.ReproRunner.Cli.Manifests; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class ListCommand : Command +{ + private readonly IAnsiConsole _console; + private readonly ReproRootLocator _rootLocator; + + /// + /// Initializes a new instance of the class. + /// + /// The console used to render output. + /// Resolves the repro root directory. + public ListCommand(IAnsiConsole console, ReproRootLocator rootLocator) + { + _console = console ?? throw new ArgumentNullException(nameof(console)); + _rootLocator = rootLocator ?? throw new ArgumentNullException(nameof(rootLocator)); + } + + /// + /// Executes the list command. + /// + /// The Spectre command context. + /// The user-provided settings. + /// The process exit code. + public override int Execute(CommandContext context, ListCommandSettings settings) + { + var repository = new ManifestRepository(_rootLocator.ResolveRoot(settings.Root)); + var manifests = repository.Discover(); + var valid = manifests.Where(x => x.IsValid).ToList(); + var invalid = manifests.Where(x => !x.IsValid).ToList(); + Regex? filter = null; + + if (!string.IsNullOrWhiteSpace(settings.Filter)) + { + try + { + filter = new Regex(settings.Filter, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + } + catch (ArgumentException ex) + { + _console.MarkupLine($"[red]Invalid --filter pattern[/]: {Markup.Escape(ex.Message)}"); + return 1; + } + } + + if (filter is not null) + { + valid = valid.Where(repro => MatchesFilter(repro, filter)).ToList(); + invalid = invalid.Where(repro => MatchesFilter(repro, filter)).ToList(); + } + + if (settings.Json) + { + WriteJson(_console, valid, invalid); + } + else + { + CliOutput.PrintList(_console, valid); + + foreach (var repro in invalid) + { + CliOutput.PrintInvalid(_console, repro); + } + } + + if (settings.Strict && invalid.Count > 0) + { + return 2; + } + + return 0; + } + + private static bool MatchesFilter(DiscoveredRepro repro, Regex filter) + { + var identifier = repro.Manifest?.Id ?? repro.RawId ?? repro.RelativeManifestPath; + return identifier is not null && filter.IsMatch(identifier); + } + + private static void WriteJson(IAnsiConsole console, IReadOnlyList valid, IReadOnlyList invalid) + { + var validEntries = valid + .Where(item => item.Manifest is not null) + .Select(item => item.Manifest!) + .Select(manifest => + { + var supports = manifest.Supports.Count > 0 ? manifest.Supports : new[] { "any" }; + object? os = null; + + if (manifest.OsConstraints is not null && + (manifest.OsConstraints.IncludePlatforms.Count > 0 || + manifest.OsConstraints.IncludeLabels.Count > 0 || + manifest.OsConstraints.ExcludePlatforms.Count > 0 || + manifest.OsConstraints.ExcludeLabels.Count > 0)) + { + os = new + { + includePlatforms = manifest.OsConstraints.IncludePlatforms, + includeLabels = manifest.OsConstraints.IncludeLabels, + excludePlatforms = manifest.OsConstraints.ExcludePlatforms, + excludeLabels = manifest.OsConstraints.ExcludeLabels + }; + } + + return new + { + name = manifest.Id, + supports, + os + }; + }) + .ToList(); + + var invalidEntries = invalid + .Select(item => new + { + name = item.Manifest?.Id ?? item.RawId ?? item.RelativeManifestPath.Replace(Path.DirectorySeparatorChar, '/'), + errors = item.Validation.Errors.ToArray() + }) + .Where(entry => entry.errors.Length > 0) + .ToList(); + + var payload = new + { + repros = validEntries, + invalid = invalidEntries.Count > 0 ? invalidEntries : null + }; + + var options = new JsonSerializerOptions + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + var json = JsonSerializer.Serialize(payload, options); + console.WriteLine(json); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ListCommandSettings.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ListCommandSettings.cs new file mode 100644 index 000000000..1d5d6164a --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ListCommandSettings.cs @@ -0,0 +1,28 @@ +using System.ComponentModel; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class ListCommandSettings : RootCommandSettings +{ + /// + /// Gets or sets a value indicating whether the command should fail when invalid manifests exist. + /// + [CommandOption("--strict")] + [Description("Return exit code 2 if any manifests are invalid.")] + public bool Strict { get; set; } + + /// + /// Gets or sets a value indicating whether output should be emitted as JSON. + /// + [CommandOption("--json")] + [Description("Emit the repro inventory as JSON instead of a rendered table.")] + public bool Json { get; set; } + + /// + /// Gets or sets an optional regular expression used to filter repro identifiers. + /// + [CommandOption("--filter ")] + [Description("Return only repros whose identifiers match the supplied regular expression.")] + public string? Filter { get; set; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RootCommandSettings.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RootCommandSettings.cs new file mode 100644 index 000000000..5d73dd030 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RootCommandSettings.cs @@ -0,0 +1,14 @@ +using System.ComponentModel; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal class RootCommandSettings : CommandSettings +{ + /// + /// Gets or sets the root directory that contains repro manifests. + /// + [CommandOption("--root ")] + [Description("Override the LiteDB.ReproRunner root directory.")] + public string? Root { get; set; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RunCommand.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RunCommand.cs new file mode 100644 index 000000000..c98e5f1cb --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RunCommand.cs @@ -0,0 +1,850 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Channels; +using System.Xml.Linq; +using LiteDB.ReproRunner.Cli.Execution; +using LiteDB.ReproRunner.Cli.Infrastructure; +using LiteDB.ReproRunner.Cli.Manifests; +using LiteDB.ReproRunner.Shared.Messaging; +using Spectre.Console; +using Spectre.Console.Cli; +using Spectre.Console.Rendering; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class RunCommand : AsyncCommand +{ + private const int MaxLogLines = 5; + + private readonly IAnsiConsole _console; + private readonly ReproRootLocator _rootLocator; + private readonly RunDirectoryPlanner _planner; + private readonly ReproBuildCoordinator _buildCoordinator; + private readonly ReproExecutor _executor; + private readonly CancellationToken _cancellationToken; + private readonly ReproOutcomeEvaluator _outcomeEvaluator = new(); + + /// + /// Initializes a new instance of the class. + /// + /// The console used to render output. + /// Resolves the repro root directory. + /// Creates deterministic run directories. + /// Builds repro variants before execution. + /// Executes repro variants. + /// Signals cancellation requests. + public RunCommand( + IAnsiConsole console, + ReproRootLocator rootLocator, + RunDirectoryPlanner planner, + ReproBuildCoordinator buildCoordinator, + ReproExecutor executor, + CancellationToken cancellationToken) + { + _console = console ?? throw new ArgumentNullException(nameof(console)); + _rootLocator = rootLocator ?? throw new ArgumentNullException(nameof(rootLocator)); + _planner = planner ?? throw new ArgumentNullException(nameof(planner)); + _buildCoordinator = buildCoordinator ?? throw new ArgumentNullException(nameof(buildCoordinator)); + _executor = executor ?? throw new ArgumentNullException(nameof(executor)); + _cancellationToken = cancellationToken; + } + + /// + /// Executes the run command. + /// + /// The Spectre command context. + /// The run settings provided by the user. + /// The process exit code. + public override async Task ExecuteAsync(CommandContext context, RunCommandSettings settings) + { + _cancellationToken.ThrowIfCancellationRequested(); + var repository = new ManifestRepository(_rootLocator.ResolveRoot(settings.Root)); + var manifests = repository.Discover(); + var selected = settings.All + ? manifests.ToList() + : manifests + .Where(x => string.Equals(x.Manifest?.Id ?? x.RawId, settings.Id, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (selected.Count == 0) + { + if (settings.All) + { + _console.MarkupLine("[yellow]No repros discovered.[/]"); + return 0; + } + + _console.MarkupLine($"[red]Repro '{Markup.Escape(settings.Id!)}' was not found.[/]"); + return 1; + } + + + var report = new RunReport { Root = repository.RootPath }; + var table = new Table() + .Border(TableBorder.Rounded) + .Expand() + .AddColumns("Repro", "Repro Version", "Reproduced", "Fixed", "Overall"); + var overallExitCode = 0; + var plannedVariants = new List(); + var buildFailures = new List(); + var useLiveDisplay = ShouldUseLiveDisplay(); + + try + { + if (useLiveDisplay) + { + var logLines = new List(); + var targetFps = settings.Fps ?? RunCommandSettings.DefaultFps; + var layout = new Layout("root") + .SplitRows( + new Layout("logs").Size(8), + new Layout("results")); + + layout["results"].Update(table); + layout["logs"].Update(CreateLogView(logLines, targetFps)); + + var uiUpdates = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + AllowSynchronousContinuations = false + }); + + try + { + await _console.Live(layout).StartAsync(async ctx => + { + var uiTask = ProcessUiUpdatesAsync(uiUpdates.Reader, table, layout, logLines, targetFps, ctx, _cancellationToken); + var writer = uiUpdates.Writer; + var rowStates = new Dictionary(); + var previousObserver = _executor.LogObserver; + var previousSuppression = _executor.SuppressConsoleLogOutput; + _executor.SuppressConsoleLogOutput = true; + _executor.LogObserver = entry => + { + var formatted = FormatLogLine(entry); + writer.TryWrite(new LogLineUpdate(formatted)); + }; + + void HandleInitialRow(ReproRowState state) + { + rowStates[state.ReproId] = state; + writer.TryWrite(new TableRowUpdate(state.ReproId, state.ReproVersion, state.Reproduced, state.Fixed, state.Overall)); + } + + void HandleRowUpdate(ReproRowState state) + { + rowStates[state.ReproId] = state; + writer.TryWrite(new TableRefreshUpdate(new Dictionary(rowStates))); + } + + void LogLine(string message) + { + writer.TryWrite(new LogLineUpdate(message)); + } + + void LogBuild(string message) + { + writer.TryWrite(new LogLineUpdate($"BUILD: {message}")); + } + + try + { + var result = await RunExecutionLoopAsync( + selected, + settings, + report, + plannedVariants, + buildFailures, + HandleInitialRow, + HandleRowUpdate, + LogLine, + LogBuild, + _cancellationToken).ConfigureAwait(false); + overallExitCode = result.ExitCode; + } + finally + { + _executor.LogObserver = previousObserver; + _executor.SuppressConsoleLogOutput = previousSuppression; + writer.TryComplete(); + + try + { + await uiTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + } + }).ConfigureAwait(false); + } + finally + { + uiUpdates.Writer.TryComplete(); + } + } + else + { + var previousObserver = _executor.LogObserver; + var previousSuppression = _executor.SuppressConsoleLogOutput; + RunExecutionResult result; + + try + { + _executor.SuppressConsoleLogOutput = true; + _executor.LogObserver = entry => + { + var formatted = FormatLogLine(entry); + _console.MarkupLine(formatted); + }; + + result = await RunExecutionLoopAsync( + selected, + settings, + report, + plannedVariants, + buildFailures, + _ => { }, + _ => { }, + message => _console.MarkupLine(message), + message => _console.MarkupLine($"BUILD: {Markup.Escape(message)}"), + _cancellationToken).ConfigureAwait(false); + overallExitCode = result.ExitCode; + } + finally + { + _executor.LogObserver = previousObserver; + _executor.SuppressConsoleLogOutput = previousSuppression; + } + + _console.WriteLine(); + var finalTable = new Table() + .Border(TableBorder.Rounded) + .Expand() + .AddColumns("Repro", "Repro Version", "Reproduced", "Fixed", "Overall"); + + foreach (var state in result.States.Values.OrderBy(s => s.ReproId)) + { + finalTable.AddRow(state.ReproId, state.ReproVersion, state.Reproduced, state.Fixed, state.Overall); + } + + _console.Write(finalTable); + } + + if (buildFailures.Count > 0) + { + _console.WriteLine(); + foreach (var failure in buildFailures) + { + _console.MarkupLine($"[red]Build failed for {Markup.Escape(failure.ManifestId)} ({Markup.Escape(failure.Variant)}).[/]"); + + if (failure.Output.Count == 0) + { + _console.MarkupLine("[yellow](No build output captured.)[/]"); + } + else + { + foreach (var line in failure.Output) + { + _console.WriteLine(line); + } + } + + _console.WriteLine(); + } + } + } + finally + { + foreach (var plan in plannedVariants) + { + plan.Dispose(); + } + } + + if (settings.ReportPath is string reportPath) + { + await WriteReportAsync(report, reportPath, settings.ReportFormat, _cancellationToken).ConfigureAwait(false); + } + + return overallExitCode; + } + + + private async Task RunExecutionLoopAsync( + IReadOnlyList selected, + RunCommandSettings settings, + RunReport report, + List plannedVariants, + List buildFailures, + Action onInitialRow, + Action onRowStateUpdated, + Action logLine, + Action logBuild, + CancellationToken cancellationToken) + { + var overallExitCode = 0; + var candidates = new List(); + var finalStates = new Dictionary(); + + foreach (var repro in selected) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!repro.IsValid) + { + if (!settings.SkipValidation) + { + var state = new ReproRowState(Markup.Escape(repro.RawId ?? "(unknown)"), "[red]n/a[/]", "[red]❌[/]", "[red]❌[/]", "[red]Invalid[/]"); + onInitialRow(state); + finalStates[state.ReproId] = state; + overallExitCode = overallExitCode == 0 ? 2 : overallExitCode; + continue; + } + + if (repro.Manifest is null) + { + var state = new ReproRowState(Markup.Escape(repro.RawId ?? "(unknown)"), "[red]n/a[/]", "[red]❌[/]", "[red]❌[/]", "[red]Invalid[/]"); + onInitialRow(state); + finalStates[state.ReproId] = state; + overallExitCode = overallExitCode == 0 ? 2 : overallExitCode; + continue; + } + } + + if (repro.Manifest is null) + { + var state = new ReproRowState(Markup.Escape(repro.RawId ?? "(unknown)"), "[red]n/a[/]", "[red]❌[/]", "[red]❌[/]", "[red]Missing[/]"); + onInitialRow(state); + finalStates[state.ReproId] = state; + overallExitCode = overallExitCode == 0 ? 2 : overallExitCode; + continue; + } + + var manifest = repro.Manifest; + var instances = settings.Instances ?? manifest.DefaultInstances; + + if (manifest.RequiresParallel && instances < 2) + { + var state = new ReproRowState(Markup.Escape(manifest.Id), "[red]n/a[/]", "[red]❌[/]", "[red]❌[/]", "[red]Config Error[/]"); + onInitialRow(state); + finalStates[state.ReproId] = state; + overallExitCode = 1; + continue; + } + + if (repro.ProjectPath is null) + { + var state = new ReproRowState(Markup.Escape(manifest.Id), "[red]n/a[/]", "[red]❌[/]", "[red]❌[/]", "[red]Project Missing[/]"); + onInitialRow(state); + finalStates[state.ReproId] = state; + overallExitCode = overallExitCode == 0 ? 2 : overallExitCode; + continue; + } + + var timeoutSeconds = settings.Timeout ?? manifest.TimeoutSeconds; + var packageVersion = TryResolvePackageVersion(repro.ProjectPath); + var packageDisplay = packageVersion ?? "NuGet"; + var packageVariantId = BuildVariantIdentifier(packageVersion); + var failingSince = string.IsNullOrWhiteSpace(manifest.FailingSince) + ? packageDisplay + : manifest.FailingSince!; + var reproVersionCell = Markup.Escape(failingSince); + var pendingOverallCell = FormatOverallPending(manifest.State); + + var packagePlan = _planner.CreateVariantPlan( + repro, + manifest.Id, + packageVariantId, + packageDisplay, + useProjectReference: false, + liteDbPackageVersion: packageVersion); + + var latestPlan = _planner.CreateVariantPlan( + repro, + manifest.Id, + "ver_latest", + "Latest", + useProjectReference: true, + liteDbPackageVersion: packageVersion); + + plannedVariants.Add(packagePlan); + plannedVariants.Add(latestPlan); + + var candidate = new RunCandidate( + manifest, + instances, + timeoutSeconds, + packageDisplay, + packagePlan, + latestPlan, + reproVersionCell); + + candidates.Add(candidate); + + var pendingState = new ReproRowState(manifest.Id, reproVersionCell, "[yellow]⏳[/]", "[yellow]⏳[/]", pendingOverallCell); + onRowStateUpdated(pendingState); + finalStates[manifest.Id] = pendingState; + + logLine($"Discovered repro: {Markup.Escape(manifest.Id)}"); + } + + if (candidates.Count == 0) + { + return new RunExecutionResult(overallExitCode, finalStates); + } + + foreach (var candidate in candidates) + { + var buildingState = new ReproRowState( + candidate.Manifest.Id, + candidate.ReproVersionCell, + "[yellow]Building...[/]", + "[yellow]⏳[/]", + FormatOverallPending(candidate.Manifest.State)); + onRowStateUpdated(buildingState); + finalStates[candidate.Manifest.Id] = buildingState; + } + + logBuild($"Starting build for {plannedVariants.Count} variants across {candidates.Count} repros"); + var buildResults = await _buildCoordinator.BuildAsync(plannedVariants, cancellationToken).ConfigureAwait(false); + logBuild($"Build completed. Processing {buildResults.Count()} results"); + var buildLookup = buildResults.ToDictionary(result => result.Plan); + + foreach (var candidate in candidates) + { + var packageBuild = buildLookup[candidate.PackagePlan]; + var latestBuild = buildLookup[candidate.LatestPlan]; + + if (!packageBuild.Succeeded) + { + logBuild($"Package build failed for {Markup.Escape(candidate.Manifest.Id)} ({Markup.Escape(candidate.PackageDisplay)})"); + var failedState = new ReproRowState(candidate.Manifest.Id, candidate.ReproVersionCell, "[red]Build Failed[/]", "[red]❌[/]", "[red]Build Failed[/]"); + onRowStateUpdated(failedState); + finalStates[candidate.Manifest.Id] = failedState; + overallExitCode = overallExitCode == 0 ? 1 : overallExitCode; + buildFailures.Add(new BuildFailure(candidate.Manifest.Id, candidate.PackageDisplay, packageBuild.Output)); + } + + if (!latestBuild.Succeeded) + { + logBuild($"Latest build failed for {Markup.Escape(candidate.Manifest.Id)}"); + if (packageBuild.Succeeded) + { + var partialState = new ReproRowState(candidate.Manifest.Id, candidate.ReproVersionCell, "[yellow]⏳[/]", "[red]Build Failed[/]", "[red]Build Failed[/]"); + onRowStateUpdated(partialState); + finalStates[candidate.Manifest.Id] = partialState; + } + + overallExitCode = overallExitCode == 0 ? 1 : overallExitCode; + buildFailures.Add(new BuildFailure(candidate.Manifest.Id, "Latest", latestBuild.Output)); + } + + ReproExecutionResult? packageResult = null; + ReproExecutionResult? latestResult = null; + + if (packageBuild.Succeeded) + { + logBuild($"Build succeeded for {Markup.Escape(candidate.Manifest.Id)} ({Markup.Escape(candidate.PackageDisplay)}), starting execution"); + var runningState = new ReproRowState(candidate.Manifest.Id, candidate.ReproVersionCell, "[yellow]Running...[/]", latestBuild.Succeeded ? "[yellow]⏳[/]" : "[yellow]⏳[/]", FormatOverallPending(candidate.Manifest.State)); + onRowStateUpdated(runningState); + finalStates[candidate.Manifest.Id] = runningState; + packageResult = await _executor.ExecuteAsync(packageBuild, candidate.Instances, candidate.TimeoutSeconds, cancellationToken).ConfigureAwait(false); + } + + if (latestBuild.Succeeded) + { + var interimPackageStatus = packageResult is null + ? (packageBuild.Succeeded ? "[yellow]⏳[/]" : "[red]Build Failed[/]") + : "[yellow]Completed[/]"; + var latestRunningState = new ReproRowState(candidate.Manifest.Id, candidate.ReproVersionCell, interimPackageStatus, "[yellow]Running...[/]", FormatOverallPending(candidate.Manifest.State)); + onRowStateUpdated(latestRunningState); + finalStates[candidate.Manifest.Id] = latestRunningState; + latestResult = await _executor.ExecuteAsync(latestBuild, candidate.Instances, candidate.TimeoutSeconds, cancellationToken).ConfigureAwait(false); + } + + var evaluation = _outcomeEvaluator.Evaluate(candidate.Manifest, packageResult, latestResult); + var packageCell = FormatVariantCell(evaluation.Package); + var latestCell = FormatVariantCell(evaluation.Latest, false); + var overallState = ComputeOverallState(evaluation); + var overallCell = FormatOverallCell(evaluation, overallState); + var finalState = new ReproRowState(candidate.Manifest.Id, candidate.ReproVersionCell, packageCell, latestCell, overallCell); + onRowStateUpdated(finalState); + finalStates[candidate.Manifest.Id] = finalState; + + if (evaluation.Package.ShouldFail && evaluation.Package.FailureReason is string packageReason) + { + logLine($"FAIL: {Markup.Escape(candidate.Manifest.Id)} package - {Markup.Escape(packageReason)}"); + } + + if (evaluation.Latest.ShouldFail && evaluation.Latest.FailureReason is string latestReason) + { + logLine($"FAIL: {Markup.Escape(candidate.Manifest.Id)} latest - {Markup.Escape(latestReason)}"); + } + else if (evaluation.Latest.ShouldWarn && evaluation.Latest.FailureReason is string latestWarning) + { + logLine($"WARN: {Markup.Escape(candidate.Manifest.Id)} latest - {Markup.Escape(latestWarning)}"); + } + + if (evaluation.ShouldFail) + { + overallExitCode = overallExitCode == 0 ? 1 : overallExitCode; + } + + report.Add(new RunReportEntry + { + Id = candidate.Manifest.Id, + State = overallState, + Failed = evaluation.ShouldFail, + Warned = evaluation.ShouldWarn, + Package = CreateReportVariant(evaluation.Package, candidate.PackagePlan.UseProjectReference), + Latest = CreateReportVariant(evaluation.Latest, candidate.LatestPlan.UseProjectReference) + }); + + logLine($"Completed execution for {Markup.Escape(candidate.Manifest.Id)}"); + } + + return new RunExecutionResult(overallExitCode, finalStates); + } + + private bool ShouldUseLiveDisplay() + { + if (IsCiEnvironment()) + { + return false; + } + + return _console.Profile.Capabilities.Interactive; + } + + private static bool IsCiEnvironment() + { + static bool IsTrue(string? value) => !string.IsNullOrEmpty(value) && string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + return IsTrue(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")) || IsTrue(Environment.GetEnvironmentVariable("CI")); + } + + private static async Task WriteReportAsync(RunReport report, string path, string? format, CancellationToken cancellationToken) + { + if (format is not null && !string.Equals(format, "json", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Unsupported report format '{format}'."); + } + + var serializerOptions = new JsonSerializerOptions + { + WriteIndented = true + }; + + var json = JsonSerializer.Serialize(report, serializerOptions); + + if (string.Equals(path, "-", StringComparison.Ordinal)) + { + await Console.Out.WriteLineAsync(json).ConfigureAwait(false); + return; + } + + var fullPath = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(fullPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + await File.WriteAllTextAsync(fullPath, json, cancellationToken).ConfigureAwait(false); + } + + private static RunReportVariant CreateReportVariant(ReproVariantEvaluation evaluation, bool useProjectReference) + { + var capturedLines = Array.Empty(); + + if (evaluation.Result is ReproExecutionResult result) + { + capturedLines = result.CapturedOutput + .Select(line => new RunReportCapturedLine + { + Stream = line.Stream == ReproExecutionStream.StandardOutput ? "stdout" : "stderr", + Text = line.Text ?? string.Empty + }) + .ToArray(); + } + + return new RunReportVariant + { + Expected = evaluation.Expectation.Kind, + ExpectedExitCode = evaluation.Expectation.ExitCode, + ExpectedLogContains = evaluation.Expectation.LogContains, + Actual = evaluation.ActualKind, + Met = evaluation.Met, + ExitCode = evaluation.Result?.ExitCode, + DurationSeconds = evaluation.Result?.Duration.TotalSeconds, + UseProjectReference = evaluation.Result?.UseProjectReference ?? useProjectReference, + FailureReason = evaluation.FailureReason, + Output = capturedLines + }; + } + + private static string FormatVariantCell(ReproVariantEvaluation evaluation, bool judgeOnMet = true) + { + var symbol1 = evaluation.Result?.Reproduced switch + { + true => "[green]✅[/]", + false => "[red]❌[/]", + null => "[yellow]⚠️[/]" + }; + + return FormatVariantCell(evaluation, symbol1); + } + + private static string FormatVariantCell(ReproVariantEvaluation evaluation, string symbol) + { + var detail = evaluation.Result is { } result + ? string.Format(CultureInfo.InvariantCulture, "exit {0}", result.ExitCode) + : "no-run"; + + var expectation = evaluation.Expectation.Kind switch + { + ReproOutcomeKind.Reproduce => "repro", + ReproOutcomeKind.NoRepro => "no-repro", + ReproOutcomeKind.HardFail => "hard-fail", + _ => evaluation.Expectation.Kind.ToString().ToLowerInvariant() + }; + + return $"{symbol} {detail} [dim](exp {expectation})[/]"; + } + + private static ReproState ComputeOverallState(ReproRunEvaluation evaluation) + { + if (evaluation.ShouldFail) + { + return ReproState.Red; + } + + if (evaluation.ShouldWarn) + { + return ReproState.Flaky; + } + + return ReproState.Green; + } + + private static string FormatOverallCell(ReproRunEvaluation evaluation, ReproState overallState) + { + var symbol = evaluation.ShouldFail + ? "[red]❌[/]" + : evaluation.ShouldWarn + ? "[yellow]⚠️[/]" + : "[green]✅[/]"; + + return $"{symbol} {FormatReproState(overallState)}"; + } + + private static string FormatOverallPending(ReproState state) + { + return $"[yellow]⏳[/] {FormatReproState(state)}"; + } + + private static string FormatReproState(ReproState state) + { + return state switch + { + ReproState.Red => "[red]red[/]", + ReproState.Green => "[green]green[/]", + ReproState.Flaky => "[yellow]flaky[/]", + _ => Markup.Escape(state.ToString().ToLowerInvariant()) + }; + } + + private static string BuildVariantIdentifier(string? packageVersion) + { + if (string.IsNullOrWhiteSpace(packageVersion)) + { + return "ver_package"; + } + + var normalized = packageVersion.Replace('.', '_').Replace('-', '_'); + return $"ver_{normalized}"; + } + + private static string? TryResolvePackageVersion(string? projectPath) + { + if (projectPath is null || !File.Exists(projectPath)) + { + return null; + } + + try + { + var document = XDocument.Load(projectPath); + var ns = document.Root?.Name.Namespace ?? XNamespace.None; + + var versionElement = document + .Descendants(ns + "LiteDBPackageVersion") + .FirstOrDefault(); + + if (versionElement is not null && !string.IsNullOrWhiteSpace(versionElement.Value)) + { + return versionElement.Value.Trim(); + } + + var packageReference = document + .Descendants(ns + "PackageReference") + .FirstOrDefault(e => string.Equals(e.Attribute("Include")?.Value, "LiteDB", StringComparison.OrdinalIgnoreCase)); + + var version = packageReference?.Attribute("Version")?.Value; + if (!string.IsNullOrWhiteSpace(version)) + { + return version!.Trim(); + } + } + catch + { + } + + return null; + } + + private sealed record RunExecutionResult(int ExitCode, Dictionary States); + + private sealed record RunCandidate( + ReproManifest Manifest, + int Instances, + int TimeoutSeconds, + string PackageDisplay, + RunVariantPlan PackagePlan, + RunVariantPlan LatestPlan, + string ReproVersionCell); + + private sealed record BuildFailure(string ManifestId, string Variant, IReadOnlyList Output); + + private static IRenderable CreateLogView(IReadOnlyList lines, decimal fps) + { + var logTable = new Table().Border(TableBorder.Rounded).Expand(); + var fpsLabel = fps <= 0 + ? "Unlimited" + : string.Format(CultureInfo.InvariantCulture, "{0:0.0}", fps); + logTable.AddColumn(new TableColumn($"[bold]Recent Logs[/] [dim](FPS: {fpsLabel})[/]").LeftAligned()); + + if (lines.Count == 0) + { + logTable.AddRow("[dim]No log entries.[/]"); + } + else + { + foreach (var line in lines) + { + logTable.AddRow(line); + } + } + + return logTable; + } + + private static string FormatLogLine(ReproExecutionLogEntry entry) + { + var levelMarkup = entry.Level switch + { + ReproHostLogLevel.Error or ReproHostLogLevel.Critical => "[red]ERR[/]", + ReproHostLogLevel.Warning => "[yellow]WRN[/]", + ReproHostLogLevel.Debug => "[grey]DBG[/]", + ReproHostLogLevel.Trace => "[grey]TRC[/]", + _ => "[grey]INF[/]" + }; + + return $"{levelMarkup} [dim]#{entry.InstanceIndex}[/] {Markup.Escape(entry.Message)}"; + } + + private static async Task ProcessUiUpdatesAsync( + ChannelReader reader, + Table table, + Layout layout, + List logLines, + decimal fps, + LiveDisplayContext context, + CancellationToken cancellationToken) + { + var refreshInterval = CalculateRefreshInterval(fps); + var nextRefreshTime = DateTimeOffset.MinValue; + var needsRefresh = false; + + try + { + await foreach (var update in reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) + { + switch (update) + { + case LogLineUpdate logUpdate: + logLines.Add(logUpdate.Line); + while (logLines.Count > MaxLogLines) + { + logLines.RemoveAt(0); + } + + layout["logs"].Update(CreateLogView(logLines, fps)); + break; + case TableRowUpdate rowUpdate: + table.AddRow(rowUpdate.ReproId, rowUpdate.ReproVersion, rowUpdate.Reproduced, rowUpdate.Fixed, rowUpdate.Overall); + break; + case TableRefreshUpdate refreshUpdate: + // Rebuild the entire table with current states + var newTable = new Table() + .Border(TableBorder.Rounded) + .Expand() + .AddColumns("Repro", "Repro Version", "Reproduced", "Fixed", "Overall"); + foreach (var state in refreshUpdate.RowStates.Values.OrderBy(s => s.ReproId)) + { + newTable.AddRow(state.ReproId, state.ReproVersion, state.Reproduced, state.Fixed, state.Overall); + } + layout["results"].Update(newTable); + break; + } + + needsRefresh = true; + + if (refreshInterval == TimeSpan.Zero || DateTimeOffset.UtcNow >= nextRefreshTime) + { + context.Refresh(); + needsRefresh = false; + + if (refreshInterval != TimeSpan.Zero) + { + nextRefreshTime = DateTimeOffset.UtcNow + refreshInterval; + } + } + } + } + catch (OperationCanceledException) + { + } + finally + { + if (needsRefresh) + { + context.Refresh(); + } + } + } + + private static TimeSpan CalculateRefreshInterval(decimal fps) + { + if (fps <= 0) + { + return TimeSpan.Zero; + } + + var secondsPerFrame = (double)(1m / fps); + return TimeSpan.FromSeconds(secondsPerFrame); + } + + private abstract record UiUpdate; + + private sealed record LogLineUpdate(string Line) : UiUpdate; + + private sealed record TableRowUpdate(string ReproId, string ReproVersion, string Reproduced, string Fixed, string Overall) : UiUpdate; + + private sealed record TableRefreshUpdate(Dictionary RowStates) : UiUpdate; + + private sealed record ReproRowState(string ReproId, string ReproVersion, string Reproduced, string Fixed, string Overall); +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RunCommandSettings.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RunCommandSettings.cs new file mode 100644 index 000000000..b1bb98a2e --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/RunCommandSettings.cs @@ -0,0 +1,114 @@ +using System; +using System.ComponentModel; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class RunCommandSettings : RootCommandSettings +{ + /// + /// Gets the default frames-per-second limit applied to the live UI. + /// + public const decimal DefaultFps = 2m; + + /// + /// Gets or sets the identifier of the repro to execute. + /// + [CommandArgument(0, "[id]")] + [Description("Identifier of the repro to run.")] + public string? Id { get; set; } + + /// + /// Gets or sets a value indicating whether all repros should be executed. + /// + [CommandOption("--all")] + [Description("Run every repro in sequence.")] + public bool All { get; set; } + + /// + /// Gets or sets the number of instances to launch for each repro. + /// + [CommandOption("--instances ")] + [Description("Override the number of instances to launch.")] + public int? Instances { get; set; } + + /// + /// Gets or sets the timeout to apply to each repro execution, in seconds. + /// + [CommandOption("--timeout ")] + [Description("Override the timeout applied to each repro.")] + public int? Timeout { get; set; } + + /// + /// Gets or sets a value indicating whether validation failures should be ignored. + /// + [CommandOption("--skipValidation")] + [Description("Allow execution even when manifest validation fails.")] + public bool SkipValidation { get; set; } + + /// + /// Gets or sets the maximum number of times the live UI refreshes per second. + /// + [CommandOption("--fps ")] + [Description("Limit the live UI refresh rate in frames per second (default: 2).")] + public decimal? Fps { get; set; } + + /// + /// Gets or sets the optional report output path. + /// + [CommandOption("--report ")] + [Description("Write a machine-readable report to the specified file or '-' for stdout.")] + public string? ReportPath { get; set; } + + /// + /// Gets or sets the report format. + /// + [CommandOption("--report-format ")] + [Description("Report format (currently only 'json').")] + public string? ReportFormat { get; set; } + + /// + /// Validates the run command settings. + /// + /// The validation result describing any errors. + public override ValidationResult Validate() + { + if (All && Id is not null) + { + return ValidationResult.Error("Cannot specify both --all and ."); + } + + if (!All && Id is null) + { + return ValidationResult.Error("run requires a repro id or --all."); + } + + if (Instances is int instances && instances < 1) + { + return ValidationResult.Error("--instances expects a positive integer."); + } + + if (Timeout is int timeout && timeout < 1) + { + return ValidationResult.Error("--timeout expects a positive integer value in seconds."); + } + + if (Fps is decimal fps && fps <= 0) + { + return ValidationResult.Error("--fps expects a positive decimal value."); + } + + if (ReportFormat is string format && !string.Equals(format, "json", StringComparison.OrdinalIgnoreCase)) + { + return ValidationResult.Error("--report-format supports only 'json'."); + } + + if (ReportPath is null && ReportFormat is not null) + { + return ValidationResult.Error("--report-format requires --report."); + } + + return base.Validate(); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ShowCommand.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ShowCommand.cs new file mode 100644 index 000000000..bb9d19f3f --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ShowCommand.cs @@ -0,0 +1,51 @@ +using LiteDB.ReproRunner.Cli.Infrastructure; +using LiteDB.ReproRunner.Cli.Manifests; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class ShowCommand : Command +{ + private readonly IAnsiConsole _console; + private readonly ReproRootLocator _rootLocator; + + /// + /// Initializes a new instance of the class. + /// + /// The console used to render output. + /// Resolves the repro root directory. + public ShowCommand(IAnsiConsole console, ReproRootLocator rootLocator) + { + _console = console ?? throw new ArgumentNullException(nameof(console)); + _rootLocator = rootLocator ?? throw new ArgumentNullException(nameof(rootLocator)); + } + + /// + /// Executes the show command. + /// + /// The Spectre command context. + /// The user-provided settings. + /// The process exit code. + public override int Execute(CommandContext context, ShowCommandSettings settings) + { + var repository = new ManifestRepository(_rootLocator.ResolveRoot(settings.Root)); + var manifests = repository.Discover(); + var repro = manifests.FirstOrDefault(x => string.Equals(x.Manifest?.Id ?? x.RawId, settings.Id, StringComparison.OrdinalIgnoreCase)); + + if (repro is null) + { + _console.MarkupLine($"[red]Repro '{Markup.Escape(settings.Id)}' was not found.[/]"); + return 1; + } + + if (!repro.IsValid) + { + CliOutput.PrintInvalid(_console, repro); + return 2; + } + + CliOutput.PrintManifest(_console, repro); + return 0; + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ShowCommandSettings.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ShowCommandSettings.cs new file mode 100644 index 000000000..3551fce6f --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ShowCommandSettings.cs @@ -0,0 +1,14 @@ +using System.ComponentModel; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class ShowCommandSettings : RootCommandSettings +{ + /// + /// Gets or sets the identifier of the repro to display. + /// + [CommandArgument(0, "")] + [Description("Identifier of the repro to show.")] + public string Id { get; set; } = string.Empty; +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ValidateCommand.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ValidateCommand.cs new file mode 100644 index 000000000..e989da833 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ValidateCommand.cs @@ -0,0 +1,57 @@ +using LiteDB.ReproRunner.Cli.Infrastructure; +using LiteDB.ReproRunner.Cli.Manifests; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class ValidateCommand : Command +{ + private readonly IAnsiConsole _console; + private readonly ReproRootLocator _rootLocator; + + /// + /// Initializes a new instance of the class. + /// + /// The console used to render output. + /// Resolves the repro root directory. + public ValidateCommand(IAnsiConsole console, ReproRootLocator rootLocator) + { + _console = console ?? throw new ArgumentNullException(nameof(console)); + _rootLocator = rootLocator ?? throw new ArgumentNullException(nameof(rootLocator)); + } + + /// + /// Executes the validate command. + /// + /// The Spectre command context. + /// The user-provided settings. + /// The process exit code. + public override int Execute(CommandContext context, ValidateCommandSettings settings) + { + var repository = new ManifestRepository(_rootLocator.ResolveRoot(settings.Root)); + var manifests = repository.Discover(); + + if (!settings.All && settings.Id is not null) + { + var repro = manifests.FirstOrDefault(x => string.Equals(x.Manifest?.Id ?? x.RawId, settings.Id, StringComparison.OrdinalIgnoreCase)); + if (repro is null) + { + _console.MarkupLine($"[red]Repro '{Markup.Escape(settings.Id)}' was not found.[/]"); + return 1; + } + + CliOutput.PrintValidationResult(_console, repro); + return repro.IsValid ? 0 : 2; + } + + var anyInvalid = false; + foreach (var repro in manifests) + { + CliOutput.PrintValidationResult(_console, repro); + anyInvalid |= !repro.IsValid; + } + + return anyInvalid ? 2 : 0; + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ValidateCommandSettings.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ValidateCommandSettings.cs new file mode 100644 index 000000000..f335fc098 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Commands/ValidateCommandSettings.cs @@ -0,0 +1,36 @@ +using System.ComponentModel; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Commands; + +internal sealed class ValidateCommandSettings : RootCommandSettings +{ + /// + /// Gets or sets a value indicating whether all manifests should be validated. + /// + [CommandOption("--all")] + [Description("Validate every repro manifest (default).")] + public bool All { get; set; } + + /// + /// Gets or sets the identifier of the repro to validate. + /// + [CommandOption("--id ")] + [Description("Validate a single repro by id.")] + public string? Id { get; set; } + + /// + /// Validates the command settings. + /// + /// The validation result describing any errors. + public override ValidationResult Validate() + { + if (All && Id is not null) + { + return ValidationResult.Error("Cannot specify both --all and --id."); + } + + return base.Validate(); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproBuildCoordinator.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproBuildCoordinator.cs new file mode 100644 index 000000000..38a7fc8ad --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproBuildCoordinator.cs @@ -0,0 +1,214 @@ +using System.Diagnostics; +using System.Text; + +namespace LiteDB.ReproRunner.Cli.Execution; + +/// +/// Coordinates building repro variants ahead of execution. +/// +internal sealed class ReproBuildCoordinator +{ + /// + /// Builds the provided repro variants sequentially. + /// + /// The variants to build. + /// The token used to observe cancellation requests. + /// The collection of build results for the supplied variants. + public async Task> BuildAsync( + IEnumerable variants, + CancellationToken cancellationToken) + { + if (variants is null) + { + throw new ArgumentNullException(nameof(variants)); + } + + var results = new List(); + + foreach (var plan in variants) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (plan.Repro.ProjectPath is null) + { + results.Add(ReproBuildResult.CreateMissingProject(plan)); + continue; + } + + var projectPath = plan.Repro.ProjectPath; + var projectDirectory = Path.GetDirectoryName(projectPath)!; + var assemblyName = Path.GetFileNameWithoutExtension(projectPath); + + var arguments = new List + { + "build", + projectPath, + "-c", + "Release", + "--nologo", + $"-p:UseProjectReference={(plan.UseProjectReference ? "true" : "false")}", + $"-p:OutputPath={plan.BuildOutputDirectory}" + }; + + if (!string.IsNullOrWhiteSpace(plan.LiteDBPackageVersion)) + { + arguments.Add($"-p:LiteDBPackageVersion={plan.LiteDBPackageVersion}"); + } + + var (exitCode, output) = await RunProcessAsync(projectDirectory, arguments, cancellationToken).ConfigureAwait(false); + + if (exitCode != 0) + { + results.Add(ReproBuildResult.CreateFailure(plan, exitCode, output)); + continue; + } + + var assemblyPath = Path.Combine(plan.BuildOutputDirectory, assemblyName + ".dll"); + + if (!File.Exists(assemblyPath)) + { + output.Add($"Assembly '{assemblyPath}' was not produced by build."); + results.Add(ReproBuildResult.CreateFailure(plan, exitCode: -1, output)); + continue; + } + + results.Add(ReproBuildResult.CreateSuccess(plan, assemblyPath, output)); + } + + return results; + } + + private static async Task<(int ExitCode, List Output)> RunProcessAsync( + string workingDirectory, + IEnumerable arguments, + CancellationToken cancellationToken) + { + var startInfo = new ProcessStartInfo("dotnet") + { + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + StandardErrorEncoding = Encoding.UTF8, + StandardOutputEncoding = Encoding.UTF8 + }; + + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + using var process = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start dotnet build process."); + + var lines = new List(); + + var outputTask = CaptureAsync(process.StandardOutput, lines, cancellationToken); + var errorTask = CaptureAsync(process.StandardError, lines, cancellationToken); + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + await Task.WhenAll(outputTask, errorTask).ConfigureAwait(false); + + return (process.ExitCode, lines); + } + + private static async Task CaptureAsync(StreamReader reader, List destination, CancellationToken cancellationToken) + { + try + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var line = await reader.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + destination.Add(line); + } + } + catch (Exception ex) when (ex is IOException or ObjectDisposedException) + { + } + } +} + +/// +/// Represents the outcome of a repro variant build. +/// +internal sealed class ReproBuildResult +{ + private ReproBuildResult( + RunVariantPlan plan, + bool succeeded, + int exitCode, + string? assemblyPath, + IReadOnlyList output) + { + Plan = plan ?? throw new ArgumentNullException(nameof(plan)); + Succeeded = succeeded; + ExitCode = exitCode; + AssemblyPath = assemblyPath; + Output = output; + } + + /// + /// Gets the plan that was built. + /// + public RunVariantPlan Plan { get; } + + /// + /// Gets a value indicating whether the build succeeded. + /// + public bool Succeeded { get; } + + /// + /// Gets the exit code reported by the build process. + /// + public int ExitCode { get; } + + /// + /// Gets the path to the built assembly when the build succeeds. + /// + public string? AssemblyPath { get; } + + /// + /// Gets the captured build output lines. + /// + public IReadOnlyList Output { get; } + + /// + /// Creates a successful build result for the specified plan. + /// + /// The plan that was built. + /// The path to the produced assembly. + /// The captured output from the build process. + /// The successful build result. + public static ReproBuildResult CreateSuccess(RunVariantPlan plan, string assemblyPath, IReadOnlyList output) + { + return new ReproBuildResult(plan, succeeded: true, exitCode: 0, assemblyPath: assemblyPath, output: output); + } + + /// + /// Creates a failed build result for the specified plan. + /// + /// The plan that was built. + /// The exit code returned by the build process. + /// The captured output from the build process. + /// The failed build result. + public static ReproBuildResult CreateFailure(RunVariantPlan plan, int exitCode, IReadOnlyList output) + { + return new ReproBuildResult(plan, succeeded: false, exitCode: exitCode, assemblyPath: null, output: output); + } + + /// + /// Creates a build result indicating that the repro project was not found. + /// + /// The plan that failed to build. + /// The build result representing the missing project. + public static ReproBuildResult CreateMissingProject(RunVariantPlan plan) + { + var output = new List { "No project file was discovered for this repro." }; + return new ReproBuildResult(plan, succeeded: false, exitCode: -2, assemblyPath: null, output); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproExecutor.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproExecutor.cs new file mode 100644 index 000000000..96af1df0f --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproExecutor.cs @@ -0,0 +1,740 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Text.Json; +using LiteDB.ReproRunner.Cli.Manifests; +using LiteDB.ReproRunner.Shared; +using LiteDB.ReproRunner.Shared.Messaging; + +namespace LiteDB.ReproRunner.Cli.Execution; + +/// +/// Executes built repro assemblies and relays their structured output. +/// +internal sealed class ReproExecutor +{ + private const int CapturedOutputLimit = 200; + + private readonly TextWriter _standardOut; + private readonly TextWriter _standardError; + private readonly object _writeLock = new(); + private readonly object _configurationLock = new(); + private readonly Dictionary _configurationStates = new(); + private ConfigurationExpectation? _configurationExpectation; + private bool _configurationMismatchDetected; + private int _expectedConfigurationInstances; + + /// + /// Initializes a new instance of the class using the console streams. + /// + public ReproExecutor() + : this(Console.Out, Console.Error) + { + } + + internal ReproExecutor(TextWriter? standardOut, TextWriter? standardError) + { + _standardOut = standardOut ?? Console.Out; + _standardError = standardError ?? Console.Error; + } + + internal Action? StructuredMessageObserver { get; set; } + + internal Action? LogObserver { get; set; } + + internal bool SuppressConsoleLogOutput { get; set; } + + internal void ConfigureExpectedConfiguration(bool useProjectReference, string? liteDbPackageVersion, int instanceCount) + { + var normalizedVersion = string.IsNullOrWhiteSpace(liteDbPackageVersion) + ? null + : liteDbPackageVersion.Trim(); + + lock (_configurationLock) + { + _configurationExpectation = new ConfigurationExpectation(useProjectReference, normalizedVersion); + _configurationStates.Clear(); + _expectedConfigurationInstances = Math.Max(instanceCount, 0); + _configurationMismatchDetected = false; + + for (var index = 0; index < _expectedConfigurationInstances; index++) + { + _configurationStates[index] = new ConfigurationState(); + } + } + } + + /// + /// Executes the provided repro build across the requested number of instances. + /// + /// The build to execute. + /// The number of instances to launch. + /// The timeout applied to the execution. + /// The token used to observe cancellation requests. + /// The execution result for the run. + public async Task ExecuteAsync( + ReproBuildResult build, + int instances, + int timeoutSeconds, + CancellationToken cancellationToken) + { + if (build is null) + { + throw new ArgumentNullException(nameof(build)); + } + + if (!build.Succeeded || string.IsNullOrWhiteSpace(build.AssemblyPath)) + { + return new ReproExecutionResult(build.Plan.UseProjectReference, false, build.ExitCode, TimeSpan.Zero, Array.Empty()); + } + + var repro = build.Plan.Repro; + + if (repro.ProjectPath is null) + { + return new ReproExecutionResult(build.Plan.UseProjectReference, false, build.ExitCode, TimeSpan.Zero, Array.Empty()); + } + + var manifest = repro.Manifest ?? throw new InvalidOperationException("Manifest is required to execute a repro."); + ConfigureExpectedConfiguration(build.Plan.UseProjectReference, build.Plan.LiteDBPackageVersion, instances); + var projectDirectory = Path.GetDirectoryName(repro.ProjectPath)!; + var stopwatch = Stopwatch.StartNew(); + + var sharedKey = !string.IsNullOrWhiteSpace(manifest.SharedDatabaseKey) + ? manifest.SharedDatabaseKey! + : manifest.Id; + + var runIdentifier = Guid.NewGuid().ToString("N"); + var sharedRoot = Path.Combine(build.Plan.ExecutionRootDirectory, Sanitize(sharedKey), runIdentifier); + Directory.CreateDirectory(sharedRoot); + + var capturedOutput = new BoundedLogBuffer(CapturedOutputLimit); + + try + { + var exitCode = await RunInstancesAsync( + manifest, + projectDirectory, + build.AssemblyPath, + instances, + timeoutSeconds, + sharedRoot, + runIdentifier, + capturedOutput, + cancellationToken).ConfigureAwait(false); + + FinalizeConfigurationValidation(); + var configurationMismatch = HasConfigurationMismatch(); + + if (configurationMismatch && exitCode == 0) + { + exitCode = -2; + } + + stopwatch.Stop(); + return new ReproExecutionResult( + build.Plan.UseProjectReference, + exitCode == 0 && !configurationMismatch, + exitCode, + stopwatch.Elapsed, + capturedOutput.ToSnapshot()); + } + finally + { + ResetConfigurationExpectation(); + } + } + + private async Task RunInstancesAsync( + ReproManifest manifest, + string projectDirectory, + string assemblyPath, + int instances, + int timeoutSeconds, + string sharedRoot, + string runIdentifier, + BoundedLogBuffer capturedOutput, + CancellationToken cancellationToken) + { + var manifestArgs = manifest.Args; + var processes = new List(); + var outputTasks = new List(); + var errorTasks = new List(); + + try + { + for (var index = 0; index < instances; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var startInfo = CreateStartInfo(projectDirectory, assemblyPath, manifestArgs); + startInfo.Environment["LITEDB_RR_SHARED_DB"] = sharedRoot; + startInfo.Environment["LITEDB_RR_INSTANCE_INDEX"] = index.ToString(); + startInfo.Environment["LITEDB_RR_TOTAL_INSTANCES"] = instances.ToString(); + startInfo.Environment["LITEDB_RR_RUN_IDENTIFIER"] = runIdentifier; + + var process = Process.Start(startInfo); + if (process is null) + { + throw new InvalidOperationException("Failed to start repro process."); + } + + processes.Add(process); + outputTasks.Add(PumpStandardOutputAsync(process, index, capturedOutput, cancellationToken)); + errorTasks.Add(PumpStandardErrorAsync(process, index, capturedOutput, cancellationToken)); + + await SendHostHandshakeAsync(process, manifest, sharedRoot, runIdentifier, index, instances, cancellationToken).ConfigureAwait(false); + } + + var timeout = TimeSpan.FromSeconds(timeoutSeconds); + var waitTasks = processes.Select(p => p.WaitForExitAsync(cancellationToken)).ToList(); + var timeoutTask = Task.Delay(timeout, cancellationToken); + var allProcessesTask = Task.WhenAll(waitTasks); + var completed = await Task.WhenAny(allProcessesTask, timeoutTask).ConfigureAwait(false); + + if (cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException(cancellationToken); + } + + if (completed == timeoutTask) + { + foreach (var process in processes) + { + TryKill(process); + } + + return 1; + } + + await allProcessesTask.ConfigureAwait(false); + await Task.WhenAll(outputTasks.Concat(errorTasks)).ConfigureAwait(false); + + var exitCode = 0; + + foreach (var process in processes) + { + if (process.ExitCode != 0 && exitCode == 0) + { + exitCode = process.ExitCode; + } + } + + return exitCode; + } + finally + { + foreach (var process in processes) + { + if (!process.HasExited) + { + TryKill(process); + } + + process.Dispose(); + } + } + } + + private static ProcessStartInfo CreateStartInfo(string workingDirectory, string assemblyPath, IEnumerable arguments) + { + var startInfo = new ProcessStartInfo("dotnet") + { + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + RedirectStandardInput = true, + StandardOutputEncoding = Encoding.UTF8, + StandardErrorEncoding = Encoding.UTF8 + }; + + startInfo.ArgumentList.Add(assemblyPath); + + foreach (var argument in arguments) + { + startInfo.ArgumentList.Add(argument); + } + + return startInfo; + } + + private async Task PumpStandardOutputAsync(Process process, int instanceIndex, BoundedLogBuffer capturedOutput, CancellationToken cancellationToken) + { + try + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var line = await process.StandardOutput.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + capturedOutput.Add(ReproExecutionStream.StandardOutput, line); + if (!TryProcessStructuredLine(line, instanceIndex)) + { + WriteOutputLine($"[{instanceIndex}] {line}"); + } + } + } + catch (Exception ex) when (ex is IOException or ObjectDisposedException) + { + } + } + + private async Task PumpStandardErrorAsync(Process process, int instanceIndex, BoundedLogBuffer capturedOutput, CancellationToken cancellationToken) + { + try + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var line = await process.StandardError.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + capturedOutput.Add(ReproExecutionStream.StandardError, line); + WriteErrorLine($"[{instanceIndex}] {line}"); + } + } + catch (Exception ex) when (ex is IOException or ObjectDisposedException) + { + } + } + + internal bool TryProcessStructuredLine(string line, int instanceIndex) + { + if (!ReproHostMessageEnvelope.TryParse(line, out var envelope, out _)) + { + return false; + } + + StructuredMessageObserver?.Invoke(instanceIndex, envelope!); + + if (!HandleConfigurationHandshake(instanceIndex, envelope!)) + { + return true; + } + + HandleStructuredMessage(instanceIndex, envelope!); + return true; + } + + private void HandleStructuredMessage(int instanceIndex, ReproHostMessageEnvelope envelope) + { + switch (envelope.Type) + { + case ReproHostMessageTypes.Log: + WriteLogMessage(instanceIndex, envelope); + break; + case ReproHostMessageTypes.Result: + WriteResultMessage(instanceIndex, envelope); + break; + case ReproHostMessageTypes.Lifecycle: + WriteOutputLine($"[{instanceIndex}] lifecycle: {envelope.Event ?? "(unknown)"}"); + break; + case ReproHostMessageTypes.Progress: + var suffix = envelope.Progress is double progress + ? $" ({progress:0.##}%)" + : string.Empty; + WriteOutputLine($"[{instanceIndex}] progress: {envelope.Event ?? "(unknown)"}{suffix}"); + break; + case ReproHostMessageTypes.Configuration: + break; + default: + WriteOutputLine($"[{instanceIndex}] {envelope.Type}: {envelope.Text ?? string.Empty}"); + break; + } + } + + private bool HandleConfigurationHandshake(int instanceIndex, ReproHostMessageEnvelope envelope) + { + string? errorMessage = null; + var shouldProcess = true; + + lock (_configurationLock) + { + if (_configurationExpectation is not { } expectation) + { + if (string.Equals(envelope.Type, ReproHostMessageTypes.Configuration, StringComparison.Ordinal)) + { + shouldProcess = false; + } + + return shouldProcess; + } + + if (!_configurationStates.TryGetValue(instanceIndex, out var state)) + { + state = new ConfigurationState(); + _configurationStates[instanceIndex] = state; + } + + if (!state.Received) + { + if (!string.Equals(envelope.Type, ReproHostMessageTypes.Configuration, StringComparison.Ordinal)) + { + errorMessage = "expected configuration handshake before other messages."; + state.Received = true; + state.IsValid = false; + _configurationMismatchDetected = true; + shouldProcess = false; + } + else + { + var payload = envelope.DeserializePayload(); + if (payload is null) + { + errorMessage = "reported configuration without a payload."; + state.Received = true; + state.IsValid = false; + _configurationMismatchDetected = true; + shouldProcess = false; + } + else + { + var actualVersion = string.IsNullOrWhiteSpace(payload.LiteDBPackageVersion) + ? null + : payload.LiteDBPackageVersion.Trim(); + + var expectedVersion = expectation.LiteDbPackageVersion; + var versionMatches = string.Equals( + actualVersion ?? string.Empty, + expectedVersion ?? string.Empty, + StringComparison.OrdinalIgnoreCase); + + if (payload.UseProjectReference != expectation.UseProjectReference || !versionMatches) + { + var expectedVersionDisplay = expectedVersion ?? "(unspecified)"; + var actualVersionDisplay = actualVersion ?? "(unspecified)"; + errorMessage = $"reported configuration UseProjectReference={payload.UseProjectReference}, LiteDBPackageVersion={actualVersionDisplay} but expected UseProjectReference={expectation.UseProjectReference}, LiteDBPackageVersion={expectedVersionDisplay}."; + state.IsValid = false; + _configurationMismatchDetected = true; + } + else + { + state.IsValid = true; + } + + state.Received = true; + shouldProcess = false; + } + } + } + else if (!state.IsValid) + { + shouldProcess = false; + } + else if (string.Equals(envelope.Type, ReproHostMessageTypes.Configuration, StringComparison.Ordinal)) + { + shouldProcess = false; + } + } + + if (errorMessage is not null) + { + WriteConfigurationError(instanceIndex, errorMessage); + } + + return shouldProcess; + } + + private void FinalizeConfigurationValidation() + { + List? missingInstances = null; + + lock (_configurationLock) + { + if (_configurationExpectation is null) + { + return; + } + + for (var index = 0; index < _expectedConfigurationInstances; index++) + { + if (!_configurationStates.TryGetValue(index, out var state)) + { + state = new ConfigurationState(); + _configurationStates[index] = state; + } + + if (!state.Received) + { + state.Received = true; + state.IsValid = false; + _configurationMismatchDetected = true; + missingInstances ??= new List(); + missingInstances.Add(index); + } + } + } + + if (missingInstances is null) + { + return; + } + + foreach (var instanceIndex in missingInstances) + { + WriteConfigurationError(instanceIndex, "did not report configuration handshake."); + } + } + + private bool HasConfigurationMismatch() + { + lock (_configurationLock) + { + if (_configurationExpectation is null) + { + return false; + } + + if (_configurationMismatchDetected) + { + return true; + } + + foreach (var state in _configurationStates.Values) + { + if (!state.IsValid) + { + return true; + } + } + + return false; + } + } + + private void ResetConfigurationExpectation() + { + lock (_configurationLock) + { + _configurationExpectation = null; + _configurationStates.Clear(); + _configurationMismatchDetected = false; + _expectedConfigurationInstances = 0; + } + } + + private void WriteConfigurationError(int instanceIndex, string message) + { + LogObserver?.Invoke(new ReproExecutionLogEntry(instanceIndex, $"configuration error: {message}", ReproHostLogLevel.Error)); + + if (SuppressConsoleLogOutput) + { + return; + } + + WriteErrorLine($"[{instanceIndex}] configuration error: {message}"); + } + + private void WriteLogMessage(int instanceIndex, ReproHostMessageEnvelope envelope) + { + var message = envelope.Text ?? string.Empty; + var level = envelope.Level ?? ReproHostLogLevel.Information; + var formatted = $"[{instanceIndex}] {message}"; + + LogObserver?.Invoke(new ReproExecutionLogEntry(instanceIndex, message, level)); + + if (SuppressConsoleLogOutput) + { + return; + } + + switch (level) + { + case ReproHostLogLevel.Error: + case ReproHostLogLevel.Critical: + WriteErrorLine(formatted); + break; + case ReproHostLogLevel.Warning: + WriteErrorLine(formatted); + break; + default: + WriteOutputLine(formatted); + break; + } + } + + private void WriteResultMessage(int instanceIndex, ReproHostMessageEnvelope envelope) + { + var success = envelope.Success is true; + var status = success ? "succeeded" : "completed"; + var summary = envelope.Text ?? $"Repro {status}."; + WriteOutputLine($"[{instanceIndex}] {summary}"); + } + + private async Task SendHostHandshakeAsync( + Process process, + ReproManifest manifest, + string sharedRoot, + string runIdentifier, + int instanceIndex, + int totalInstances, + CancellationToken cancellationToken) + { + try + { + var envelope = ReproInputEnvelope.CreateHostReady(runIdentifier, sharedRoot, instanceIndex, totalInstances, manifest.Id); + var json = JsonSerializer.Serialize(envelope, ReproJsonOptions.Default); + var writer = process.StandardInput; + writer.AutoFlush = true; + cancellationToken.ThrowIfCancellationRequested(); + await writer.WriteLineAsync(json).ConfigureAwait(false); + await writer.FlushAsync().ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or InvalidOperationException or ObjectDisposedException) + { + } + } + + private sealed class BoundedLogBuffer + { + private readonly int _capacity; + private readonly Queue _buffer; + private readonly object _sync = new(); + + public BoundedLogBuffer(int capacity) + { + _capacity = Math.Max(1, capacity); + _buffer = new Queue(_capacity); + } + + public void Add(ReproExecutionStream stream, string text) + { + if (text is null) + { + return; + } + + var entry = new ReproExecutionCapturedLine(stream, text); + + lock (_sync) + { + _buffer.Enqueue(entry); + while (_buffer.Count > _capacity) + { + _buffer.Dequeue(); + } + } + } + + public IReadOnlyList ToSnapshot() + { + lock (_sync) + { + return _buffer.ToArray(); + } + } + } + + private void WriteOutputLine(string message) + { + if (SuppressConsoleLogOutput) + { + return; + } + + lock (_writeLock) + { + _standardOut.WriteLine(message); + _standardOut.Flush(); + } + } + + private void WriteErrorLine(string message) + { + if (SuppressConsoleLogOutput) + { + return; + } + + lock (_writeLock) + { + _standardError.WriteLine(message); + _standardError.Flush(); + } + } + + private static void TryKill(Process process) + { + try + { + if (!process.HasExited) + { + process.Kill(true); + } + } + catch + { + } + } + + private static string Sanitize(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var ch in value) + { + if (Array.IndexOf(Path.GetInvalidFileNameChars(), ch) >= 0) + { + builder.Append('_'); + } + else + { + builder.Append(ch); + } + } + + return builder.Length == 0 ? "shared" : builder.ToString(); + } + + private sealed class ConfigurationState + { + public bool Received { get; set; } + + public bool IsValid { get; set; } = true; + } + + private readonly record struct ConfigurationExpectation(bool UseProjectReference, string? LiteDbPackageVersion); +} + +/// +/// Represents the result of executing a repro variant. +/// +/// Indicates whether the run targeted the source project build. +/// Indicates whether the repro successfully reproduced the issue. +/// The exit code reported by the repro host. +/// The elapsed time for the execution. +/// The captured standard output and error lines. +internal readonly record struct ReproExecutionResult(bool UseProjectReference, bool Reproduced, int ExitCode, TimeSpan Duration, IReadOnlyList CapturedOutput); + +/// +/// Represents a structured log entry emitted during repro execution. +/// +/// The zero-based instance index originating the log entry. +/// The log message text. +/// The severity associated with the log entry. +internal readonly record struct ReproExecutionLogEntry(int InstanceIndex, string Message, ReproHostLogLevel Level); + +/// +/// Identifies the stream that produced a captured line of output. +/// +internal enum ReproExecutionStream +{ + StandardOutput, + StandardError +} + +/// +/// Represents a captured line of standard output or error for report generation. +/// +/// The source stream for the line. +/// The raw text captured from the process. +internal readonly record struct ReproExecutionCapturedLine(ReproExecutionStream Stream, string Text); diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproOutcomeEvaluator.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproOutcomeEvaluator.cs new file mode 100644 index 000000000..03d7b3ebc --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/ReproOutcomeEvaluator.cs @@ -0,0 +1,200 @@ +using System; +using System.Collections.Generic; +using LiteDB.ReproRunner.Cli.Manifests; + +namespace LiteDB.ReproRunner.Cli.Execution; + +internal sealed class ReproOutcomeEvaluator +{ + public ReproRunEvaluation Evaluate(ReproManifest manifest, ReproExecutionResult? packageResult, ReproExecutionResult? latestResult) + { + if (manifest is null) + { + throw new ArgumentNullException(nameof(manifest)); + } + + var packageExpectation = manifest.ExpectedOutcomes.Package ?? new ReproOutcomeExpectation(ReproOutcomeKind.Reproduce, null, null); + var latestExpectation = manifest.ExpectedOutcomes.Latest ?? CreateDefaultLatestExpectation(manifest.State); + + var package = EvaluateVariant(packageExpectation, packageResult, isLatest: false, manifest.State); + var latest = EvaluateVariant(latestExpectation, latestResult, isLatest: true, manifest.State); + + var shouldFail = package.ShouldFail || latest.ShouldFail; + var shouldWarn = package.ShouldWarn || latest.ShouldWarn; + + return new ReproRunEvaluation(manifest.State, package, latest, shouldFail, shouldWarn); + } + + private static ReproVariantEvaluation EvaluateVariant( + ReproOutcomeExpectation expectation, + ReproExecutionResult? result, + bool isLatest, + ReproState state) + { + var actualKind = ComputeActualKind(result); + var (met, failureReason) = MatchesExpectation(expectation, result); + + var shouldFail = false; + var shouldWarn = false; + + if (!met) + { + if (isLatest) + { + if (state == ReproState.Flaky) + { + shouldWarn = true; + } + else + { + shouldFail = true; + } + } + else + { + shouldFail = true; + } + } + + return new ReproVariantEvaluation(expectation, actualKind, result, met, shouldFail, shouldWarn, failureReason); + } + + private static (bool Met, string? FailureReason) MatchesExpectation(ReproOutcomeExpectation expectation, ReproExecutionResult? result) + { + if (result is null) + { + return (false, "Variant did not execute."); + } + + var exitCode = result.Value.ExitCode; + + switch (expectation.Kind) + { + case ReproOutcomeKind.Reproduce: + if (exitCode != 0) + { + return (false, $"Expected exit code 0 but observed {exitCode}."); + } + break; + case ReproOutcomeKind.NoRepro: + case ReproOutcomeKind.HardFail: + if (exitCode == 0) + { + return (false, "Expected non-zero exit code."); + } + break; + default: + return (false, $"Unsupported expectation kind: {expectation.Kind}."); + } + + if (expectation.ExitCode.HasValue && exitCode != expectation.ExitCode.Value) + { + return (false, $"Expected exit code {expectation.ExitCode.Value} but observed {exitCode}."); + } + + if (!MatchesLog(expectation.LogContains, result.Value.CapturedOutput)) + { + return (false, $"Expected output containing '{expectation.LogContains}'."); + } + + return (true, null); + } + + private static bool MatchesLog(string? expected, IReadOnlyList lines) + { + if (string.IsNullOrWhiteSpace(expected)) + { + return true; + } + + foreach (var line in lines) + { + if (line.Text?.IndexOf(expected, StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + + return false; + } + + private static ReproOutcomeKind ComputeActualKind(ReproExecutionResult? result) + { + if (result is null) + { + return ReproOutcomeKind.NoRepro; + } + + return result.Value.ExitCode == 0 + ? ReproOutcomeKind.Reproduce + : ReproOutcomeKind.NoRepro; + } + + private static ReproOutcomeExpectation CreateDefaultLatestExpectation(ReproState state) + { + return state switch + { + ReproState.Red => new ReproOutcomeExpectation(ReproOutcomeKind.Reproduce, null, null), + ReproState.Green => new ReproOutcomeExpectation(ReproOutcomeKind.NoRepro, null, null), + ReproState.Flaky => new ReproOutcomeExpectation(ReproOutcomeKind.Reproduce, null, null), + _ => new ReproOutcomeExpectation(ReproOutcomeKind.Reproduce, null, null) + }; + } +} + +internal sealed class ReproRunEvaluation +{ + public ReproRunEvaluation(ReproState state, ReproVariantEvaluation package, ReproVariantEvaluation latest, bool shouldFail, bool shouldWarn) + { + State = state; + Package = package; + Latest = latest; + ShouldFail = shouldFail; + ShouldWarn = shouldWarn; + } + + public ReproState State { get; } + + public ReproVariantEvaluation Package { get; } + + public ReproVariantEvaluation Latest { get; } + + public bool ShouldFail { get; } + + public bool ShouldWarn { get; } +} + +internal sealed class ReproVariantEvaluation +{ + public ReproVariantEvaluation( + ReproOutcomeExpectation expectation, + ReproOutcomeKind actualKind, + ReproExecutionResult? result, + bool met, + bool shouldFail, + bool shouldWarn, + string? failureReason) + { + Expectation = expectation; + ActualKind = actualKind; + Result = result; + Met = met; + ShouldFail = shouldFail; + ShouldWarn = shouldWarn; + FailureReason = failureReason; + } + + public ReproOutcomeExpectation Expectation { get; } + + public ReproOutcomeKind ActualKind { get; } + + public ReproExecutionResult? Result { get; } + + public bool Met { get; } + + public bool ShouldFail { get; } + + public bool ShouldWarn { get; } + + public string? FailureReason { get; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/RunDirectoryPlanner.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/RunDirectoryPlanner.cs new file mode 100644 index 000000000..158b7fda7 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/RunDirectoryPlanner.cs @@ -0,0 +1,215 @@ +using System.Text; +using LiteDB.ReproRunner.Cli.Manifests; + +namespace LiteDB.ReproRunner.Cli.Execution; + +/// +/// Produces deterministic run directories for repro build and execution artifacts. +/// +internal sealed class RunDirectoryPlanner +{ + private readonly string _runsRoot; + + /// + /// Initializes a new instance of the class. + /// + public RunDirectoryPlanner() + { + _runsRoot = Path.Combine(AppContext.BaseDirectory, "runs"); + } + + /// + /// Creates a plan that describes where a repro variant should build and execute. + /// + /// The repro being executed. + /// The identifier to use for the manifest directory. + /// The identifier to use for the variant directory. + /// The display label shown to the user. + /// Indicates whether the repro should build against the source project. + /// The LiteDB package version associated with the variant. + /// The planned variant with prepared directories. + public RunVariantPlan CreateVariantPlan( + DiscoveredRepro repro, + string manifestIdentifier, + string variantIdentifier, + string displayName, + bool useProjectReference, + string? liteDbPackageVersion) + { + if (repro is null) + { + throw new ArgumentNullException(nameof(repro)); + } + + if (string.IsNullOrWhiteSpace(manifestIdentifier)) + { + manifestIdentifier = repro.Manifest?.Id ?? repro.RawId ?? "repro"; + } + + if (string.IsNullOrWhiteSpace(variantIdentifier)) + { + variantIdentifier = useProjectReference ? "ver_latest" : "ver_package"; + } + + var manifestSegment = Sanitize(manifestIdentifier); + var variantSegment = Sanitize(variantIdentifier); + + var variantRoot = Path.Combine(_runsRoot, manifestSegment, variantSegment); + PurgeDirectory(variantRoot); + Directory.CreateDirectory(variantRoot); + + var buildOutput = Path.Combine(variantRoot, "build"); + Directory.CreateDirectory(buildOutput); + + var executionRoot = Path.Combine(variantRoot, "run"); + Directory.CreateDirectory(executionRoot); + + return new RunVariantPlan( + repro, + displayName, + useProjectReference, + liteDbPackageVersion, + manifestIdentifier, + variantIdentifier, + variantRoot, + buildOutput, + executionRoot, + static path => PurgeDirectory(path)); + } + + private static string Sanitize(string value) + { + var buffer = new StringBuilder(value.Length); + foreach (var ch in value) + { + if (Array.IndexOf(Path.GetInvalidFileNameChars(), ch) >= 0) + { + buffer.Append('_'); + } + else + { + buffer.Append(ch); + } + } + + return buffer.ToString(); + } + + private static void PurgeDirectory(string path) + { + try + { + if (Directory.Exists(path)) + { + Directory.Delete(path, recursive: true); + } + } + catch + { + } + } +} + +/// +/// Represents a planned repro variant along with its associated directories. +/// +internal sealed class RunVariantPlan : IDisposable +{ + private readonly Action _cleanup; + + /// + /// Initializes a new instance of the class. + /// + /// The repro that produced this plan. + /// The friendly name presented to the user. + /// Indicates whether a project reference build should be used. + /// The LiteDB package version associated with the plan. + /// The identifier used for the manifest directory. + /// The identifier used for the variant directory. + /// The root directory allocated for the variant. + /// The directory where build outputs are written. + /// The directory where execution artifacts are stored. + /// The cleanup delegate invoked when the plan is disposed. + public RunVariantPlan( + DiscoveredRepro repro, + string displayName, + bool useProjectReference, + string? liteDbPackageVersion, + string manifestIdentifier, + string variantIdentifier, + string rootDirectory, + string buildOutputDirectory, + string executionRootDirectory, + Action cleanup) + { + Repro = repro ?? throw new ArgumentNullException(nameof(repro)); + DisplayName = displayName; + UseProjectReference = useProjectReference; + LiteDBPackageVersion = liteDbPackageVersion; + ManifestIdentifier = manifestIdentifier; + VariantIdentifier = variantIdentifier; + RootDirectory = rootDirectory ?? throw new ArgumentNullException(nameof(rootDirectory)); + BuildOutputDirectory = buildOutputDirectory ?? throw new ArgumentNullException(nameof(buildOutputDirectory)); + ExecutionRootDirectory = executionRootDirectory ?? throw new ArgumentNullException(nameof(executionRootDirectory)); + _cleanup = cleanup ?? throw new ArgumentNullException(nameof(cleanup)); + } + + /// + /// Gets the repro that produced this plan. + /// + public DiscoveredRepro Repro { get; } + + /// + /// Gets the friendly name presented to the user. + /// + public string DisplayName { get; } + + /// + /// Gets a value indicating whether the build uses project references. + /// + public bool UseProjectReference { get; } + + /// + /// Gets the LiteDB package version associated with the plan, when applicable. + /// + public string? LiteDBPackageVersion { get; } + + /// + /// Gets the identifier used for the manifest directory. + /// + public string ManifestIdentifier { get; } + + /// + /// Gets the identifier used for the variant directory. + /// + public string VariantIdentifier { get; } + + /// + /// Gets the root directory allocated for the variant. + /// + public string RootDirectory { get; } + + /// + /// Gets the directory where build outputs are written. + /// + public string BuildOutputDirectory { get; } + + /// + /// Gets the directory where execution artifacts are stored. + /// + public string ExecutionRootDirectory { get; } + + /// + /// Cleans up the planned directories when the plan is disposed. + /// + public void Dispose() + { + try + { + _cleanup(RootDirectory); + } + catch + { + } + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/RunReportModels.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/RunReportModels.cs new file mode 100644 index 000000000..9de7b9506 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Execution/RunReportModels.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using LiteDB.ReproRunner.Cli.Manifests; + +namespace LiteDB.ReproRunner.Cli.Execution; + +internal sealed class RunReport +{ + public DateTimeOffset GeneratedAt { get; init; } = DateTimeOffset.UtcNow; + + public string? Root { get; init; } + + public IReadOnlyList Repros => _repros; + + private readonly List _repros = new(); + + public void Add(RunReportEntry entry) + { + if (entry is null) + { + throw new ArgumentNullException(nameof(entry)); + } + + _repros.Add(entry); + } +} + +internal sealed class RunReportEntry +{ + public string Id { get; init; } = string.Empty; + + public ReproState State { get; init; } + + public bool Failed { get; init; } + + public bool Warned { get; init; } + + public RunReportVariant Package { get; init; } = new(); + + public RunReportVariant Latest { get; init; } = new(); +} + +internal sealed class RunReportVariant +{ + public ReproOutcomeKind Expected { get; init; } + + public int? ExpectedExitCode { get; init; } + + public string? ExpectedLogContains { get; init; } + + public ReproOutcomeKind Actual { get; init; } + + public bool Met { get; init; } + + public int? ExitCode { get; init; } + + public double? DurationSeconds { get; init; } + + public bool UseProjectReference { get; init; } + + public string? FailureReason { get; init; } + + public IReadOnlyList Output { get; init; } = Array.Empty(); +} + +internal sealed class RunReportCapturedLine +{ + public string Stream { get; init; } = string.Empty; + + public string Text { get; init; } = string.Empty; +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/CliOutput.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/CliOutput.cs new file mode 100644 index 000000000..1cbac87ba --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/CliOutput.cs @@ -0,0 +1,126 @@ +using LiteDB.ReproRunner.Cli.Manifests; +using Spectre.Console; + +namespace LiteDB.ReproRunner.Cli.Infrastructure; + +/// +/// Provides helper methods for rendering CLI output. +/// +internal static class CliOutput +{ + /// + /// Prints the validation errors for an invalid repro manifest. + /// + /// The console used for rendering output. + /// The repro that failed validation. + public static void PrintInvalid(IAnsiConsole console, DiscoveredRepro repro) + { + console.MarkupLine($"[red]INVALID[/] {Markup.Escape(NormalizePath(repro.RelativeManifestPath))}"); + foreach (var error in repro.Validation.Errors) + { + console.MarkupLine($" - {Markup.Escape(error)}"); + } + } + + /// + /// Prints the manifest details for a repro in a table format. + /// + /// The console used for rendering output. + /// The repro whose manifest should be displayed. + public static void PrintManifest(IAnsiConsole console, DiscoveredRepro repro) + { + if (repro.Manifest is null) + { + return; + } + + var manifest = repro.Manifest; + var table = new Table().Border(TableBorder.Rounded).AddColumns("Field", "Value"); + table.AddRow("Id", Markup.Escape(manifest.Id)); + table.AddRow("Title", Markup.Escape(manifest.Title)); + table.AddRow("State", Markup.Escape(FormatState(manifest.State))); + table.AddRow("TimeoutSeconds", Markup.Escape(manifest.TimeoutSeconds.ToString())); + table.AddRow("RequiresParallel", Markup.Escape(manifest.RequiresParallel.ToString())); + table.AddRow("DefaultInstances", Markup.Escape(manifest.DefaultInstances.ToString())); + table.AddRow("SharedDatabaseKey", Markup.Escape(manifest.SharedDatabaseKey ?? "-")); + table.AddRow("FailingSince", Markup.Escape(manifest.FailingSince ?? "-")); + table.AddRow("Tags", Markup.Escape(manifest.Tags.Count > 0 ? string.Join(", ", manifest.Tags) : "-")); + table.AddRow("Args", Markup.Escape(manifest.Args.Count > 0 ? string.Join(" ", manifest.Args) : "-")); + + if (manifest.Issues.Count > 0) + { + table.AddRow("Issues", Markup.Escape(string.Join(Environment.NewLine, manifest.Issues))); + } + + console.Write(table); + } + + /// + /// Prints the validation result for a repro manifest. + /// + /// The console used for rendering output. + /// The repro whose validation status should be displayed. + public static void PrintValidationResult(IAnsiConsole console, DiscoveredRepro repro) + { + if (repro.IsValid) + { + console.MarkupLine($"[green]VALID[/] {Markup.Escape(NormalizePath(repro.RelativeManifestPath))}"); + } + else + { + PrintInvalid(console, repro); + } + } + + /// + /// Prints a table listing the discovered valid repro manifests. + /// + /// The console used for rendering output. + /// The list of valid repros to display. + public static void PrintList(IAnsiConsole console, IReadOnlyList valid) + { + if (valid.Count == 0) + { + console.MarkupLine("[yellow]No valid repro manifests found.[/]"); + return; + } + + var table = new Table().Border(TableBorder.Rounded); + table.AddColumns("Id", "State", "Timeout", "Failing Since", "Tags", "Title"); + + foreach (var repro in valid) + { + if (repro.Manifest is null) + { + continue; + } + + var manifest = repro.Manifest; + table.AddRow( + Markup.Escape(manifest.Id), + Markup.Escape(FormatState(manifest.State)), + Markup.Escape($"{manifest.TimeoutSeconds}s"), + Markup.Escape(manifest.FailingSince ?? "-"), + Markup.Escape(manifest.Tags.Count > 0 ? string.Join(",", manifest.Tags) : "-"), + Markup.Escape(manifest.Title)); + } + + console.Write(table); + } + + private static string NormalizePath(string path) + { + return path.Replace(Path.DirectorySeparatorChar, '/'); + } + + private static string FormatState(ReproState state) + { + return state switch + { + ReproState.Red => "red", + ReproState.Green => "green", + ReproState.Flaky => "flaky", + _ => state.ToString().ToLowerInvariant() + }; + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/ReproRootLocator.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/ReproRootLocator.cs new file mode 100644 index 000000000..5bc26e2df --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/ReproRootLocator.cs @@ -0,0 +1,86 @@ +namespace LiteDB.ReproRunner.Cli.Infrastructure; + +/// +/// Locates the root directory containing repro definitions. +/// +internal sealed class ReproRootLocator +{ + private readonly string? _defaultRoot; + + /// + /// Initializes a new instance of the class. + /// + /// The optional default root path to use when discovery fails. + public ReproRootLocator(string? defaultRoot = null) + { + _defaultRoot = string.IsNullOrWhiteSpace(defaultRoot) ? null : defaultRoot; + } + + /// + /// Resolves the repro root directory using the supplied override or discovery heuristics. + /// + /// The optional root path override supplied by the user. + /// The resolved repro root directory. + /// Thrown when a repro root cannot be located. + public string ResolveRoot(string? rootOverride) + { + var candidateRoot = string.IsNullOrWhiteSpace(rootOverride) ? _defaultRoot : rootOverride; + + if (!string.IsNullOrEmpty(candidateRoot)) + { + var candidate = Path.GetFullPath(candidateRoot!); + var resolved = TryResolveRoot(candidate); + if (resolved is null) + { + throw new InvalidOperationException($"Unable to locate a Repros directory under '{candidate}'."); + } + + return resolved; + } + + var searchRoots = new List + { + new DirectoryInfo(Directory.GetCurrentDirectory()) + }; + + var baseDirectory = new DirectoryInfo(AppContext.BaseDirectory); + if (!searchRoots.Any(d => string.Equals(d.FullName, baseDirectory.FullName, StringComparison.Ordinal))) + { + searchRoots.Add(baseDirectory); + } + + foreach (var start in searchRoots) + { + var current = start; + + while (current is not null) + { + var resolved = TryResolveRoot(current.FullName); + if (resolved is not null) + { + return resolved; + } + + current = current.Parent; + } + } + + throw new InvalidOperationException("Unable to locate the LiteDB.ReproRunner directory. Use --root to specify the path."); + } + + private static string? TryResolveRoot(string path) + { + if (Directory.Exists(Path.Combine(path, "Repros"))) + { + return Path.GetFullPath(path); + } + + var candidate = Path.Combine(path, "LiteDB.ReproRunner"); + if (Directory.Exists(Path.Combine(candidate, "Repros"))) + { + return Path.GetFullPath(candidate); + } + + return null; + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/TypeRegistrar.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/TypeRegistrar.cs new file mode 100644 index 000000000..015c2ed5d --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Infrastructure/TypeRegistrar.cs @@ -0,0 +1,113 @@ +using Microsoft.Extensions.DependencyInjection; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli.Infrastructure; + +/// +/// Bridges Spectre.Console type registration to Microsoft.Extensions.DependencyInjection. +/// +internal sealed class TypeRegistrar : ITypeRegistrar +{ + private readonly IServiceCollection _services = new ServiceCollection(); + + /// + /// Registers a service implementation type. + /// + /// The service contract. + /// The implementation type. + public void Register(Type service, Type implementation) + { + if (service is null) + { + throw new ArgumentNullException(nameof(service)); + } + + if (implementation is null) + { + throw new ArgumentNullException(nameof(implementation)); + } + + _services.AddSingleton(service, implementation); + } + + /// + /// Registers a specific service instance. + /// + /// The service contract. + /// The implementation instance. + public void RegisterInstance(Type service, object implementation) + { + if (service is null) + { + throw new ArgumentNullException(nameof(service)); + } + + if (implementation is null) + { + throw new ArgumentNullException(nameof(implementation)); + } + + _services.AddSingleton(service, implementation); + } + + /// + /// Registers a lazily constructed service implementation. + /// + /// The service contract. + /// The factory used to create the implementation. + public void RegisterLazy(Type service, Func factory) + { + if (service is null) + { + throw new ArgumentNullException(nameof(service)); + } + + if (factory is null) + { + throw new ArgumentNullException(nameof(factory)); + } + + _services.AddSingleton(service, _ => factory()); + } + + /// + /// Builds the resolver that Spectre.Console uses to resolve services. + /// + /// The type resolver backed by the configured service provider. + public ITypeResolver Build() + { + return new TypeResolver(_services.BuildServiceProvider()); + } + + private sealed class TypeResolver : ITypeResolver, IDisposable + { + private readonly ServiceProvider _provider; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying service provider. + public TypeResolver(ServiceProvider provider) + { + _provider = provider ?? throw new ArgumentNullException(nameof(provider)); + } + + /// + /// Resolves an instance of the requested service type. + /// + /// The service type to resolve. + /// The resolved instance or null when unavailable. + public object? Resolve(Type? type) + { + return type is null ? null : _provider.GetService(type); + } + + /// + /// Releases resources associated with the service provider. + /// + public void Dispose() + { + _provider.Dispose(); + } + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/LiteDB.ReproRunner.Cli.csproj b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/LiteDB.ReproRunner.Cli.csproj new file mode 100644 index 000000000..0248fb3aa --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/LiteDB.ReproRunner.Cli.csproj @@ -0,0 +1,16 @@ + + + Exe + net8.0 + enable + enable + + + + + + + + + + diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/DiscoveredRepro.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/DiscoveredRepro.cs new file mode 100644 index 000000000..884d7e676 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/DiscoveredRepro.cs @@ -0,0 +1,83 @@ +namespace LiteDB.ReproRunner.Cli.Manifests; + +/// +/// Represents a repro manifest discovered on disk along with its metadata. +/// +internal sealed class DiscoveredRepro +{ + /// + /// Initializes a new instance of the class. + /// + /// The resolved root directory that contains all repros. + /// The directory where the repro resides. + /// The full path to the manifest file. + /// The manifest path relative to the repro root. + /// The optional project file path declared by the manifest. + /// The parsed manifest payload, if validation succeeded. + /// The validation results produced while parsing. + /// The manifest identifier captured prior to validation. + public DiscoveredRepro( + string rootPath, + string directoryPath, + string manifestPath, + string relativeManifestPath, + string? projectPath, + ReproManifest? manifest, + ManifestValidationResult validation, + string? rawId) + { + RootPath = rootPath; + DirectoryPath = directoryPath; + ManifestPath = manifestPath; + RelativeManifestPath = relativeManifestPath; + ProjectPath = projectPath; + Manifest = manifest; + Validation = validation; + RawId = rawId; + } + + /// + /// Gets the resolved root directory that contains all repros. + /// + public string RootPath { get; } + + /// + /// Gets the full path to the directory that contains the repro manifest. + /// + public string DirectoryPath { get; } + + /// + /// Gets the full path to the manifest file on disk. + /// + public string ManifestPath { get; } + + /// + /// Gets the manifest path relative to the repro root. + /// + public string RelativeManifestPath { get; } + + /// + /// Gets the optional project file path declared in the manifest. + /// + public string? ProjectPath { get; } + + /// + /// Gets the parsed manifest payload, if validation succeeded. + /// + public ReproManifest? Manifest { get; } + + /// + /// Gets the validation results collected while processing the manifest. + /// + public ManifestValidationResult Validation { get; } + + /// + /// Gets the manifest identifier captured prior to validation. + /// + public string? RawId { get; } + + /// + /// Gets a value indicating whether the manifest was parsed successfully. + /// + public bool IsValid => Manifest is not null && Validation.IsValid; +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestRepository.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestRepository.cs new file mode 100644 index 000000000..2a14eac3a --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestRepository.cs @@ -0,0 +1,133 @@ +using System.Linq; +using System.Text.Json; + +namespace LiteDB.ReproRunner.Cli.Manifests; + +/// +/// Discovers and loads repro manifests from the configured root directory. +/// +internal sealed class ManifestRepository +{ + private readonly string _rootPath; + private readonly string _reprosPath; + private readonly ManifestValidator _validator = new(); + + /// + /// Initializes a new instance of the class. + /// + /// The root directory that contains the repro definitions. + public ManifestRepository(string rootPath) + { + _rootPath = Path.GetFullPath(rootPath); + _reprosPath = Path.Combine(_rootPath, "Repros"); + } + + /// + /// Gets the absolute root path where manifests are searched. + /// + public string RootPath => _rootPath; + + /// + /// Discovers repro manifests beneath the configured root directory. + /// + /// The list of discovered repros with their validation results. + public IReadOnlyList Discover() + { + if (!Directory.Exists(_reprosPath)) + { + return Array.Empty(); + } + + var repros = new List(); + + foreach (var manifestPath in Directory.EnumerateFiles(_reprosPath, "repro.json", SearchOption.AllDirectories)) + { + repros.Add(LoadManifest(manifestPath)); + } + + ApplyDuplicateValidation(repros); + + repros.Sort((left, right) => + { + var leftKey = left.Manifest?.Id ?? left.RawId ?? left.RelativeManifestPath; + var rightKey = right.Manifest?.Id ?? right.RawId ?? right.RelativeManifestPath; + return string.Compare(leftKey, rightKey, StringComparison.OrdinalIgnoreCase); + }); + + return repros; + } + + private DiscoveredRepro LoadManifest(string manifestPath) + { + var validation = new ManifestValidationResult(); + ReproManifest? manifest = null; + string? rawId = null; + + try + { + using var stream = File.OpenRead(manifestPath); + using var document = JsonDocument.Parse(stream); + manifest = _validator.Validate(document.RootElement, validation, out rawId); + } + catch (JsonException ex) + { + validation.AddError($"Invalid JSON: {ex.Message}"); + } + catch (IOException ex) + { + validation.AddError($"Unable to read manifest: {ex.Message}"); + } + catch (UnauthorizedAccessException ex) + { + validation.AddError($"Unable to read manifest: {ex.Message}"); + } + + var directory = Path.GetDirectoryName(manifestPath)!; + var relativeManifestPath = Path.GetRelativePath(_rootPath, manifestPath); + + var projectFiles = Directory.GetFiles(directory, "*.csproj", SearchOption.TopDirectoryOnly); + string? projectPath = null; + + if (projectFiles.Length == 0) + { + validation.AddError("No project file (*.csproj) found in repro directory."); + } + else if (projectFiles.Length > 1) + { + validation.AddError("Multiple project files found in repro directory. Only one project is supported."); + } + else + { + projectPath = projectFiles[0]; + } + + return new DiscoveredRepro(_rootPath, directory, manifestPath, relativeManifestPath, projectPath, manifest, validation, rawId); + } + + private void ApplyDuplicateValidation(List repros) + { + var groups = repros + .Where(r => !string.IsNullOrWhiteSpace(r.Manifest?.Id ?? r.RawId)) + .GroupBy(r => (r.Manifest?.Id ?? r.RawId)!, StringComparer.OrdinalIgnoreCase); + + foreach (var group in groups) + { + if (group.Count() <= 1) + { + continue; + } + + var allPaths = group + .Select(r => r.RelativeManifestPath.Replace(Path.DirectorySeparatorChar, '/')) + .ToList(); + + foreach (var repro in group) + { + var currentPath = repro.RelativeManifestPath.Replace(Path.DirectorySeparatorChar, '/'); + var others = allPaths.Where(path => !string.Equals(path, currentPath, StringComparison.OrdinalIgnoreCase)); + var othersList = string.Join(", ", others); + repro.Validation.AddError($"$.id: duplicate identifier also defined in {othersList}"); + } + } + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestValidationResult.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestValidationResult.cs new file mode 100644 index 000000000..61e4dd6f4 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestValidationResult.cs @@ -0,0 +1,31 @@ +namespace LiteDB.ReproRunner.Cli.Manifests; + +/// +/// Collects validation errors produced while parsing a repro manifest. +/// +internal sealed class ManifestValidationResult +{ + private readonly List _errors = new(); + + /// + /// Gets the collection of validation errors discovered for the manifest. + /// + public IReadOnlyList Errors => _errors; + + /// + /// Gets a value indicating whether the manifest passed validation. + /// + public bool IsValid => _errors.Count == 0; + + /// + /// Records a new validation error for the manifest. + /// + /// The message describing the validation failure. + public void AddError(string message) + { + if (!string.IsNullOrWhiteSpace(message)) + { + _errors.Add(message.Trim()); + } + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestValidator.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestValidator.cs new file mode 100644 index 000000000..6358fd9d9 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ManifestValidator.cs @@ -0,0 +1,750 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace LiteDB.ReproRunner.Cli.Manifests; + +/// +/// Validates repro manifest documents and produces strongly typed models. +/// +internal sealed class ManifestValidator +{ + private static readonly Dictionary AllowedStates = new(StringComparer.OrdinalIgnoreCase) + { + ["red"] = ReproState.Red, + ["green"] = ReproState.Green, + ["flaky"] = ReproState.Flaky + }; + private static readonly Regex IdPattern = new("^[A-Za-z0-9_]+$", RegexOptions.Compiled); + + /// + /// Validates the supplied JSON manifest and produces a instance. + /// + /// The root JSON element to evaluate. + /// The validation result collector that will receive any errors. + /// When this method returns, contains the identifier parsed before validation succeeded. + /// The parsed manifest when validation succeeds; otherwise, null. + public ReproManifest? Validate(JsonElement root, ManifestValidationResult validation, out string? rawId) + { + rawId = null; + + if (root.ValueKind != JsonValueKind.Object) + { + validation.AddError("Manifest root must be a JSON object."); + return null; + } + + var map = new Dictionary(StringComparer.Ordinal); + foreach (var property in root.EnumerateObject()) + { + map[property.Name] = property.Value; + } + + var allowed = new HashSet(StringComparer.Ordinal) + { + "id", + "title", + "issues", + "failingSince", + "timeoutSeconds", + "requiresParallel", + "defaultInstances", + "sharedDatabaseKey", + "args", + "tags", + "state", + "expectedOutcomes", + "supports", + "os" + }; + + foreach (var name in map.Keys) + { + if (!allowed.Contains(name)) + { + validation.AddError($"$.{name}: unknown property."); + } + } + + string? id = null; + if (map.TryGetValue("id", out var idElement)) + { + if (idElement.ValueKind == JsonValueKind.String) + { + var value = idElement.GetString(); + rawId = value; + + if (string.IsNullOrWhiteSpace(value)) + { + validation.AddError("$.id: value must not be empty."); + } + else if (!IdPattern.IsMatch(value)) + { + validation.AddError($"$.id: must match ^[A-Za-z0-9_]+$ (got: {value})."); + } + else + { + id = value; + } + } + else + { + validation.AddError($"$.id: expected string (got: {DescribeKind(idElement.ValueKind)})."); + } + } + else + { + validation.AddError("$.id: property is required."); + } + + string? title = null; + if (map.TryGetValue("title", out var titleElement)) + { + if (titleElement.ValueKind == JsonValueKind.String) + { + var value = titleElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + validation.AddError("$.title: value must not be empty."); + } + else + { + title = value!; + } + } + else + { + validation.AddError($"$.title: expected string (got: {DescribeKind(titleElement.ValueKind)})."); + } + } + else + { + validation.AddError("$.title: property is required."); + } + + var issues = new List(); + if (map.TryGetValue("issues", out var issuesElement)) + { + if (issuesElement.ValueKind == JsonValueKind.Array) + { + var index = 0; + foreach (var item in issuesElement.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + validation.AddError($"$.issues[{index}]: expected string (got: {DescribeKind(item.ValueKind)})."); + } + else + { + var url = item.GetString(); + var trimmed = url?.Trim(); + if (string.IsNullOrWhiteSpace(trimmed) || !Uri.TryCreate(trimmed, UriKind.Absolute, out _)) + { + validation.AddError($"$.issues[{index}]: '{url}' is not a valid absolute URL."); + } + else + { + issues.Add(trimmed); + } + } + + index++; + } + } + else + { + validation.AddError("$.issues: expected an array of strings."); + } + } + + string? failingSince = null; + if (map.TryGetValue("failingSince", out var failingElement)) + { + if (failingElement.ValueKind == JsonValueKind.String) + { + var value = failingElement.GetString(); + var trimmed = value?.Trim(); + if (!string.IsNullOrWhiteSpace(trimmed)) + { + failingSince = trimmed; + } + else + { + validation.AddError("$.failingSince: value must not be empty when provided."); + } + } + else + { + validation.AddError("$.failingSince: expected string value."); + } + } + + int? timeoutSeconds = null; + if (map.TryGetValue("timeoutSeconds", out var timeoutElement)) + { + if (timeoutElement.ValueKind == JsonValueKind.Number && timeoutElement.TryGetInt32(out var timeout)) + { + if (timeout < 1 || timeout > 36000) + { + validation.AddError("$.timeoutSeconds: expected integer between 1 and 36000."); + } + else + { + timeoutSeconds = timeout; + } + } + else + { + validation.AddError("$.timeoutSeconds: expected integer value."); + } + } + else + { + validation.AddError("$.timeoutSeconds: property is required."); + } + + bool? requiresParallel = null; + if (map.TryGetValue("requiresParallel", out var parallelElement)) + { + if (parallelElement.ValueKind == JsonValueKind.True || parallelElement.ValueKind == JsonValueKind.False) + { + requiresParallel = parallelElement.GetBoolean(); + } + else + { + validation.AddError("$.requiresParallel: expected boolean value."); + } + } + else + { + validation.AddError("$.requiresParallel: property is required."); + } + + int? defaultInstances = null; + if (map.TryGetValue("defaultInstances", out var instancesElement)) + { + if (instancesElement.ValueKind == JsonValueKind.Number && instancesElement.TryGetInt32(out var instances)) + { + if (instances < 1) + { + validation.AddError("$.defaultInstances: expected integer >= 1."); + } + else + { + defaultInstances = instances; + } + } + else + { + validation.AddError("$.defaultInstances: expected integer value."); + } + } + else + { + validation.AddError("$.defaultInstances: property is required."); + } + + string? sharedDatabaseKey = null; + if (map.TryGetValue("sharedDatabaseKey", out var sharedElement)) + { + if (sharedElement.ValueKind == JsonValueKind.String) + { + var value = sharedElement.GetString(); + var trimmed = value?.Trim(); + if (string.IsNullOrWhiteSpace(trimmed)) + { + validation.AddError("$.sharedDatabaseKey: value must not be empty when provided."); + } + else + { + sharedDatabaseKey = trimmed; + } + } + else + { + validation.AddError("$.sharedDatabaseKey: expected string value."); + } + } + + var args = new List(); + if (map.TryGetValue("args", out var argsElement)) + { + if (argsElement.ValueKind == JsonValueKind.Array) + { + var index = 0; + foreach (var item in argsElement.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + validation.AddError($"$.args[{index}]: expected string value."); + } + else + { + args.Add(item.GetString()!); + } + + index++; + } + } + else + { + validation.AddError("$.args: expected an array of strings."); + } + } + + var tags = new List(); + if (map.TryGetValue("tags", out var tagsElement)) + { + if (tagsElement.ValueKind == JsonValueKind.Array) + { + var index = 0; + foreach (var item in tagsElement.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + validation.AddError($"$.tags[{index}]: expected string value."); + } + else + { + var tag = item.GetString(); + var trimmed = tag?.Trim(); + if (!string.IsNullOrWhiteSpace(trimmed)) + { + tags.Add(trimmed!); + } + else + { + validation.AddError($"$.tags[{index}]: value must not be empty."); + } + } + + index++; + } + } + else + { + validation.AddError("$.tags: expected an array of strings."); + } + } + + var supports = new List(); + if (map.TryGetValue("supports", out var supportsElement)) + { + if (supportsElement.ValueKind == JsonValueKind.Array) + { + var index = 0; + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var item in supportsElement.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + validation.AddError($"$.supports[{index}]: expected string value."); + } + else + { + var raw = item.GetString(); + var trimmed = raw?.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + validation.AddError($"$.supports[{index}]: value must not be empty."); + } + else + { + var normalized = trimmed.ToLowerInvariant(); + if (normalized != "windows" && normalized != "linux" && normalized != "any") + { + validation.AddError($"$.supports[{index}]: expected one of windows, linux, any."); + } + else if (normalized == "any" && seen.Count > 0) + { + validation.AddError("$.supports: 'any' cannot be combined with other platform values."); + } + else if (normalized != "any" && seen.Contains("any")) + { + validation.AddError("$.supports: 'any' cannot be combined with other platform values."); + } + else if (seen.Add(normalized)) + { + supports.Add(normalized); + } + } + } + + index++; + } + } + else if (supportsElement.ValueKind != JsonValueKind.Null) + { + validation.AddError("$.supports: expected an array of strings."); + } + } + + ReproOsConstraints? osConstraints = null; + if (map.TryGetValue("os", out var osElement)) + { + if (osElement.ValueKind == JsonValueKind.Object) + { + osConstraints = ParseOsConstraints(osElement, validation); + } + else if (osElement.ValueKind != JsonValueKind.Null) + { + validation.AddError("$.os: expected object value."); + } + } + + ReproState? state = null; + if (map.TryGetValue("state", out var stateElement)) + { + if (stateElement.ValueKind == JsonValueKind.String) + { + var value = stateElement.GetString()?.Trim(); + if (string.IsNullOrEmpty(value) || !AllowedStates.TryGetValue(value, out var parsedState)) + { + validation.AddError("$.state: expected one of red, green, flaky."); + } + else + { + state = parsedState; + } + } + else + { + validation.AddError("$.state: expected string value."); + } + } + else + { + validation.AddError("$.state: property is required."); + } + + ReproVariantOutcomeExpectations? expectedOutcomes = ReproVariantOutcomeExpectations.Empty; + if (map.TryGetValue("expectedOutcomes", out var expectedOutcomesElement)) + { + expectedOutcomes = ParseExpectedOutcomes(expectedOutcomesElement, validation); + } + + if (requiresParallel == true) + { + if (defaultInstances.HasValue && defaultInstances.Value < 2) + { + validation.AddError("$.defaultInstances: must be >= 2 when requiresParallel is true."); + } + + if (string.IsNullOrWhiteSpace(sharedDatabaseKey)) + { + validation.AddError("$.sharedDatabaseKey: required when requiresParallel is true."); + } + } + + if (id is null || title is null || timeoutSeconds is null || requiresParallel is null || defaultInstances is null || state is null || expectedOutcomes is null) + { + return null; + } + + var issuesArray = issues.Count > 0 ? issues.ToArray() : Array.Empty(); + var argsArray = args.Count > 0 ? args.ToArray() : Array.Empty(); + var tagsArray = tags.Count > 0 ? tags.ToArray() : Array.Empty(); + var supportsArray = supports.Count > 0 ? supports.ToArray() : Array.Empty(); + + return new ReproManifest( + id, + title, + issuesArray, + failingSince, + timeoutSeconds.Value, + requiresParallel.Value, + defaultInstances.Value, + sharedDatabaseKey, + argsArray, + tagsArray, + state.Value, + expectedOutcomes, + supportsArray, + osConstraints); + } + + private static ReproOsConstraints? ParseOsConstraints(JsonElement root, ManifestValidationResult validation) + { + var allowed = new HashSet(StringComparer.Ordinal) + { + "includePlatforms", + "includeLabels", + "excludePlatforms", + "excludeLabels" + }; + + foreach (var property in root.EnumerateObject()) + { + if (!allowed.Contains(property.Name)) + { + validation.AddError($"$.os.{property.Name}: unknown property."); + } + } + + var includePlatforms = ParsePlatformArray(root, "includePlatforms", validation); + var includeLabels = ParseLabelArray(root, "includeLabels", validation); + var excludePlatforms = ParsePlatformArray(root, "excludePlatforms", validation); + var excludeLabels = ParseLabelArray(root, "excludeLabels", validation); + + if (includePlatforms is null || includeLabels is null || excludePlatforms is null || excludeLabels is null) + { + return null; + } + + return new ReproOsConstraints(includePlatforms, includeLabels, excludePlatforms, excludeLabels); + } + + private static IReadOnlyList? ParsePlatformArray(JsonElement root, string propertyName, ManifestValidationResult validation) + { + if (!root.TryGetProperty(propertyName, out var element) || element.ValueKind == JsonValueKind.Null) + { + return Array.Empty(); + } + + if (element.ValueKind != JsonValueKind.Array) + { + validation.AddError($"$.os.{propertyName}: expected an array of strings."); + return null; + } + + var values = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var index = 0; + + foreach (var item in element.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + validation.AddError($"$.os.{propertyName}[{index}]: expected string value."); + } + else + { + var value = item.GetString(); + var trimmed = value?.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + validation.AddError($"$.os.{propertyName}[{index}]: value must not be empty."); + } + else + { + var normalized = trimmed.ToLowerInvariant(); + if (normalized != "windows" && normalized != "linux") + { + validation.AddError($"$.os.{propertyName}[{index}]: expected one of windows, linux."); + } + else if (seen.Add(normalized)) + { + values.Add(normalized); + } + } + } + + index++; + } + + return values; + } + + private static IReadOnlyList? ParseLabelArray(JsonElement root, string propertyName, ManifestValidationResult validation) + { + if (!root.TryGetProperty(propertyName, out var element) || element.ValueKind == JsonValueKind.Null) + { + return Array.Empty(); + } + + if (element.ValueKind != JsonValueKind.Array) + { + validation.AddError($"$.os.{propertyName}: expected an array of strings."); + return null; + } + + var values = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var index = 0; + + foreach (var item in element.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.String) + { + validation.AddError($"$.os.{propertyName}[{index}]: expected string value."); + } + else + { + var value = item.GetString(); + var trimmed = value?.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + validation.AddError($"$.os.{propertyName}[{index}]: value must not be empty."); + } + else if (seen.Add(trimmed)) + { + values.Add(trimmed); + } + } + + index++; + } + + return values; + } + + private static string DescribeKind(JsonValueKind kind) + { + return kind switch + { + JsonValueKind.Array => "array", + JsonValueKind.Object => "object", + JsonValueKind.String => "string", + JsonValueKind.Number => "number", + JsonValueKind.True => "boolean", + JsonValueKind.False => "boolean", + JsonValueKind.Null => "null", + _ => kind.ToString().ToLowerInvariant() + }; + } + + private static ReproVariantOutcomeExpectations? ParseExpectedOutcomes(JsonElement root, ManifestValidationResult validation) + { + if (root.ValueKind != JsonValueKind.Object) + { + validation.AddError("$.expectedOutcomes: expected object value."); + return null; + } + + var allowed = new HashSet(StringComparer.Ordinal) + { + "package", + "latest" + }; + + foreach (var property in root.EnumerateObject()) + { + if (!allowed.Contains(property.Name)) + { + validation.AddError($"$.expectedOutcomes.{property.Name}: unknown property."); + } + } + + ReproOutcomeExpectation? package = null; + ReproOutcomeExpectation? latest = null; + + if (root.TryGetProperty("package", out var packageElement)) + { + package = ParseOutcomeExpectation(packageElement, "$.expectedOutcomes.package", validation); + } + + if (root.TryGetProperty("latest", out var latestElement)) + { + latest = ParseOutcomeExpectation(latestElement, "$.expectedOutcomes.latest", validation); + if (latest?.Kind == ReproOutcomeKind.HardFail) + { + validation.AddError("$.expectedOutcomes.latest.kind: hardFail is only supported for the package variant."); + } + } + + if (package is null && root.TryGetProperty("package", out _)) + { + return null; + } + + if (latest is null && root.TryGetProperty("latest", out _)) + { + return null; + } + + return new ReproVariantOutcomeExpectations(package, latest); + } + + private static ReproOutcomeExpectation? ParseOutcomeExpectation(JsonElement element, string path, ManifestValidationResult validation) + { + if (element.ValueKind != JsonValueKind.Object) + { + validation.AddError($"{path}: expected object value."); + return null; + } + + var allowed = new HashSet(StringComparer.Ordinal) + { + "kind", + "exitCode", + "logContains" + }; + + foreach (var property in element.EnumerateObject()) + { + if (!allowed.Contains(property.Name)) + { + validation.AddError($"{path}.{property.Name}: unknown property."); + } + } + + if (!element.TryGetProperty("kind", out var kindElement) || kindElement.ValueKind != JsonValueKind.String) + { + validation.AddError($"{path}.kind: expected string value."); + return null; + } + + var kindText = kindElement.GetString()?.Trim(); + if (string.IsNullOrEmpty(kindText)) + { + validation.AddError($"{path}.kind: value must not be empty."); + return null; + } + + ReproOutcomeKind kind; + switch (kindText.ToLowerInvariant()) + { + case "reproduce": + kind = ReproOutcomeKind.Reproduce; + break; + case "norepro": + kind = ReproOutcomeKind.NoRepro; + break; + case "hardfail": + kind = ReproOutcomeKind.HardFail; + break; + default: + validation.AddError($"{path}.kind: expected one of reproduce, norepro, hardFail."); + return null; + } + + int? exitCode = null; + if (element.TryGetProperty("exitCode", out var exitCodeElement)) + { + if (exitCodeElement.ValueKind == JsonValueKind.Number && exitCodeElement.TryGetInt32(out var parsed)) + { + exitCode = parsed; + } + else + { + validation.AddError($"{path}.exitCode: expected integer value."); + return null; + } + } + + string? logContains = null; + if (element.TryGetProperty("logContains", out var logElement)) + { + if (logElement.ValueKind == JsonValueKind.String) + { + var value = logElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + validation.AddError($"{path}.logContains: value must not be empty when provided."); + return null; + } + + logContains = value; + } + else + { + validation.AddError($"{path}.logContains: expected string value."); + return null; + } + } + + return new ReproOutcomeExpectation(kind, exitCode, logContains); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproManifest.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproManifest.cs new file mode 100644 index 000000000..e1c68338e --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproManifest.cs @@ -0,0 +1,128 @@ +using System.Collections.Generic; + +namespace LiteDB.ReproRunner.Cli.Manifests; + +/// +/// Describes a reproducible scenario definition consumed by the CLI. +/// +internal sealed class ReproManifest +{ + /// + /// Initializes a new instance of the class. + /// + /// The unique identifier for the repro. + /// A friendly title describing the repro. + /// Issue tracker links associated with the repro. + /// The date or version when the repro began failing. + /// The timeout for the repro in seconds. + /// Indicates whether the repro requires parallel instances. + /// The default number of instances to launch. + /// The key used to share database state between instances. + /// Additional command-line arguments passed to the repro host. + /// Tags describing the repro characteristics. + /// The current state of the repro (e.g., red, green). + /// The optional expected outcomes per variant. + /// Optional collection declaring supported platform families. + /// Optional OS constraint overrides controlling runner labels. + public ReproManifest( + string id, + string title, + IReadOnlyList issues, + string? failingSince, + int timeoutSeconds, + bool requiresParallel, + int defaultInstances, + string? sharedDatabaseKey, + IReadOnlyList args, + IReadOnlyList tags, + ReproState state, + ReproVariantOutcomeExpectations expectedOutcomes, + IReadOnlyList? supports = null, + ReproOsConstraints? osConstraints = null) + { + Id = id; + Title = title; + Issues = issues; + FailingSince = failingSince; + TimeoutSeconds = timeoutSeconds; + RequiresParallel = requiresParallel; + DefaultInstances = defaultInstances; + SharedDatabaseKey = sharedDatabaseKey; + Args = args; + Tags = tags; + State = state; + ExpectedOutcomes = expectedOutcomes ?? ReproVariantOutcomeExpectations.Empty; + Supports = supports ?? Array.Empty(); + OsConstraints = osConstraints; + } + + /// + /// Gets the unique identifier for the repro. + /// + public string Id { get; } + + /// + /// Gets the human-readable title for the repro. + /// + public string Title { get; } + + /// + /// Gets the issue tracker links associated with the repro. + /// + public IReadOnlyList Issues { get; } + + /// + /// Gets the optional date or version indicating when the repro started failing. + /// + public string? FailingSince { get; } + + /// + /// Gets the timeout for the repro, in seconds. + /// + public int TimeoutSeconds { get; } + + /// + /// Gets a value indicating whether the repro requires parallel execution. + /// + public bool RequiresParallel { get; } + + /// + /// Gets the default number of instances to execute. + /// + public int DefaultInstances { get; } + + /// + /// Gets the shared database key used to coordinate state between instances. + /// + public string? SharedDatabaseKey { get; } + + /// + /// Gets the additional command-line arguments passed to the repro host. + /// + public IReadOnlyList Args { get; } + + /// + /// Gets the descriptive tags applied to the repro. + /// + public IReadOnlyList Tags { get; } + + /// + /// Gets the declared state of the repro (for example, ). + /// + public ReproState State { get; } + + /// + /// Gets the expected outcomes for the package and latest variants. + /// + public ReproVariantOutcomeExpectations ExpectedOutcomes { get; } + + /// + /// Gets the declared platform families supported by this repro. + /// + public IReadOnlyList Supports { get; } + + /// + /// Gets the runner label constraints declared by the manifest. + /// + public ReproOsConstraints? OsConstraints { get; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOsConstraints.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOsConstraints.cs new file mode 100644 index 000000000..b09b55b1f --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOsConstraints.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; + +namespace LiteDB.ReproRunner.Cli.Manifests; + +/// +/// Represents advanced operating system constraints declared by a repro manifest. +/// +internal sealed class ReproOsConstraints +{ + /// + /// Initializes a new instance of the class. + /// + /// Platform families that must be included. + /// Specific runner labels that must be included. + /// Platform families that must be excluded. + /// Specific runner labels that must be excluded. + public ReproOsConstraints( + IReadOnlyList includePlatforms, + IReadOnlyList includeLabels, + IReadOnlyList excludePlatforms, + IReadOnlyList excludeLabels) + { + IncludePlatforms = includePlatforms ?? Array.Empty(); + IncludeLabels = includeLabels ?? Array.Empty(); + ExcludePlatforms = excludePlatforms ?? Array.Empty(); + ExcludeLabels = excludeLabels ?? Array.Empty(); + } + + /// + /// Gets the platform families that must be included. + /// + public IReadOnlyList IncludePlatforms { get; } + + /// + /// Gets the runner labels that must be included. + /// + public IReadOnlyList IncludeLabels { get; } + + /// + /// Gets the platform families that must be excluded. + /// + public IReadOnlyList ExcludePlatforms { get; } + + /// + /// Gets the runner labels that must be excluded. + /// + public IReadOnlyList ExcludeLabels { get; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOutcomeExpectation.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOutcomeExpectation.cs new file mode 100644 index 000000000..aacc934ba --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOutcomeExpectation.cs @@ -0,0 +1,17 @@ +namespace LiteDB.ReproRunner.Cli.Manifests; + +internal sealed class ReproOutcomeExpectation +{ + public ReproOutcomeExpectation(ReproOutcomeKind kind, int? exitCode, string? logContains) + { + Kind = kind; + ExitCode = exitCode; + LogContains = logContains; + } + + public ReproOutcomeKind Kind { get; } + + public int? ExitCode { get; } + + public string? LogContains { get; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOutcomeKind.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOutcomeKind.cs new file mode 100644 index 000000000..d4128fe10 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproOutcomeKind.cs @@ -0,0 +1,8 @@ +namespace LiteDB.ReproRunner.Cli.Manifests; + +internal enum ReproOutcomeKind +{ + Reproduce, + NoRepro, + HardFail +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproState.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproState.cs new file mode 100644 index 000000000..fc03dfc7d --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproState.cs @@ -0,0 +1,8 @@ +namespace LiteDB.ReproRunner.Cli.Manifests; + +internal enum ReproState +{ + Red, + Green, + Flaky +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproVariantOutcomeExpectations.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproVariantOutcomeExpectations.cs new file mode 100644 index 000000000..64cc82116 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Manifests/ReproVariantOutcomeExpectations.cs @@ -0,0 +1,16 @@ +namespace LiteDB.ReproRunner.Cli.Manifests; + +internal sealed class ReproVariantOutcomeExpectations +{ + public static readonly ReproVariantOutcomeExpectations Empty = new(null, null); + + public ReproVariantOutcomeExpectations(ReproOutcomeExpectation? package, ReproOutcomeExpectation? latest) + { + Package = package; + Latest = latest; + } + + public ReproOutcomeExpectation? Package { get; } + + public ReproOutcomeExpectation? Latest { get; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Program.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Program.cs new file mode 100644 index 000000000..d9e2daeac --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Program.cs @@ -0,0 +1,121 @@ +using System.Threading; +using LiteDB.ReproRunner.Cli.Commands; +using LiteDB.ReproRunner.Cli.Execution; +using LiteDB.ReproRunner.Cli.Infrastructure; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace LiteDB.ReproRunner.Cli; + +/// +/// Entry point for the repro runner CLI application. +/// +internal static class Program +{ + /// + /// Application entry point. + /// + /// The command-line arguments provided by the user. + /// The process exit code. + public static async Task Main(string[] args) + { + using var cts = new CancellationTokenSource(); + + Console.CancelKeyPress += (_, eventArgs) => + { + eventArgs.Cancel = true; + cts.Cancel(); + }; + + var (filteredArgs, rootOverride) = ExtractGlobalOptions(args); + var console = AnsiConsole.Create(new AnsiConsoleSettings()); + var registrar = new TypeRegistrar(); + registrar.RegisterInstance(typeof(IAnsiConsole), console); + registrar.RegisterInstance(typeof(ReproExecutor), new ReproExecutor()); + registrar.RegisterInstance(typeof(RunDirectoryPlanner), new RunDirectoryPlanner()); + registrar.RegisterInstance(typeof(ReproBuildCoordinator), new ReproBuildCoordinator()); + registrar.RegisterInstance(typeof(ReproRootLocator), new ReproRootLocator(rootOverride)); + registrar.RegisterInstance(typeof(CancellationToken), cts.Token); + + var app = new CommandApp(registrar); + app.Configure(config => + { + config.SetApplicationName("repro-runner"); + config.AddCommand("list").WithDescription("List discovered repros and highlight invalid manifests."); + config.AddCommand("show").WithDescription("Display manifest metadata for a repro."); + config.AddCommand("validate").WithDescription("Validate repro manifests."); + config.AddCommand("run").WithDescription("Execute repros against package and source builds."); + }); + + try + { + return await app.RunAsync(filteredArgs).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + console.MarkupLine("[yellow]Execution cancelled.[/]"); + return 1; + } + catch (CommandRuntimeException ex) + { + console.WriteException(ex, ExceptionFormats.ShortenEverything); + return 1; + } + catch (Exception ex) + { + console.WriteException(ex, ExceptionFormats.ShortenEverything); + return 1; + } + } + + private static (string[] FilteredArgs, string? RootOverride) ExtractGlobalOptions(string[] args) + { + if (args.Length == 0) + { + return (Array.Empty(), null); + } + + var filtered = new List(); + string? rootOverride = null; + var index = 0; + + while (index < args.Length) + { + var token = args[index]; + + if (!token.StartsWith("-", StringComparison.Ordinal)) + { + break; + } + + if (token == "--root") + { + index++; + if (index >= args.Length) + { + throw new InvalidOperationException("--root requires a path."); + } + + rootOverride = args[index]; + index++; + continue; + } + + if (token == "--") + { + filtered.Add(token); + index++; + break; + } + + break; + } + + for (; index < args.Length; index++) + { + filtered.Add(args[index]); + } + + return (filtered.ToArray(), rootOverride); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Properties/AssemblyInfo.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..952babc51 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("LiteDB.ReproRunner.Tests")] diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/LiteDB.ReproRunner.Shared.csproj b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/LiteDB.ReproRunner.Shared.csproj new file mode 100644 index 000000000..e8cd59923 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/LiteDB.ReproRunner.Shared.csproj @@ -0,0 +1,7 @@ + + + net8.0 + enable + enable + + diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostClient.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostClient.cs new file mode 100644 index 000000000..1de3fb54d --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostClient.cs @@ -0,0 +1,222 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; + +namespace LiteDB.ReproRunner.Shared.Messaging; + +/// +/// Coordinates structured communication between repro processes and the host CLI. +/// +public sealed class ReproHostClient +{ + private readonly TextWriter _writer; + private readonly TextReader _reader; + private readonly JsonSerializerOptions _options; + + /// + /// Initializes a new instance of the class. + /// + /// The writer used to send messages to the host. + /// The reader used to receive messages from the host. + /// Optional serializer options to customize message encoding. + public ReproHostClient(TextWriter writer, TextReader reader, JsonSerializerOptions? options = null) + { + _writer = writer ?? throw new ArgumentNullException(nameof(writer)); + _reader = reader ?? throw new ArgumentNullException(nameof(reader)); + _options = options ?? ReproJsonOptions.Default; + } + + /// + /// Creates a client that communicates over the console standard streams. + /// + /// The created client instance. + public static ReproHostClient CreateDefault() + { + return new ReproHostClient(Console.Out, Console.In); + } + + /// + /// Sends a structured envelope to the host asynchronously. + /// + /// The envelope to send. + /// The token used to observe cancellation requests. + /// A task that completes when the message has been written. + public Task SendAsync(ReproHostMessageEnvelope envelope, CancellationToken cancellationToken = default) + { + if (envelope is null) + { + throw new ArgumentNullException(nameof(envelope)); + } + + cancellationToken.ThrowIfCancellationRequested(); + var json = JsonSerializer.Serialize(envelope, _options); + return WriteAsync(json, cancellationToken); + } + + /// + /// Sends a structured log message to the host. + /// + /// The log message text. + /// The severity associated with the log message. + /// The token used to observe cancellation requests. + /// A task that completes when the message has been written. + public Task SendLogAsync(string message, ReproHostLogLevel level = ReproHostLogLevel.Information, CancellationToken cancellationToken = default) + { + return SendAsync(ReproHostMessageEnvelope.CreateLog(message, level), cancellationToken); + } + + /// + /// Sends a structured result message to the host. + /// + /// Indicates whether the repro succeeded. + /// An optional summary describing the result. + /// Optional payload accompanying the result. + /// The token used to observe cancellation requests. + /// A task that completes when the message has been written. + public Task SendResultAsync(bool success, string? summary = null, object? payload = null, CancellationToken cancellationToken = default) + { + return SendAsync(ReproHostMessageEnvelope.CreateResult(success, summary, payload), cancellationToken); + } + + /// + /// Sends a lifecycle notification to the host. + /// + /// The lifecycle stage being reported. + /// Optional payload accompanying the notification. + /// The token used to observe cancellation requests. + /// A task that completes when the message has been written. + public Task SendLifecycleAsync(string stage, object? payload = null, CancellationToken cancellationToken = default) + { + return SendAsync(ReproHostMessageEnvelope.CreateLifecycle(stage, payload), cancellationToken); + } + + /// + /// Sends a progress update to the host. + /// + /// The progress stage being reported. + /// The optional percentage complete. + /// Optional payload accompanying the notification. + /// The token used to observe cancellation requests. + /// A task that completes when the message has been written. + public Task SendProgressAsync(string stage, double? percentComplete = null, object? payload = null, CancellationToken cancellationToken = default) + { + return SendAsync(ReproHostMessageEnvelope.CreateProgress(stage, percentComplete, payload), cancellationToken); + } + + /// + /// Sends a configuration handshake to the host. + /// + /// Indicates whether the repro was built against the source project. + /// The LiteDB package version referenced by the repro, when applicable. + /// The token used to observe cancellation requests. + /// A task that completes when the message has been written. + public Task SendConfigurationAsync(bool useProjectReference, string? liteDbPackageVersion, CancellationToken cancellationToken = default) + { + return SendAsync(ReproHostMessageEnvelope.CreateConfiguration(useProjectReference, liteDbPackageVersion), cancellationToken); + } + + /// + /// Sends a structured log message to the host synchronously. + /// + /// The log message text. + /// The severity associated with the log message. + public void SendLog(string message, ReproHostLogLevel level = ReproHostLogLevel.Information) + { + SendLogAsync(message, level).GetAwaiter().GetResult(); + } + + /// + /// Sends a structured result message to the host synchronously. + /// + /// Indicates whether the repro succeeded. + /// An optional summary describing the result. + /// Optional payload accompanying the result. + public void SendResult(bool success, string? summary = null, object? payload = null) + { + SendResultAsync(success, summary, payload).GetAwaiter().GetResult(); + } + + /// + /// Sends a lifecycle notification to the host synchronously. + /// + /// The lifecycle stage being reported. + /// Optional payload accompanying the notification. + public void SendLifecycle(string stage, object? payload = null) + { + SendLifecycleAsync(stage, payload).GetAwaiter().GetResult(); + } + + /// + /// Sends a progress update to the host synchronously. + /// + /// The progress stage being reported. + /// The optional percentage complete. + /// Optional payload accompanying the notification. + public void SendProgress(string stage, double? percentComplete = null, object? payload = null) + { + SendProgressAsync(stage, percentComplete, payload).GetAwaiter().GetResult(); + } + + /// + /// Sends a configuration handshake to the host synchronously. + /// + /// Indicates whether the repro was built against the source project. + /// The LiteDB package version referenced by the repro, when applicable. + public void SendConfiguration(bool useProjectReference, string? liteDbPackageVersion) + { + SendConfigurationAsync(useProjectReference, liteDbPackageVersion).GetAwaiter().GetResult(); + } + + /// + /// Reads a single input envelope from the host. + /// + /// The token used to observe cancellation requests. + /// The next input envelope, or null when the stream ends. + public async Task ReadAsync(CancellationToken cancellationToken = default) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var line = await _reader.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + return null; + } + + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + if (ReproInputEnvelope.TryParse(line, out var envelope, out _)) + { + return envelope; + } + } + } + + /// + /// Reads input envelopes from the host until the stream is exhausted. + /// + /// The token used to observe cancellation requests. + /// An async stream of input envelopes. + public async IAsyncEnumerable ReadAllAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + while (true) + { + var envelope = await ReadAsync(cancellationToken).ConfigureAwait(false); + if (envelope is null) + { + yield break; + } + + yield return envelope; + } + } + + private async Task WriteAsync(string json, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await _writer.WriteLineAsync(json).ConfigureAwait(false); + await _writer.FlushAsync().ConfigureAwait(false); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostConfigurationPayload.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostConfigurationPayload.cs new file mode 100644 index 000000000..3b0529df5 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostConfigurationPayload.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace LiteDB.ReproRunner.Shared.Messaging; + +/// +/// Represents the configuration handshake payload emitted by repro processes. +/// +public sealed record class ReproHostConfigurationPayload +{ + /// + /// Gets or sets a value indicating whether the repro was built against the source project. + /// + [JsonPropertyName("useProjectReference")] + public required bool UseProjectReference { get; init; } + + /// + /// Gets or sets the LiteDB package version used by the repro when built from NuGet. + /// + [JsonPropertyName("liteDBPackageVersion")] + public string? LiteDBPackageVersion { get; init; } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostLogLevel.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostLogLevel.cs new file mode 100644 index 000000000..73169fb30 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostLogLevel.cs @@ -0,0 +1,37 @@ +namespace LiteDB.ReproRunner.Shared.Messaging; + +/// +/// Represents the severity of a structured repro log message. +/// +public enum ReproHostLogLevel +{ + /// + /// Verbose diagnostic messages useful for tracing execution. + /// + Trace, + + /// + /// Low-level diagnostic messages used for debugging. + /// + Debug, + + /// + /// Informational messages describing normal operation. + /// + Information, + + /// + /// Indicates a non-fatal issue that may require attention. + /// + Warning, + + /// + /// Indicates that an error occurred during execution. + /// + Error, + + /// + /// Indicates a critical failure that prevents continued execution. + /// + Critical +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostMessageEnvelope.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostMessageEnvelope.cs new file mode 100644 index 000000000..133da6bab --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostMessageEnvelope.cs @@ -0,0 +1,212 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace LiteDB.ReproRunner.Shared.Messaging; + +/// +/// Represents a structured message emitted by a repro process. +/// +public sealed class ReproHostMessageEnvelope +{ + /// + /// Gets or sets the message type identifier. + /// + [JsonPropertyName("type")] + public required string Type { get; init; } + + /// + /// Gets or sets the timestamp associated with the message. + /// + [JsonPropertyName("timestamp")] + public DateTimeOffset Timestamp { get; init; } = DateTimeOffset.UtcNow; + + /// + /// Gets or sets the log level for log messages. + /// + [JsonPropertyName("level")] + public ReproHostLogLevel? Level { get; init; } + + /// + /// Gets or sets the textual payload associated with the message. + /// + [JsonPropertyName("text")] + public string? Text { get; init; } + + /// + /// Gets or sets the success flag for result messages. + /// + [JsonPropertyName("success")] + public bool? Success { get; init; } + + /// + /// Gets or sets the lifecycle or progress event name. + /// + [JsonPropertyName("event")] + public string? Event { get; init; } + + /// + /// Gets or sets the percentage complete for progress messages. + /// + [JsonPropertyName("progress")] + public double? Progress { get; init; } + + /// + /// Gets or sets the optional payload serialized with the message. + /// + [JsonPropertyName("payload")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public JsonElement? Payload { get; init; } + + /// + /// Creates a log message envelope. + /// + /// The log message text. + /// The severity associated with the log message. + /// An optional timestamp to associate with the message. + /// The constructed log message envelope. + public static ReproHostMessageEnvelope CreateLog(string message, ReproHostLogLevel level = ReproHostLogLevel.Information, DateTimeOffset? timestamp = null) + { + if (message is null) + { + throw new ArgumentNullException(nameof(message)); + } + + return new ReproHostMessageEnvelope + { + Type = ReproHostMessageTypes.Log, + Timestamp = timestamp ?? DateTimeOffset.UtcNow, + Level = level, + Text = message + }; + } + + /// + /// Creates a result message envelope. + /// + /// Indicates whether the repro succeeded. + /// An optional summary describing the result. + /// Optional payload associated with the result. + /// An optional timestamp to associate with the message. + /// The constructed result message envelope. + public static ReproHostMessageEnvelope CreateResult(bool success, string? summary = null, object? payload = null, DateTimeOffset? timestamp = null) + { + return new ReproHostMessageEnvelope + { + Type = ReproHostMessageTypes.Result, + Timestamp = timestamp ?? DateTimeOffset.UtcNow, + Success = success, + Text = summary, + Payload = payload is null ? null : JsonSerializer.SerializeToElement(payload, ReproJsonOptions.Default) + }; + } + + /// + /// Creates a lifecycle message envelope. + /// + /// The lifecycle stage being reported. + /// Optional payload associated with the lifecycle event. + /// An optional timestamp to associate with the message. + /// The constructed lifecycle message envelope. + public static ReproHostMessageEnvelope CreateLifecycle(string stage, object? payload = null, DateTimeOffset? timestamp = null) + { + if (string.IsNullOrWhiteSpace(stage)) + { + throw new ArgumentException("Lifecycle stage must be provided.", nameof(stage)); + } + + return new ReproHostMessageEnvelope + { + Type = ReproHostMessageTypes.Lifecycle, + Timestamp = timestamp ?? DateTimeOffset.UtcNow, + Event = stage, + Payload = payload is null ? null : JsonSerializer.SerializeToElement(payload, ReproJsonOptions.Default) + }; + } + + /// + /// Creates a progress message envelope. + /// + /// The progress stage being reported. + /// The optional percentage complete. + /// Optional payload associated with the progress update. + /// An optional timestamp to associate with the message. + /// The constructed progress message envelope. + public static ReproHostMessageEnvelope CreateProgress(string stage, double? percentComplete = null, object? payload = null, DateTimeOffset? timestamp = null) + { + return new ReproHostMessageEnvelope + { + Type = ReproHostMessageTypes.Progress, + Timestamp = timestamp ?? DateTimeOffset.UtcNow, + Event = stage, + Progress = percentComplete, + Payload = payload is null ? null : JsonSerializer.SerializeToElement(payload, ReproJsonOptions.Default) + }; + } + + /// + /// Creates a configuration handshake message envelope. + /// + /// Indicates whether the repro was built against the source project. + /// The LiteDB package version referenced by the repro, when applicable. + /// An optional timestamp to associate with the message. + /// The constructed configuration message envelope. + public static ReproHostMessageEnvelope CreateConfiguration(bool useProjectReference, string? liteDbPackageVersion, DateTimeOffset? timestamp = null) + { + var payload = new ReproHostConfigurationPayload + { + UseProjectReference = useProjectReference, + LiteDBPackageVersion = liteDbPackageVersion + }; + + return new ReproHostMessageEnvelope + { + Type = ReproHostMessageTypes.Configuration, + Timestamp = timestamp ?? DateTimeOffset.UtcNow, + Payload = JsonSerializer.SerializeToElement(payload, ReproJsonOptions.Default) + }; + } + + /// + /// Deserializes the payload to a strongly typed value. + /// + /// The payload type. + /// The deserialized payload, or null when no payload is available. + public T? DeserializePayload() + { + if (Payload is not { } payload) + { + return default; + } + + return payload.Deserialize(ReproJsonOptions.Default); + } + + /// + /// Attempts to parse a message envelope from its JSON representation. + /// + /// The JSON text to parse. + /// When this method returns, contains the parsed envelope if successful. + /// When this method returns, contains the parsing error if parsing failed. + /// true if the envelope was parsed successfully; otherwise, false. + public static bool TryParse(string? line, out ReproHostMessageEnvelope? envelope, out JsonException? error) + { + envelope = null; + error = null; + + if (string.IsNullOrWhiteSpace(line)) + { + return false; + } + + try + { + envelope = JsonSerializer.Deserialize(line, ReproJsonOptions.Default); + return envelope is not null; + } + catch (JsonException jsonException) + { + error = jsonException; + return false; + } + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostMessageTypes.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostMessageTypes.cs new file mode 100644 index 000000000..9e94f5f81 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproHostMessageTypes.cs @@ -0,0 +1,32 @@ +namespace LiteDB.ReproRunner.Shared.Messaging; + +/// +/// Provides the well-known structured message kinds emitted by repros. +/// +public static class ReproHostMessageTypes +{ + /// + /// Identifies structured log messages emitted by repros. + /// + public const string Log = "log"; + + /// + /// Identifies structured result messages emitted by repros. + /// + public const string Result = "result"; + + /// + /// Identifies lifecycle notifications emitted by repros. + /// + public const string Lifecycle = "lifecycle"; + + /// + /// Identifies progress updates emitted by repros. + /// + public const string Progress = "progress"; + + /// + /// Identifies configuration handshakes emitted by repros. + /// + public const string Configuration = "configuration"; +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproInputEnvelope.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproInputEnvelope.cs new file mode 100644 index 000000000..ae1bc7581 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproInputEnvelope.cs @@ -0,0 +1,143 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace LiteDB.ReproRunner.Shared.Messaging; + +/// +/// Represents a structured input sent to a repro process via STDIN. +/// +public sealed class ReproInputEnvelope +{ + /// + /// Gets or sets the message type identifier. + /// + [JsonPropertyName("type")] + public required string Type { get; init; } + + /// + /// Gets or sets the time the envelope was created. + /// + [JsonPropertyName("timestamp")] + public DateTimeOffset Timestamp { get; init; } = DateTimeOffset.UtcNow; + + /// + /// Gets or sets the optional payload associated with the envelope. + /// + [JsonPropertyName("payload")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public JsonElement? Payload { get; init; } + + /// + /// Creates a host-ready envelope that signals the repro host is ready to receive work. + /// + /// The unique identifier for the run. + /// The shared database directory assigned by the host. + /// The index of the current instance. + /// The total number of instances participating in the run. + /// The identifier of the manifest being executed. + /// The constructed host-ready envelope. + public static ReproInputEnvelope CreateHostReady(string runIdentifier, string sharedDatabaseRoot, int instanceIndex, int totalInstances, string? manifestId) + { + var payload = new ReproHostReadyPayload + { + RunIdentifier = runIdentifier, + SharedDatabaseRoot = sharedDatabaseRoot, + InstanceIndex = instanceIndex, + TotalInstances = totalInstances, + ManifestId = manifestId + }; + + return new ReproInputEnvelope + { + Type = ReproInputTypes.HostReady, + Timestamp = DateTimeOffset.UtcNow, + Payload = JsonSerializer.SerializeToElement(payload, ReproJsonOptions.Default) + }; + } + + /// + /// Deserializes the payload to a strongly typed value. + /// + /// The payload type. + /// The deserialized payload, or null when no payload is available. + public T? DeserializePayload() + { + if (Payload is not { } payload) + { + return default; + } + + return payload.Deserialize(ReproJsonOptions.Default); + } + + /// + /// Attempts to parse an input envelope from its JSON representation. + /// + /// The JSON text to parse. + /// When this method returns, contains the parsed envelope if successful. + /// When this method returns, contains the parsing error if parsing failed. + /// true if the envelope was parsed successfully; otherwise, false. + public static bool TryParse(string? line, out ReproInputEnvelope? envelope, out JsonException? error) + { + envelope = null; + error = null; + + if (string.IsNullOrWhiteSpace(line)) + { + return false; + } + + try + { + envelope = JsonSerializer.Deserialize(line, ReproJsonOptions.Default); + return envelope is not null; + } + catch (JsonException jsonException) + { + error = jsonException; + return false; + } + } +} + +/// +/// Describes the payload attached to . +/// +public sealed record ReproHostReadyPayload +{ + /// + /// Gets or sets the unique run identifier assigned by the host. + /// + public required string RunIdentifier { get; init; } + + /// + /// Gets or sets the shared database directory assigned by the host. + /// + public required string SharedDatabaseRoot { get; init; } + + /// + /// Gets or sets the zero-based instance index. + /// + public int InstanceIndex { get; init; } + + /// + /// Gets or sets the total number of instances participating in the run. + /// + public int TotalInstances { get; init; } + + /// + /// Gets or sets the identifier of the manifest being executed. + /// + public string? ManifestId { get; init; } +} + +/// +/// Well-known input envelope types understood by repros. +/// +public static class ReproInputTypes +{ + /// + /// Identifies an envelope notifying that the host is ready. + /// + public const string HostReady = "host-ready"; +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproJsonOptions.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproJsonOptions.cs new file mode 100644 index 000000000..2ed9ddb3e --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/Messaging/ReproJsonOptions.cs @@ -0,0 +1,27 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace LiteDB.ReproRunner.Shared.Messaging; + +/// +/// Provides JSON serialization options shared between repro hosts and the CLI. +/// +public static class ReproJsonOptions +{ + /// + /// Gets the default serializer options for repro messaging. + /// + public static JsonSerializerOptions Default { get; } = Create(); + + private static JsonSerializerOptions Create() + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false + }; + + options.Converters.Add(new JsonStringEnumConverter()); + return options; + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/ReproConfigurationReporter.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/ReproConfigurationReporter.cs new file mode 100644 index 000000000..b31886f5c --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/ReproConfigurationReporter.cs @@ -0,0 +1,90 @@ +using System; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using LiteDB.ReproRunner.Shared.Messaging; + +namespace LiteDB.ReproRunner.Shared; + +/// +/// Provides helpers for reporting the repro build configuration back to the host. +/// +public static class ReproConfigurationReporter +{ + private const string UseProjectReferenceMetadataKey = "LiteDB.ReproRunner.UseProjectReference"; + private const string LiteDbPackageVersionMetadataKey = "LiteDB.ReproRunner.LiteDBPackageVersion"; + + /// + /// Reads the repro configuration metadata from the entry assembly and sends it to the host synchronously. + /// + /// The client used to communicate with the host. + public static void SendConfiguration(ReproHostClient client) + { + if (client is null) + { + throw new ArgumentNullException(nameof(client)); + } + + var (useProjectReference, packageVersion) = ReadConfiguration(); + client.SendConfiguration(useProjectReference, packageVersion); + } + + /// + /// Reads the repro configuration metadata from the entry assembly and sends it to the host asynchronously. + /// + /// The client used to communicate with the host. + /// The token used to observe cancellation requests. + /// A task that completes when the configuration has been transmitted. + public static Task SendConfigurationAsync(ReproHostClient client, CancellationToken cancellationToken = default) + { + if (client is null) + { + throw new ArgumentNullException(nameof(client)); + } + + var (useProjectReference, packageVersion) = ReadConfiguration(); + return client.SendConfigurationAsync(useProjectReference, packageVersion, cancellationToken); + } + + private static (bool UseProjectReference, string? LiteDbPackageVersion) ReadConfiguration() + { + var assembly = Assembly.GetEntryAssembly() ?? Assembly.GetExecutingAssembly(); + var metadata = assembly.GetCustomAttributes(); + + string? useProjectReferenceValue = null; + string? packageVersionValue = null; + + foreach (var attribute in metadata) + { + if (string.Equals(attribute.Key, UseProjectReferenceMetadataKey, StringComparison.Ordinal)) + { + useProjectReferenceValue = attribute.Value; + } + else if (string.Equals(attribute.Key, LiteDbPackageVersionMetadataKey, StringComparison.Ordinal)) + { + packageVersionValue = attribute.Value; + } + } + + var useProjectReference = ParseBoolean(useProjectReferenceValue); + var packageVersion = string.IsNullOrWhiteSpace(packageVersionValue) ? null : packageVersionValue; + return (useProjectReference, packageVersion); + } + + private static bool ParseBoolean(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (bool.TryParse(value, out var parsed)) + { + return parsed; + } + + return string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "yes", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "on", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/ReproContext.cs b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/ReproContext.cs new file mode 100644 index 000000000..6cfaeab71 --- /dev/null +++ b/LiteDB.ReproRunner/LiteDB.ReproRunner.Shared/ReproContext.cs @@ -0,0 +1,113 @@ +using System.Globalization; + +namespace LiteDB.ReproRunner.Shared; + +/// +/// Provides access to the execution context passed to repro processes by the host. +/// +public sealed class ReproContext +{ + private const string SharedDatabaseVariable = "LITEDB_RR_SHARED_DB"; + private const string InstanceIndexVariable = "LITEDB_RR_INSTANCE_INDEX"; + private const string TotalInstancesVariable = "LITEDB_RR_TOTAL_INSTANCES"; + + private ReproContext(string? sharedDatabaseRoot, int instanceIndex, int totalInstances) + { + SharedDatabaseRoot = string.IsNullOrWhiteSpace(sharedDatabaseRoot) ? null : sharedDatabaseRoot; + InstanceIndex = instanceIndex; + TotalInstances = Math.Max(totalInstances, 1); + } + + /// + /// Gets the optional shared database root configured by the host. When null, repros should + /// fall back to or another suitable location. + /// + public string? SharedDatabaseRoot { get; } + + /// + /// Gets the index assigned to the current repro instance. The first instance is 0. + /// + public int InstanceIndex { get; } + + /// + /// Gets the total number of repro instances that were launched for the current run. + /// + public int TotalInstances { get; } + + /// + /// Resolves a from the current process environment. + /// + public static ReproContext FromEnvironment() + { + return FromEnvironment(Environment.GetEnvironmentVariable); + } + + /// + /// Resolves a from the provided environment accessor. + /// + /// A delegate that retrieves environment variables by name. + public static ReproContext FromEnvironment(Func resolver) + { + if (resolver is null) + { + throw new ArgumentNullException(nameof(resolver)); + } + + var sharedDatabaseRoot = resolver(SharedDatabaseVariable); + var instanceIndex = ParseOrDefault(resolver(InstanceIndexVariable)); + var totalInstances = ParseOrDefault(resolver(TotalInstancesVariable), 1); + + return new ReproContext(sharedDatabaseRoot, instanceIndex, totalInstances); + } + + /// + /// Attempts to resolve a from the current environment. + /// + public static bool TryFromEnvironment(out ReproContext? context) + { + try + { + context = FromEnvironment(); + return true; + } + catch + { + context = null; + return false; + } + } + + /// + /// Returns a dictionary representation of the environment variables required to recreate the context. + /// + public IReadOnlyDictionary ToEnvironmentVariables() + { + var variables = new Dictionary + { + [InstanceIndexVariable] = InstanceIndex.ToString(CultureInfo.InvariantCulture), + [TotalInstancesVariable] = TotalInstances.ToString(CultureInfo.InvariantCulture) + }; + + if (!string.IsNullOrWhiteSpace(SharedDatabaseRoot)) + { + variables[SharedDatabaseVariable] = SharedDatabaseRoot!; + } + + return variables; + } + + private static int ParseOrDefault(string? value, int defaultValue = 0) + { + if (string.IsNullOrWhiteSpace(value)) + { + return defaultValue; + } + + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + return defaultValue; + } +} diff --git a/LiteDB.ReproRunner/README.md b/LiteDB.ReproRunner/README.md new file mode 100644 index 000000000..112c7202e --- /dev/null +++ b/LiteDB.ReproRunner/README.md @@ -0,0 +1,310 @@ +# LiteDB ReproRunner + +The LiteDB ReproRunner is a small command-line harness that discovers, validates, and executes the +**reproduction projects** ("repros") that live in this repository. Each repro is an isolated console +application that demonstrates a past or current LiteDB issue. ReproRunner standardises the folder +layout, metadata, and execution model so that contributors and CI machines can list, validate, and run +repros deterministically. + +## Repository layout + +``` +LiteDB.ReproRunner/ + README.md # You are here + LiteDB.ReproRunner.Cli/ # The command-line runner project + Repros/ # One subfolder per repro + Issue_2561_TransactionMonitor/ + Issue_2561_TransactionMonitor.csproj + Program.cs + repro.json + README.md +``` + +Each repro folder contains: + +* A `.csproj` console project that can target the NuGet package or the in-repo sources. +* A `Program.cs` implementing the actual reproduction logic. +* A `repro.json` manifest with metadata required by the CLI. +* A short `README.md` describing the scenario and expected outcome. + +## Quick start + +Build the CLI and list the currently registered repros: + +```bash +dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- list +``` + +Show the manifest for a particular repro: + +```bash +dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- show Issue_2561_TransactionMonitor +``` + +Validate all manifests (exit code `2` means one or more manifests are invalid): + +```bash +dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- validate +``` + +Run a repro against both the packaged and source builds: + +```bash +dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- run Issue_2561_TransactionMonitor +``` + +## Writing a new repro + +1. **Create the folder** under `LiteDB.ReproRunner/Repros/`. The folder name should match the manifest + identifier, for example `Issue_1234_MyScenario/`. + +2. **Scaffold a console project** targeting `.NET 8` that toggles between the NuGet package and the + in-repo project using the `UseProjectReference` property: + + ```xml + + + Exe + net8.0 + enable + enable + false + 5.0.20 + + + + + + + + + + + ``` + +3. **Implement `Program.cs`** so that it: + + * Performs the minimal work necessary to reproduce the issue. + * Exits with code `0` when the expected (possibly buggy) behaviour is observed. + * Exits non-zero when the repro cannot trigger the behaviour. + * Reads the orchestration environment variables: + * `LITEDB_RR_SHARED_DB` – path to the shared working directory for multi-process repros. + * `LITEDB_RR_INSTANCE_INDEX` – zero-based index for the current process. + +4. **Author `repro.json`** with the required metadata (see schema below). The CLI refuses to run + invalid manifests unless `--skipValidation` is provided. + +5. **Add a `README.md`** to document the scenario, relevant issue links, the expected outcome, and any + local troubleshooting tips. + +6. **Update CI** if the repro should be part of the smoke suite (typically red or flaky repros). + +## Manifest schema + +Every repro ships with a `repro.json` manifest. The CLI validates the manifest at discovery time and +exposes the metadata in the `list`, `show`, and `run` commands. + +| Field | Type | Required | Notes | +| -------------------- | ------------- | -------- | ----- | +| `id` | string | yes | Unique identifier matching `^[A-Za-z0-9_]+$` and the folder name. | +| `title` | string | yes | Human readable summary. | +| `issues` | string[] | no | Absolute URLs to related GitHub issues. | +| `failingSince` | string | no | First version that exhibited the failure (e.g. `5.0.x`). | +| `timeoutSeconds` | int | yes | Global timeout (1–36000 seconds). | +| `requiresParallel` | bool | yes | `true` when the repro needs multiple processes. | +| `defaultInstances` | int | yes | Default process count (`≥ 2` when `requiresParallel=true`). | +| `sharedDatabaseKey` | string | conditional | Required when `requiresParallel=true`, used to derive a shared working directory. | +| `args` | string[] | no | Extra arguments passed to `dotnet run --`. | +| `tags` | string[] | no | Arbitrary labels for filtering (`list` prints them). | +| `state` | enum | yes | One of `red`, `green`, or `flaky`. | +| `expectedOutcomes` | object | no | Optional overrides for package/latest expectations (see below). | + +Unknown properties are rejected to keep the schema tight. The CLI also validates that each repro +folder contains exactly one `.csproj` file. + +### Expected outcomes + +The optional `expectedOutcomes` block lets a manifest describe how each variant should behave. When +absent, the CLI assumes the package build reproduces the issue (`kind: reproduce`) while the latest +build matches the manifest state (`red` expects another reproduction, `green` expects `noRepro`, and +`flaky` tolerates either outcome but emits a warning when it deviates). + +```json +"expectedOutcomes": { + "package": { + "kind": "hardFail", + "exitCode": -5, + "logContains": "NetworkException" + }, + "latest": { + "kind": "noRepro" + } +} +``` + +Each variant supports the following `kind` values: + +* `reproduce` – Exit code `0` indicates success. +* `noRepro` – Any non-zero exit code is treated as expected, signalling that the repro no longer + manifests the issue. +* `hardFail` – Reserved for the package build. Use this when reproduction requires the host to crash + with a specific exit code or log message. The latest build must always exit cleanly. + +`exitCode` constrains the expected exit value, while `logContains` (case-insensitive) ensures the +captured stdout/stderr output contains a specific substring. Leave them unset to accept any value. + +### Validation output + +`repro-runner validate` prints `VALID` or `INVALID` lines for each manifest. Errors are grouped under +an `INVALID` header that includes the relative path. Example: + +``` +INVALID Repros/Issue_999_Bad/repro.json + - $.timeoutSeconds: expected integer between 1 and 36000. + - $.id: must match ^[A-Za-z0-9_]+$ (got: Issue 999 Bad) +``` + +`repro-runner list --strict` returns exit code `2` when any manifest is invalid; without `--strict` +it merely prints the warnings. + +## CLI usage + +``` +Usage: repro-runner [--root ] [options] +``` + +Global option: + +* `--root ` – Override the discovery root. Defaults to the nearest `LiteDB.ReproRunner` folder. + +Commands: + +* `list [--strict]` – Lists all discovered repros with their state, timeout, tags, and titles. +* `show ` – Dumps the full manifest for a repro. +* `validate [--all|--id ]` – Validates manifests. Exit code `2` indicates invalid manifests. +* `run [options]` – Executes a repro (see below). + +### Running repros + +``` +repro-runner run [--instances N] [--timeout S] [--skipValidation] +``` + +* `--instances` – Override the number of processes to spawn. Must be ≥ 1 and ≥ 2 when the manifest + requires parallel execution. +* `--timeout` – Override the manifest timeout (seconds). +* `--skipValidation` – Run even when the manifest is invalid (execution still requires the manifest to + parse successfully). +* `--report ` – Emit a JSON summary of the run to the specified file (use `-` for stdout). +* `--report-format json` – Explicitly request JSON output. Additional formats may be added later. + +Each invocation plans deterministic run directories under the CLI’s output folder +(`bin///runs//`), cleans any leftover artifacts, and prepares all +builds before execution begins. Package and source variants are compiled in Release mode with +`UseProjectReference` flipped appropriately, and their artifacts are placed inside the planned run +directories. Execution then launches the built assemblies directly so that run output stays within the +per-variant folder. All run directories are removed once the command completes, even when failures or +cancellations occur. + +Timeouts are enforced across all processes. When the timeout elapses the runner terminates all child +processes and returns exit code `1`. Build failures surface in the run table and return a non-zero exit +code even when execution is skipped. + +### Machine-readable reports + +Pass `--report ` (or `--report -` to stream to stdout) to persist the run summary as JSON. Each +entry captures the manifest state, expectation result, exit codes, durations, and a trimmed copy of the +stdout/stderr output: + +```json +{ + "generatedAt": "2024-03-31T12:34:56.7891234+00:00", + "repros": [ + { + "id": "Issue_2561_TransactionMonitor", + "state": "Red", + "failed": false, + "package": { + "expected": "Reproduce", + "met": true, + "exitCode": 0, + "durationSeconds": 8.2 + }, + "latest": { + "expected": "Reproduce", + "met": true, + "exitCode": 0, + "durationSeconds": 8.1 + } + } + ] +} +``` + +The workflow publishes this file as an artifact so that other tools (for example, dashboards or +auto-triage bots) can reason about the latest run without scraping console output. + +## Parallel repros + +Set `requiresParallel` to `true` and choose a descriptive `sharedDatabaseKey`. The key is used to build +a deterministic folder for shared resources (for example, a LiteDB data file). Each process receives: + +* `LITEDB_RR_SHARED_DB` – The shared directory path rooted under the variant’s run folder (for example + `runs/Issue_1234_Sample/ver_latest/run/`). +* `LITEDB_RR_INSTANCE_INDEX` – The zero-based process index. +* `LITEDB_RR_TOTAL_INSTANCES` – Total instances spawned for this run. + +The repro code is responsible for coordinating work across instances, often by using the shared +folder to create or open the same database file. + +## CI integration + +The GitHub Actions workflow builds the repository in Release mode and then exercises the CLI in three +steps: + +1. `repro-runner list --strict` +2. `repro-runner validate` +3. `repro-runner run --all --report repro-summary.json` + +Execution honours the manifest state and expectations: + +* **Package variant** – Must reproduce the issue. If the manifest declares a `hardFail` expectation, + the runner accepts the specified crash signature instead. +* **Latest variant (`red`)** – Still expected to reproduce. CI stays green even though the repro + remains failing. +* **Latest variant (`green`)** – Must *not* reproduce. When the repro finally passes, CI turns green + automatically. +* **Latest variant (`flaky`)** – Deviations emit warnings in the JSON report and table output but do + not fail the build. + +The JSON summary (`repro-summary.json`) is uploaded as an artifact so downstream automation can +inspect exit codes, durations, and captured logs. + +## Troubleshooting + +* **Manifest fails validation** – Run `repro-runner show ` to inspect the parsed metadata. Most + errors reference the JSON path that needs attention. +* **Build failures** – When `repro-runner run` fails to build, the `dotnet build` diagnostics for each + variant are printed after the run table. Use the variant label (package vs latest) to pinpoint which + configuration needs attention. +* **Timeouts** – Use `--timeout` to temporarily raise the limit while debugging long-running repros. +* **Custom LiteDB package version** – Pass `-p:LiteDBPackageVersion=` through the CLI to + target a specific NuGet version. The property is forwarded to both build and run invocations. + +## FAQ + +**Why not use xUnit?** Repros often require multi-process coordination, timeouts, or package/project +switching. The CLI gives us deterministic orchestration without constraining the repro to the xUnit +lifecycle. Test suites can still shell out to `repro-runner` if desired. + +**How do I mark a repro as fixed?** Update `state` to `green`, adjust the README, and make sure the +`expectedOutcomes.latest` entry (or the implicit default) expects `noRepro`. Hard-fail overrides for the +package build can stay in place. Keeping the repro around prevents regressions. + +**Can I share helper code?** Keep repros isolated so they are self-documenting. If you must share +helpers, place them next to the CLI and avoid coupling repros together. + +## Further reading + +* [Issue 2561 – TransactionMonitor finalizer crash](https://github.com/litedb-org/LiteDB/issues/2561) +* `LiteDB.ReproRunner.Cli` source for the discovery/validation logic. diff --git a/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/Issue_2561_TransactionMonitor.csproj b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/Issue_2561_TransactionMonitor.csproj new file mode 100644 index 000000000..3b4602acc --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/Issue_2561_TransactionMonitor.csproj @@ -0,0 +1,29 @@ + + + + Exe + net8.0 + enable + enable + false + 5.0.20 + + + + + + + + + + + + + + + + + + + + diff --git a/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/Program.cs b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/Program.cs new file mode 100644 index 000000000..4cde1b59d --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/Program.cs @@ -0,0 +1,168 @@ +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using LiteDB; +using LiteDB.ReproRunner.Shared; +using LiteDB.ReproRunner.Shared.Messaging; + +namespace Issue_2561_TransactionMonitor; + +internal static class Program +{ + private const string DefaultDatabaseName = "issue2561.db"; + + private static int Main() + { + var host = ReproHostClient.CreateDefault(); + ReproConfigurationReporter.SendConfiguration(host); + var context = ReproContext.FromEnvironment(); + + host.SendLifecycle("starting", new + { + context.InstanceIndex, + context.TotalInstances, + context.SharedDatabaseRoot + }); + + try + { + Run(host, context); + host.SendResult(true, "Repro succeeded: ReleaseTransaction threw on the wrong thread."); + host.SendLifecycle("completed", new { Success = true }); + return 0; + } + catch (Exception ex) + { + host.SendLog($"Reproduction failed: {ex}", ReproHostLogLevel.Error); + host.SendResult(false, "Reproduction failed.", new { Exception = ex.ToString() }); + host.SendLifecycle("completed", new { Success = false }); + Console.Error.WriteLine("Reproduction failed: {0}", ex); + return 1; + } + } + + private static void Run(ReproHostClient host, ReproContext context) + { + var databaseDirectory = string.IsNullOrWhiteSpace(context.SharedDatabaseRoot) + ? AppContext.BaseDirectory + : context.SharedDatabaseRoot; + + Directory.CreateDirectory(databaseDirectory); + + var databasePath = Path.Combine(databaseDirectory, DefaultDatabaseName); + if (File.Exists(databasePath)) + { + File.Delete(databasePath); + } + + Log(host, $"Using database: {databasePath}"); + + var connectionString = new ConnectionString + { + Filename = databasePath, + Connection = ConnectionType.Direct + }; + + using var database = new LiteDatabase(connectionString); + var collection = database.GetCollection("docs"); + + var engineField = typeof(LiteDatabase).GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("LiteDatabase._engine field not found."); + var engine = engineField.GetValue(database) ?? throw new InvalidOperationException("LiteDatabase engine unavailable."); + var engineType = engine.GetType(); + + var beginTrans = engineType.GetMethod("BeginTrans", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("LiteEngine.BeginTrans method not found."); + var rollback = engineType.GetMethod("Rollback", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("LiteEngine.Rollback method not found."); + var monitorField = engineType.GetField("_monitor", BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("LiteEngine._monitor field not found."); + var monitor = monitorField.GetValue(engine) ?? throw new InvalidOperationException("Transaction monitor unavailable."); + var monitorType = monitor.GetType(); + + var transactionsProperty = monitorType.GetProperty("Transactions", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("TransactionMonitor.Transactions property not found."); + var releaseTransaction = monitorType.GetMethod("ReleaseTransaction", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("TransactionMonitor.ReleaseTransaction method not found."); + + Log(host, "Opening explicit transaction on main thread..."); + + if (!(beginTrans.Invoke(engine, Array.Empty()) is bool began) || !began) + { + throw new InvalidOperationException("BeginTrans returned false; the transaction was not created."); + } + + collection.Insert(new BsonDocument + { + ["_id"] = ObjectId.NewObjectId(), + ["description"] = "Issue 2561 transaction monitor repro", + ["createdAt"] = DateTime.UtcNow + }); + + var transactions = (IEnumerable)(transactionsProperty.GetValue(monitor) + ?? throw new InvalidOperationException("Transaction list is unavailable.")); + + var capturedTransaction = transactions.Single(); + var reproObserved = new ManualResetEventSlim(false); + LiteException? observedException = null; + + var worker = new Thread(() => + { + try + { + releaseTransaction.Invoke(monitor, new[] { capturedTransaction }); + Log(host, "ReleaseTransaction completed without throwing.", ReproHostLogLevel.Warning); + } + catch (TargetInvocationException invocationException) when (invocationException.InnerException is LiteException liteException) + { + observedException = liteException; + Log(host, "Observed LiteException from ReleaseTransaction on worker thread:"); + Log(host, liteException.ToString()); + } + finally + { + reproObserved.Set(); + } + }) + { + IsBackground = true, + Name = "Issue2561-Repro-Worker" + }; + + worker.Start(); + + if (!reproObserved.Wait(TimeSpan.FromSeconds(10))) + { + throw new TimeoutException("Timed out waiting for ReleaseTransaction to finish."); + } + + rollback.Invoke(engine, Array.Empty()); + + if (observedException is null) + { + throw new InvalidOperationException("Expected LiteException was not observed."); + } + + if (!observedException.Message.Contains("current thread must contains transaction parameter", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"LiteException did not contain expected message. Actual: {observedException.Message}"); + } + + Log(host, "Repro succeeded: ReleaseTransaction threw on the wrong thread."); + } + + private static void Log(ReproHostClient host, string message, ReproHostLogLevel level = ReproHostLogLevel.Information) + { + host.SendLog(message, level); + + if (level >= ReproHostLogLevel.Warning) + { + Console.Error.WriteLine(message); + } + else + { + Console.WriteLine(message); + } + } +} diff --git a/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/README.md b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/README.md new file mode 100644 index 000000000..aa62df11d --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/README.md @@ -0,0 +1,36 @@ +# Issue 2561 – TransactionMonitor finalizer crash + +This repro demonstrates [LiteDB #2561](https://github.com/litedb-org/LiteDB/issues/2561). The +`TransactionMonitor` finalizer (`TransactionService.Dispose(false)`) executes on the GC finalizer thread, +which violates the monitor’s expectation that the transaction belongs to the current thread. The +resulting `LiteException` brings the process down with the message `current thread must contains +transaction parameter`. + +## Scenario + +1. Open a `LiteDatabase` connection against a clean data file. +2. Begin an explicit transaction on the main thread and perform a write. +3. Use reflection to grab the `TransactionMonitor` and the `TransactionService` tracked for the main thread. +4. Invoke `TransactionMonitor.ReleaseTransaction` from a different thread to mimic the finalizer’s behavior. + +The monitor throws the same `LiteException` that was captured in the original crash, proving that the +guard fails when the finalizer thread releases the transaction. + +## Running the repro + +```bash +dotnet run --project LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/Issue_2561_TransactionMonitor.csproj -c Release -p:UseProjectReference=true +``` + +By default the project references the NuGet package (`LiteDBPackageVersion` defaults to `5.0.20`). Pass +`-p:LiteDBPackageVersion=` to pin a different package, or `-p:UseProjectReference=true` to link +against the in-repo sources. The ReproRunner CLI orchestrates those switches automatically. + +The repro now consumes `LiteDB.ReproRunner.Shared` helpers. `ReproContext` resolves the +`LITEDB_RR_*` environment variables supplied by the CLI, while `ReproHostClient` emits structured +JSON messages so progress and results surface cleanly in the host output. + +## Expected outcome + +The repro prints the captured `LiteException` and exits with code `0` once the message containing +`current thread must contains transaction parameter` is observed. diff --git a/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/repro.json b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/repro.json new file mode 100644 index 000000000..cf6d51a41 --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2561_TransactionMonitor/repro.json @@ -0,0 +1,13 @@ +{ + "id": "Issue_2561_TransactionMonitor", + "title": "Transaction monitor finalizer runs on wrong thread", + "issues": ["https://github.com/litedb-org/LiteDB/issues/2561"], + "failingSince": "5.0.x", + "timeoutSeconds": 120, + "requiresParallel": false, + "defaultInstances": 1, + "sharedDatabaseKey": "issue2561", + "args": [], + "tags": ["transaction", "monitor", "finalizer"], + "state": "red" +} diff --git a/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/Issue_2586_RollbackTransaction.csproj b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/Issue_2586_RollbackTransaction.csproj new file mode 100644 index 000000000..695dbb5c0 --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/Issue_2586_RollbackTransaction.csproj @@ -0,0 +1,27 @@ + + + Exe + net8.0 + enable + enable + false + 5.0.20 + + + + + + + + + + + + + + + + + + + diff --git a/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/Program.cs b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/Program.cs new file mode 100644 index 000000000..a54b7083b --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/Program.cs @@ -0,0 +1,605 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using LiteDB; +using LiteDB.ReproRunner.Shared; +using LiteDB.ReproRunner.Shared.Messaging; + +namespace Issue_2586_RollbackTransaction; + +/// +/// Repro of LiteDB issue #2586. The repro returns exit code 0 when the rollback throws the expected +/// LiteException and non-zero when the bug fails to reproduce. +/// +internal static class Program +{ + private const int HolderTransactionCount = 99; + private const int DocumentWriteCount = 10_000; + + private static int Main() + { + var host = ReproHostClient.CreateDefault(); + ReproConfigurationReporter.SendConfiguration(host); + var context = ReproContext.FromEnvironment(); + + host.SendLifecycle("starting", new + { + context.InstanceIndex, + context.TotalInstances, + context.SharedDatabaseRoot + }); + + try + { + var reproduced = Run(host, context); + host.SendResult(reproduced, reproduced + ? "Repro succeeded: rollback threw with exhausted transaction pool." + : "Repro did not reproduce the rollback exception."); + host.SendLifecycle("completed", new { Success = reproduced }); + return reproduced ? 0 : 1; + } + catch (Exception ex) + { + host.SendLog($"Reproduction failed: {ex}", ReproHostLogLevel.Error); + host.SendResult(false, "Reproduction failed.", new { Exception = ex.ToString() }); + host.SendLifecycle("completed", new { Success = false }); + Console.Error.WriteLine(ex); + return 1; + } + } + + private static bool Run(ReproHostClient host, ReproContext context) + { + var stopwatch = Stopwatch.StartNew(); + + var databaseDirectory = string.IsNullOrWhiteSpace(context.SharedDatabaseRoot) + ? AppContext.BaseDirectory + : context.SharedDatabaseRoot; + + Directory.CreateDirectory(databaseDirectory); + + var databasePath = Path.Combine(databaseDirectory, "rollback-crash.db"); + Log(host, $"Database path: {databasePath}"); + + if (File.Exists(databasePath)) + { + Log(host, "Deleting previous database file."); + File.Delete(databasePath); + } + + var connectionString = new ConnectionString + { + Filename = databasePath, + Connection = ConnectionType.Direct + }; + + using var db = new LiteDatabase(connectionString); + var collection = db.GetCollection("documents"); + + using var releaseHolders = new ManualResetEventSlim(false); + using var holdersReady = new CountdownEvent(HolderTransactionCount); + + var holderThreads = StartGuardTransactions(host, db, holdersReady, releaseHolders); + + holdersReady.Wait(); + Log(host, $"Spawned {HolderTransactionCount} background transactions to exhaust the shared transaction memory pool."); + + try + { + return RunFailingTransaction(host, db, collection); + } + finally + { + releaseHolders.Set(); + + foreach (var thread in holderThreads) + { + thread.Join(); + } + + stopwatch.Stop(); + Log(host, $"Total elapsed time: {stopwatch.Elapsed}."); + } + } + + private static IReadOnlyList StartGuardTransactions(ReproHostClient host, LiteDatabase db, CountdownEvent ready, ManualResetEventSlim release) + { + var threads = new List(HolderTransactionCount); + + for (var i = 0; i < HolderTransactionCount; i++) + { + var thread = new Thread(() => HoldTransaction(host, db, ready, release)) + { + IsBackground = true, + Name = $"Holder-{i:D2}" + }; + + thread.Start(); + threads.Add(thread); + } + + return threads; + } + + private static void HoldTransaction(ReproHostClient host, LiteDatabase db, CountdownEvent ready, ManualResetEventSlim release) + { + var threadId = Thread.CurrentThread.ManagedThreadId; + var began = false; + + try + { + began = db.BeginTrans(); + if (!began) + { + Log(host, $"[{threadId}] BeginTrans returned false for holder transaction.", ReproHostLogLevel.Warning); + } + } + catch (LiteException ex) + { + Log(host, $"[{threadId}] Failed to start holder transaction: {ex.Message}", ReproHostLogLevel.Warning); + } + finally + { + ready.Signal(); + } + + if (!began) + { + return; + } + + try + { + release.Wait(); + } + finally + { + try + { + db.Rollback(); + } + catch (LiteException ex) + { + Log(host, $"[{threadId}] Holder rollback threw: {ex.Message}", ReproHostLogLevel.Warning); + } + } + } + + private static bool RunFailingTransaction(ReproHostClient host, LiteDatabase db, ILiteCollection collection) + { + Console.WriteLine(); + Log(host, $"Starting write transaction on thread {Thread.CurrentThread.ManagedThreadId}."); + + if (!db.BeginTrans()) + { + throw new InvalidOperationException("Failed to begin primary transaction for reproduction."); + } + + TransactionInspector? inspector = null; + var maxSize = 0; + var safepointTriggered = false; + var shouldTriggerSafepoint = false; + + var payloadA = new string('A', 4_096); + var payloadB = new string('B', 4_096); + var payloadC = new string('C', 2_048); + var largeBinary = new byte[128 * 1024]; + + try + { + for (var i = 0; i < DocumentWriteCount; i++) + { + if (shouldTriggerSafepoint && !safepointTriggered) + { + inspector ??= TransactionInspector.Attach(db, collection.Name); + + Console.WriteLine(); + Log(host, $"Manually invoking safepoint before processing document #{i:N0}."); + + inspector.InvokeSafepoint(); + // Safepoint transitions all dirty buffers into the readable cache. Manually + // mark the collection page as readable to mirror the race condition described + // in the bug investigation before triggering the rollback path. + inspector.ForceCollectionPageShareCounter(1); + + safepointTriggered = true; + + throw new InvalidOperationException("Simulating transaction failure after safepoint flush."); + } + + var document = new LargeDocument + { + Id = i, + BatchId = Guid.NewGuid(), + CreatedUtc = DateTime.UtcNow, + Description = $"Large document #{i:N0}", + Payload1 = payloadA, + Payload2 = payloadB, + Payload3 = payloadC, + LargePayload = largeBinary + }; + + collection.Upsert(document); + + if (i % 100 == 0) + { + Log(host, $"Upserted {i:N0} documents..."); + } + + inspector ??= TransactionInspector.Attach(db, collection.Name); + + var currentSize = inspector.CurrentSize; + maxSize = Math.Max(maxSize, inspector.MaxSize); + + if (!shouldTriggerSafepoint && currentSize >= maxSize) + { + shouldTriggerSafepoint = true; + Log(host, $"Queued safepoint after reaching transaction size {currentSize} at document #{i + 1:N0}."); + } + } + + Console.WriteLine(); + Log(host, "Simulating failure after safepoint flush."); + throw new InvalidOperationException("Simulating transaction failure after safepoint flush."); + } + catch (Exception ex) when (ex is not LiteException) + { + Log(host, $"Caught application exception: {ex.Message}"); + Log(host, "Requesting rollback — this should trigger 'discarded page must be writable'."); + + var shareCounter = inspector?.GetCollectionShareCounter(); + if (shareCounter.HasValue) + { + Log(host, $"Collection page share counter before rollback: {shareCounter.Value}."); + } + + if (inspector is not null) + { + foreach (var (pageId, pageType, counter) in inspector.EnumerateWritablePages()) + { + Log(host, $"Writable page {pageId} ({pageType}) share counter: {counter}."); + } + } + + try + { + db.Rollback(); + + var previous = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("Rollback returned without throwing — the bug did not reproduce."); + Console.ForegroundColor = previous; + + Log(host, "Rollback returned without throwing — the bug did not reproduce.", ReproHostLogLevel.Warning); + return false; + } + catch (LiteException liteException) + { + Console.WriteLine(); + Console.WriteLine("Captured expected LiteDB.LiteException:"); + Console.WriteLine(liteException); + + Log(host, "Captured expected LiteDB.LiteException:"); + Log(host, liteException.ToString(), ReproHostLogLevel.Warning); + + var color = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Red; + Console.WriteLine("Rollback threw LiteException — the bug reproduced."); + Console.ForegroundColor = color; + + Log(host, "Rollback threw LiteException — the bug reproduced.", ReproHostLogLevel.Error); + return true; + } + } + } + + private sealed class LargeDocument + { + public int Id { get; set; } + public Guid BatchId { get; set; } + public DateTime CreatedUtc { get; set; } + public string Description { get; set; } = string.Empty; + public string Payload1 { get; set; } = string.Empty; + public string Payload2 { get; set; } = string.Empty; + public string Payload3 { get; set; } = string.Empty; + public byte[] LargePayload { get; set; } = Array.Empty(); + } + + private sealed class TransactionInspector + { + private readonly object _transaction; + private readonly object _transactionPages; + private readonly object _snapshot; + private readonly PropertyInfo _transactionSizeProperty; + private readonly PropertyInfo _maxTransactionSizeProperty; + private readonly PropertyInfo _collectionPageProperty; + private readonly PropertyInfo _bufferProperty; + private readonly FieldInfo _shareCounterField; + private readonly MethodInfo _safepointMethod; + + private TransactionInspector( + object transaction, + object transactionPages, + object snapshot, + PropertyInfo transactionSizeProperty, + PropertyInfo maxTransactionSizeProperty, + PropertyInfo collectionPageProperty, + PropertyInfo bufferProperty, + FieldInfo shareCounterField, + MethodInfo safepointMethod) + { + _transaction = transaction; + _transactionPages = transactionPages; + _snapshot = snapshot; + _transactionSizeProperty = transactionSizeProperty; + _maxTransactionSizeProperty = maxTransactionSizeProperty; + _collectionPageProperty = collectionPageProperty; + _bufferProperty = bufferProperty; + _shareCounterField = shareCounterField; + _safepointMethod = safepointMethod; + } + + public int CurrentSize => (int)_transactionSizeProperty.GetValue(_transactionPages)!; + + public int MaxSize => (int)_maxTransactionSizeProperty.GetValue(_transaction)!; + + public int? GetCollectionShareCounter() + { + var collectionPage = _collectionPageProperty.GetValue(_snapshot); + + if (collectionPage is null) + { + return null; + } + + var buffer = _bufferProperty.GetValue(collectionPage); + + return buffer is null ? null : (int?)_shareCounterField.GetValue(buffer); + } + + public void InvokeSafepoint() + { + _safepointMethod.Invoke(_transaction, Array.Empty()); + } + + public void ForceCollectionPageShareCounter(int shareCounter) + { + var collectionPage = _collectionPageProperty.GetValue(_snapshot); + + if (collectionPage is null) + { + return; + } + + var buffer = _bufferProperty.GetValue(collectionPage); + + if (buffer is null) + { + return; + } + + _shareCounterField.SetValue(buffer, shareCounter); + } + + public IEnumerable<(uint PageId, string PageType, int ShareCounter)> EnumerateWritablePages() + { + var getPagesMethod = _snapshot.GetType().GetMethod( + "GetWritablePages", + BindingFlags.Public | BindingFlags.Instance, + binder: null, + new[] { typeof(bool), typeof(bool) }, + modifiers: null) + ?? throw new InvalidOperationException("GetWritablePages method not found on snapshot."); + + if (getPagesMethod.Invoke(_snapshot, new object[] { true, true }) is not IEnumerable pages) + { + yield break; + } + + foreach (var page in pages) + { + var pageIdProperty = page.GetType().GetProperty("PageID", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("PageID property not found on page."); + + var pageTypeProperty = page.GetType().GetProperty("PageType", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("PageType property not found on page."); + + var bufferProperty = page.GetType().GetProperty("Buffer", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("Buffer property not found on page."); + + var buffer = bufferProperty.GetValue(page); + + if (buffer is null) + { + continue; + } + + var pageId = (uint)pageIdProperty.GetValue(page)!; + var pageTypeName = pageTypeProperty.GetValue(page)?.ToString() ?? ""; + var shareCounter = (int)_shareCounterField.GetValue(buffer)!; + + yield return (pageId, pageTypeName, shareCounter); + } + } + + public static TransactionInspector Attach(LiteDatabase db, string collectionName) + { + var engineField = typeof(LiteDatabase).GetField("_engine", BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("Unable to locate LiteDatabase engine field."); + + var engine = engineField.GetValue(db) ?? throw new InvalidOperationException("LiteDatabase engine is not initialized."); + + var monitorField = engine.GetType().GetField("_monitor", BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("Unable to locate TransactionMonitor field."); + + var monitor = monitorField.GetValue(engine) ?? throw new InvalidOperationException("TransactionMonitor is unavailable."); + + var getThreadTransaction = monitor.GetType().GetMethod("GetThreadTransaction", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("GetThreadTransaction method not found."); + + var transaction = getThreadTransaction.Invoke(monitor, Array.Empty()) + ?? throw new InvalidOperationException("Thread transaction is not available."); + + var transactionType = transaction.GetType(); + + var transactionPages = GetTransactionPages(transactionType, transaction) + ?? throw new InvalidOperationException("Transaction pages are unavailable."); + + var snapshot = GetSnapshot(transactionType, transaction, collectionName) + ?? throw new InvalidOperationException("Transaction snapshot is unavailable."); + + var transactionSizeProperty = transactionPages.GetType().GetProperty("TransactionSize", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("TransactionSize property not found."); + + var maxTransactionSizeProperty = transaction.GetType().GetProperty("MaxTransactionSize", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("MaxTransactionSize property not found."); + + var collectionPageProperty = snapshot.GetType().GetProperty("CollectionPage", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("CollectionPage property not found on snapshot."); + + var bufferProperty = collectionPageProperty.PropertyType.GetProperty("Buffer", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("Buffer property not found on collection page."); + + var shareCounterField = bufferProperty.PropertyType.GetField("ShareCounter", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("ShareCounter field not found on buffer."); + + var safepointMethod = transaction.GetType().GetMethod("Safepoint", BindingFlags.Public | BindingFlags.Instance) + ?? throw new InvalidOperationException("Safepoint method not found on transaction."); + + return new TransactionInspector( + transaction, + transactionPages, + snapshot, + transactionSizeProperty, + maxTransactionSizeProperty, + collectionPageProperty, + bufferProperty, + shareCounterField, + safepointMethod); + } + + private static object? GetTransactionPages(Type transactionType, object transaction) + { + var field = transactionType.GetField("_transactionPages", BindingFlags.NonPublic | BindingFlags.Instance) + ?? transactionType.GetField("_transPages", BindingFlags.NonPublic | BindingFlags.Instance); + + if (field?.GetValue(transaction) is { } pages) + { + return pages; + } + + var property = transactionType.GetProperty("Pages", BindingFlags.Public | BindingFlags.Instance); + + return property?.GetValue(transaction); + } + + private static object? GetSnapshot(Type transactionType, object transaction, string collectionName) + { + var snapshot = transactionType.GetField("_snapshot", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(transaction); + + if (snapshot is not null) + { + return snapshot; + } + + var snapshotsField = transactionType.GetField("_snapshots", BindingFlags.NonPublic | BindingFlags.Instance); + + if (snapshotsField?.GetValue(transaction) is IDictionary dictionary) + { + snapshot = FindSnapshot(dictionary, collectionName); + + if (snapshot is not null) + { + return snapshot; + } + } + + var snapshotsProperty = transactionType.GetProperty("Snapshots", BindingFlags.Public | BindingFlags.Instance); + + if (snapshotsProperty?.GetValue(transaction) is IEnumerable enumerable) + { + return FindSnapshot(enumerable, collectionName); + } + + return null; + } + + private static object? FindSnapshot(IDictionary dictionary, string collectionName) + { + if (!string.IsNullOrWhiteSpace(collectionName) && dictionary.Contains(collectionName)) + { + var value = dictionary[collectionName]; + + if (value is not null) + { + return value; + } + } + + foreach (DictionaryEntry entry in dictionary) + { + if (entry.Value is null) + { + continue; + } + + if (MatchesCollection(entry.Value, collectionName)) + { + return entry.Value; + } + } + + return null; + } + + private static object? FindSnapshot(IEnumerable snapshots, string collectionName) + { + foreach (var snapshot in snapshots) + { + if (snapshot is null) + { + continue; + } + + if (MatchesCollection(snapshot, collectionName)) + { + return snapshot; + } + } + + return null; + } + + private static bool MatchesCollection(object snapshot, string collectionName) + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return true; + } + + var nameProperty = snapshot.GetType().GetProperty("CollectionName", BindingFlags.Public | BindingFlags.Instance); + var name = nameProperty?.GetValue(snapshot)?.ToString(); + + return string.Equals(name, collectionName, StringComparison.OrdinalIgnoreCase); + } + } + + private static void Log(ReproHostClient host, string message, ReproHostLogLevel level = ReproHostLogLevel.Information) + { + host.SendLog(message, level); + + if (level >= ReproHostLogLevel.Warning) + { + Console.Error.WriteLine(message); + } + else + { + Console.WriteLine(message); + } + } +} diff --git a/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/README.md b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/README.md new file mode 100644 index 000000000..575823a4c --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/README.md @@ -0,0 +1,26 @@ +# Issue 2586 – Rollback fails after safepoint flush + +This repro demonstrates [LiteDB issue #2586](https://github.com/litedb-org/LiteDB/issues/2586) +where a write transaction rollback can throw `LiteDB.LiteException` with the message +`"discarded page must be writable"` when many guard transactions exhaust the shared transaction +memory pool. + +## Expected outcome + +Running the repro against LiteDB 5.0.20 should print the captured `LiteException` and exit with +code `0`. A fixed build allows the rollback to complete without throwing, causing the repro to +exit with a non-zero code. + +## Running the repro + +```bash +# Run against the published LiteDB 5.0.20 package (default) +dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- run Issue_2586_RollbackTransaction + +# Run against the in-repo LiteDB sources +dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- run Issue_2586_RollbackTransaction --useProjectRef +``` + +The repro integrates the `LiteDB.ReproRunner.Shared` library for its execution context and +message pipeline. `ReproContext` resolves the CLI-provided environment variables, and +`ReproHostClient` streams JSON-formatted progress and results back to the host. diff --git a/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/repro.json b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/repro.json new file mode 100644 index 000000000..368381fd4 --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2586_RollbackTransaction/repro.json @@ -0,0 +1,12 @@ +{ + "id": "Issue_2586_RollbackTransaction", + "title": "Rollback throws 'discarded page must be writable'", + "issues": ["https://github.com/litedb-org/LiteDB/issues/2586"], + "failingSince": "5.0.20", + "timeoutSeconds": 300, + "requiresParallel": false, + "defaultInstances": 1, + "args": [], + "tags": ["transaction", "rollback", "safepoint"], + "state": "green" +} diff --git a/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/Issue_2614_DiskServiceDispose.csproj b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/Issue_2614_DiskServiceDispose.csproj new file mode 100644 index 000000000..ace789af2 --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/Issue_2614_DiskServiceDispose.csproj @@ -0,0 +1,29 @@ + + + + Exe + net8.0 + enable + enable + false + 5.0.21 + + + + + + + + + + + + + + + + + + + + diff --git a/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/Program.cs b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/Program.cs new file mode 100644 index 000000000..69cedd1ce --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/Program.cs @@ -0,0 +1,232 @@ +using System.Runtime.InteropServices; +using LiteDB; +using LiteDB.ReproRunner.Shared; +using LiteDB.ReproRunner.Shared.Messaging; + +namespace Issue_2614_DiskServiceDispose; + +internal static class Program +{ + private const ulong FileSizeLimitBytes = 4096; + private const string DatabaseFileName = "issue2614-disk.db"; + + private static int Main() + { + var host = ReproHostClient.CreateDefault(); + ReproConfigurationReporter.SendConfiguration(host); + var context = ReproContext.FromEnvironment(); + + host.SendLifecycle("starting", new + { + context.InstanceIndex, + context.TotalInstances, + context.SharedDatabaseRoot + }); + + try + { + if (!OperatingSystem.IsLinux() && !OperatingSystem.IsMacOS()) + { + const string message = "This repro requires a Unix-like environment to adjust RLIMIT_FSIZE."; + host.SendLog(message, ReproHostLogLevel.Error); + host.SendResult(false, message); + host.SendLifecycle("completed", new { Success = false }); + return 1; + } + + Run(host, context); + + host.SendResult(true, "DiskService left the database file locked after initialization failure."); + host.SendLifecycle("completed", new { Success = true }); + return 0; + } + catch (Exception ex) + { + host.SendLog($"Reproduction failed: {ex}", ReproHostLogLevel.Error); + host.SendResult(false, "Reproduction failed unexpectedly.", new { Exception = ex.ToString() }); + host.SendLifecycle("completed", new { Success = false }); + Console.Error.WriteLine(ex); + return 1; + } + } + + private static void Run(ReproHostClient host, ReproContext context) + { + NativeMethods.IgnoreSigxfsz(); + + var rootDirectory = string.IsNullOrWhiteSpace(context.SharedDatabaseRoot) + ? Path.Combine(AppContext.BaseDirectory, "issue2614") + : context.SharedDatabaseRoot; + + Directory.CreateDirectory(rootDirectory); + + var databasePath = Path.Combine(rootDirectory, DatabaseFileName); + + if (File.Exists(databasePath)) + { + host.SendLog($"Removing pre-existing database file: {databasePath}"); + File.Delete(databasePath); + } + + var limitScope = FileSizeLimitScope.Apply(host, FileSizeLimitBytes); + + var connectionString = new ConnectionString + { + Filename = databasePath, + Connection = ConnectionType.Direct + }; + + host.SendLog($"Attempting to create database at {databasePath}"); + + Exception? observedException = null; + + try + { + try + { + using var database = new LiteDatabase(connectionString); + throw new InvalidOperationException("LiteDatabase constructor succeeded unexpectedly."); + } + catch (Exception ex) + { + observedException = ex; + host.SendLog($"Observed exception while opening database: {ex.GetType().FullName}: {ex.Message}"); + } + + if (observedException is null) + { + throw new InvalidOperationException("No exception observed while creating the database."); + } + } + finally + { + limitScope.Dispose(); + } + + host.SendLog("Restored original file size limit. Checking for lingering file handles..."); + + var lockObserved = false; + + try + { + using var stream = File.Open(databasePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + host.SendLog("Successfully reopened the database file with exclusive access.", ReproHostLogLevel.Error); + } + catch (IOException ioException) + { + lockObserved = true; + host.SendLog($"Failed to reopen database file due to lingering handle: {ioException.Message}", ReproHostLogLevel.Information); + } + + if (!lockObserved) + { + throw new InvalidOperationException("Database file reopened successfully; the repro did not observe the lock."); + } + + try + { + host.SendLog("Attempting to delete the locked database file."); + File.Delete(databasePath); + } + catch (Exception deleteException) + { + host.SendLog($"Expected failure deleting locked file: {deleteException.Message}"); + } + } + + private sealed class FileSizeLimitScope : IDisposable + { + private readonly NativeMethods.RLimit _original; + private bool _restored; + + private FileSizeLimitScope(NativeMethods.RLimit original) + { + _original = original; + } + + public static FileSizeLimitScope Apply(ReproHostClient host, ulong requestedLimit) + { + var original = NativeMethods.GetFileSizeLimit(); + host.SendLog($"Original RLIMIT_FSIZE: cur={original.rlim_cur}, max={original.rlim_max}"); + + var newLimit = original; + var effectiveLimit = Math.Min(original.rlim_max, requestedLimit); + + if (effectiveLimit == 0) + { + throw new InvalidOperationException($"Unable to set RLIMIT_FSIZE to zero. Original max={original.rlim_max}."); + } + + newLimit.rlim_cur = effectiveLimit; + NativeMethods.SetFileSizeLimit(newLimit); + + host.SendLog($"Applied RLIMIT_FSIZE cur={newLimit.rlim_cur}"); + + return new FileSizeLimitScope(original); + } + + public void Dispose() + { + if (_restored) + { + return; + } + + NativeMethods.SetFileSizeLimit(_original); + _restored = true; + } + } + + private static class NativeMethods + { + private const int RLimitFileSize = 1; + private const int Sigxfsz = 25; + private static readonly IntPtr SigIgn = (IntPtr)1; + private static readonly IntPtr SigErr = (IntPtr)(-1); + + [StructLayout(LayoutKind.Sequential)] + internal struct RLimit + { + public ulong rlim_cur; + public ulong rlim_max; + } + + [DllImport("libc", SetLastError = true)] + private static extern int getrlimit(int resource, out RLimit rlim); + + [DllImport("libc", SetLastError = true)] + private static extern int setrlimit(int resource, ref RLimit rlim); + + [DllImport("libc", SetLastError = true)] + private static extern IntPtr signal(int signum, IntPtr handler); + + public static void IgnoreSigxfsz() + { + var previous = signal(Sigxfsz, SigIgn); + + if (previous == SigErr) + { + throw new InvalidOperationException($"signal failed (errno={Marshal.GetLastWin32Error()})."); + } + } + + public static RLimit GetFileSizeLimit() + { + ThrowOnError(getrlimit(RLimitFileSize, out var limit)); + return limit; + } + + public static void SetFileSizeLimit(RLimit limit) + { + ThrowOnError(setrlimit(RLimitFileSize, ref limit)); + } + + private static void ThrowOnError(int result) + { + if (result != 0) + { + throw new InvalidOperationException($"Native call failed (errno={Marshal.GetLastWin32Error()})."); + } + } + } +} diff --git a/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/README.md b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/README.md new file mode 100644 index 000000000..b5d332007 --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/README.md @@ -0,0 +1,27 @@ +# Issue 2614 – DiskService leaks file handles on initialization failure + +This repro enforces a very small per-process file size limit using `setrlimit(RLIMIT_FSIZE)` +and then attempts to create a new database file. LiteDB writes the header page during +`DiskService` construction, so exceeding the limit triggers an exception before the engine +finishes initializing. When the constructor throws, the `DiskService` instance is never +assigned to the engine field and therefore never disposed. + +The expected behaviour is that all file handles opened during the initialization attempt +are released so the caller can retry after freeing disk space. Instead, LiteDB leaves the +writer handle open, preventing any subsequent attempt to reopen or delete the data file +within the same process. + +## Running the repro + +```bash +dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- run Issue_2614_DiskServiceDispose +``` + +The process must run on a Unix-like operating system where `setrlimit` is available. +The repro succeeds when it observes: + +* The initial `LiteDatabase` constructor throws due to the enforced file size limit. +* The follow-up attempt to reopen the database file exclusively fails because a lingering + handle is still holding it open. + +Reference: diff --git a/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/repro.json b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/repro.json new file mode 100644 index 000000000..b1a54f54f --- /dev/null +++ b/LiteDB.ReproRunner/Repros/Issue_2614_DiskServiceDispose/repro.json @@ -0,0 +1,14 @@ +{ + "id": "Issue_2614_DiskServiceDispose", + "title": "DiskService leaks handles when initialization fails", + "issues": ["https://github.com/litedb-org/LiteDB/issues/2614"], + "failingSince": "5.0.21", + "timeoutSeconds": 120, + "requiresParallel": false, + "defaultInstances": 1, + "sharedDatabaseKey": "issue2614-disk", + "args": [], + "tags": ["disk", "rlimit", "platform:unix"], + "supports": ["linux"], + "state": "red" +} diff --git a/LiteDB.Shell/LiteDB.Shell.csproj b/LiteDB.Shell/LiteDB.Shell.csproj index 4b361b0f7..56d24672b 100644 --- a/LiteDB.Shell/LiteDB.Shell.csproj +++ b/LiteDB.Shell/LiteDB.Shell.csproj @@ -1,14 +1,11 @@  - net6 + net8.0 LiteDB.Shell LiteDB.Shell Exe LiteDB.Shell - 5.0.6.0 - 5.0.6 - 5.0.6 Maurício David MIT en-US diff --git a/LiteDB.Stress/LiteDB.Stress.csproj b/LiteDB.Stress/LiteDB.Stress.csproj index f34f8fcb8..dd5ee37f9 100644 --- a/LiteDB.Stress/LiteDB.Stress.csproj +++ b/LiteDB.Stress/LiteDB.Stress.csproj @@ -2,7 +2,7 @@ Exe - net8 + net8.0 diff --git a/LiteDB.Stress/Test/TestExecution.cs b/LiteDB.Stress/Test/TestExecution.cs index e2ad943c6..3af18f4a9 100644 --- a/LiteDB.Stress/Test/TestExecution.cs +++ b/LiteDB.Stress/Test/TestExecution.cs @@ -47,7 +47,10 @@ public void Execute() this.CreateThreads(); // start report thread - var t = new Thread(() => this.ReportThread()); + var t = new Thread(() => this.ReportThread()) + { + IsBackground = true + }; t.Name = "REPORT"; t.Start(); } @@ -100,7 +103,10 @@ private void CreateThreads() info.Counter++; info.LastRun = DateTime.Now; } - }); + }) + { + IsBackground = true + }; _threads[thread.ManagedThreadId] = new ThreadInfo { diff --git a/LiteDB.Tests.SharedMutexHarness/Program.cs b/LiteDB.Tests.SharedMutexHarness/Program.cs new file mode 100644 index 000000000..8ceecc8cf --- /dev/null +++ b/LiteDB.Tests.SharedMutexHarness/Program.cs @@ -0,0 +1,341 @@ +using System.Diagnostics; +using System.IO; +using System.Reflection; +using System.Text; +using System.Threading; +using LiteDB; + +var executablePath = Environment.ProcessPath ?? throw new InvalidOperationException("ProcessPath could not be determined."); +var options = HarnessOptions.Parse(args, executablePath); + +if (options.Mode == HarnessMode.Child) +{ + RunChild(options); + return; +} + +RunParent(options); +return; + +void RunParent(HarnessOptions options) +{ + Console.WriteLine($"[parent] creating shared mutex '{options.MutexName}'"); + if (options.UsePsExec) + { + Console.WriteLine($"[parent] PsExec mode enabled (session {options.SessionId}, tool: {options.PsExecPath})"); + } + + Directory.CreateDirectory(options.LogDirectory); + + using var mutex = CreateSharedMutex(options.MutexName); + + Console.WriteLine("[parent] acquiring mutex"); + mutex.WaitOne(); + Console.WriteLine("[parent] mutex acquired"); + + Console.WriteLine("[parent] spawning child with 2s timeout while mutex is held"); + var probeResult = StartChildProcess(options, waitMilliseconds: 2000, "probe"); + Console.WriteLine(probeResult); + + Console.WriteLine("[parent] releasing mutex"); + mutex.ReleaseMutex(); + + Console.WriteLine("[parent] spawning child waiting without timeout after release"); + var acquireResult = StartChildProcess(options, waitMilliseconds: -1, "post-release"); + Console.WriteLine(acquireResult); + + Console.WriteLine("[parent] experiment finished"); +} + +string StartChildProcess(HarnessOptions options, int waitMilliseconds, string label) +{ + var logPath = Path.Combine(options.LogDirectory, $"{label}-{DateTimeOffset.UtcNow:yyyyMMddHHmmssfff}-{Guid.NewGuid():N}.log"); + + var psi = BuildChildStartInfo(options, waitMilliseconds, logPath); + + using var process = Process.Start(psi) ?? throw new InvalidOperationException("Failed to spawn child process."); + + var output = new StringBuilder(); + process.OutputDataReceived += (_, e) => + { + if (e.Data != null) + { + output.AppendLine(e.Data); + } + }; + process.ErrorDataReceived += (_, e) => + { + if (e.Data != null) + { + output.AppendLine("[stderr] " + e.Data); + } + }; + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + if (!process.WaitForExit(10000)) + { + process.Kill(entireProcessTree: true); + throw new TimeoutException("Child process exceeded wait timeout."); + } + + process.WaitForExit(); + + if (File.Exists(logPath)) + { + output.AppendLine("[child log]"); + output.Append(File.ReadAllText(logPath)); + File.Delete(logPath); + } + else + { + output.AppendLine("[child log missing]"); + } + + return output.ToString(); +} + +ProcessStartInfo BuildChildStartInfo(HarnessOptions options, int waitMilliseconds, string logPath) +{ + if (!options.UsePsExec) + { + var directStartInfo = new ProcessStartInfo(options.ExecutablePath) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + directStartInfo.ArgumentList.Add("child"); + directStartInfo.ArgumentList.Add(options.MutexName); + directStartInfo.ArgumentList.Add(waitMilliseconds.ToString()); + directStartInfo.ArgumentList.Add(logPath); + + return directStartInfo; + } + + if (!File.Exists(options.PsExecPath)) + { + throw new FileNotFoundException("PsExec executable could not be located.", options.PsExecPath); + } + + var psi = new ProcessStartInfo(options.PsExecPath) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + psi.ArgumentList.Add("-accepteula"); + psi.ArgumentList.Add("-nobanner"); + psi.ArgumentList.Add("-i"); + psi.ArgumentList.Add(options.SessionId.ToString()); + + if (options.RunAsSystem) + { + psi.ArgumentList.Add("-s"); + } + + psi.ArgumentList.Add(options.ExecutablePath); + psi.ArgumentList.Add("child"); + psi.ArgumentList.Add(options.MutexName); + psi.ArgumentList.Add(waitMilliseconds.ToString()); + psi.ArgumentList.Add(logPath); + + return psi; +} + +void RunChild(HarnessOptions options) +{ + if (!string.IsNullOrEmpty(options.LogPath)) + { + var directory = Path.GetDirectoryName(options.LogPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + } + + void Log(string message) + { + Console.WriteLine(message); + if (!string.IsNullOrEmpty(options.LogPath)) + { + File.AppendAllText(options.LogPath, message + Environment.NewLine); + } + } + + using var mutex = CreateSharedMutex(options.MutexName); + Log($"[child {Environment.ProcessId}] attempting to acquire mutex '{options.MutexName}' (wait={options.ChildWaitMilliseconds}ms)"); + + var sw = Stopwatch.StartNew(); + bool acquired; + + if (options.ChildWaitMilliseconds >= 0) + { + acquired = mutex.WaitOne(options.ChildWaitMilliseconds); + } + else + { + acquired = mutex.WaitOne(); + } + + sw.Stop(); + + Log($"[child {Environment.ProcessId}] acquired={acquired} after {sw.ElapsedMilliseconds}ms"); + + if (acquired) + { + mutex.ReleaseMutex(); + Log($"[child {Environment.ProcessId}] released mutex"); + } +} + +static Mutex CreateSharedMutex(string name) +{ + var liteDbAssembly = typeof(SharedEngine).Assembly; + var factoryType = liteDbAssembly.GetType("LiteDB.SharedMutexFactory", throwOnError: true) + ?? throw new InvalidOperationException("Could not locate SharedMutexFactory."); + + var createMethod = factoryType.GetMethod("Create", BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Could not resolve the Create method on SharedMutexFactory."); + + var mutex = createMethod.Invoke(null, new object[] { name }) as Mutex; + + if (mutex is null) + { + throw new InvalidOperationException("SharedMutexFactory.Create returned null."); + } + + return mutex; +} + +internal sealed record HarnessOptions( + HarnessMode Mode, + string MutexName, + bool UsePsExec, + int SessionId, + bool RunAsSystem, + string PsExecPath, + string ExecutablePath, + int ChildWaitMilliseconds, + string? LogPath, + string LogDirectory) +{ + public static HarnessOptions Parse(string[] args, string executablePath) + { + bool usePsExec = false; + int sessionId = 0; + string? psExecPath = null; + bool runAsSystem = false; + string? logDirectory = null; + var positional = new List(); + + for (var i = 0; i < args.Length; i++) + { + var arg = args[i]; + + if (string.Equals(arg, "--use-psexec", StringComparison.OrdinalIgnoreCase)) + { + usePsExec = true; + continue; + } + + if (string.Equals(arg, "--session", StringComparison.OrdinalIgnoreCase)) + { + if (i + 1 >= args.Length) + { + throw new ArgumentException("Missing session identifier after --session."); + } + + sessionId = int.Parse(args[++i]); + continue; + } + + if (arg.StartsWith("--psexec-path=", StringComparison.OrdinalIgnoreCase)) + { + psExecPath = arg.Substring("--psexec-path=".Length); + continue; + } + + if (string.Equals(arg, "--system", StringComparison.OrdinalIgnoreCase)) + { + runAsSystem = true; + continue; + } + + if (arg.StartsWith("--log-dir=", StringComparison.OrdinalIgnoreCase)) + { + logDirectory = arg.Substring("--log-dir=".Length); + continue; + } + + positional.Add(arg); + } + + var mutexName = positional.Count > 1 + ? positional[1] + : positional.Count == 1 && !string.Equals(positional[0], "child", StringComparison.OrdinalIgnoreCase) + ? positional[0] + : "LiteDB_SharedMutexHarness"; + + if (positional.Count > 0 && string.Equals(positional[0], "child", StringComparison.OrdinalIgnoreCase)) + { + if (positional.Count < 3) + { + throw new ArgumentException("Child invocation expects mutex name and wait duration."); + } + + var waitMilliseconds = int.Parse(positional[2]); + var logPath = positional.Count >= 4 ? positional[3] : null; + var childLogDirectory = logDirectory + ?? (logPath != null + ? Path.GetDirectoryName(logPath) ?? DefaultLogDirectory() + : DefaultLogDirectory()); + + return new HarnessOptions( + HarnessMode.Child, + positional[1], + UsePsExec: false, + sessionId, + runAsSystem, + psExecPath ?? DefaultPsExecPath(), + executablePath, + waitMilliseconds, + logPath, + childLogDirectory); + } + + var resolvedLogDirectory = logDirectory ?? DefaultLogDirectory(); + + return new HarnessOptions( + HarnessMode.Parent, + mutexName, + usePsExec, + sessionId, + runAsSystem, + psExecPath ?? DefaultPsExecPath(), + executablePath, + ChildWaitMilliseconds: -1, + LogPath: null, + LogDirectory: resolvedLogDirectory); + } + + private static string DefaultPsExecPath() + { + var userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + return Path.Combine(userProfile, "tools", "Sysinternals", "PsExec.exe"); + } + + private static string DefaultLogDirectory() + { + return Path.Combine(Path.GetTempPath(), "SharedMutexHarnessLogs"); + } +} + +internal enum HarnessMode +{ + Parent, + Child +} diff --git a/LiteDB.Tests.SharedMutexHarness/README.md b/LiteDB.Tests.SharedMutexHarness/README.md new file mode 100644 index 000000000..8793db3fe --- /dev/null +++ b/LiteDB.Tests.SharedMutexHarness/README.md @@ -0,0 +1,25 @@ +# SharedMutexHarness + +Console harness that stress-tests LiteDB’s `SharedMutexFactory` across processes and Windows sessions. + +## Getting Started + +```bash +dotnet run --project SharedMutexHarness/SharedMutexHarness.csproj +``` + +The parent process acquires the shared mutex, spawns a child that times out, releases the mutex, and spawns a second child that succeeds. + +## Cross-Session Probe (PsExec) + +Run the parent from an elevated PowerShell so PsExec can install `PSEXESVC`: + +```powershell +dotnet run --project SharedMutexHarness/SharedMutexHarness.csproj -- --use-psexec --session 0 +``` + +- `--session ` targets a specific interactive session (see `qwinsta` output). +- Add `--system` to launch the child as SYSTEM (optional). +- Use `--log-dir=` to override the default `%TEMP%\SharedMutexHarnessLogs` location. + +Each child writes its progress to stdout and a per-run log file; the parent echoes that log when the child completes so you can confirm whether the mutex was acquired. diff --git a/LiteDB.Tests.SharedMutexHarness/SharedMutexHarness.csproj b/LiteDB.Tests.SharedMutexHarness/SharedMutexHarness.csproj new file mode 100644 index 000000000..b52723fd4 --- /dev/null +++ b/LiteDB.Tests.SharedMutexHarness/SharedMutexHarness.csproj @@ -0,0 +1,12 @@ + + + Exe + net8.0 + enable + enable + + + + + + diff --git a/LiteDB.Tests/BsonValue/BsonVector_Tests.cs b/LiteDB.Tests/BsonValue/BsonVector_Tests.cs new file mode 100644 index 000000000..19dd0916a --- /dev/null +++ b/LiteDB.Tests/BsonValue/BsonVector_Tests.cs @@ -0,0 +1,323 @@ +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using LiteDB.Vector; +using Xunit; + +namespace LiteDB.Tests.BsonValue_Types; + +public class BsonVector_Tests +{ + + private static readonly Collation _collation = Collation.Binary; + private static readonly BsonDocument _root = new BsonDocument(); + + [Fact] + public void BsonVector_RoundTrip_Success() + { + var original = new BsonDocument + { + ["vec"] = new BsonVector(new float[] { 1.0f, 2.5f, -3.75f }) + }; + + var bytes = BsonSerializer.Serialize(original); + var deserialized = BsonSerializer.Deserialize(bytes); + + var vec = deserialized["vec"].AsVector; + Assert.Equal(3, vec.Length); + Assert.Equal(1.0f, vec[0]); + Assert.Equal(2.5f, vec[1]); + Assert.Equal(-3.75f, vec[2]); + } + + [Fact] + public void BsonVector_RoundTrip_UInt16Limit() + { + var values = Enumerable.Range(0, ushort.MaxValue).Select(i => (float)(i % 32)).ToArray(); + + var original = new BsonDocument + { + ["vec"] = new BsonVector(values) + }; + + var bytes = BsonSerializer.Serialize(original); + var deserialized = BsonSerializer.Deserialize(bytes); + + deserialized["vec"].AsVector.Should().Equal(values); + } + + private class VectorDoc + { + public int Id { get; set; } + public float[] Embedding { get; set; } + } + + [Fact] + public void VectorSim_Query_ReturnsExpectedNearest() + { + using var db = new LiteDatabase(":memory:"); + var col = db.GetCollection("vectors"); + + // Insert vectorized documents + col.Insert(new VectorDoc { Id = 1, Embedding = new float[] { 1.0f, 0.0f } }); + col.Insert(new VectorDoc { Id = 2, Embedding = new float[] { 0.0f, 1.0f } }); + col.Insert(new VectorDoc { Id = 3, Embedding = new float[] { 1.0f, 1.0f } }); + + // Create index on the embedding field (if applicable to your implementation) + col.EnsureIndex("Embedding", "Embedding"); + + // Query: Find vectors nearest to [1, 0] + var target = new float[] { 1.0f, 0.0f }; + var results = col.Query() + .WhereNear(r => r.Embedding, [1.0f, 0.0f], maxDistance: .28) + .ToList(); + + results.Should().NotBeEmpty(); + results.Select(x => x.Id).Should().Contain(1); + results.Select(x => x.Id).Should().NotContain(2); + results.Select(x => x.Id).Should().NotContain(3); // too far away + } + + [Fact] + public void VectorSim_Query_WhereVectorSimilar_AppliesAlias() + { + using var db = new LiteDatabase(":memory:"); + var col = db.GetCollection("vectors"); + + col.Insert(new VectorDoc { Id = 1, Embedding = new float[] { 1.0f, 0.0f } }); + col.Insert(new VectorDoc { Id = 2, Embedding = new float[] { 0.0f, 1.0f } }); + col.Insert(new VectorDoc { Id = 3, Embedding = new float[] { 1.0f, 1.0f } }); + + var target = new float[] { 1.0f, 0.0f }; + + var nearResults = col.Query() + .WhereNear(r => r.Embedding, target, maxDistance: .28) + .ToList() + .Select(r => r.Id) + .OrderBy(id => id) + .ToList(); + + var similarResults = col.Query() + .WhereNear(r => r.Embedding, target, maxDistance: .28) + .ToList() + .Select(r => r.Id) + .OrderBy(id => id) + .ToList(); + + similarResults.Should().Equal(nearResults); + } + + [Fact] + public void VectorSim_Query_BsonExpressionOverload_ReturnsExpectedNearest() + { + using var db = new LiteDatabase(":memory:"); + var col = db.GetCollection("vectors"); + + col.Insert(new VectorDoc { Id = 1, Embedding = new float[] { 1.0f, 0.0f } }); + col.Insert(new VectorDoc { Id = 2, Embedding = new float[] { 0.0f, 1.0f } }); + col.Insert(new VectorDoc { Id = 3, Embedding = new float[] { 1.0f, 1.0f } }); + + var target = new float[] { 1.0f, 0.0f }; + var fieldExpr = BsonExpression.Create("$.Embedding"); + + var results = col.Query() + .WhereNear(fieldExpr, target, maxDistance: .28) + .ToList(); + + results.Select(x => x.Id).Should().ContainSingle(id => id == 1); + } + + [Fact] + public void VectorSim_ExpressionQuery_WorksViaSQL() + { + using var db = new LiteDatabase(":memory:"); + var col = db.GetCollection("vectors"); + + col.Insert(new BsonDocument + { + ["_id"] = 1, + ["Embedding"] = new BsonVector(new float[] { 1.0f, 0.0f }) + }); + col.Insert(new BsonDocument + { + ["_id"] = 2, + ["Embedding"] = new BsonVector(new float[] { 0.0f, 1.0f }) + }); + col.Insert(new BsonDocument + { + ["_id"] = 3, + ["Embedding"] = new BsonVector(new float[] { 1.0f, 1.0f }) + }); + + var query = "SELECT * FROM vectors WHERE $.Embedding VECTOR_SIM [1.0, 0.0] <= 0.25"; + var rawResults = db.Execute(query).ToList(); + + var docs = rawResults + .Where(r => r.IsDocument) + .SelectMany(r => + { + var doc = r.AsDocument; + if (doc.TryGetValue("expr", out var expr) && expr.IsArray) + { + return expr.AsArray + .Where(x => x.IsDocument) + .Select(x => x.AsDocument); + } + + return new[] { doc }; + }) + .ToList(); + + docs.Select(d => d["_id"].AsInt32).Should().Contain(1); + docs.Select(d => d["_id"].AsInt32).Should().NotContain(2); + docs.Select(d => d["_id"].AsInt32).Should().NotContain(3); // cosine ~ 0.293 + } + + [Fact] + public void VectorSim_InfixExpression_ParsesAndEvaluates() + { + var expr = BsonExpression.Create("$.Embedding VECTOR_SIM [1.0, 0.0]"); + + expr.Type.Should().Be(BsonExpressionType.VectorSim); + + var doc = new BsonDocument + { + ["Embedding"] = new BsonArray { 1.0, 0.0 } + }; + + var result = expr.ExecuteScalar(doc); + + result.IsDouble.Should().BeTrue(); + result.AsDouble.Should().BeApproximately(0.0, 1e-6); + } + + [Fact] + public void VectorSim_FunctionCall_ParsesAndEvaluates() + { + var expr = BsonExpression.Create("VECTOR_SIM($.Embedding, [1.0, 0.0])"); + + expr.Type.Should().Be(BsonExpressionType.VectorSim); + + var doc = new BsonDocument + { + ["Embedding"] = new BsonArray { 1.0, 0.0 } + }; + + var result = expr.ExecuteScalar(doc); + + result.IsDouble.Should().BeTrue(); + result.AsDouble.Should().BeApproximately(0.0, 1e-6); + } + + [Fact] + public void VectorSim_ReturnsZero_ForIdenticalVectors() + { + var left = new BsonArray { 1.0, 0.0 }; + var right = new BsonVector(new float[] { 1.0f, 0.0f }); + + var result = BsonExpressionMethods.VECTOR_SIM(left, right); + + Assert.NotNull(result); + Assert.True(result.IsDouble); + Assert.Equal(0.0, result.AsDouble, 6); // Cosine distance = 0.0 + } + + [Fact] + public void VectorSim_ReturnsOne_ForOrthogonalVectors() + { + var left = new BsonArray { 1.0, 0.0 }; + var right = new BsonVector(new float[] { 0.0f, 1.0f }); + + var result = BsonExpressionMethods.VECTOR_SIM(left, right); + + Assert.NotNull(result); + Assert.True(result.IsDouble); + Assert.Equal(1.0, result.AsDouble, 6); // Cosine distance = 1.0 + } + + [Fact] + public void VectorSim_ReturnsNull_ForInvalidInput() + { + var left = new BsonArray { "a", "b" }; + var right = new BsonVector(new float[] { 1.0f, 0.0f }); + + var result = BsonExpressionMethods.VECTOR_SIM(left, right); + + Assert.True(result.IsNull); + } + + [Fact] + public void VectorSim_ReturnsNull_ForMismatchedLengths() + { + var left = new BsonArray { 1.0, 2.0, 3.0 }; + var right = new BsonVector(new float[] { 1.0f, 2.0f }); + + var result = BsonExpressionMethods.VECTOR_SIM(left, right); + + Assert.True(result.IsNull); + } + + + [Fact] + public void VectorSim_TopK_ReturnsCorrectOrder() + { + using var db = new LiteDatabase(":memory:"); + var col = db.GetCollection("vectors"); + + col.Insert(new VectorDoc { Id = 1, Embedding = new float[] { 1.0f, 0.0f } }); // sim = 0.0 + col.Insert(new VectorDoc { Id = 2, Embedding = new float[] { 0.0f, 1.0f } }); // sim = 1.0 + col.Insert(new VectorDoc { Id = 3, Embedding = new float[] { 1.0f, 1.0f } }); // sim ≈ 0.293 + + var target = new float[] { 1.0f, 0.0f }; + + var results = col.Query() + .TopKNear(x => x.Embedding, target, 2) + .ToList(); + + var ids = results.Select(r => r.Id).ToList(); + ids.Should().BeEquivalentTo(new[] { 1, 3 }, options => options.WithStrictOrdering()); + } + + [Fact] + public void BsonVector_CompareTo_SortsLexicographically() + { + var values = new List + { + new BsonVector(new float[] { 1.0f }), + new BsonVector(new float[] { 0.0f, 2.0f }), + new BsonVector(new float[] { 0.0f, 1.0f, 0.5f }), + new BsonVector(new float[] { 0.0f, 1.0f }) + }; + + values.Sort(); + + values.Should().Equal( + new BsonVector(new float[] { 0.0f, 1.0f }), + new BsonVector(new float[] { 0.0f, 1.0f, 0.5f }), + new BsonVector(new float[] { 0.0f, 2.0f }), + new BsonVector(new float[] { 1.0f })); + } + + [Fact] + public void BsonVector_Index_OrderIsDeterministic() + { + using var db = new LiteDatabase(":memory:"); + var col = db.GetCollection("vectors"); + + var docs = new[] + { + new VectorDoc { Id = 1, Embedding = new float[] { 0.0f, 1.0f } }, + new VectorDoc { Id = 2, Embedding = new float[] { 0.0f, 1.0f, 0.5f } }, + new VectorDoc { Id = 3, Embedding = new float[] { 0.0f, 2.0f } }, + new VectorDoc { Id = 4, Embedding = new float[] { 1.0f } } + }; + + col.InsertBulk(docs); + + col.EnsureIndex(x => x.Embedding); + + var ordered = col.Query().OrderBy(x => x.Embedding).ToList(); + + ordered.Select(x => x.Id).Should().Equal(1, 2, 3, 4); + } +} diff --git a/LiteDB.Tests/Database/AutoId_Tests.cs b/LiteDB.Tests/Database/AutoId_Tests.cs index bd615d8d4..0e2eed9f8 100644 --- a/LiteDB.Tests/Database/AutoId_Tests.cs +++ b/LiteDB.Tests/Database/AutoId_Tests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using LiteDB; +using LiteDB.Tests.Utils; using FluentAssertions; using Xunit; @@ -268,7 +269,7 @@ public void AutoId_No_Duplicate_After_Delete() [Fact] public void AutoId_Zero_Int() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { var test = db.GetCollection("Test", BsonAutoId.Int32); var doc = new BsonDocument() { ["_id"] = 0, ["p1"] = 1 }; diff --git a/LiteDB.Tests/Database/Create_Database_Tests.cs b/LiteDB.Tests/Database/Create_Database_Tests.cs index 169ad6ebd..a8c945ade 100644 --- a/LiteDB.Tests/Database/Create_Database_Tests.cs +++ b/LiteDB.Tests/Database/Create_Database_Tests.cs @@ -3,6 +3,7 @@ using System.Linq; using FluentAssertions; using LiteDB.Engine; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -17,7 +18,7 @@ public void Create_Database_With_Initial_Size() using (var file = new TempFile()) { - using (var db = new LiteDatabase("filename=" + file.Filename + ";initial size=" + initial)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, "filename=" + file.Filename + ";initial size=" + initial)) { var col = db.GetCollection("col"); diff --git a/LiteDB.Tests/Database/Database_Pragmas_Tests.cs b/LiteDB.Tests/Database/Database_Pragmas_Tests.cs index e21041bfb..1d9092cbe 100644 --- a/LiteDB.Tests/Database/Database_Pragmas_Tests.cs +++ b/LiteDB.Tests/Database/Database_Pragmas_Tests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using LiteDB; +using LiteDB.Tests.Utils; using FluentAssertions; using Xunit; using System.Globalization; @@ -13,7 +14,7 @@ public class Database_Pragmas_Tests [Fact] public void Database_Pragmas_Get_Set() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { db.Timeout.TotalSeconds.Should().Be(60.0); db.UtcDate.Should().Be(false); diff --git a/LiteDB.Tests/Database/DeleteMany_Tests.cs b/LiteDB.Tests/Database/DeleteMany_Tests.cs index f7de356bd..cf80be741 100644 --- a/LiteDB.Tests/Database/DeleteMany_Tests.cs +++ b/LiteDB.Tests/Database/DeleteMany_Tests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using LiteDB; +using LiteDB.Tests.Utils; using FluentAssertions; using Xunit; @@ -12,7 +13,7 @@ public class DeleteMany_Tests [Fact] public void DeleteMany_With_Arguments() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { var c1 = db.GetCollection("Test"); diff --git a/LiteDB.Tests/Database/Delete_By_Name_Tests.cs b/LiteDB.Tests/Database/Delete_By_Name_Tests.cs index ed206f7b0..711c400d2 100644 --- a/LiteDB.Tests/Database/Delete_By_Name_Tests.cs +++ b/LiteDB.Tests/Database/Delete_By_Name_Tests.cs @@ -1,7 +1,7 @@ using System; -using System.IO; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -21,8 +21,7 @@ public class Person [Fact] public void Delete_By_Name() { - using (var f = new TempFile()) - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create()) { var col = db.GetCollection("Person"); diff --git a/LiteDB.Tests/Database/Document_Size_Tests.cs b/LiteDB.Tests/Database/Document_Size_Tests.cs index 6c25112cb..f19de2871 100644 --- a/LiteDB.Tests/Database/Document_Size_Tests.cs +++ b/LiteDB.Tests/Database/Document_Size_Tests.cs @@ -1,9 +1,9 @@ using System; using System.Diagnostics; -using System.IO; using System.Linq; using FluentAssertions; using LiteDB.Engine; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -15,8 +15,7 @@ public class Document_Size_Tests [Fact] public void Very_Large_Single_Document_Support_With_Partial_Load_Memory_Usage() { - using (var file = new TempFile()) - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create()) { var col = db.GetCollection("col"); diff --git a/LiteDB.Tests/Database/FindAll_Tests.cs b/LiteDB.Tests/Database/FindAll_Tests.cs index 7f0f2b9a0..11b194ec9 100644 --- a/LiteDB.Tests/Database/FindAll_Tests.cs +++ b/LiteDB.Tests/Database/FindAll_Tests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -23,7 +24,7 @@ public void FindAll() { using (var f = new TempFile()) { - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, f.Filename)) { var col = db.GetCollection("Person"); @@ -34,7 +35,7 @@ public void FindAll() } // close datafile - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, f.Filename)) { var p = db.GetCollection("Person").Find(Query.All("Fullname", Query.Ascending)); diff --git a/LiteDB.Tests/Database/IndexSortAndFilter_Tests.cs b/LiteDB.Tests/Database/IndexSortAndFilter_Tests.cs index 1e2f780fe..bc407a588 100644 --- a/LiteDB.Tests/Database/IndexSortAndFilter_Tests.cs +++ b/LiteDB.Tests/Database/IndexSortAndFilter_Tests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -19,13 +20,11 @@ public class Item #endregion private readonly ILiteCollection _collection; - private readonly TempFile _tempFile; private readonly ILiteDatabase _database; public IndexSortAndFilterTest() { - _tempFile = new TempFile(); - _database = new LiteDatabase(_tempFile.Filename); + _database = DatabaseFactory.Create(); _collection = _database.GetCollection("items"); _collection.Upsert(new Item() { Id = "C", Value = "Value 1" }); @@ -38,7 +37,6 @@ public IndexSortAndFilterTest() public void Dispose() { _database.Dispose(); - _tempFile.Dispose(); } [Fact] diff --git a/LiteDB.Tests/Database/MultiKey_Mapper_Tests.cs b/LiteDB.Tests/Database/MultiKey_Mapper_Tests.cs index b4d0ae131..b791bfbdb 100644 --- a/LiteDB.Tests/Database/MultiKey_Mapper_Tests.cs +++ b/LiteDB.Tests/Database/MultiKey_Mapper_Tests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -28,7 +29,7 @@ public class Customer [Fact] public void MultiKey_Mapper() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { var col = db.GetCollection("col"); diff --git a/LiteDB.Tests/Database/NonIdPoco_Tests.cs b/LiteDB.Tests/Database/NonIdPoco_Tests.cs index 4ba0b29d6..2983f5bd9 100644 --- a/LiteDB.Tests/Database/NonIdPoco_Tests.cs +++ b/LiteDB.Tests/Database/NonIdPoco_Tests.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; -using System.IO; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -21,8 +21,7 @@ public class MissingIdDoc [Fact] public void MissingIdDoc_Test() { - using (var file = new TempFile()) - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create()) { var col = db.GetCollection("col"); diff --git a/LiteDB.Tests/Database/Query_Min_Max_Tests.cs b/LiteDB.Tests/Database/Query_Min_Max_Tests.cs index 46511594a..6134d0a8e 100644 --- a/LiteDB.Tests/Database/Query_Min_Max_Tests.cs +++ b/LiteDB.Tests/Database/Query_Min_Max_Tests.cs @@ -1,7 +1,7 @@ using System; -using System.IO; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -24,8 +24,7 @@ public class EntityMinMax [Fact] public void Query_Min_Max() { - using (var f = new TempFile()) - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create()) { var c = db.GetCollection("col"); diff --git a/LiteDB.Tests/Database/Site_Tests.cs b/LiteDB.Tests/Database/Site_Tests.cs index 2ab05b6d2..743666f6e 100644 --- a/LiteDB.Tests/Database/Site_Tests.cs +++ b/LiteDB.Tests/Database/Site_Tests.cs @@ -1,8 +1,8 @@ using System; -using System.IO; using System.Linq; using System.Security.Cryptography; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -12,8 +12,7 @@ public class Site_Tests [Fact] public void Home_Example() { - using (var f = new TempFile()) - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create()) { // Get customer collection var customers = db.GetCollection("customers"); diff --git a/LiteDB.Tests/Database/Snapshot_Upgrade_Tests.cs b/LiteDB.Tests/Database/Snapshot_Upgrade_Tests.cs index 07ec94af3..1a8285dcf 100644 --- a/LiteDB.Tests/Database/Snapshot_Upgrade_Tests.cs +++ b/LiteDB.Tests/Database/Snapshot_Upgrade_Tests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -11,7 +12,7 @@ public class Snapshot_Upgrade_Tests [Fact] public void Transaction_Update_Upsert() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var col = db.GetCollection("test"); bool transactionCreated = db.BeginTrans(); diff --git a/LiteDB.Tests/Database/Storage_Tests.cs b/LiteDB.Tests/Database/Storage_Tests.cs index b54d5f395..de861d20c 100644 --- a/LiteDB.Tests/Database/Storage_Tests.cs +++ b/LiteDB.Tests/Database/Storage_Tests.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Security.Cryptography; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database @@ -30,8 +31,7 @@ public Storage_Tests() [Fact] public void Storage_Upload_Download() { - using (var f = new TempFile()) - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create()) //using (var db = new LiteDatabase(@"c:\temp\file.db")) { var fs = db.GetStorage("_files", "_chunks"); diff --git a/LiteDB.Tests/Database/Upgrade_Tests.cs b/LiteDB.Tests/Database/Upgrade_Tests.cs index b1fd21761..a6605984f 100644 --- a/LiteDB.Tests/Database/Upgrade_Tests.cs +++ b/LiteDB.Tests/Database/Upgrade_Tests.cs @@ -2,10 +2,9 @@ using System.IO; using System.Linq; using LiteDB; +using LiteDB.Tests.Utils; using FluentAssertions; using Xunit; -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel; - namespace LiteDB.Tests.Database { public class Upgrade_Tests @@ -16,7 +15,7 @@ public void Migrage_From_V4() // v5 upgrades only from v4! using(var tempFile = new TempFile("../../../Resources/v4.db")) { - using (var db = new LiteDatabase($"filename={tempFile};upgrade=true")) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, $"filename={tempFile};upgrade=true")) { // convert and open database var col1 = db.GetCollection("col1"); @@ -24,7 +23,7 @@ public void Migrage_From_V4() col1.Count().Should().Be(3); } - using (var db = new LiteDatabase($"filename={tempFile};upgrade=true")) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, $"filename={tempFile};upgrade=true")) { // database already converted var col1 = db.GetCollection("col1"); @@ -40,7 +39,7 @@ public void Migrage_From_V4_No_FileExtension() // v5 upgrades only from v4! using (var tempFile = new TempFile("../../../Resources/v4.db")) { - using (var db = new LiteDatabase($"filename={tempFile};upgrade=true")) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, $"filename={tempFile};upgrade=true")) { // convert and open database var col1 = db.GetCollection("col1"); @@ -48,7 +47,7 @@ public void Migrage_From_V4_No_FileExtension() col1.Count().Should().Be(3); } - using (var db = new LiteDatabase($"filename={tempFile};upgrade=true")) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, $"filename={tempFile};upgrade=true")) { // database already converted var col1 = db.GetCollection("col1"); diff --git a/LiteDB.Tests/Database/Writing_While_Reading_Test.cs b/LiteDB.Tests/Database/Writing_While_Reading_Test.cs index fc2c8f86d..69dda1db6 100644 --- a/LiteDB.Tests/Database/Writing_While_Reading_Test.cs +++ b/LiteDB.Tests/Database/Writing_While_Reading_Test.cs @@ -1,4 +1,5 @@ using System.IO; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Database; @@ -9,7 +10,7 @@ public class Writing_While_Reading_Test public void Test() { using var f = new TempFile(); - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, f.Filename)) { var col = db.GetCollection("col"); col.Insert(new MyClass { Name = "John", Description = "Doe" }); @@ -18,7 +19,7 @@ public void Test() } - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, f.Filename)) { var col = db.GetCollection("col"); foreach (var item in col.FindAll()) @@ -31,7 +32,7 @@ public void Test() } - using (var db = new LiteDatabase(f.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, f.Filename)) { var col = db.GetCollection("col"); foreach (var item in col.FindAll()) diff --git a/LiteDB.Tests/Document/Decimal_Tests.cs b/LiteDB.Tests/Document/Decimal_Tests.cs index c29d2a063..682712f40 100644 --- a/LiteDB.Tests/Document/Decimal_Tests.cs +++ b/LiteDB.Tests/Document/Decimal_Tests.cs @@ -10,8 +10,8 @@ public void BsonValue_New_Decimal_Type() { var d0 = 0m; var d1 = 1m; - var dmin = new BsonValue(decimal.MinValue); - var dmax = new BsonValue(decimal.MaxValue); + var dmin = new LiteDB.BsonValue(decimal.MinValue); + var dmax = new LiteDB.BsonValue(decimal.MaxValue); JsonSerializer.Serialize(d0).Should().Be("{\"$numberDecimal\":\"0\"}"); JsonSerializer.Serialize(d1).Should().Be("{\"$numberDecimal\":\"1\"}"); diff --git a/LiteDB.Tests/Document/Implicit_Tests.cs b/LiteDB.Tests/Document/Implicit_Tests.cs index b81468e3c..51ce3b9a6 100644 --- a/LiteDB.Tests/Document/Implicit_Tests.cs +++ b/LiteDB.Tests/Document/Implicit_Tests.cs @@ -13,9 +13,9 @@ public void BsonValue_Implicit_Convert() long l = long.MaxValue; ulong u = ulong.MaxValue; - BsonValue bi = i; - BsonValue bl = l; - BsonValue bu = u; + LiteDB.BsonValue bi = i; + LiteDB.BsonValue bl = l; + LiteDB.BsonValue bu = u; bi.IsInt32.Should().BeTrue(); bl.IsInt64.Should().BeTrue(); @@ -35,7 +35,7 @@ public void BsonDocument_Inner() customer["CreateDate"] = DateTime.Now; customer["Phones"] = new BsonArray { "8000-0000", "9000-000" }; customer["IsActive"] = true; - customer["IsAdmin"] = new BsonValue(true); + customer["IsAdmin"] = new LiteDB.BsonValue(true); customer["Address"] = new BsonDocument { ["Street"] = "Av. Protasio Alves" diff --git a/LiteDB.Tests/Document/Json_Tests.cs b/LiteDB.Tests/Document/Json_Tests.cs index 154e79532..2bf162caf 100644 --- a/LiteDB.Tests/Document/Json_Tests.cs +++ b/LiteDB.Tests/Document/Json_Tests.cs @@ -94,5 +94,18 @@ public void Json_DoubleNaN_Tests() Assert.False(double.IsNegativeInfinity(bson["doubleNegativeInfinity"].AsDouble)); Assert.False(double.IsPositiveInfinity(bson["doublePositiveInfinity"].AsDouble)); } + + [Fact] + public void Json_Writes_BsonVector_As_Array() + { + var document = new BsonDocument + { + ["Embedding"] = new BsonVector(new float[] { 1.0f, 2.5f, -3.75f }) + }; + + var json = JsonSerializer.Serialize(document); + + json.Should().Contain("\"Embedding\":[1.0,2.5,-3.75]"); + } } } \ No newline at end of file diff --git a/LiteDB.Tests/Engine/Collation_Tests.cs b/LiteDB.Tests/Engine/Collation_Tests.cs index 02e21b727..558d87b4a 100644 --- a/LiteDB.Tests/Engine/Collation_Tests.cs +++ b/LiteDB.Tests/Engine/Collation_Tests.cs @@ -33,7 +33,10 @@ public void Culture_Ordinal_Sort() e.Insert("col1", names.Select(x => new BsonDocument { ["name"] = x }), BsonAutoId.Int32); // sort by merge sort - var sortByOrderByName = e.Query("col1", new Query { OrderBy = "name" }) + var orderQuery = new Query(); + orderQuery.OrderBy.Add(new QueryOrder("name", Query.Ascending)); + + var sortByOrderByName = e.Query("col1", orderQuery) .ToEnumerable() .Select(x => x["name"].AsString) .ToArray(); @@ -53,8 +56,11 @@ public void Culture_Ordinal_Sort() // index test e.EnsureIndex("col1", "idx_name", "name", false); + var indexOrderQuery = new Query(); + indexOrderQuery.OrderBy.Add(new QueryOrder("name", Query.Ascending)); + // sort by index - var sortByIndexName = e.Query("col1", new Query { OrderBy = "name" }) + var sortByIndexName = e.Query("col1", indexOrderQuery) .ToEnumerable() .Select(x => x["name"].AsString) .ToArray(); diff --git a/LiteDB.Tests/Engine/CrossProcess_Shared_Tests.cs b/LiteDB.Tests/Engine/CrossProcess_Shared_Tests.cs new file mode 100644 index 000000000..f9b3c6505 --- /dev/null +++ b/LiteDB.Tests/Engine/CrossProcess_Shared_Tests.cs @@ -0,0 +1,280 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace LiteDB.Tests.Engine; + +public class CrossProcess_Shared_Tests : IDisposable +{ + private readonly ITestOutputHelper _output; + private readonly string _dbPath; + private readonly string _testId; + + public CrossProcess_Shared_Tests(ITestOutputHelper output) + { + _output = output; + _testId = Guid.NewGuid().ToString("N"); + _dbPath = Path.Combine(Path.GetTempPath(), $"litedb_crossprocess_{_testId}.db"); + + // Clean up any existing test database + TryDeleteDatabase(); + } + + public void Dispose() + { + TryDeleteDatabase(); + } + + private void TryDeleteDatabase() + { + try + { + if (File.Exists(_dbPath)) + { + File.Delete(_dbPath); + } + var logPath = _dbPath + "-log"; + if (File.Exists(logPath)) + { + File.Delete(logPath); + } + } + catch + { + // Ignore cleanup errors + } + } + + [Fact] + public async Task CrossProcess_Shared_MultipleProcesses_CanAccessSameDatabase() + { + // This test verifies that multiple concurrent connections can access the same database in shared mode + // Each Task simulates a separate process/application accessing the database + const int processCount = 3; + const int documentsPerProcess = 10; + + _output.WriteLine($"Starting shared mode concurrent access test with {processCount} tasks"); + _output.WriteLine($"Database path: {_dbPath}"); + + // Initialize the database in the main process + using (var db = new LiteDatabase(new ConnectionString + { + Filename = _dbPath, + Connection = ConnectionType.Shared + })) + { + var col = db.GetCollection("cross_process_test"); + col.Insert(new BsonDocument { ["_id"] = 0, ["source"] = "main_process", ["timestamp"] = DateTime.UtcNow }); + } + + // Spawn multiple concurrent tasks that will access the database via shared mode + var tasks = new List(); + for (int i = 1; i <= processCount; i++) + { + var processId = i; + tasks.Add(Task.Run(() => RunChildProcess(processId, documentsPerProcess))); + } + + // Wait for all tasks to complete + await Task.WhenAll(tasks); + + // Verify all documents were written + using (var db = new LiteDatabase(new ConnectionString + { + Filename = _dbPath, + Connection = ConnectionType.Shared + })) + { + var col = db.GetCollection("cross_process_test"); + var allDocs = col.FindAll().ToList(); + + _output.WriteLine($"Total documents found: {allDocs.Count}"); + + // Should have 1 (main) + (processCount * documentsPerProcess) documents + var expectedCount = 1 + (processCount * documentsPerProcess); + allDocs.Count.Should().Be(expectedCount, + $"Expected {expectedCount} documents (1 main + {processCount} processes × {documentsPerProcess} docs each)"); + + // Verify documents from each concurrent connection + for (int i = 1; i <= processCount; i++) + { + var processSource = $"process_{i}"; + var processDocs = allDocs.Where(d => d["source"].AsString == processSource).ToList(); + processDocs.Count.Should().Be(documentsPerProcess, + $"Task {i} should have written {documentsPerProcess} documents"); + } + } + + _output.WriteLine("Shared mode concurrent access test completed successfully"); + } + + [Fact] + public async Task CrossProcess_Shared_ConcurrentWrites_InsertDocuments() + { + // This test verifies that concurrent inserts from multiple connections work correctly + // Each task inserts unique documents to test concurrent write capability + const int taskCount = 5; + const int documentsPerTask = 20; + + _output.WriteLine($"Starting concurrent insert test with {taskCount} tasks"); + + // Initialize collection + using (var db = new LiteDatabase(new ConnectionString + { + Filename = _dbPath, + Connection = ConnectionType.Shared + })) + { + var col = db.GetCollection("concurrent_inserts"); + col.EnsureIndex("task_id"); + } + + // Spawn concurrent tasks that will insert documents + var tasks = new List(); + for (int i = 1; i <= taskCount; i++) + { + var taskId = i; + tasks.Add(Task.Run(() => RunInsertTask(taskId, documentsPerTask))); + } + + await Task.WhenAll(tasks); + + // Verify all documents were inserted + using (var db = new LiteDatabase(new ConnectionString + { + Filename = _dbPath, + Connection = ConnectionType.Shared + })) + { + var col = db.GetCollection("concurrent_inserts"); + var totalDocs = col.Count(); + + var expectedCount = taskCount * documentsPerTask; + totalDocs.Should().Be(expectedCount, + $"Expected {expectedCount} documents ({taskCount} tasks × {documentsPerTask} docs each)"); + + // Verify each task inserted the correct number + for (int i = 1; i <= taskCount; i++) + { + var taskDocs = col.Count(Query.EQ("task_id", i)); + taskDocs.Should().Be(documentsPerTask, + $"Task {i} should have inserted {documentsPerTask} documents"); + } + } + + _output.WriteLine("Concurrent insert test completed successfully"); + } + + private void RunInsertTask(int taskId, int documentCount) + { + var task = Task.Run(() => + { + try + { + _output.WriteLine($"Insert task {taskId} starting with {documentCount} documents"); + + using var db = new LiteDatabase(new ConnectionString + { + Filename = _dbPath, + Connection = ConnectionType.Shared + }); + + var col = db.GetCollection("concurrent_inserts"); + + for (int i = 0; i < documentCount; i++) + { + var doc = new BsonDocument + { + ["task_id"] = taskId, + ["doc_number"] = i, + ["timestamp"] = DateTime.UtcNow, + ["data"] = $"Data from task {taskId}, document {i}" + }; + + col.Insert(doc); + + // Small delay to ensure concurrent access + Thread.Sleep(2); + } + + _output.WriteLine($"Insert task {taskId} completed {documentCount} insertions"); + } + catch (Exception ex) + { + _output.WriteLine($"Insert task {taskId} ERROR: {ex.Message}"); + throw; + } + }); + + if (!task.Wait(30000)) // 30 second timeout + { + throw new TimeoutException($"Insert task {taskId} timed out"); + } + + if (task.IsFaulted) + { + throw new Exception($"Insert task {taskId} faulted", task.Exception); + } + } + + private void RunChildProcess(int processId, int documentCount) + { + // Instead of spawning actual processes, we'll use Tasks to simulate concurrent access + // This is safer for CI environments and still tests the shared mode locking + var task = Task.Run(() => + { + try + { + _output.WriteLine($"Task {processId} starting with {documentCount} documents to write"); + + using var db = new LiteDatabase(new ConnectionString + { + Filename = _dbPath, + Connection = ConnectionType.Shared + }); + + var col = db.GetCollection("cross_process_test"); + + for (int i = 0; i < documentCount; i++) + { + var doc = new BsonDocument + { + ["source"] = $"process_{processId}", + ["doc_number"] = i, + ["timestamp"] = DateTime.UtcNow, + ["thread_id"] = Thread.CurrentThread.ManagedThreadId + }; + + col.Insert(doc); + + // Small delay to ensure concurrent access + Thread.Sleep(10); + } + + _output.WriteLine($"Task {processId} completed writing {documentCount} documents"); + } + catch (Exception ex) + { + _output.WriteLine($"Task {processId} ERROR: {ex.Message}"); + throw; + } + }); + + if (!task.Wait(30000)) // 30 second timeout + { + throw new TimeoutException($"Task {processId} timed out"); + } + + if (task.IsFaulted) + { + throw new Exception($"Task {processId} faulted", task.Exception); + } + } +} diff --git a/LiteDB.Tests/Engine/DropCollection_Tests.cs b/LiteDB.Tests/Engine/DropCollection_Tests.cs index 13be40d58..ded4d9435 100644 --- a/LiteDB.Tests/Engine/DropCollection_Tests.cs +++ b/LiteDB.Tests/Engine/DropCollection_Tests.cs @@ -1,22 +1,41 @@ -using System.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; using FluentAssertions; +using LiteDB; +using LiteDB.Engine; +using LiteDB.Tests.Utils; +using LiteDB.Vector; using Xunit; namespace LiteDB.Tests.Engine { public class DropCollection_Tests { + private class VectorDocument + { + public int Id { get; set; } + + public float[] Embedding { get; set; } + } + + private const string VectorIndexName = "embedding_idx"; + + private static readonly FieldInfo EngineField = typeof(LiteDatabase).GetField("_engine", BindingFlags.NonPublic | BindingFlags.Instance); + private static readonly MethodInfo AutoTransactionMethod = typeof(LiteEngine).GetMethod("AutoTransaction", BindingFlags.NonPublic | BindingFlags.Instance); + [Fact] public void DropCollection() { - using (var file = new TempFile()) - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create()) { db.GetCollectionNames().Should().NotContain("col"); var col = db.GetCollection("col"); - col.Insert(new BsonDocument {["a"] = 1}); + col.Insert(new BsonDocument { ["a"] = 1 }); db.GetCollectionNames().Should().Contain("col"); @@ -31,7 +50,7 @@ public void InsertDropCollection() { using (var file = new TempFile()) { - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { var col = db.GetCollection("test"); col.Insert(new BsonDocument { ["_id"] = 1 }); @@ -39,12 +58,292 @@ public void InsertDropCollection() db.Rebuild(); } - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { var col = db.GetCollection("test"); col.Insert(new BsonDocument { ["_id"] = 1 }); } } } + + [Fact] + public void DropCollection_WithVectorIndex_ReclaimsPages_ByCounting() + { + using var file = new TempFile(); + + const ushort dimensions = 6; + + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) + { + var collection = db.GetCollection("docs"); + + collection.EnsureIndex( + VectorIndexName, + BsonExpression.Create("$.embedding"), + new VectorIndexOptions(dimensions, VectorDistanceMetric.Cosine)); + + for (var i = 0; i < 8; i++) + { + var embedding = new BsonArray(Enumerable.Range(0, dimensions) + .Select(j => new BsonValue(i + (j * 0.1)))); + + collection.Insert(new BsonDocument + { + ["_id"] = i + 1, + ["embedding"] = embedding + }); + } + + db.Checkpoint(); + } + + var beforeCounts = CountPagesByType(file.Filename); + beforeCounts.TryGetValue(PageType.VectorIndex, out var vectorPagesBefore); + vectorPagesBefore.Should().BeGreaterThan(0, "creating a vector index should allocate vector pages"); + + var drop = () => + { + using var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename); + db.DropCollection("docs"); + db.Checkpoint(); + }; + + drop.Should().NotThrow(); + + var afterCounts = CountPagesByType(file.Filename); + afterCounts.TryGetValue(PageType.VectorIndex, out var vectorPagesAfter); + vectorPagesAfter.Should().BeLessThan(vectorPagesBefore, "dropping the collection should reclaim vector pages"); + } + + [Fact] + public void DropCollection_WithVectorIndex_ReclaimsPages_SimpleVectors() + { + using var file = new TempFile(); + + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) + { + var collection = db.GetCollection("vectors"); + var options = new VectorIndexOptions(8, VectorDistanceMetric.Cosine); + + collection.Insert(new List + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0.5f, -0.25f, 0.75f, 1.5f, -0.5f, 0.25f, -1f } }, + new VectorDocument { Id = 2, Embedding = new[] { -0.5f, 0.25f, 0.75f, -1.5f, 1f, 0.5f, -0.25f, 0.125f } }, + new VectorDocument { Id = 3, Embedding = new[] { 0.5f, -0.75f, 1.25f, 0.875f, -0.375f, 0.625f, -1.125f, 0.25f } } + }); + + collection.EnsureIndex(VectorIndexName, x => x.Embedding, options); + + db.Checkpoint(); + + Action drop = () => db.DropCollection("vectors"); + + drop.Should().NotThrow( + "dropping a collection with vector indexes should release vector index pages instead of treating them like skip-list indexes"); + + db.Checkpoint(); + } + + using (var reopened = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) + { + reopened.GetCollectionNames().Should().NotContain("vectors"); + } + } + + [Fact] + public void DropCollection_WithVectorIndex_ReclaimsTrackedPages() + { + using var file = new TempFile(); + + HashSet vectorPages; + HashSet vectorDataPages; + + var dimensions = (DataService.MAX_DATA_BYTES_PER_PAGE / sizeof(float)) + 64; + dimensions.Should().BeLessThan(ushort.MaxValue); + + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) + { + var collection = db.GetCollection("docs"); + var documents = Enumerable.Range(1, 6) + .Select(i => new VectorDocument + { + Id = i, + Embedding = CreateLargeVector(i, dimensions) + }) + .ToList(); + + collection.Insert(documents); + + var indexOptions = new VectorIndexOptions((ushort)dimensions, VectorDistanceMetric.Euclidean); + collection.EnsureIndex(VectorIndexName, BsonExpression.Create("$.Embedding"), indexOptions); + + (vectorPages, vectorDataPages) = CollectVectorPageUsage(db, "docs"); + + vectorPages.Should().NotBeEmpty(); + vectorDataPages.Should().NotBeEmpty(); + + db.Checkpoint(); + } + + Action drop = () => + { + using var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename); + db.DropCollection("docs"); + db.Checkpoint(); + }; + + drop.Should().NotThrow(); + + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) + { + var vectorPageTypes = GetPageTypes(db, vectorPages); + foreach (var kvp in vectorPageTypes) + { + kvp.Value.Should().Be(PageType.Empty, $"vector index page {kvp.Key} should be reclaimed after dropping the collection"); + } + + var dataPageTypes = GetPageTypes(db, vectorDataPages); + foreach (var kvp in dataPageTypes) + { + kvp.Value.Should().Be(PageType.Empty, $"vector data page {kvp.Key} should be reclaimed after dropping the collection"); + } + + db.GetCollectionNames().Should().NotContain("docs"); + } + } + + private static Dictionary CountPagesByType(string filename) + { + var counts = new Dictionary(); + var buffer = new byte[Constants.PAGE_SIZE]; + + using var stream = File.OpenRead(filename); + + while (stream.Read(buffer, 0, buffer.Length) == buffer.Length) + { + var pageType = (PageType)buffer[BasePage.P_PAGE_TYPE]; + counts.TryGetValue(pageType, out var current); + counts[pageType] = current + 1; + } + + return counts; + } + + private static T ExecuteInTransaction(LiteDatabase db, Func action) + { + var engine = (LiteEngine)EngineField.GetValue(db); + var method = AutoTransactionMethod.MakeGenericMethod(typeof(T)); + return (T)method.Invoke(engine, new object[] { action }); + } + + private static T InspectVectorIndex(LiteDatabase db, string collection, Func selector) + { + return ExecuteInTransaction(db, transaction => + { + var snapshot = transaction.CreateSnapshot(LockMode.Read, collection, false); + var metadata = snapshot.CollectionPage.GetVectorIndexMetadata(VectorIndexName); + + if (metadata == null) + { + return default; + } + + return selector(snapshot, metadata); + }); + } + + private static (HashSet VectorPages, HashSet DataPages) CollectVectorPageUsage(LiteDatabase db, string collection) + { + var (vectorPages, dataPages) = InspectVectorIndex(db, collection, (snapshot, metadata) => + { + var trackedVectorPages = new HashSet(); + var trackedDataPages = new HashSet(); + + if (metadata.Root.IsEmpty) + { + return (trackedVectorPages, trackedDataPages); + } + + var queue = new Queue(); + var visited = new HashSet(); + queue.Enqueue(metadata.Root); + + while (queue.Count > 0) + { + var address = queue.Dequeue(); + if (!visited.Add(address)) + { + continue; + } + + var page = snapshot.GetPage(address.PageID); + var node = page.GetNode(address.Index); + + trackedVectorPages.Add(page.PageID); + + for (var level = 0; level < node.LevelCount; level++) + { + foreach (var neighbor in node.GetNeighbors(level)) + { + if (!neighbor.IsEmpty) + { + queue.Enqueue(neighbor); + } + } + } + + if (!node.HasInlineVector) + { + var block = node.ExternalVector; + while (!block.IsEmpty) + { + trackedDataPages.Add(block.PageID); + + var dataPage = snapshot.GetPage(block.PageID); + var dataBlock = dataPage.GetBlock(block.Index); + block = dataBlock.NextBlock; + } + } + } + + return (trackedVectorPages, trackedDataPages); + }); + + if (vectorPages == null || dataPages == null) + { + return (new HashSet(), new HashSet()); + } + + return (vectorPages, dataPages); + } + + private static Dictionary GetPageTypes(LiteDatabase db, IEnumerable pageIds) + { + return ExecuteInTransaction(db, transaction => + { + var snapshot = transaction.CreateSnapshot(LockMode.Read, "$", false); + var map = new Dictionary(); + + foreach (var pageID in pageIds.Distinct()) + { + var page = snapshot.GetPage(pageID); + map[pageID] = page.PageType; + } + + return map; + }); + } + + private static float[] CreateLargeVector(int seed, int dimensions) + { + var vector = new float[dimensions]; + + for (var i = 0; i < dimensions; i++) + { + vector[i] = (float)Math.Sin((seed * 0.37) + (i * 0.11)); + } + + return vector; + } } -} \ No newline at end of file +} diff --git a/LiteDB.Tests/Engine/Index_Tests.cs b/LiteDB.Tests/Engine/Index_Tests.cs index bdd468c17..16c522c1c 100644 --- a/LiteDB.Tests/Engine/Index_Tests.cs +++ b/LiteDB.Tests/Engine/Index_Tests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Engine @@ -10,7 +11,7 @@ public class Index_Tests [Fact] public void Index_With_No_Name() { - using (var db = new LiteDatabase("filename=:memory:")) + using (var db = DatabaseFactory.Create(connectionString: "filename=:memory:")) { var users = db.GetCollection("users"); var indexes = db.GetCollection("$indexes"); @@ -31,7 +32,7 @@ public void Index_With_No_Name() [Fact] public void Index_Order() { - using (var db = new LiteDatabase("filename=:memory:")) + using (var db = DatabaseFactory.Create(connectionString: "filename=:memory:")) { var col = db.GetCollection("col"); var indexes = db.GetCollection("$indexes"); @@ -68,7 +69,7 @@ public void Index_Order() [Fact] public void Index_With_Like() { - using (var db = new LiteDatabase("filename=:memory:")) + using (var db = DatabaseFactory.Create(connectionString: "filename=:memory:")) { var col = db.GetCollection("names", BsonAutoId.Int32); @@ -118,7 +119,7 @@ public void Index_With_Like() [Fact] public void EnsureIndex_Invalid_Arguments() { - using var db = new LiteDatabase("filename=:memory:"); + using var db = DatabaseFactory.Create(connectionString: "filename=:memory:"); var test = db.GetCollection("test"); // null name @@ -143,7 +144,7 @@ public void EnsureIndex_Invalid_Arguments() [Fact] public void MultiKey_Index_Test() { - using var db = new LiteDatabase("filename=:memory:"); + using var db = DatabaseFactory.Create(connectionString: "filename=:memory:"); var col = db.GetCollection("customers", BsonAutoId.Int32); col.EnsureIndex("$.Phones[*].Type"); diff --git a/LiteDB.Tests/Engine/Rebuild_Crash_Tests.cs b/LiteDB.Tests/Engine/Rebuild_Crash_Tests.cs index 7bce95d7a..c5f7ea8af 100644 --- a/LiteDB.Tests/Engine/Rebuild_Crash_Tests.cs +++ b/LiteDB.Tests/Engine/Rebuild_Crash_Tests.cs @@ -1,92 +1,122 @@ -using FluentAssertions; +using FluentAssertions; using LiteDB.Engine; using System; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Xunit; +using Xunit.Abstractions; #if DEBUG namespace LiteDB.Tests.Engine { public class Rebuild_Crash_Tests { + private readonly ITestOutputHelper _output; + + public Rebuild_Crash_Tests(ITestOutputHelper output) + { + _output = output; + } [Fact] public void Rebuild_Crash_IO_Write_Error() { - var N = 1_000; - - using (var file = new TempFile()) + _output.WriteLine("Running Rebuild_Crash_IO_Write_Error"); + try { - var settings = new EngineSettings - { - AutoRebuild = true, - Filename = file.Filename, - Password = "46jLz5QWd5fI3m4LiL2r" - }; + var N = 1_000; - var data = Enumerable.Range(1, N).Select(i => new BsonDocument - { - ["_id"] = i, - ["name"] = Faker.Fullname(), - ["age"] = Faker.Age(), - ["created"] = Faker.Birthday(), - ["lorem"] = Faker.Lorem(5, 25) - }).ToArray(); - - try + using (var file = new TempFile()) { - using (var db = new LiteEngine(settings)) + var settings = new EngineSettings { - db.SimulateDiskWriteFail = (page) => + AutoRebuild = true, + Filename = file.Filename, + Password = "46jLz5QWd5fI3m4LiL2r" + }; + + var data = Enumerable.Range(1, N).Select(i => new BsonDocument + { + ["_id"] = i, + ["name"] = Faker.Fullname(), + ["age"] = Faker.Age(), + ["created"] = Faker.Birthday(), + ["lorem"] = Faker.Lorem(5, 25) + }).ToArray(); + + var faultInjected = 0; + + try + { + using (var db = new LiteEngine(settings)) { - var p = new BasePage(page); + var writeHits = 0; - if (p.PageID == 28) + db.SimulateDiskWriteFail = (page) => { - p.ColID.Should().Be(1); - p.PageType.Should().Be(PageType.Data); + var p = new BasePage(page); - page.Write((uint)123123123, 8192 - 4); - } - }; + if (p.PageType == PageType.Data && p.ColID == 1) + { + var hit = Interlocked.Increment(ref writeHits); - db.Pragma("USER_VERSION", 123); + if (hit == 10) + { + p.PageType.Should().Be(PageType.Data); + p.ColID.Should().Be(1); - db.EnsureIndex("col1", "idx_age", "$.age", false); + page.Write((uint)123123123, 8192 - 4); - db.Insert("col1", data, BsonAutoId.Int32); - db.Insert("col2", data, BsonAutoId.Int32); + Interlocked.Exchange(ref faultInjected, 1); + } + } + }; - db.Checkpoint(); + db.Pragma("USER_VERSION", 123); - // will fail - var col1 = db.Query("col1", Query.All()).ToList().Count; + db.EnsureIndex("col1", "idx_age", "$.age", false); + + db.Insert("col1", data, BsonAutoId.Int32); + db.Insert("col2", data, BsonAutoId.Int32); + + db.Checkpoint(); - // never run here - Assert.Fail("should get error in query"); + // will fail + var col1 = db.Query("col1", Query.All()).ToList().Count; + + // never run here + Assert.Fail("should get error in query"); + } } - } - catch (Exception ex) - { - Assert.True(ex is LiteException lex && lex.ErrorCode == 999); - } + catch (Exception ex) + { + faultInjected.Should().Be(1, "the simulated disk write fault should have triggered"); - //Console.WriteLine("Recovering database..."); + Assert.True(ex is LiteException lex && lex.ErrorCode == 999); + } - using (var db = new LiteEngine(settings)) - { - var col1 = db.Query("col1", Query.All()).ToList().Count; - var col2 = db.Query("col2", Query.All()).ToList().Count; - var errors = db.Query("_rebuild_errors", Query.All()).ToList().Count; + //Console.WriteLine("Recovering database..."); - col1.Should().Be(N - 1); - col2.Should().Be(N); - errors.Should().Be(1); + using (var db = new LiteEngine(settings)) + { + var col1 = db.Query("col1", Query.All()).ToList().Count; + var col2 = db.Query("col2", Query.All()).ToList().Count; + var errors = db.Query("_rebuild_errors", Query.All()).ToList().Count; + + col1.Should().Be(N - 1); + col2.Should().Be(N); + errors.Should().Be(1); + } } } + finally + { + _output.WriteLine("Finished running Rebuild_Crash_IO_Write_Error"); + } } } } diff --git a/LiteDB.Tests/Engine/Rebuild_Tests.cs b/LiteDB.Tests/Engine/Rebuild_Tests.cs index 8de711bf2..6a979bcb4 100644 --- a/LiteDB.Tests/Engine/Rebuild_Tests.cs +++ b/LiteDB.Tests/Engine/Rebuild_Tests.cs @@ -1,6 +1,8 @@ using FluentAssertions; using LiteDB.Engine; +using LiteDB.Tests.Utils; using System; +using System.Collections.Generic; using System.IO; using System.Linq; @@ -14,11 +16,11 @@ public class Rebuild_Tests public void Rebuild_After_DropCollection() { using (var file = new TempFile()) - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { var col = db.GetCollection("zip"); - col.Insert(DataGen.Zip()); + col.Insert(CreateSyntheticZipData(200, SurvivorId)); db.DropCollection("zip"); @@ -46,7 +48,7 @@ void DoTest(ILiteDatabase db, ILiteCollection col) using (var file = new TempFile()) { - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { var col = db.GetCollection(); @@ -54,31 +56,33 @@ void DoTest(ILiteDatabase db, ILiteCollection col) col.EnsureIndex("city", false); - var inserted = col.Insert(DataGen.Zip()); // 29.353 docs - var deleted = col.DeleteMany(x => x.Id != "01001"); // delete 29.352 docs + const int documentCount = 200; - Assert.Equal(29353, inserted); - Assert.Equal(29352, deleted); + var inserted = col.Insert(CreateSyntheticZipData(documentCount, SurvivorId)); + var deleted = col.DeleteMany(x => x.Id != SurvivorId); + + Assert.Equal(documentCount, inserted); + Assert.Equal(documentCount - 1, deleted); Assert.Equal(1, col.Count()); // must checkpoint db.Checkpoint(); - // file still large than 5mb (even with only 1 document) - Assert.True(file.Size > 5 * 1024 * 1024); + // file still larger than 1 MB (even with only 1 document) + Assert.True(file.Size > 1 * 1024 * 1024); // reduce datafile var reduced = db.Rebuild(); - // now file are small than 50kb - Assert.True(file.Size < 50 * 1024); + // now file should be small again + Assert.True(file.Size < 256 * 1024); DoTest(db, col); } // re-open and rebuild again - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { var col = db.GetCollection(); @@ -91,11 +95,48 @@ void DoTest(ILiteDatabase db, ILiteCollection col) } } + private const string SurvivorId = "01001"; + + private static IEnumerable CreateSyntheticZipData(int totalCount, string survivingId) + { + if (totalCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(totalCount)); + } + + const int payloadLength = 32 * 1024; // 32 KB payload to force file growth + + for (var i = 0; i < totalCount; i++) + { + var id = (20000 + i).ToString("00000"); + + if (!string.IsNullOrEmpty(survivingId) && i == 0) + { + id = survivingId; + } + + var payload = new byte[payloadLength]; + for (var j = 0; j < payload.Length; j++) + { + payload[j] = (byte)(i % 256); + } + + yield return new Zip + { + Id = id, + City = $"City {i:D4}", + Loc = new[] { (double)i, (double)i + 0.5 }, + State = "ST", + Payload = payload + }; + } + } + [Fact (Skip = "Not supported yet")] public void Rebuild_Change_Culture_Error() { using (var file = new TempFile()) - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { // remove string comparer ignore case db.Rebuild(new RebuildOptions { Collation = new Collation("en-US/None") }); diff --git a/LiteDB.Tests/Engine/Recursion_Tests.cs b/LiteDB.Tests/Engine/Recursion_Tests.cs index 4a9fdb3c7..f4fa85d78 100644 --- a/LiteDB.Tests/Engine/Recursion_Tests.cs +++ b/LiteDB.Tests/Engine/Recursion_Tests.cs @@ -3,6 +3,7 @@ namespace LiteDB.Tests.Engine; +[Collection("SharedDemoDatabase")] public class Recursion_Tests { [Fact] diff --git a/LiteDB.Tests/Engine/Transactions_Tests.cs b/LiteDB.Tests/Engine/Transactions_Tests.cs index 9fd567a34..3fc266151 100644 --- a/LiteDB.Tests/Engine/Transactions_Tests.cs +++ b/LiteDB.Tests/Engine/Transactions_Tests.cs @@ -1,10 +1,14 @@ -using System.IO; +using System.Diagnostics; +using System.IO; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using FluentAssertions; using LiteDB.Engine; +using LiteDB.Tests.Utils; using Xunit; +using Xunit.Sdk; namespace LiteDB.Tests.Engine { @@ -12,16 +16,19 @@ namespace LiteDB.Tests.Engine public class Transactions_Tests { - [Fact] + const int MIN_CPU_COUNT = 2; + + [CpuBoundFact(MIN_CPU_COUNT)] public async Task Transaction_Write_Lock_Timeout() { var data1 = DataGen.Person(1, 100).ToArray(); var data2 = DataGen.Person(101, 200).ToArray(); - using (var db = new LiteDatabase("filename=:memory:")) + using (var db = DatabaseFactory.Create(connectionString: "filename=:memory:")) { - // small timeout + // configure the minimal pragma timeout and then override the engine to a few milliseconds db.Pragma(Pragmas.TIMEOUT, 1); + SetEngineTimeout(db, TimeSpan.FromMilliseconds(20)); var person = db.GetCollection(); @@ -31,8 +38,8 @@ public async Task Transaction_Write_Lock_Timeout() var taskASemaphore = new SemaphoreSlim(0, 1); var taskBSemaphore = new SemaphoreSlim(0, 1); - // task A will open transaction and will insert +100 documents - // but will commit only 2s later + // task A will open transaction and will insert +100 documents + // but will commit only after task B observes the timeout var ta = Task.Run(() => { db.BeginTrans(); @@ -49,7 +56,7 @@ public async Task Transaction_Write_Lock_Timeout() db.Commit(); }); - // task B will try delete all documents but will be locked during 1 second + // task B will try delete all documents but will be locked until the short timeout is hit var tb = Task.Run(() => { taskBSemaphore.Wait(); @@ -68,8 +75,8 @@ public async Task Transaction_Write_Lock_Timeout() } } - - [Fact] + + [CpuBoundFact(MIN_CPU_COUNT)] public async Task Transaction_Avoid_Dirty_Read() { var data1 = DataGen.Person(1, 100).ToArray(); @@ -128,8 +135,9 @@ public async Task Transaction_Avoid_Dirty_Read() await Task.WhenAll(ta, tb); } } + - [Fact] + [CpuBoundFact(MIN_CPU_COUNT)] public async Task Transaction_Read_Version() { var data1 = DataGen.Person(1, 100).ToArray(); @@ -186,7 +194,7 @@ public async Task Transaction_Read_Version() } } - [Fact] + [CpuBoundFact(MIN_CPU_COUNT)] public void Test_Transaction_States() { var data0 = DataGen.Person(1, 10).ToArray(); @@ -225,6 +233,110 @@ public void Test_Transaction_States() } } +#if DEBUG || TESTING + [Fact] + public void Transaction_Rollback_Should_Skip_ReadOnly_Buffers_From_Safepoint() + { + using var db = DatabaseFactory.Create(); + var collection = db.GetCollection("docs"); + + db.BeginTrans().Should().BeTrue(); + + for (var i = 0; i < 10; i++) + { + collection.Insert(new BsonDocument + { + ["_id"] = i, + ["value"] = $"value-{i}" + }); + } + + var engine = GetLiteEngine(db); + var monitor = GetTransactionMonitor(engine); + var transaction = monitor.GetThreadTransaction(); + + transaction.Should().NotBeNull(); + + var transactionService = transaction!; + transactionService.Pages.TransactionSize.Should().BeGreaterThan(0); + + transactionService.MaxTransactionSize = Math.Max(1, transactionService.Pages.TransactionSize); + SetMonitorFreePages(monitor, 0); + + transactionService.Safepoint(); + transactionService.Pages.TransactionSize.Should().Be(0); + + var snapshot = transactionService.Snapshots.Single(); + snapshot.CollectionPage.Should().NotBeNull(); + + var collectionPage = snapshot.CollectionPage!; + collectionPage.IsDirty = true; + + var buffer = collectionPage.Buffer; + + try + { + buffer.ShareCounter = 1; + + var shareCounters = snapshot + .GetWritablePages(true, true) + .Select(page => page.Buffer.ShareCounter) + .ToList(); + + shareCounters.Should().NotBeEmpty(); + shareCounters.Should().Contain(counter => counter != Constants.BUFFER_WRITABLE); + + db.Rollback().Should().BeTrue(); + } + finally + { + buffer.ShareCounter = 0; + } + + collection.Count().Should().Be(0); + } + + [Fact] + public void Transaction_Rollback_Should_Discard_Writable_Dirty_Pages() + { + using var db = DatabaseFactory.Create(); + var collection = db.GetCollection("docs"); + + db.BeginTrans().Should().BeTrue(); + + for (var i = 0; i < 3; i++) + { + collection.Insert(new BsonDocument + { + ["_id"] = i, + ["value"] = $"value-{i}" + }); + } + + var engine = GetLiteEngine(db); + var monitor = GetTransactionMonitor(engine); + var transaction = monitor.GetThreadTransaction(); + + transaction.Should().NotBeNull(); + + var transactionService = transaction!; + var snapshot = transactionService.Snapshots.Single(); + + var shareCounters = snapshot + .GetWritablePages(true, true) + .Select(page => page.Buffer.ShareCounter) + .ToList(); + + shareCounters.Should().NotBeEmpty(); + shareCounters.Should().OnlyContain(counter => counter == Constants.BUFFER_WRITABLE); + + db.Rollback().Should().BeTrue(); + + collection.Count().Should().Be(0); + } + +#endif + private class BlockingStream : MemoryStream { public readonly AutoResetEvent Blocked = new AutoResetEvent(false); @@ -243,11 +355,12 @@ public override void Write(byte[] buffer, int offset, int count) } } - [Fact] + [CpuBoundFact(MIN_CPU_COUNT)] public void Test_Transaction_ReleaseWhenFailToStart() { var blockingStream = new BlockingStream(); - var db = new LiteDatabase(blockingStream) { Timeout = TimeSpan.FromSeconds(1) }; + var db = new LiteDatabase(blockingStream); + SetEngineTimeout(db, TimeSpan.FromMilliseconds(50)); Thread lockerThread = null; try { @@ -257,9 +370,12 @@ public void Test_Transaction_ReleaseWhenFailToStart() blockingStream.ShouldBlock = true; db.Checkpoint(); db.Dispose(); - }); + }) + { + IsBackground = true + }; lockerThread.Start(); - blockingStream.Blocked.WaitOne(1000).Should().BeTrue(); + blockingStream.Blocked.WaitOne(200).Should().BeTrue(); Assert.Throws(() => db.GetCollection().Insert(new Person())).Message.Should().Contain("timeout"); Assert.Throws(() => db.GetCollection().Insert(new Person())).Message.Should().Contain("timeout"); } @@ -269,5 +385,60 @@ public void Test_Transaction_ReleaseWhenFailToStart() lockerThread?.Join(); } } + + private static LiteEngine GetLiteEngine(LiteDatabase database) + { + var engineField = typeof(LiteDatabase).GetField("_engine", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to locate LiteDatabase engine field."); + + if (engineField.GetValue(database) is not LiteEngine engine) + { + throw new InvalidOperationException("LiteDatabase engine is not initialized."); + } + + return engine; + } + + private static TransactionMonitor GetTransactionMonitor(LiteEngine engine) + { + var getter = typeof(LiteEngine).GetMethod("GetMonitor", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + + if (getter != null && getter.ReturnType == typeof(TransactionMonitor)) + { + return (TransactionMonitor)(getter.Invoke(engine, Array.Empty()) ?? throw new InvalidOperationException("LiteEngine monitor accessor returned null.")); + } + + var monitorField = typeof(LiteEngine).GetField("_monitor", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to locate LiteEngine monitor field."); + + if (monitorField.GetValue(engine) is not TransactionMonitor monitor) + { + throw new InvalidOperationException("LiteEngine monitor instance is not available."); + } + + return monitor; + } + + private static void SetMonitorFreePages(TransactionMonitor monitor, int value) + { + var freePagesField = typeof(TransactionMonitor).GetField("_freePages", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to locate TransactionMonitor free pages field."); + + freePagesField.SetValue(monitor, value); + } + + private static void SetEngineTimeout(LiteDatabase database, TimeSpan timeout) + { + var engine = GetLiteEngine(database); + + var headerField = typeof(LiteEngine).GetField("_header", BindingFlags.Instance | BindingFlags.NonPublic); + var header = headerField?.GetValue(engine) ?? throw new InvalidOperationException("LiteEngine header not available."); + var pragmasProp = header.GetType().GetProperty("Pragmas", BindingFlags.Instance | BindingFlags.Public) ?? throw new InvalidOperationException("Engine pragmas not accessible."); + var pragmas = pragmasProp.GetValue(header) ?? throw new InvalidOperationException("Engine pragmas not available."); + var timeoutProp = pragmas.GetType().GetProperty("Timeout", BindingFlags.Instance | BindingFlags.Public) ?? throw new InvalidOperationException("Timeout property not found."); + var setter = timeoutProp.GetSetMethod(true) ?? throw new InvalidOperationException("Timeout setter not accessible."); + + setter.Invoke(pragmas, new object[] { timeout }); + } } -} \ No newline at end of file +} diff --git a/LiteDB.Tests/Engine/UserVersion_Tests.cs b/LiteDB.Tests/Engine/UserVersion_Tests.cs index 5ac937052..410dec974 100644 --- a/LiteDB.Tests/Engine/UserVersion_Tests.cs +++ b/LiteDB.Tests/Engine/UserVersion_Tests.cs @@ -1,4 +1,5 @@ using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Engine @@ -10,14 +11,14 @@ public void UserVersion_Get_Set() { using (var file = new TempFile()) { - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { db.UserVersion.Should().Be(0); db.UserVersion = 5; db.Checkpoint(); } - using (var db = new LiteDatabase(file.Filename)) + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, file.Filename)) { db.UserVersion.Should().Be(5); } diff --git a/LiteDB.Tests/Internals/BufferWriter_Tests.cs b/LiteDB.Tests/Internals/BufferWriter_Tests.cs index 4bdd0146a..da5d3366f 100644 --- a/LiteDB.Tests/Internals/BufferWriter_Tests.cs +++ b/LiteDB.Tests/Internals/BufferWriter_Tests.cs @@ -87,6 +87,60 @@ public void Buffer_Write_String() ((char) source.ReadByte(10)).Should().Be('\0'); } + [Fact] + public void Buffer_Write_CString_MultiSegment_Writes_Terminator() + { + var buffer = new byte[32]; + + var slice0 = new BufferSlice(buffer, 0, 3); + var slice1 = new BufferSlice(buffer, 3, 4); + var slice2 = new BufferSlice(buffer, 7, 8); + var slice3 = new BufferSlice(buffer, 15, 10); + + using (var writer = new BufferWriter(new[] { slice0, slice1, slice2, slice3 })) + { + writer.WriteCString("abcdefghi"); + writer.Position.Should().Be(10); + } + + buffer[9].Should().Be((byte)0x00); + + using (var reader = new BufferReader(new[] { slice0, slice1, slice2, slice3 })) + { + reader.ReadCString().Should().Be("abcdefghi"); + reader.Position.Should().Be(10); + } + } + + [Fact] + public void Buffer_Write_String_Specs_MultiSegment_Writes_Terminator() + { + var buffer = new byte[48]; + + var slice0 = new BufferSlice(buffer, 0, 5); + var slice1 = new BufferSlice(buffer, 5, 4); + var slice2 = new BufferSlice(buffer, 9, 7); + var slice3 = new BufferSlice(buffer, 16, 12); + var slice4 = new BufferSlice(buffer, 28, 20); + + var value = "segment-boundary"; + + using (var writer = new BufferWriter(new[] { slice0, slice1, slice2, slice3, slice4 })) + { + writer.WriteString(value, true); + writer.Position.Should().Be(value.Length + 5); + } + + using (var reader = new BufferReader(new[] { slice0, slice1, slice2, slice3, slice4 })) + { + reader.ReadInt32().Should().Be(value.Length + 1); + reader.ReadCString().Should().Be(value); + reader.Position.Should().Be(value.Length + 5); + } + + buffer[4 + value.Length].Should().Be((byte)0x00); + } + [Fact] public void Buffer_Write_Numbers() { diff --git a/LiteDB.Tests/Internals/Document_Tests.cs b/LiteDB.Tests/Internals/Document_Tests.cs index d350ea13a..8cf129c5f 100644 --- a/LiteDB.Tests/Internals/Document_Tests.cs +++ b/LiteDB.Tests/Internals/Document_Tests.cs @@ -30,7 +30,7 @@ public void Document_Copies_Properties_To_KeyValue_Array() // ACT // copy all properties to destination array - var result = new KeyValuePair[document.Count()]; + var result = new KeyValuePair[document.Count]; document.CopyTo(result, 0); } diff --git a/LiteDB.Tests/Internals/ExtendedLength_Tests.cs b/LiteDB.Tests/Internals/ExtendedLength_Tests.cs index 6e0b0fdb7..054ad19bb 100644 --- a/LiteDB.Tests/Internals/ExtendedLength_Tests.cs +++ b/LiteDB.Tests/Internals/ExtendedLength_Tests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Text; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Internals @@ -22,7 +23,7 @@ public void ExtendedLengthHelper_Tests() [Fact] public void IndexExtendedLength_Tests() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var col = db.GetCollection("customers", BsonAutoId.Int32); col.EnsureIndex("$.Name"); col.Insert(new BsonDocument { ["Name"] = new string('A', 1010) }); diff --git a/LiteDB.Tests/Internals/Sort_Tests.cs b/LiteDB.Tests/Internals/Sort_Tests.cs index 8e55aa5df..f6837fd2f 100644 --- a/LiteDB.Tests/Internals/Sort_Tests.cs +++ b/LiteDB.Tests/Internals/Sort_Tests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Linq; using System.Collections.Generic; @@ -24,7 +24,7 @@ public void Sort_String_Asc() pragmas.Set(Pragmas.COLLATION, Collation.Binary.ToString(), false); using (var tempDisk = new SortDisk(_factory, 10 * 8192, pragmas)) - using (var s = new SortService(tempDisk, Query.Ascending, pragmas)) + using (var s = new SortService(tempDisk, new[] { Query.Ascending }, pragmas)) { s.Insert(source); @@ -43,25 +43,24 @@ public void Sort_String_Asc() [Fact] public void Sort_Int_Desc() { - var rnd = new Random(); - var source = Enumerable.Range(0, 20000) - .Select(x => new KeyValuePair(rnd.Next(1, 30000), PageAddress.Empty)) + var source = Enumerable.Range(0, 900) + .Select(x => (x * 37) % 1000) + .Select(x => new KeyValuePair(x, PageAddress.Empty)) .ToArray(); var pragmas = new EnginePragmas(null); pragmas.Set(Pragmas.COLLATION, Collation.Binary.ToString(), false); - using (var tempDisk = new SortDisk(_factory, 10 * 8192, pragmas)) - using (var s = new SortService(tempDisk, Query.Descending, pragmas)) + using (var tempDisk = new SortDisk(_factory, 8192, pragmas)) + using (var s = new SortService(tempDisk, [Query.Descending], pragmas)) { s.Insert(source); - s.Count.Should().Be(20000); - s.Containers.Count.Should().Be(3); + s.Count.Should().Be(900); + s.Containers.Count.Should().Be(2); - s.Containers.ElementAt(0).Count.Should().Be(8192); - s.Containers.ElementAt(1).Count.Should().Be(8192); - s.Containers.ElementAt(2).Count.Should().Be(3616); + s.Containers.ElementAt(0).Count.Should().Be(819); + s.Containers.ElementAt(1).Count.Should().Be(81); var output = s.Sort().ToArray(); diff --git a/LiteDB.Tests/Issues/Issue1651_Tests.cs b/LiteDB.Tests/Issues/Issue1651_Tests.cs index 0b71dcce2..eccc545cc 100644 --- a/LiteDB.Tests/Issues/Issue1651_Tests.cs +++ b/LiteDB.Tests/Issues/Issue1651_Tests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Xunit; using System.Linq; +using LiteDB.Tests.Utils; namespace LiteDB.Tests.Issues { @@ -28,7 +29,7 @@ public void Find_ByRelationId_Success() { BsonMapper.Global.Entity().DbRef(order => order.Customer); - using var _database = new LiteDatabase(":memory:"); + using var _database = DatabaseFactory.Create(); var _orderCollection = _database.GetCollection("Order"); var _customerCollection = _database.GetCollection("Customer"); diff --git a/LiteDB.Tests/Issues/Issue1695_Tests.cs b/LiteDB.Tests/Issues/Issue1695_Tests.cs index c4ee06122..de3504f04 100644 --- a/LiteDB.Tests/Issues/Issue1695_Tests.cs +++ b/LiteDB.Tests/Issues/Issue1695_Tests.cs @@ -4,6 +4,7 @@ using System.Linq; using FluentAssertions; using LiteDB.Engine; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Issues @@ -19,7 +20,7 @@ public class StateModel [Fact] public void ICollection_Parameter_Test() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var col = db.GetCollection("col"); ICollection ids = new List(); diff --git a/LiteDB.Tests/Issues/Issue1701_Tests.cs b/LiteDB.Tests/Issues/Issue1701_Tests.cs index edd2c609a..16db55ced 100644 --- a/LiteDB.Tests/Issues/Issue1701_Tests.cs +++ b/LiteDB.Tests/Issues/Issue1701_Tests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Xunit; using System.Linq; +using LiteDB.Tests.Utils; namespace LiteDB.Tests.Issues { @@ -10,7 +11,7 @@ public class Issue1701_Tests [Fact] public void Deleted_Index_Slot_Test() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var col = db.GetCollection("col", BsonAutoId.Int32); var id = col.Insert(new BsonDocument { ["attr1"] = "attr", ["attr2"] = "attr", ["attr3"] = "attr" }); diff --git a/LiteDB.Tests/Issues/Issue1838_Tests.cs b/LiteDB.Tests/Issues/Issue1838_Tests.cs index 5d3c47507..7934bd07a 100644 --- a/LiteDB.Tests/Issues/Issue1838_Tests.cs +++ b/LiteDB.Tests/Issues/Issue1838_Tests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Xunit; using System.Linq; +using LiteDB.Tests.Utils; namespace LiteDB.Tests.Issues { @@ -10,7 +11,7 @@ public class Issue1838_Tests [Fact] public void Find_ByDatetime_Offset() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var collection = db.GetCollection(nameof(TestType)); // sample data diff --git a/LiteDB.Tests/Issues/Issue1860_Tests.cs b/LiteDB.Tests/Issues/Issue1860_Tests.cs index f34f5cc67..a869960ce 100644 --- a/LiteDB.Tests/Issues/Issue1860_Tests.cs +++ b/LiteDB.Tests/Issues/Issue1860_Tests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Xunit; using System.Linq; +using LiteDB.Tests.Utils; namespace LiteDB.Tests.Issues { @@ -10,7 +11,7 @@ public class Issue1860_Tests [Fact] public void Constructor_has_enum_bsonctor() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); // Get a collection (or create, if doesn't exist) var col1 = db.GetCollection("c1"); @@ -44,7 +45,7 @@ public void Constructor_has_enum_bsonctor() [Fact] public void Constructor_has_enum() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); // Get a collection (or create, if doesn't exist) var col1 = db.GetCollection("c1"); @@ -78,7 +79,7 @@ public void Constructor_has_enum() [Fact] public void Constructor_has_enum_asint() { - using var db = new LiteDatabase(":memory:", new BsonMapper { EnumAsInteger = true }); + using var db = DatabaseFactory.Create(mapper: new BsonMapper { EnumAsInteger = true }); // Get a collection (or create, if doesn't exist) var col1 = db.GetCollection("c1"); diff --git a/LiteDB.Tests/Issues/Issue1865_Tests.cs b/LiteDB.Tests/Issues/Issue1865_Tests.cs index 0d9ef62e3..90e53662e 100644 --- a/LiteDB.Tests/Issues/Issue1865_Tests.cs +++ b/LiteDB.Tests/Issues/Issue1865_Tests.cs @@ -3,6 +3,7 @@ using Xunit; using System.Linq; using System.Security.Cryptography; +using LiteDB.Tests.Utils; namespace LiteDB.Tests.Issues { @@ -37,7 +38,7 @@ public void Incluced_document_types_should_be_reald() //BsonMapper.Global.ResolveCollectionName = (s) => "activity"; - using var _database = new LiteDatabase(":memory:"); + using var _database = DatabaseFactory.Create(); var projectsCol = _database.GetCollection("activity"); var pointsCol = _database.GetCollection("activity"); diff --git a/LiteDB.Tests/Issues/Issue2127_Tests.cs b/LiteDB.Tests/Issues/Issue2127_Tests.cs index 4a2320c5b..60bd8fac2 100644 --- a/LiteDB.Tests/Issues/Issue2127_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2127_Tests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Text; using System.Threading; +using LiteDB.Tests.Utils; namespace LiteDB.Tests.Issues { @@ -53,8 +54,12 @@ public void InsertItemBackToBack_Test() var liteDbContent = File.ReadAllText(copiedLiteDbPath, Encoding.UTF8); liteDbContent += File.ReadAllText(copiedLiteDbLogPath, Encoding.UTF8); - Assert.True(liteDbContent.Contains(item1.SomeProperty, StringComparison.OrdinalIgnoreCase), $"Could not find item 1 property. {item1.SomeProperty}, Iteration: {i}"); - Assert.True(liteDbContent.Contains(item2.SomeProperty, StringComparison.OrdinalIgnoreCase), $"Could not find item 2 property. {item2.SomeProperty}, Iteration: {i}"); + Assert.True( + liteDbContent.IndexOf(item1.SomeProperty, StringComparison.OrdinalIgnoreCase) >= 0, + $"Could not find item 1 property. {item1.SomeProperty}, Iteration: {i}"); + Assert.True( + liteDbContent.IndexOf(item2.SomeProperty, StringComparison.OrdinalIgnoreCase) >= 0, + $"Could not find item 2 property. {item2.SomeProperty}, Iteration: {i}"); } } } @@ -81,7 +86,7 @@ public ExampleItemRepository(string databasePath) Connection = ConnectionType.Direct }; - _liteDb = new LiteDatabase(connectionString); + _liteDb = DatabaseFactory.Create(TestDatabaseType.Disk, connectionString.ToString()); } public void Insert(ExampleItem item) diff --git a/LiteDB.Tests/Issues/Issue2129_Tests.cs b/LiteDB.Tests/Issues/Issue2129_Tests.cs index 1dce48cd1..3779191bf 100644 --- a/LiteDB.Tests/Issues/Issue2129_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2129_Tests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Issues @@ -10,7 +11,7 @@ public class Issue2129_Tests [Fact] public void TestInsertAfterDeleteAll() { - var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var col = db.GetCollection(nameof(SwapChance)); col.EnsureIndex(x => x.Accounts1to2); col.EnsureIndex(x => x.Accounts2to1); diff --git a/LiteDB.Tests/Issues/Issue2265_Tests.cs b/LiteDB.Tests/Issues/Issue2265_Tests.cs index 347753348..57257c12c 100644 --- a/LiteDB.Tests/Issues/Issue2265_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2265_Tests.cs @@ -1,5 +1,6 @@ using System; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Issues; @@ -31,7 +32,7 @@ public Weights() [Fact] public void Test() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { var c = db.GetCollection("weights"); Weights? w = c.FindOne(x => true); diff --git a/LiteDB.Tests/Issues/Issue2298_Tests.cs b/LiteDB.Tests/Issues/Issue2298_Tests.cs index c4d4c5a97..a952b443f 100644 --- a/LiteDB.Tests/Issues/Issue2298_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2298_Tests.cs @@ -2,66 +2,77 @@ using System.Collections.Generic; using System.Linq; using System.Text; +#if NETCOREAPP using System.Text.Json; +#endif using System.Threading.Tasks; +using LiteDB.Tests.Utils; using Xunit; -namespace LiteDB.Tests.Issues; - -public class Issue2298_Tests +namespace LiteDB.Tests.Issues { - public struct Mass + public class Issue2298_Tests { - public enum Units - { Pound, Kilogram } +#if !NETCOREAPP + [Fact(Skip = "System.Text.Json is not supported on this target framework for this scenario.")] + public void We_Dont_Need_Ctor() + { + } +#else + public struct Mass + { + public enum Units + { Pound, Kilogram } - public Mass(double value, Units unit) - { Value = value; Unit = unit; } + public Mass(double value, Units unit) + { Value = value; Unit = unit; } - public double Value { get; init; } - public Units Unit { get; init; } - } + public double Value { get; init; } + public Units Unit { get; init; } + } - public class QuantityRange - { - public QuantityRange(double min, double max, Enum unit) - { Min = min; Max = max; Unit = unit; } + public class QuantityRange + { + public QuantityRange(double min, double max, Enum unit) + { Min = min; Max = max; Unit = unit; } - public double Min { get; init; } - public double Max { get; init; } - public Enum Unit { get; init; } - } + public double Min { get; init; } + public double Max { get; init; } + public Enum Unit { get; init; } + } - public static QuantityRange MassRangeBuilder(BsonDocument document) - { - var doc = JsonDocument.Parse(document.ToString()).RootElement; - var min = doc.GetProperty(nameof(QuantityRange.Min)).GetDouble(); - var max = doc.GetProperty(nameof(QuantityRange.Max)).GetDouble(); - var unit = Enum.Parse(doc.GetProperty(nameof(QuantityRange.Unit)).GetString()); + public static QuantityRange MassRangeBuilder(BsonDocument document) + { + var doc = JsonDocument.Parse(document.ToString()).RootElement; + var min = doc.GetProperty(nameof(QuantityRange.Min)).GetDouble(); + var max = doc.GetProperty(nameof(QuantityRange.Max)).GetDouble(); + var unit = Enum.Parse(doc.GetProperty(nameof(QuantityRange.Unit)).GetString()); - var restored = new QuantityRange(min, max, unit); - return restored; - } + var restored = new QuantityRange(min, max, unit); + return restored; + } - [Fact] - public void We_Dont_Need_Ctor() - { - BsonMapper.Global.RegisterType>( - serialize: (range) => new BsonDocument - { - { nameof(QuantityRange.Min), range.Min }, - { nameof(QuantityRange.Max), range.Max }, - { nameof(QuantityRange.Unit), range.Unit.ToString() } - }, - deserialize: (document) => MassRangeBuilder(document as BsonDocument) - ); + [Fact] + public void We_Dont_Need_Ctor() + { + BsonMapper.Global.RegisterType>( + serialize: (range) => new BsonDocument + { + { nameof(QuantityRange.Min), range.Min }, + { nameof(QuantityRange.Max), range.Max }, + { nameof(QuantityRange.Unit), range.Unit.ToString() } + }, + deserialize: (document) => MassRangeBuilder(document as BsonDocument) + ); - var range = new QuantityRange(100, 500, Mass.Units.Pound); - var filename = "Demo.DB"; - var DB = new LiteDatabase(filename); - var collection = DB.GetCollection>("DEMO"); - collection.Insert(range); - var restored = collection.FindAll().First(); + var range = new QuantityRange(100, 500, Mass.Units.Pound); + using var db = DatabaseFactory.Create(); + var collection = db.GetCollection>("DEMO"); + collection.Insert(range); + var restored = collection.FindAll().First(); + } + +#endif } -} \ No newline at end of file +} diff --git a/LiteDB.Tests/Issues/Issue2458_Tests.cs b/LiteDB.Tests/Issues/Issue2458_Tests.cs index 113132f6c..2cf0b8f4e 100644 --- a/LiteDB.Tests/Issues/Issue2458_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2458_Tests.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Issues; @@ -9,7 +10,7 @@ public class Issue2458_Tests [Fact] public void NegativeSeekFails() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var fs = db.FileStorage; AddTestFile("test", 1, fs); using Stream stream = fs.OpenRead("test"); @@ -21,7 +22,7 @@ public void NegativeSeekFails() [Fact] public void SeekPastFileSucceds() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var fs = db.FileStorage; AddTestFile("test", 1, fs); using Stream stream = fs.OpenRead("test"); @@ -31,7 +32,7 @@ public void SeekPastFileSucceds() [Fact] public void SeekShortChunks() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var fs = db.FileStorage; using(Stream writeStream = fs.OpenWrite("test", "test")) { @@ -49,6 +50,7 @@ public void SeekShortChunks() private void AddTestFile(string id, long length, ILiteStorage fs) { using Stream writeStream = fs.OpenWrite(id, id); - writeStream.Write(new byte[length]); + var buffer = new byte[length]; + writeStream.Write(buffer, 0, buffer.Length); } -} \ No newline at end of file +} diff --git a/LiteDB.Tests/Issues/Issue2471_Test.cs b/LiteDB.Tests/Issues/Issue2471_Test.cs index 1f50e1aff..1c7aa3ed2 100644 --- a/LiteDB.Tests/Issues/Issue2471_Test.cs +++ b/LiteDB.Tests/Issues/Issue2471_Test.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Issues; @@ -17,7 +18,7 @@ public class Issue2471_Test [Fact] public void TestFragmentDB_FindByIDException() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var collection = db.GetCollection("fragtest"); var fragment = new object { }; @@ -36,7 +37,7 @@ public void TestFragmentDB_FindByIDException() [Fact] public void MultipleReadCleansUpTransaction() { - using var database = new LiteDatabase(":memory:"); + using var database = DatabaseFactory.Create(); var collection = database.GetCollection("test"); collection.Insert(new BsonDocument { ["_id"] = 1 }); diff --git a/LiteDB.Tests/Issues/Issue2494_Tests.cs b/LiteDB.Tests/Issues/Issue2494_Tests.cs index e57917497..88a4df7f2 100644 --- a/LiteDB.Tests/Issues/Issue2494_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2494_Tests.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading.Tasks; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Issues; @@ -23,7 +24,7 @@ public static void Test() Upgrade = true, }; - using (var db = new LiteDatabase(connectionString)) // <= throws as of version 5.0.18 + using (var db = DatabaseFactory.Create(TestDatabaseType.Disk, connectionString.ToString())) // <= throws as of version 5.0.18 { var col = db.GetCollection(); col.FindAll(); diff --git a/LiteDB.Tests/Issues/Issue2523_ReadFull_Tests.cs b/LiteDB.Tests/Issues/Issue2523_ReadFull_Tests.cs new file mode 100644 index 000000000..66b229e99 --- /dev/null +++ b/LiteDB.Tests/Issues/Issue2523_ReadFull_Tests.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using FluentAssertions; +using LiteDB.Engine; +using Xunit; + +namespace LiteDB.Tests.Issues; + +public class Issue2523_ReadFull_Tests +{ + [Fact] + public void ReadFull_Must_See_Log_Page_Written_In_Same_Tick() + { + using var logStream = new DelayedPublishLogStream(); // <-- changed + using var dataStream = new MemoryStream(); + + var settings = new EngineSettings + { + DataStream = dataStream, + LogStream = logStream + }; + + var state = new EngineState(null, settings); + var disk = new DiskService(settings, state, new[] { 10 }); + + try + { + // Arrange: create a single, full page + var page = disk.NewPage(); + page.Fill(0xAC); + + // Act: write the page to the WAL/log + disk.WriteLogDisk(new[] { page }); + + // Assert: immediately read the log back fully. + // Pre-fix: throws (ReadFull must read PAGE_SIZE bytes) + // Post-fix: returns 1 page, filled with 0xAC + var logPages = disk.ReadFull(FileOrigin.Log).ToList(); + + logPages.Should().HaveCount(1); + logPages[0].All(0xAC).Should().BeTrue(); + } + finally + { + disk.Dispose(); + } + } + + /// + /// Stream that "accepts" writes (increases Length as the writer would see it), + /// but hides the bytes from readers until Flush/FlushAsync publishes them. + /// This mirrors the visibility gap the fix (stream.Flush()) closes. + /// + private sealed class DelayedPublishLogStream : Stream + { + private readonly MemoryStream _committed = new(); // bytes visible to readers + private readonly List<(long Position, byte[] Data)> _pending = new(); + + private long _writerLength; // total bytes "written" by Write(...) + private long _visibleLength; // committed length visible to Read(...) + private long _position; // logical cursor for both read/write + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => true; + + // IMPORTANT: advertise writer's view of length (includes pending). + public override long Length => _writerLength; + + public override long Position + { + get => _position; + set => _position = value; + } + + public override void Flush() + { + // Publish pending bytes + foreach (var (pos, data) in _pending) + { + _committed.Position = pos; + _committed.Write(data, 0, data.Length); + } + _pending.Clear(); + _committed.Flush(); + + // Make everything visible + _visibleLength = _writerLength; + } + + public override System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) + { + Flush(); + return System.Threading.Tasks.Task.CompletedTask; + } + + public override int Read(byte[] buffer, int offset, int count) + { + // Serve only what has been published (visibleLength) + if (_position >= _visibleLength) return 0; + + var available = (int)Math.Min(count, _visibleLength - _position); + _committed.Position = _position; + var read = _committed.Read(buffer, offset, available); + _position += read; + return read; + } + + public override long Seek(long offset, SeekOrigin origin) + { + _position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + // IMPORTANT: base End on the *advertised* writer length, not committed length + SeekOrigin.End => _writerLength + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)) + }; + + if (_position < 0) throw new IOException("Negative position."); + return _position; + } + + public override void SetLength(long value) + { + if (value < 0) throw new IOException("Negative length."); + // Adjust both writer length and (if shrinking) visible length. + _writerLength = value; + if (_visibleLength > value) _visibleLength = value; + if (_committed.Length < value) _committed.SetLength(value); + if (_position > value) _position = value; + } + + public override void Write(byte[] buffer, int offset, int count) + { + if (buffer is null) throw new ArgumentNullException(nameof(buffer)); + if ((uint)offset > buffer.Length) throw new ArgumentOutOfRangeException(nameof(offset)); + if ((uint)count > buffer.Length - offset) throw new ArgumentOutOfRangeException(nameof(count)); + + // Capture write into pending (not visible yet) + var copy = new byte[count]; + Buffer.BlockCopy(buffer, offset, copy, 0, count); + _pending.Add((_position, copy)); + + _position += count; + if (_position > _writerLength) _writerLength = _position; + // NOTE: _visibleLength is NOT updated here; only Flush() publishes writes. + } + + protected override void Dispose(bool disposing) + { + if (disposing) _committed.Dispose(); + base.Dispose(disposing); + } + } +} diff --git a/LiteDB.Tests/Issues/Issue2534_Tests.cs b/LiteDB.Tests/Issues/Issue2534_Tests.cs index 725a276d3..7abd9e88a 100644 --- a/LiteDB.Tests/Issues/Issue2534_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2534_Tests.cs @@ -2,6 +2,7 @@ namespace LiteDB.Tests.Issues; +[Collection("SharedDemoDatabase")] public class Issue2534_Tests { [Fact] diff --git a/LiteDB.Tests/Issues/Issue2570_Tests.cs b/LiteDB.Tests/Issues/Issue2570_Tests.cs index 1051f2bf6..621fafb2b 100644 --- a/LiteDB.Tests/Issues/Issue2570_Tests.cs +++ b/LiteDB.Tests/Issues/Issue2570_Tests.cs @@ -1,4 +1,5 @@ using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Issues; @@ -15,7 +16,7 @@ public class Person [Fact] public void Issue2570_Tuples() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { var col = db.GetCollection("Person"); @@ -46,7 +47,7 @@ public class PersonWithStruct [Fact] public void Issue2570_Structs() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { var col = db.GetCollection("Person"); diff --git a/LiteDB.Tests/Issues/Issue546_Tests.cs b/LiteDB.Tests/Issues/Issue546_Tests.cs new file mode 100644 index 000000000..a5f134c11 --- /dev/null +++ b/LiteDB.Tests/Issues/Issue546_Tests.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using Xunit; + +namespace LiteDB.Tests.Issues; + +public class Issue546_Tests +{ + [Fact] + public void Test() + { + using LiteDatabase dataBase = new("demo.db"); + ILiteCollection guidDictCollection = dataBase.GetCollection("Issue546_Guid_Keys"); + + guidDictCollection.DeleteAll(); + guidDictCollection.Insert(new GuidDictContainer()); + + Assert.Single(guidDictCollection.FindAll()); + } + + private class GuidDictContainer + { + public Dictionary GuidDict { get; set; } = new() + { + [Guid.NewGuid()] = "test", + }; + public Dictionary EnumDict { get; set; } = new() + { + [TestEnum.ThingA] = "test", + }; + } + private enum TestEnum + { + ThingA, + ThingB, + } +} \ No newline at end of file diff --git a/LiteDB.Tests/Issues/IssueCheckpointFlush_Tests.cs b/LiteDB.Tests/Issues/IssueCheckpointFlush_Tests.cs new file mode 100644 index 000000000..e8e66b8b2 --- /dev/null +++ b/LiteDB.Tests/Issues/IssueCheckpointFlush_Tests.cs @@ -0,0 +1,104 @@ +using System; +using System.IO; +using FluentAssertions; +using LiteDB; +using LiteDB.Tests; +using Xunit; + +namespace LiteDB.Tests.Issues +{ + public class IssueCheckpointFlush_Tests + { + private class Entity + { + public int Id { get; set; } + + public string Value { get; set; } = string.Empty; + } + + [Fact] + public void CommittedChangesAreLostWhenClosingExternalStreamWithoutCheckpoint() + { + using var tempFile = new TempFile(); + + using (var createStream = new FileStream(tempFile.Filename, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) + { + using var createDb = new LiteDatabase(createStream); + var collection = createDb.GetCollection("entities"); + + collection.Upsert(new Entity { Id = 1, Value = "initial" }); + + createDb.Commit(); + createStream.Flush(true); + } + + var updateStream = new FileStream(tempFile.Filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); + var updateDb = new LiteDatabase(updateStream); + var updateCollection = updateDb.GetCollection("entities"); + + updateCollection.Upsert(new Entity { Id = 1, Value = "updated" }); + + updateDb.Commit(); + updateStream.Flush(true); + updateStream.Dispose(); + updateDb = null; + + GC.Collect(); + GC.WaitForPendingFinalizers(); + + using (var verifyStream = new FileStream(tempFile.Filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)) + using (var verifyDb = new LiteDatabase(verifyStream)) + { + var document = verifyDb.GetCollection("entities").FindById(1); + + document.Should().NotBeNull(); + document!.Value.Should().Be("updated"); + } + } + + [Fact] + public void StreamConstructorRestoresCheckpointSizeAfterDisposal() + { + using var tempFile = new TempFile(); + + using (var fileDb = new LiteDatabase(tempFile.Filename)) + { + fileDb.CheckpointSize.Should().Be(1000); + } + + using (var stream = new FileStream(tempFile.Filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)) + using (var streamDb = new LiteDatabase(stream)) + { + streamDb.CheckpointSize.Should().Be(1); + } + + using (var reopened = new LiteDatabase(tempFile.Filename)) + { + reopened.CheckpointSize.Should().Be(1000); + } + } + + [Fact] + public void StreamConstructorAllowsReadOnlyStreams() + { + using var tempFile = new TempFile(); + + using (var setup = new LiteDatabase(tempFile.Filename)) + { + var collection = setup.GetCollection("entities"); + + collection.Insert(new Entity { Id = 1, Value = "initial" }); + + setup.Checkpoint(); + } + + using var readOnlyStream = new FileStream(tempFile.Filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var readOnlyDb = new LiteDatabase(readOnlyStream); + + var document = readOnlyDb.GetCollection("entities").FindById(1); + + document.Should().NotBeNull(); + document!.Value.Should().Be("initial"); + } + } +} diff --git a/LiteDB.Tests/Issues/Pull2468_Tests.cs b/LiteDB.Tests/Issues/Pull2468_Tests.cs index 5e5f35244..ad358000b 100644 --- a/LiteDB.Tests/Issues/Pull2468_Tests.cs +++ b/LiteDB.Tests/Issues/Pull2468_Tests.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading.Tasks; +using LiteDB.Tests.Utils; using Xunit; using static LiteDB.Tests.Issues.Issue1838_Tests; @@ -18,7 +19,7 @@ public class Pull2468_Tests [Fact] public void Supports_LowerInvariant() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var collection = db.GetCollection(nameof(TestType)); collection.Insert(new TestType() @@ -45,7 +46,7 @@ public void Supports_LowerInvariant() [Fact] public void Supports_UpperInvariant() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); var collection = db.GetCollection(nameof(TestType)); collection.Insert(new TestType() diff --git a/LiteDB.Tests/LiteDB.Tests.csproj b/LiteDB.Tests/LiteDB.Tests.csproj index 47c3bd292..da911494e 100644 --- a/LiteDB.Tests/LiteDB.Tests.csproj +++ b/LiteDB.Tests/LiteDB.Tests.csproj @@ -1,50 +1,70 @@ - - - - net8 - LiteDB.Tests - LiteDB.Tests - Maurício David - MIT - en-US - false - 1701;1702;1705;1591;0618 - True - ..\LiteDB\LiteDB.snk - - - - - PreserveNewest - - - - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - \ No newline at end of file + + + + false + net461;net481;net8.0 + $(TargetFrameworks);net10.0 + latest + LiteDB.Tests + LiteDB.Tests + Maurício David + MIT + en-US + false + 1701;1702;1705;1591;0618 + $(DefineConstants);NETFRAMEWORK + $(DefineConstants);NETFRAMEWORK + $(DefineConstants);NETFRAMEWORK + $(DefineConstants);TESTING + + + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/LiteDB.Tests/Mapper/Mapper_Tests.cs b/LiteDB.Tests/Mapper/Mapper_Tests.cs index a1fb989b7..abf9eb645 100644 --- a/LiteDB.Tests/Mapper/Mapper_Tests.cs +++ b/LiteDB.Tests/Mapper/Mapper_Tests.cs @@ -1,6 +1,7 @@ using FluentAssertions; using System; using System.Reflection; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.Mapper @@ -23,7 +24,7 @@ public void ToDocument_ReturnsNull_WhenFail() [Fact] public void Class_Not_Assignable() { - using (var db = new LiteDatabase(":memory:")) + using (var db = DatabaseFactory.Create()) { var col = db.GetCollection("Test"); col.Insert(new MyClass { Id = 1, Member = null }); diff --git a/LiteDB.Tests/Query/Data/PersonQueryData.cs b/LiteDB.Tests/Query/Data/PersonQueryData.cs index 2e266b0cf..16b4d22ca 100644 --- a/LiteDB.Tests/Query/Data/PersonQueryData.cs +++ b/LiteDB.Tests/Query/Data/PersonQueryData.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using LiteDB.Tests.Utils; namespace LiteDB.Tests.QueryTest { @@ -13,7 +14,7 @@ public PersonQueryData() { _local = DataGen.Person().ToArray(); - _db = new LiteDatabase(":memory:"); + _db = DatabaseFactory.Create(); _collection = _db.GetCollection("person"); _collection.Insert(this._local); } diff --git a/LiteDB.Tests/Query/GroupBy_Tests.cs b/LiteDB.Tests/Query/GroupBy_Tests.cs index 8f84dd430..65c28ae83 100644 --- a/LiteDB.Tests/Query/GroupBy_Tests.cs +++ b/LiteDB.Tests/Query/GroupBy_Tests.cs @@ -1,104 +1,412 @@ -using FluentAssertions; -using System; -using System.IO; +using System.IO; using System.Linq; + +using FluentAssertions; +using LiteDB; +using LiteDB.Engine; +using LiteDB.Tests; using Xunit; namespace LiteDB.Tests.QueryTest { public class GroupBy_Tests { - [Fact(Skip = "Missing implement LINQ for GroupBy")] + [Fact] + [Trait("Area", "GroupBy")] + public void Having_Sees_Group_Key() + { + using var stream = new MemoryStream(); + using var db = new LiteDatabase(stream); + var col = db.GetCollection("numbers"); + + for (var i = 1; i <= 5; i++) + { + col.Insert(new GroupItem { Id = i, Value = i, Parity = i % 2 }); + } + + var results = col.Query() + .GroupBy(x => x.Parity) + .Having(BsonExpression.Create("@key = 1")) + .Select(g => new { g.Key, Count = g.Count() }) + .ToArray(); + + var because = $"HAVING must see @key; got {results.Length} groups."; + results.Should().HaveCount(1, because); + results[0].Key.Should().Be(1, "HAVING must filter on the grouping key value 1."); + results[0].Count.Should().Be(3, "HAVING must aggregate the odd values (1, 3, 5)."); + } + + [Fact] + [Trait("Area", "GroupBy")] + public void GroupBy_Respects_Collation_For_Key_Equality() + { + using var dataStream = new MemoryStream(); + using var engine = new LiteEngine(new EngineSettings + { + DataStream = dataStream, + Collation = new Collation("en-US/IgnoreCase") + }); + using var db = new LiteDatabase(engine, disposeOnClose: false); + var col = db.GetCollection("names"); + + col.Insert(new NamedItem { Id = 1, Name = "ALICE" }); + col.Insert(new NamedItem { Id = 2, Name = "alice" }); + col.Insert(new NamedItem { Id = 3, Name = "Alice" }); + + var results = col.Query() + .GroupBy(x => x.Name) + .Select(g => new { g.Key, Count = g.Count() }) + .ToArray(); + + var because = $"Grouping equality must honor the configured case-insensitive collation; got {results.Length} groups."; + results.Should().HaveCount(1, because); + results[0].Count.Should().Be(3, "Grouping equality must honor the configured case-insensitive collation."); + } + + [Fact] + [Trait("Area", "GroupBy")] + public void OrderBy_Count_Before_Select_Uses_Group_Not_Projection() + { + using var stream = new MemoryStream(); + using var db = new LiteDatabase(stream); + var col = db.GetCollection("items"); + + var items = new[] + { + new CategoryItem { Id = 1, Category = "A" }, + new CategoryItem { Id = 2, Category = "A" }, + new CategoryItem { Id = 3, Category = "A" }, + new CategoryItem { Id = 4, Category = "B" }, + new CategoryItem { Id = 5, Category = "B" }, + new CategoryItem { Id = 6, Category = "C" } + }; + + col.InsertBulk(items); + + var ascending = col.Query() + .GroupBy(x => x.Category) + .OrderBy(g => g.Count()) + .Select(g => new { g.Key, Size = g.Count() }) + .ToArray(); + + var expectedAscending = items + .GroupBy(x => x.Category) + .OrderBy(g => g.Count()) + .Select(g => new { g.Key, Size = g.Count() }) + .ToArray(); + + var ascendingSizes = ascending.Select(x => x.Size).ToArray(); + var expectedAscendingSizes = expectedAscending.Select(x => x.Size).ToArray(); + ascendingSizes.Should().Equal(expectedAscendingSizes, //new[] { 1, 2, 3 } + "OrderBy aggregate must be evaluated over the group source before projection (ascending). Actual: {0}", + string.Join(", ", ascendingSizes)); + + var ascendingKeys = ascending.Select(x => x.Key).ToArray(); + var expectedAscendingKeys = expectedAscending.Select(x => x.Key).ToArray(); + ascendingKeys.Should().Equal(expectedAscendingKeys, //new[] { "C", "B", "A" } + "OrderBy aggregate must order groups by their aggregated size (ascending). Actual: {0}", + string.Join(", ", ascendingKeys)); + + var descending = col.Query() + .GroupBy(x => x.Category) + .OrderByDescending(g => g.Count()) + .Select(g => new { g.Key, Size = g.Count() }) + .ToArray(); + + var expectedDescending = items + .GroupBy(x => x.Category) + .OrderByDescending(g => g.Count()) + .Select(g => new { g.Key, Size = g.Count() }) + .ToArray(); + + var descendingSizes = descending.Select(x => x.Size).ToArray(); + var expectedDescendingSizes = expectedDescending.Select(x => x.Size).ToArray(); + descendingSizes.Should().Equal(expectedDescendingSizes, // new[] { 3, 2, 1 } + "OrderBy aggregate must be evaluated over the group source before projection (descending). Actual: {0}", + string.Join(", ", descendingSizes)); + + var descendingKeys = descending.Select(x => x.Key).ToArray(); + var expectedDescendingKeys = expectedDescending.Select(x => x.Key).ToArray(); + descendingKeys.Should().Equal(expectedDescendingKeys, //new[] { "A", "B", "C" } + "OrderBy aggregate must order groups by their aggregated size (descending). Actual: {0}", + string.Join(", ", descendingKeys)); + } + + [Fact] public void Query_GroupBy_Age_With_Count() { - //**using var db = new PersonGroupByData(); - //**var (collection, local) = db.GetData(); - //** - //**var r0 = local - //** .GroupBy(x => x.Age) - //** .Select(x => new { Age = x.Key, Count = x.Count() }) - //** .OrderBy(x => x.Age) - //** .ToArray(); - //** - //**var r1 = collection.Query() - //** .GroupBy("$.Age") - //** .Select(x => new { Age = x.Key, Count = x.Count() }) - //** .ToArray(); - //** - //**foreach (var r in r0.Zip(r1, (l, r) => new { left = l, right = r })) - //**{ - //** r.left.Age.Should().Be(r.right.Age); - //** r.left.Count.Should().Be(r.right.Count); - //**} + using var db = new PersonGroupByData(); + var (collection, local) = db.GetData(); + + var expected = local + .GroupBy(x => x.Age) + .Select(g => (Age: g.Key, Count: g.Count())) + .OrderBy(x => x.Age) + .ToArray(); + + var actual = collection.Query() + .GroupBy(x => x.Age) + .Select + ( + g => new + { + Age = g.Key, + Count = g.Count() + } + ) + .OrderBy(x => x.Age) + .ToArray() + .Select(x => (x.Age, x.Count)) + .ToArray(); + + actual.Should().Equal(expected); } - [Fact(Skip = "Missing implement LINQ for GroupBy")] + [Fact] public void Query_GroupBy_Year_With_Sum_Age() { - //** var r0 = local - //** .GroupBy(x => x.Date.Year) - //** .Select(x => new { Year = x.Key, Sum = x.Sum(q => q.Age) }) - //** .OrderBy(x => x.Year) - //** .ToArray(); - //** - //** var r1 = collection.Query() - //** .GroupBy(x => x.Date.Year) - //** .Select(x => new { Year = x.Key, Sum = x.Sum(q => q.Age) }) - //** .ToArray(); - //** - //** foreach (var r in r0.Zip(r1, (l, r) => new { left = l, right = r })) - //** { - //** r.left.Year.Should().Be(r.right.Year); - //** r.left.Sum.Should().Be(r.right.Sum); - //** } + using var db = new PersonGroupByData(); + var (collection, local) = db.GetData(); + + var expected = local + .GroupBy(x => x.Date.Year) + .Select(g => (Year: g.Key, Sum: g.Sum(p => p.Age))) + .OrderBy(x => x.Year) + .ToArray(); + + var actual = collection.Query() + .GroupBy(x => x.Date.Year) + .Select + ( + g => new + { + Year = g.Key, + Sum = g.Sum(p => p.Age) + } + ) + .OrderBy(x => x.Year) + .ToArray() + .Select(x => (x.Year, x.Sum)) + .ToArray(); + + actual.Should().Equal(expected); + } + + [Fact] + public void Query_GroupBy_Order_And_Limit() + { + using var db = new PersonGroupByData(); + var (collection, local) = db.GetData(); + + var expected = local + .GroupBy(x => x.Age) + .Select + ( + g => new + { + Age = g.Key, + Count = g.Count() + } + ) + .OrderByDescending(x => x.Count) + .ThenBy(x => x.Age) + .Skip(5) + .Take(3) + .Select(x => (x.Age, x.Count)) + .ToArray(); + + var actual = collection.Query() + .GroupBy(x => x.Age) + .Select + ( + g => new + { + Age = g.Key, + Count = g.Count() + } + ) + .OrderByDescending(x => x.Count) + .ThenBy(x => x.Age) + .Skip(5) + .Limit(3) + .ToArray() + .Select(x => (x.Age, x.Count)) + .ToArray(); + + actual.Should().Equal(expected); + } + + [Fact] + public void Query_GroupBy_ToList_Materializes_Groupings() + { + using var db = new PersonGroupByData(); + var (collection, local) = db.GetData(); + + var expected = local + .GroupBy(x => x.Age) + .OrderBy(g => g.Key) + .Select + ( + g => new + { + Key = g.Key, + Names = g.OrderBy(p => p.Name).Select(p => p.Name).ToArray() + } + ) + .ToArray(); + + var groupings = collection.Query() + .GroupBy(x => x.Age) + .ToList(); + + groupings.Should().AllBeAssignableTo>(); + + var actual = groupings + .OrderBy(g => g.Key) + .Select + ( + g => new + { + g.Key, + Names = g.OrderBy(p => p.Name).Select(p => p.Name).ToArray() + } + ) + .ToArray(); + + actual.Should().BeEquivalentTo(expected, options => options.WithStrictOrdering()); + } + + [Fact] + public void Query_GroupBy_OrderBy_Key_Before_Select_Should_Work() + { + using var db = new PersonGroupByData(); + var (collection, local) = db.GetData(); + + // This test specifically targets the bug where OrderBy(g => g.Key) before Select() + // causes issues with @key parameter binding in the GroupByPipe + var expected = local + .GroupBy(x => x.Age) + .OrderBy(g => g.Key) // Order by key BEFORE projection + .Select + ( + g => new + { + Age = g.Key, + Count = g.Count() + } + ) + .ToArray(); + + var actual = collection.Query() + .GroupBy(x => x.Age) + .OrderBy(g => g.Key) // This should work but currently fails due to @key not being bound + .Select + ( + g => new + { + Age = g.Key, + Count = g.Count() + } + ) + .ToArray(); + + actual.Should().BeEquivalentTo(expected, options => options.WithStrictOrdering()); + } + + [Fact] + public void Query_GroupBy_OrderByDescending_Key_Before_Select_Should_Work() + { + using var db = new PersonGroupByData(); + var (collection, local) = db.GetData(); + + // Test descending order as well to ensure the fix works for both directions + var expected = local + .GroupBy(x => x.Age) + .OrderByDescending(g => g.Key) // Order by key descending BEFORE projection + .Select + ( + g => new + { + Age = g.Key, + Count = g.Count() + } + ) + .ToArray(); + + var actual = collection.Query() + .GroupBy(x => x.Age) + .OrderByDescending(g => g.Key) // This should work but currently fails + .Select + ( + g => new + { + Age = g.Key, + Count = g.Count() + } + ) + .ToArray(); + + actual.Should().BeEquivalentTo(expected, options => options.WithStrictOrdering()); } - [Fact(Skip = "Missing implement LINQ for GroupBy")] - public void Query_GroupBy_Func() + [Fact] + public void Query_GroupBy_Complex_Key_OrderBy_Before_Select_Should_Work() { - //** var r0 = local - //** .GroupBy(x => x.Date.Year) - //** .Select(x => new { Year = x.Key, Count = x.Count() }) - //** .OrderBy(x => x.Year) - //** .ToArray(); - //** - //** var r1 = collection.Query() - //** .GroupBy(x => x.Date.Year) - //** .Select(x => new { x.Date.Year, Count = x }) - //** .ToArray(); - //** - //** foreach (var r in r0.Zip(r1, (l, r) => new { left = l, right = r })) - //** { - //** Assert.Equal(r.left.Year, r.right.Year); - //** Assert.Equal(r.left.Count, r.right.Count); - //** } + using var db = new PersonGroupByData(); + var (collection, local) = db.GetData(); + + // Test with a more complex grouping key to ensure the fix works with different key types + var expected = local + .GroupBy(x => x.Date.Year) + .OrderBy(g => g.Key) // Order by year key BEFORE projection + .Select + ( + g => new + { + Year = g.Key, + TotalAge = g.Sum(p => p.Age) + } + ) + .ToArray(); + + var actual = collection.Query() + .GroupBy(x => x.Date.Year) + .OrderBy(g => g.Key) // This should work but currently fails + .Select + ( + g => new + { + Year = g.Key, + TotalAge = g.Sum(p => p.Age) + } + ) + .ToArray(); + + actual.Should().BeEquivalentTo(expected, options => options.WithStrictOrdering()); } - [Fact(Skip = "Missing implement LINQ for GroupBy")] - public void Query_GroupBy_With_Array_Aggregation() + private class GroupItem { - //** // quite complex group by query - //** var r = collection.Query() - //** .GroupBy(x => x.Email.Substring(x.Email.IndexOf("@") + 1)) - //** .Select(x => new - //** { - //** Domain = x.Email.Substring(x.Email.IndexOf("@") + 1), - //** Users = Sql.ToArray(new - //** { - //** Login = x.Email.Substring(0, x.Email.IndexOf("@")).ToLower(), - //** x.Name, - //** x.Age - //** }) - //** }) - //** .Limit(10) - //** .ToArray(); - //** - //** // test first only - //** Assert.Equal(5, r[0].Users.Length); - //** Assert.Equal("imperdiet.us", r[0].Domain); - //** Assert.Equal("delilah", r[0].Users[0].Login); - //** Assert.Equal("Dahlia Warren", r[0].Users[0].Name); - //** Assert.Equal(24, r[0].Users[0].Age); + public int Id { get; set; } + + public int Value { get; set; } + + public int Parity { get; set; } } + + private class NamedItem + { + public int Id { get; set; } + + public string Name { get; set; } = string.Empty; + } + + private class CategoryItem + { + public int Id { get; set; } + + public string Category { get; set; } = string.Empty; + } + } -} \ No newline at end of file +} diff --git a/LiteDB.Tests/Query/OrderBy_Tests.cs b/LiteDB.Tests/Query/OrderBy_Tests.cs index 83fb6c8e7..67df882c2 100644 --- a/LiteDB.Tests/Query/OrderBy_Tests.cs +++ b/LiteDB.Tests/Query/OrderBy_Tests.cs @@ -1,5 +1,9 @@ -using System.Linq; +using System; +using System.Collections.Generic; +using System.Linq; using FluentAssertions; +using LiteDB.Engine; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.QueryTest @@ -16,12 +20,12 @@ public void Query_OrderBy_Using_Index() var r0 = local .OrderBy(x => x.Name) - .Select(x => new {x.Name}) + .Select(x => new { x.Name }) .ToArray(); var r1 = collection.Query() .OrderBy(x => x.Name) - .Select(x => new {x.Name}) + .Select(x => new { x.Name }) .ToArray(); r0.Should().Equal(r1); @@ -37,12 +41,12 @@ public void Query_OrderBy_Using_Index_Desc() var r0 = local .OrderByDescending(x => x.Name) - .Select(x => new {x.Name}) + .Select(x => new { x.Name }) .ToArray(); var r1 = collection.Query() .OrderByDescending(x => x.Name) - .Select(x => new {x.Name}) + .Select(x => new { x.Name }) .ToArray(); r0.Should().Equal(r1); @@ -58,12 +62,12 @@ public void Query_OrderBy_With_Func() var r0 = local .OrderBy(x => x.Date.Day) - .Select(x => new {d = x.Date.Day}) + .Select(x => new { d = x.Date.Day }) .ToArray(); var r1 = collection.Query() .OrderBy(x => x.Date.Day) - .Select(x => new {d = x.Date.Day}) + .Select(x => new { d = x.Date.Day }) .ToArray(); r0.Should().Equal(r1); @@ -106,5 +110,486 @@ public void Query_Asc_Desc() asc[0].Id.Should().Be(1); desc[0].Id.Should().Be(1000); } + + [Fact] + public void Query_OrderBy_ThenBy_Multiple_Keys() + { + using var db = new PersonQueryData(); + var (collection, local) = db.GetData(); + + collection.EnsureIndex(x => x.Age); + + var expected = local + .OrderBy(x => x.Age) + .ThenByDescending(x => x.Name) + .Select + ( + x => new + { + x.Age, + x.Name + } + ) + .ToArray(); + + var actual = collection.Query() + .OrderBy(x => x.Age) + .ThenByDescending(x => x.Name) + .Select + ( + x => new + { + x.Age, + x.Name + } + ) + .ToArray(); + + actual.Should().Equal(expected); + + var plan = collection.Query() + .OrderBy(x => x.Age) + .ThenByDescending(x => x.Name) + .GetPlan(); + + plan["index"]["order"].AsInt32.Should().Be(Query.Ascending); + + var orderBy = plan["orderBy"].AsArray; + + orderBy.Count.Should().Be(2); + orderBy[0]["expr"].AsString.Should().Be("$.Age"); + orderBy[0]["order"].AsInt32.Should().Be(Query.Ascending); + orderBy[1]["expr"].AsString.Should().Be("$.Name"); + orderBy[1]["order"].AsInt32.Should().Be(Query.Descending); + } + + [Fact] + public void Query_OrderByDescending_ThenBy_Index_Order_Applied() + { + using var db = new PersonQueryData(); + var (collection, local) = db.GetData(); + + collection.EnsureIndex(x => x.Name); + collection.EnsureIndex(x => x.Age); + + var expected = local + .OrderByDescending(x => x.Age) + .ThenBy(x => x.Name) + .Select + ( + x => new + { + x.Name, + x.Age + } + ) + .ToArray(); + + var actual = collection.Query() + .OrderByDescending(x => x.Age) + .ThenBy(x => x.Name) + .Select + ( + x => new + { + x.Name, + x.Age + } + ) + .ToArray(); + + actual.Should().Equal(expected); + + var plan = collection.Query() + .OrderByDescending(x => x.Name) + .ThenBy(x => x.Age) + .GetPlan(); + + plan["index"]["order"].AsInt32.Should().Be(Query.Descending); + + var orderBy = plan["orderBy"].AsArray; + + orderBy.Count.Should().Be(2); + orderBy[0]["order"].AsInt32.Should().Be(Query.Descending); + orderBy[1]["order"].AsInt32.Should().Be(Query.Ascending); + } + + public record Data(int Id, int Value); + + [Fact] + public void Query_OrderByDescending_ThenBy_Index_Order_Applied_Data2() + { + var data = Enumerable + .Range(1, 1000) + .Select(x => new Data(x, new Random(x).Next())) // pseudo random + .ToArray(); + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Value); + col.EnsureIndex(x => x.Id); + col.Insert(data); + + var expected = data + .OrderByDescending(x => x.Value) + .ThenBy(x => x.Id) + .ToArray(); + + var actual = col.Query() + .OrderByDescending(x => x.Value) + .ThenBy(x => x.Id) + .ToArray(); + + expected.Should().Equal(actual); + } + + [Fact] + public void Query_OrderByAscending_ThenByDescending_Index_Order_Applied_Data2() + { + var data = Enumerable + .Range(1, 1000) + .Select(x => new Data(x, new Random(x).Next())) // pseudo random + .ToArray(); + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Value); + col.EnsureIndex(x => x.Id); + col.Insert(data); + + var expected = data + .OrderBy(x => x.Value) + .ThenByDescending(x => x.Id) + .ToArray(); + + var actual = col.Query() + .OrderBy(x => x.Value) + .ThenByDescending(x => x.Id) + .ToArray(); + + expected.Should().Equal(actual); + } + + [Fact] + public void Query_OrderByAscending_ThenByAscending_Index_Order_Applied_Data2() + { + var data = Enumerable + .Range(1, 1000) + .Select(x => new Data(x, new Random(x).Next())) // pseudo random + .ToArray(); + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Value); + col.EnsureIndex(x => x.Id); + col.Insert(data); + + var expected = data + .OrderBy(x => x.Value) + .ThenBy(x => x.Id) + .ToArray(); + + var actual = col.Query() + .OrderBy(x => x.Value) + .ThenBy(x => x.Id) + .ToArray(); + + expected.Should().Equal(actual); + } + + [Fact] + public void Query_OrderByDescending_ThenByDescending_Index_Order_Applied_Data2() + { + var data = Enumerable + .Range(1, 1000) + .Select(x => new Data(x, new Random(x).Next())) // pseudo random + .ToArray(); + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Value); + col.EnsureIndex(x => x.Id); + col.Insert(data); + + var expected = data + .OrderByDescending(x => x.Value) + .ThenByDescending(x => x.Id) + .ToArray(); + + var actual = col.Query() + .OrderByDescending(x => x.Value) + .ThenByDescending(x => x.Id) + .ToArray(); + + expected.Should().Equal(actual); + } + + [Fact] + public void Query_Missmatch_Order() + { + var data = Enumerable + .Range(1, 1000) + .Select(x => new Data(x, new Random(x).Next())) // pseudo random + .ToArray(); + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Value); + col.EnsureIndex(x => x.Id); + col.Insert(data); + + var expected = data + .OrderBy(x => x.Value) + .ThenByDescending(x => x.Id) + .ToArray(); + + var actual = col.Query() + .OrderByDescending(x => x.Value) + .ThenBy(x => x.Id) + .ToArray(); + + expected.Should().NotEqual(actual); + } + + public record ThreeLayerData(int Id, int Category, string Name, int Priority); + + [Fact] + public void Query_ThreeLayer_Ascending_Ascending_Ascending() + { + var data = new[] + { + new ThreeLayerData(1, 1, "C", 3), + new ThreeLayerData(2, 1, "A", 2), + new ThreeLayerData(3, 1, "A", 1), + new ThreeLayerData(4, 2, "B", 2), + new ThreeLayerData(5, 2, "B", 1), + new ThreeLayerData(6, 1, "B", 1) + }; + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Category); + col.EnsureIndex(x => x.Name); + col.EnsureIndex(x => x.Priority); + col.Insert(data); + + var expected = data + .OrderBy(x => x.Category) + .ThenBy(x => x.Name) + .ThenBy(x => x.Priority) + .ToArray(); + + var actual = col.Query() + .OrderBy(x => x.Category) + .ThenBy(x => x.Name) + .ThenBy(x => x.Priority) + .ToArray(); + + actual.Should().Equal(expected); + } + + [Fact] + public void Query_ThreeLayer_Descending_Ascending_Descending() + { + var data = new[] + { + new ThreeLayerData(1, 1, "C", 3), + new ThreeLayerData(2, 1, "A", 2), + new ThreeLayerData(3, 1, "A", 1), + new ThreeLayerData(4, 2, "B", 2), + new ThreeLayerData(5, 2, "B", 1), + new ThreeLayerData(6, 1, "B", 1), + new ThreeLayerData(7, 3, "A", 2), + new ThreeLayerData(8, 3, "A", 3) + }; + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Category); + col.EnsureIndex(x => x.Name); + col.EnsureIndex(x => x.Priority); + col.Insert(data); + + var expected = data + .OrderByDescending(x => x.Category) + .ThenBy(x => x.Name) + .ThenByDescending(x => x.Priority) + .ToArray(); + + var actual = col.Query() + .OrderByDescending(x => x.Category) + .ThenBy(x => x.Name) + .ThenByDescending(x => x.Priority) + .ToArray(); + + actual.Should().Equal(expected); + } + + [Fact] + public void Query_ThreeLayer_Ascending_Descending_Ascending() + { + var data = new[] + { + new ThreeLayerData(1, 1, "C", 3), + new ThreeLayerData(2, 1, "A", 2), + new ThreeLayerData(3, 1, "A", 1), + new ThreeLayerData(4, 2, "B", 2), + new ThreeLayerData(5, 2, "B", 1), + new ThreeLayerData(6, 1, "B", 1), + new ThreeLayerData(7, 2, "A", 3), + new ThreeLayerData(8, 2, "C", 1) + }; + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Category); + col.EnsureIndex(x => x.Name); + col.EnsureIndex(x => x.Priority); + col.Insert(data); + + var expected = data + .OrderBy(x => x.Category) + .ThenByDescending(x => x.Name) + .ThenBy(x => x.Priority) + .ToArray(); + + var actual = col.Query() + .OrderBy(x => x.Category) + .ThenByDescending(x => x.Name) + .ThenBy(x => x.Priority) + .ToArray(); + + actual.Should().Equal(expected); + } + + [Fact] + public void Query_ThreeLayer_Descending_Descending_Descending() + { + var data = new[] + { + new ThreeLayerData(1, 1, "C", 3), + new ThreeLayerData(2, 1, "A", 2), + new ThreeLayerData(3, 1, "A", 1), + new ThreeLayerData(4, 2, "B", 2), + new ThreeLayerData(5, 2, "B", 1), + new ThreeLayerData(6, 1, "B", 1), + new ThreeLayerData(7, 3, "A", 2), + new ThreeLayerData(8, 3, "D", 4) + }; + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Category); + col.EnsureIndex(x => x.Name); + col.EnsureIndex(x => x.Priority); + col.Insert(data); + + var expected = data + .OrderByDescending(x => x.Category) + .ThenByDescending(x => x.Name) + .ThenByDescending(x => x.Priority) + .ToArray(); + + var actual = col.Query() + .OrderByDescending(x => x.Category) + .ThenByDescending(x => x.Name) + .ThenByDescending(x => x.Priority) + .ToArray(); + + actual.Should().Equal(expected); + } + + [Fact] + public void Query_ThreeLayer_With_Large_Dataset() + { + var random = new Random(42); // Fixed seed for reproducible results + var data = Enumerable + .Range(1, 1000) + .Select(x => new ThreeLayerData( + 0, + random.Next(1, 5), // Category 1-4 + ((char)('A' + random.Next(0, 5))).ToString(), // Name A-E + random.Next(1, 4) // Priority 1-3 + )) + .ToArray(); + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Category); + col.EnsureIndex(x => x.Name); + col.EnsureIndex(x => x.Priority); + col.Insert(data); + + data = col.FindAll().ToArray(); + + var expected = data + .OrderByDescending(x => x.Category) + .ThenBy(x => x.Name) + .ThenByDescending(x => x.Priority) + .ThenByDescending(x => x.Id) + .ToArray(); + + var actual = col.Query() + .OrderByDescending(x => x.Category) + .ThenBy(x => x.Name) + .ThenByDescending(x => x.Priority) + .ThenByDescending(x => x.Id) + .ToArray(); + + actual.Should().Equal(expected); + + // Verify the query plan shows all three ordering segments + var plan = col.Query() + .OrderByDescending(x => x.Category) + .ThenBy(x => x.Name) + .ThenByDescending(x => x.Priority) + .GetPlan(); + + var orderBy = plan["orderBy"].AsArray; + orderBy.Count.Should().Be(3); + orderBy[0]["expr"].AsString.Should().Be("$.Category"); + orderBy[0]["order"].AsInt32.Should().Be(Query.Descending); + orderBy[1]["expr"].AsString.Should().Be("$.Name"); + orderBy[1]["order"].AsInt32.Should().Be(Query.Ascending); + orderBy[2]["expr"].AsString.Should().Be("$.Priority"); + orderBy[2]["order"].AsInt32.Should().Be(Query.Descending); + } + + [Fact] + public void Query_ThreeLayer_Edge_Case_With_Nulls_And_Duplicates() + { + var data = new[] + { + new ThreeLayerData(1, 1, "A", 1), + new ThreeLayerData(2, 1, "A", 1), // Duplicate + new ThreeLayerData(3, 1, "A", 2), + new ThreeLayerData(4, 2, "A", 1), + new ThreeLayerData(5, 2, "B", 1), + new ThreeLayerData(6, 1, "B", 1), + new ThreeLayerData(7, 1, "B", 2), + new ThreeLayerData(8, 2, "A", 1) // Another duplicate of Id 4 + }; + + using var db = DatabaseFactory.Create(); + var col = db.GetCollection("data"); + col.EnsureIndex(x => x.Category); + col.EnsureIndex(x => x.Name); + col.EnsureIndex(x => x.Priority); + col.Insert(data); + + var expected = data + .OrderBy(x => x.Category) + .ThenByDescending(x => x.Name) + .ThenBy(x => x.Priority) + .ToArray(); + + var actual = col.Query() + .OrderBy(x => x.Category) + .ThenByDescending(x => x.Name) + .ThenBy(x => x.Priority) + .ToArray(); + + actual.Should().Equal(expected); + } } } \ No newline at end of file diff --git a/LiteDB.Tests/Query/Select_Tests.cs b/LiteDB.Tests/Query/Select_Tests.cs index 6049ad0ac..bd98497cd 100644 --- a/LiteDB.Tests/Query/Select_Tests.cs +++ b/LiteDB.Tests/Query/Select_Tests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using FluentAssertions; +using LiteDB.Tests.Utils; using Xunit; namespace LiteDB.Tests.QueryTest @@ -78,7 +79,7 @@ public void Query_Find_All_Predicate() [Fact] public void Query_With_No_Collection() { - using var db = new LiteDatabase(":memory:"); + using var db = DatabaseFactory.Create(); using (var r = db.Execute("SELECT DAY(NOW()) as DIA")) { diff --git a/LiteDB.Tests/Query/VectorExtensionSurface_Tests.cs b/LiteDB.Tests/Query/VectorExtensionSurface_Tests.cs new file mode 100644 index 000000000..80af3bad8 --- /dev/null +++ b/LiteDB.Tests/Query/VectorExtensionSurface_Tests.cs @@ -0,0 +1,52 @@ +using FluentAssertions; +using LiteDB; +using LiteDB.Vector; +using Xunit; + +namespace LiteDB.Tests.QueryTest +{ + public class VectorExtensionSurface_Tests + { + private class VectorDocument + { + public int Id { get; set; } + + public float[] Embedding { get; set; } + } + + [Fact] + public void Collection_Extension_Produces_Vector_Index_Plan() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f } }); + collection.Insert(new VectorDocument { Id = 2, Embedding = new[] { 0f, 1f } }); + + collection.EnsureIndex(x => x.Embedding, new VectorIndexOptions(2)); + + var plan = collection.Query() + .WhereNear(x => x.Embedding, new[] { 1f, 0f }, maxDistance: 0.25) + .GetPlan(); + + plan["index"]["mode"].AsString.Should().Be("VECTOR INDEX SEARCH"); + plan["index"]["expr"].AsString.Should().Be("$.Embedding"); + } + + [Fact] + public void Repository_Extension_Delegates_To_Vector_Index_Implementation() + { + using var db = new LiteDatabase(":memory:"); + ILiteRepository repository = new LiteRepository(db); + + repository.EnsureIndex(x => x.Embedding, new VectorIndexOptions(2)); + + var plan = repository.Query() + .WhereNear(x => x.Embedding, new[] { 1f, 0f }, maxDistance: 0.25) + .GetPlan(); + + plan["index"]["mode"].AsString.Should().Be("VECTOR INDEX SEARCH"); + plan["index"]["expr"].AsString.Should().Be("$.Embedding"); + } + } +} diff --git a/LiteDB.Tests/Query/VectorIndex_Tests.cs b/LiteDB.Tests/Query/VectorIndex_Tests.cs new file mode 100644 index 000000000..13ea6a6d4 --- /dev/null +++ b/LiteDB.Tests/Query/VectorIndex_Tests.cs @@ -0,0 +1,999 @@ +using FluentAssertions; +using LiteDB; +using LiteDB.Engine; +using LiteDB.Tests; +using LiteDB.Tests.Utils; +using LiteDB.Vector; +using MathNet.Numerics.LinearAlgebra; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using Xunit; + +namespace LiteDB.Tests.QueryTest +{ + public class VectorIndex_Tests + { + private class VectorDocument + { + public int Id { get; set; } + public float[] Embedding { get; set; } + public bool Flag { get; set; } + } + + private static readonly FieldInfo EngineField = typeof(LiteDatabase).GetField("_engine", BindingFlags.NonPublic | BindingFlags.Instance); + private static readonly FieldInfo HeaderField = typeof(LiteEngine).GetField("_header", BindingFlags.NonPublic | BindingFlags.Instance); + private static readonly MethodInfo AutoTransactionMethod = typeof(LiteEngine).GetMethod("AutoTransaction", BindingFlags.NonPublic | BindingFlags.Instance); + private static readonly MethodInfo ReadExternalVectorMethod = typeof(VectorIndexService).GetMethod("ReadExternalVector", BindingFlags.NonPublic | BindingFlags.Instance); + + private static T InspectVectorIndex(LiteDatabase db, string collection, Func selector) + { + var engine = (LiteEngine)EngineField.GetValue(db); + var header = (HeaderPage)HeaderField.GetValue(engine); + var collation = header.Pragmas.Collation; + var method = AutoTransactionMethod.MakeGenericMethod(typeof(T)); + + return (T)method.Invoke(engine, new object[] + { + new Func(transaction => + { + var snapshot = transaction.CreateSnapshot(LockMode.Read, collection, false); + var metadata = snapshot.CollectionPage.GetVectorIndexMetadata("embedding_idx"); + + return metadata == null ? default : selector(snapshot, collation, metadata); + }) + }); + } + + private static int CountNodes(Snapshot snapshot, PageAddress root) + { + if (root.IsEmpty) + { + return 0; + } + + var visited = new HashSet(); + var queue = new Queue(); + queue.Enqueue(root); + + var count = 0; + + while (queue.Count > 0) + { + var address = queue.Dequeue(); + if (!visited.Add(address)) + { + continue; + } + + var node = snapshot.GetPage(address.PageID).GetNode(address.Index); + count++; + + for (var level = 0; level < node.LevelCount; level++) + { + foreach (var neighbor in node.GetNeighbors(level)) + { + if (!neighbor.IsEmpty) + { + queue.Enqueue(neighbor); + } + } + } + } + + return count; + } + + private static float[] CreateVector(Random random, int dimensions) + { + var vector = new float[dimensions]; + var hasNonZero = false; + + for (var i = 0; i < dimensions; i++) + { + var value = (float)(random.NextDouble() * 2d - 1d); + vector[i] = value; + + if (!hasNonZero && Math.Abs(value) > 1e-6f) + { + hasNonZero = true; + } + } + + if (!hasNonZero) + { + vector[random.Next(dimensions)] = 1f; + } + + return vector; + } + + private static float[] ReadExternalVector(DataService dataService, PageAddress start, int dimensions, out int blocksRead) + { + var totalBytes = dimensions * sizeof(float); + var bytesCopied = 0; + var vector = new float[dimensions]; + blocksRead = 0; + + foreach (var slice in dataService.Read(start)) + { + blocksRead++; + + if (bytesCopied >= totalBytes) + { + break; + } + + var available = Math.Min(slice.Count, totalBytes - bytesCopied); + Buffer.BlockCopy(slice.Array, slice.Offset, vector, bytesCopied, available); + bytesCopied += available; + } + + if (bytesCopied != totalBytes) + { + throw new InvalidOperationException("Vector data block is incomplete."); + } + + return vector; + } + + private static (double Distance, double Similarity) ComputeReferenceMetrics(float[] candidate, float[] target, VectorDistanceMetric metric) + { + var builder = Vector.Build; + var candidateVector = builder.DenseOfEnumerable(candidate.Select(v => (double)v)); + var targetVector = builder.DenseOfEnumerable(target.Select(v => (double)v)); + + switch (metric) + { + case VectorDistanceMetric.Cosine: + var candidateNorm = candidateVector.L2Norm(); + var targetNorm = targetVector.L2Norm(); + + if (candidateNorm == 0d || targetNorm == 0d) + { + return (double.NaN, double.NaN); + } + + var cosineSimilarity = candidateVector.DotProduct(targetVector) / (candidateNorm * targetNorm); + return (1d - cosineSimilarity, double.NaN); + + case VectorDistanceMetric.Euclidean: + return ((candidateVector - targetVector).L2Norm(), double.NaN); + + case VectorDistanceMetric.DotProduct: + var dot = candidateVector.DotProduct(targetVector); + return (-dot, dot); + + default: + throw new ArgumentOutOfRangeException(nameof(metric), metric, null); + } + } + + private static List<(int Id, double Distance, double Similarity)> ComputeExpectedRanking( + IEnumerable documents, + float[] target, + VectorDistanceMetric metric, + int? limit = null) + { + var ordered = documents + .Select(doc => + { + var (distance, similarity) = ComputeReferenceMetrics(doc.Embedding, target, metric); + return (doc.Id, Distance: distance, Similarity: similarity); + }) + .Where(result => metric == VectorDistanceMetric.DotProduct + ? !double.IsNaN(result.Similarity) + : !double.IsNaN(result.Distance)) + .OrderBy(result => metric == VectorDistanceMetric.DotProduct ? -result.Similarity : result.Distance) + .ThenBy(result => result.Id) + .ToList(); + + if (limit.HasValue) + { + ordered = ordered.Take(limit.Value).ToList(); + } + + return ordered; + } + + + + + [Fact] + public void EnsureVectorIndex_CreatesAndReuses() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f }, Flag = true }, + new VectorDocument { Id = 2, Embedding = new[] { 0f, 1f }, Flag = false } + }); + + var expression = BsonExpression.Create("$.Embedding"); + var options = new VectorIndexOptions(2, VectorDistanceMetric.Cosine); + + collection.EnsureIndex("embedding_idx", expression, options).Should().BeTrue(); + collection.EnsureIndex("embedding_idx", expression, options).Should().BeFalse(); + + Action conflicting = () => collection.EnsureIndex("embedding_idx", expression, new VectorIndexOptions(2, VectorDistanceMetric.Euclidean)); + + conflicting.Should().Throw(); + } + + [Fact] + public void EnsureVectorIndex_PreservesEnumerableExpressionsForVectorIndexes() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("documents"); + + var resourcePath = Path.Combine(AppContext.BaseDirectory, "Resources", "ingest-20250922-234735.json"); + var json = File.ReadAllText(resourcePath); + + using var parsed = JsonDocument.Parse(json); + var embedding = parsed.RootElement + .GetProperty("Embedding") + .EnumerateArray() + .Select(static value => value.GetSingle()) + .ToArray(); + + var options = new VectorIndexOptions((ushort)embedding.Length, VectorDistanceMetric.Cosine); + + collection.EnsureIndex(x => x.Embedding, options); + + var document = new VectorDocument + { + Id = 1, + Embedding = embedding, + Flag = false + }; + + Action act = () => collection.Upsert(document); + + act.Should().NotThrow(); + + var stored = collection.FindById(1); + + stored.Should().NotBeNull(); + stored.Embedding.Should().Equal(embedding); + + var storesInline = InspectVectorIndex(db, "documents", (snapshot, collation, metadata) => + { + if (metadata.Root.IsEmpty) + { + return true; + } + + var page = snapshot.GetPage(metadata.Root.PageID); + var node = page.GetNode(metadata.Root.Index); + return node.HasInlineVector; + }); + + storesInline.Should().BeFalse(); + } + + [Fact] + public void WhereNear_UsesVectorIndex_WhenAvailable() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f }, Flag = true }, + new VectorDocument { Id = 2, Embedding = new[] { 0f, 1f }, Flag = false }, + new VectorDocument { Id = 3, Embedding = new[] { 1f, 1f }, Flag = false } + }); + + collection.EnsureIndex("embedding_idx", BsonExpression.Create("$.Embedding"), new VectorIndexOptions(2, VectorDistanceMetric.Cosine)); + + var query = collection.Query() + .WhereNear(x => x.Embedding, new[] { 1f, 0f }, maxDistance: 0.25); + + var plan = query.GetPlan(); + + plan["index"]["mode"].AsString.Should().Be("VECTOR INDEX SEARCH"); + plan["index"]["expr"].AsString.Should().Be("$.Embedding"); + plan.ContainsKey("filters").Should().BeFalse(); + + var results = query.ToArray(); + + results.Select(x => x.Id).Should().Equal(new[] { 1 }); + } + + [Fact] + public void WhereNear_FallsBack_WhenNoVectorIndexExists() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f }, Flag = true }, + new VectorDocument { Id = 2, Embedding = new[] { 0f, 1f }, Flag = false } + }); + + var query = collection.Query() + .WhereNear(x => x.Embedding, new[] { 1f, 0f }, maxDistance: 0.25); + + var plan = query.GetPlan(); + + plan["index"]["mode"].AsString.Should().StartWith("FULL INDEX SCAN"); + plan["index"]["name"].AsString.Should().Be("_id"); + plan["filters"].AsArray.Count.Should().Be(1); + + var results = query.ToArray(); + + results.Select(x => x.Id).Should().Equal(new[] { 1 }); + } + + [Fact] + public void WhereNear_FallsBack_WhenDimensionMismatch() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f, 0f }, Flag = true }, + new VectorDocument { Id = 2, Embedding = new[] { 0f, 1f, 0f }, Flag = false } + }); + + collection.EnsureIndex("embedding_idx", BsonExpression.Create("$.Embedding"), new VectorIndexOptions(3, VectorDistanceMetric.Cosine)); + + var query = collection.Query() + .WhereNear(x => x.Embedding, new[] { 1f, 0f }, maxDistance: 0.25); + + var plan = query.GetPlan(); + + plan["index"]["mode"].AsString.Should().StartWith("FULL INDEX SCAN"); + plan["index"]["name"].AsString.Should().Be("_id"); + + query.ToArray(); + } + + [Fact] + public void TopKNear_UsesVectorIndex() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f }, Flag = true }, + new VectorDocument { Id = 2, Embedding = new[] { 0f, 1f }, Flag = false }, + new VectorDocument { Id = 3, Embedding = new[] { 1f, 1f }, Flag = false } + }); + + collection.EnsureIndex("embedding_idx", BsonExpression.Create("$.Embedding"), new VectorIndexOptions(2, VectorDistanceMetric.Cosine)); + + var query = collection.Query() + .TopKNear(x => x.Embedding, new[] { 1f, 0f }, k: 2); + + var plan = query.GetPlan(); + + plan["index"]["mode"].AsString.Should().Be("VECTOR INDEX SEARCH"); + plan.ContainsKey("orderBy").Should().BeFalse(); + + var results = query.ToArray(); + + results.Select(x => x.Id).Should().Equal(new[] { 1, 3 }); + } + + [Fact] + public void OrderBy_VectorSimilarity_WithCompositeOrdering_UsesVectorIndex() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f }, Flag = true }, + new VectorDocument { Id = 2, Embedding = new[] { 1f, 0f }, Flag = false }, + new VectorDocument { Id = 3, Embedding = new[] { 0f, 1f }, Flag = true } + }); + + collection.EnsureIndex( + "embedding_idx", + BsonExpression.Create("$.Embedding"), + new VectorIndexOptions(2, VectorDistanceMetric.Cosine)); + + var similarity = BsonExpression.Create("VECTOR_SIM($.Embedding, [1.0, 0.0])"); + + var query = (LiteQueryable)collection.Query() + .OrderBy(similarity, Query.Ascending) + .ThenBy(x => x.Flag); + + var queryField = typeof(LiteQueryable).GetField("_query", BindingFlags.NonPublic | BindingFlags.Instance); + var definition = (Query)queryField.GetValue(query); + + definition.OrderBy.Should().HaveCount(2); + definition.OrderBy[0].Expression.Type.Should().Be(BsonExpressionType.VectorSim); + + definition.VectorField = "$.Embedding"; + definition.VectorTarget = new[] { 1f, 0f }; + definition.VectorMaxDistance = double.MaxValue; + + var plan = query.GetPlan(); + + plan["index"]["mode"].AsString.Should().Be("VECTOR INDEX SEARCH"); + plan["index"]["expr"].AsString.Should().Be("$.Embedding"); + plan.ContainsKey("orderBy").Should().BeFalse(); + + var results = query.ToArray(); + + results.Should().HaveCount(3); + results.Select(x => x.Id).Should().BeEquivalentTo(new[] { 1, 2, 3 }); + } + + [Fact] + public void WhereNear_DotProductHonorsMinimumSimilarity() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f } }, + new VectorDocument { Id = 2, Embedding = new[] { 0.6f, 0.6f } }, + new VectorDocument { Id = 3, Embedding = new[] { 0f, 1f } } + }); + + collection.EnsureIndex( + "embedding_idx", + BsonExpression.Create("$.Embedding"), + new VectorIndexOptions(2, VectorDistanceMetric.DotProduct)); + + var highThreshold = collection.Query() + .WhereNear(x => x.Embedding, new[] { 1f, 0f }, maxDistance: 0.75) + .ToArray(); + + highThreshold.Select(x => x.Id).Should().Equal(new[] { 1 }); + + var mediumThreshold = collection.Query() + .WhereNear(x => x.Embedding, new[] { 1f, 0f }, maxDistance: 0.4) + .ToArray(); + + mediumThreshold.Select(x => x.Id).Should().Equal(new[] { 1, 2 }); + } + + [Fact] + public void VectorIndex_Search_Prunes_Node_Visits() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + const int nearClusterSize = 64; + const int farClusterSize = 64; + + var documents = new List(); + + for (var i = 0; i < nearClusterSize; i++) + { + documents.Add(new VectorDocument + { + Id = i + 1, + Embedding = new[] { 1f, i / 100f }, + Flag = true + }); + } + + for (var i = 0; i < farClusterSize; i++) + { + documents.Add(new VectorDocument + { + Id = i + nearClusterSize + 1, + Embedding = new[] { -1f, 2f + i / 100f }, + Flag = false + }); + } + + collection.Insert(documents); + collection.Count().Should().Be(documents.Count); + + collection.EnsureIndex( + "embedding_idx", + BsonExpression.Create("$.Embedding"), + new VectorIndexOptions(2, VectorDistanceMetric.Euclidean)); + + var stats = InspectVectorIndex( + db, + "vectors", + (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + var matches = service.Search(metadata, new[] { 1f, 0f }, maxDistance: 0.25, limit: 5).ToList(); + var total = CountNodes(snapshot, metadata.Root); + + return (Visited: service.LastVisitedCount, Total: total, Matches: matches.Select(x => x.Document["Id"].AsInt32).ToArray()); + }); + + stats.Total.Should().BeGreaterThan(stats.Visited); + stats.Total.Should().BeGreaterOrEqualTo(nearClusterSize); + stats.Matches.Should().OnlyContain(id => id <= nearClusterSize); + } + + [Fact] + public void VectorIndex_PersistsNodes_WhenDocumentsChange() + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + collection.Insert(new[] + { + new VectorDocument { Id = 1, Embedding = new[] { 1f, 0f }, Flag = true }, + new VectorDocument { Id = 2, Embedding = new[] { 0f, 1f }, Flag = false }, + new VectorDocument { Id = 3, Embedding = new[] { 1f, 1f }, Flag = true } + }); + + collection.EnsureIndex("embedding_idx", BsonExpression.Create("$.Embedding"), new VectorIndexOptions(2, VectorDistanceMetric.Euclidean)); + + InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + metadata.Root.IsEmpty.Should().BeFalse(); + + var service = new VectorIndexService(snapshot, collation); + var target = new[] { 1f, 1f }; + service.Search(metadata, target, double.MaxValue, null).Count().Should().Be(3); + + return 0; + }); + + collection.Update(new VectorDocument { Id = 2, Embedding = new[] { 1f, 2f }, Flag = false }); + + InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + var target = new[] { 1f, 1f }; + service.Search(metadata, target, double.MaxValue, null).Count().Should().Be(3); + + return 0; + }); + + collection.Update(new VectorDocument { Id = 3, Embedding = null, Flag = true }); + + InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + var target = new[] { 1f, 1f }; + service.Search(metadata, target, double.MaxValue, null).Select(x => x.Document["_id"].AsInt32).Should().BeEquivalentTo(new[] { 1, 2 }); + + return 0; + }); + + collection.DeleteMany(x => x.Id == 1); + + InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + var target = new[] { 1f, 1f }; + var results = service.Search(metadata, target, double.MaxValue, null).ToArray(); + + results.Select(x => x.Document["_id"].AsInt32).Should().BeEquivalentTo(new[] { 2 }); + metadata.Root.IsEmpty.Should().BeFalse(); + + return 0; + }); + + collection.DeleteAll(); + + InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + var target = new[] { 1f, 1f }; + service.Search(metadata, target, double.MaxValue, null).Should().BeEmpty(); + metadata.Root.IsEmpty.Should().BeTrue(); + metadata.Reserved.Should().Be(uint.MaxValue); + + return 0; + }); + } + + [Theory] + [InlineData(VectorDistanceMetric.Cosine)] + [InlineData(VectorDistanceMetric.Euclidean)] + [InlineData(VectorDistanceMetric.DotProduct)] + public void VectorDistance_Computation_MatchesMathNet(VectorDistanceMetric metric) + { + var random = new Random(1789); + const int dimensions = 6; + + for (var i = 0; i < 20; i++) + { + var candidate = CreateVector(random, dimensions); + var target = CreateVector(random, dimensions); + + var distance = VectorIndexService.ComputeDistance(candidate, target, metric, out var similarity); + var (expectedDistance, expectedSimilarity) = ComputeReferenceMetrics(candidate, target, metric); + + if (double.IsNaN(expectedDistance)) + { + double.IsNaN(distance).Should().BeTrue(); + } + else + { + distance.Should().BeApproximately(expectedDistance, 1e-6); + } + + if (double.IsNaN(expectedSimilarity)) + { + double.IsNaN(similarity).Should().BeTrue(); + } + else + { + similarity.Should().BeApproximately(expectedSimilarity, 1e-6); + } + } + + if (metric == VectorDistanceMetric.Cosine) + { + var zero = new float[dimensions]; + var other = CreateVector(random, dimensions); + + var distance = VectorIndexService.ComputeDistance(zero, other, metric, out var similarity); + + double.IsNaN(distance).Should().BeTrue(); + double.IsNaN(similarity).Should().BeTrue(); + } + } + + [Theory] + [InlineData(VectorDistanceMetric.Cosine)] + [InlineData(VectorDistanceMetric.Euclidean)] + [InlineData(VectorDistanceMetric.DotProduct)] + public void VectorIndex_Search_MatchesReferenceRanking(VectorDistanceMetric metric) + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + var random = new Random(4242); + const int dimensions = 6; + + var documents = Enumerable.Range(1, 32) + .Select(i => new VectorDocument + { + Id = i, + Embedding = CreateVector(random, dimensions), + Flag = i % 2 == 0 + }) + .ToList(); + + collection.Insert(documents); + + collection.EnsureIndex( + "embedding_idx", + BsonExpression.Create("$.Embedding"), + new VectorIndexOptions((ushort)dimensions, metric)); + + var target = CreateVector(random, dimensions); + foreach (var limit in new[] { 5, 12 }) + { + var expectedTop = ComputeExpectedRanking(documents, target, metric, limit); + + var actual = InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + return service.Search(metadata, target, double.MaxValue, limit) + .Select(result => + { + var mapped = BsonMapper.Global.ToObject(result.Document); + return (Id: mapped.Id, Score: result.Distance); + }) + .ToList(); + }); + + actual.Should().HaveCount(expectedTop.Count); + + for (var i = 0; i < expectedTop.Count; i++) + { + actual[i].Id.Should().Be(expectedTop[i].Id); + + if (metric == VectorDistanceMetric.DotProduct) + { + actual[i].Score.Should().BeApproximately(expectedTop[i].Similarity, 1e-6); + } + else + { + actual[i].Score.Should().BeApproximately(expectedTop[i].Distance, 1e-6); + } + } + } + } + + [Theory] + [InlineData(VectorDistanceMetric.Cosine)] + [InlineData(VectorDistanceMetric.Euclidean)] + [InlineData(VectorDistanceMetric.DotProduct)] + public void WhereNear_MatchesReferenceOrdering(VectorDistanceMetric metric) + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + var random = new Random(9182); + const int dimensions = 6; + + var documents = Enumerable.Range(1, 40) + .Select(i => new VectorDocument + { + Id = i, + Embedding = CreateVector(random, dimensions), + Flag = i % 3 == 0 + }) + .ToList(); + + collection.Insert(documents); + + collection.EnsureIndex( + "embedding_idx", + BsonExpression.Create("$.Embedding"), + new VectorIndexOptions((ushort)dimensions, metric)); + + var target = CreateVector(random, dimensions); + const int limit = 12; + + var query = collection.Query() + .WhereNear(x => x.Embedding, target, double.MaxValue) + .Limit(limit); + + var plan = query.GetPlan(); + plan["index"]["mode"].AsString.Should().Be("VECTOR INDEX SEARCH"); + + var results = query.ToArray(); + + results.Should().HaveCount(limit); + + var searchIds = InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + return service.Search(metadata, target, double.MaxValue, limit) + .Select(result => BsonMapper.Global.ToObject(result.Document).Id) + .ToArray(); + }); + + results.Select(x => x.Id).Should().Equal(searchIds); + } + + [Theory] + [InlineData(VectorDistanceMetric.Cosine)] + [InlineData(VectorDistanceMetric.Euclidean)] + [InlineData(VectorDistanceMetric.DotProduct)] + public void TopKNear_MatchesReferenceOrdering(VectorDistanceMetric metric) + { + using var db = new LiteDatabase(":memory:"); + var collection = db.GetCollection("vectors"); + + var random = new Random(5461); + const int dimensions = 6; + + var documents = Enumerable.Range(1, 48) + .Select(i => new VectorDocument + { + Id = i, + Embedding = CreateVector(random, dimensions), + Flag = i % 4 == 0 + }) + .ToList(); + + collection.Insert(documents); + + collection.EnsureIndex( + "embedding_idx", + BsonExpression.Create("$.Embedding"), + new VectorIndexOptions((ushort)dimensions, metric)); + + var target = CreateVector(random, dimensions); + const int limit = 7; + var expected = ComputeExpectedRanking(documents, target, metric, limit); + + var results = collection.Query() + .TopKNear(x => x.Embedding, target, limit) + .ToArray(); + + results.Should().HaveCount(expected.Count); + results.Select(x => x.Id).Should().Equal(expected.Select(x => x.Id)); + } + + [Fact(Skip = "Skip for now cause flaky test. Feature is moved in the future so fixing now is not priority for now.")] + public void VectorIndex_HandlesVectorsSpanningMultipleDataBlocks_PersistedUpdate() + { + using var file = new MemoryStream(); + + var dimensions = ((DataService.MAX_DATA_BYTES_PER_PAGE / sizeof(float)) * 10) + 16; + dimensions.Should().BeLessThan(ushort.MaxValue); + + var random = new Random(7321); + var originalDocuments = Enumerable.Range(1, 12) + .Select(i => new VectorDocument + { + Id = i, + Embedding = CreateVector(random, dimensions), + Flag = i % 2 == 0 + }) + .ToList(); + + var updateRandom = new Random(9813); + var documents = originalDocuments + .Select(doc => new VectorDocument + { + Id = doc.Id, + Embedding = CreateVector(updateRandom, dimensions), + Flag = doc.Flag + }) + .ToList(); + + using (var setup = new LiteDatabase(file)) + { + var setupCollection = setup.GetCollection("vectors"); + setupCollection.Insert(originalDocuments); + + var indexOptions = new VectorIndexOptions((ushort)dimensions, VectorDistanceMetric.Euclidean); + setupCollection.EnsureIndex("embedding_idx", BsonExpression.Create("$.Embedding"), indexOptions); + + foreach (var doc in documents) + { + setupCollection.Update(doc); + } + + setup.Checkpoint(); + } + + using var db = new LiteDatabase(file); + var collection = db.GetCollection("vectors"); + + var (inlineDetected, mismatches) = InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + metadata.Should().NotBeNull(); + metadata.Dimensions.Should().Be((ushort)dimensions); + + var dataService = new DataService(snapshot, uint.MaxValue); + var queue = new Queue(); + var visited = new HashSet(); + var collectedMismatches = new List(); + var inlineSeen = false; + + if (!metadata.Root.IsEmpty) + { + queue.Enqueue(metadata.Root); + } + + while (queue.Count > 0) + { + var address = queue.Dequeue(); + if (!visited.Add(address)) + { + continue; + } + + var node = snapshot.GetPage(address.PageID).GetNode(address.Index); + inlineSeen |= node.HasInlineVector; + + for (var level = 0; level < node.LevelCount; level++) + { + foreach (var neighbor in node.GetNeighbors(level)) + { + if (!neighbor.IsEmpty) + { + queue.Enqueue(neighbor); + } + } + } + + var storedVector = node.HasInlineVector + ? node.ReadVector() + : ReadExternalVector(dataService, node.ExternalVector, metadata.Dimensions); + + using var reader = new BufferReader(dataService.Read(node.DataBlock)); + var document = reader.ReadDocument().GetValue(); + var typed = BsonMapper.Global.ToObject(document); + + var expected = documents.Single(d => d.Id == typed.Id).Embedding; + if (!VectorsMatch(expected, storedVector)) + { + collectedMismatches.Add(typed.Id); + } + } + + return (inlineSeen, collectedMismatches); + }); + + Assert.False(inlineDetected); + mismatches.Should().BeEmpty(); + + foreach (var doc in documents) + { + var persisted = collection.FindById(doc.Id); + Assert.NotNull(persisted); + Assert.True(VectorsMatch(doc.Embedding, persisted.Embedding)); + + var result = InspectVectorIndex(db, "vectors", (snapshot, collation, metadata) => + { + var service = new VectorIndexService(snapshot, collation); + return service.Search(metadata, doc.Embedding, double.MaxValue, 1).FirstOrDefault(); + }); + + Assert.NotNull(result.Document); + var mapped = BsonMapper.Global.ToObject(result.Document); + mapped.Id.Should().Be(doc.Id); + result.Distance.Should().BeApproximately(0d, 1e-6); + } + } + + private static bool VectorsMatch(float[] expected, float[] actual) + { + if (expected == null || actual == null) + { + return false; + } + + if (expected.Length != actual.Length) + { + return false; + } + + for (var i = 0; i < expected.Length; i++) + { +#if NETFRAMEWORK + var expectedBits = BitConverter.ToInt32(BitConverter.GetBytes(expected[i]), 0); + var actualBits = BitConverter.ToInt32(BitConverter.GetBytes(actual[i]), 0); + if (expectedBits != actualBits) + { + return false; + } +#else + if (BitConverter.SingleToInt32Bits(expected[i]) != BitConverter.SingleToInt32Bits(actual[i])) + { + return false; + } +#endif + } + + return true; + } + + private static float[] ReadExternalVector(DataService dataService, PageAddress start, int dimensions) + { + Assert.False(start.IsEmpty); + Assert.True(dimensions > 0); + + var totalBytes = dimensions * sizeof(float); + var vector = new float[dimensions]; + var bytesCopied = 0; + + foreach (var slice in dataService.Read(start)) + { + if (bytesCopied >= totalBytes) + { + break; + } + + var available = Math.Min(slice.Count, totalBytes - bytesCopied); + Assert.Equal(0, available % sizeof(float)); + + Buffer.BlockCopy(slice.Array, slice.Offset, vector, bytesCopied, available); + bytesCopied += available; + } + + Assert.Equal(totalBytes, bytesCopied); + + return vector; + } + + } +} +// #else +// using Xunit; +// +// namespace LiteDB.Tests.QueryTest +// { +// public class VectorIndex_Tests +// { +// [Fact(Skip = "Vector index tests are not supported on this target framework.")] +// public void Vector_Index_Not_Supported_On_NetFramework() +// { +// } +// } +// } diff --git a/LiteDB.Tests/Query/Where_Tests.cs b/LiteDB.Tests/Query/Where_Tests.cs index 00869f139..8d83ceb27 100644 --- a/LiteDB.Tests/Query/Where_Tests.cs +++ b/LiteDB.Tests/Query/Where_Tests.cs @@ -1,109 +1,182 @@ -using FluentAssertions; -using System.Linq; +using System.Linq; +using System.Threading.Tasks; using Xunit; +using Xunit.Abstractions; namespace LiteDB.Tests.QueryTest { public class Where_Tests : PersonQueryData { + private readonly ITestOutputHelper _output; + + public Where_Tests(ITestOutputHelper output) + { + _output = output; + } + class Entity { public string Name { get; set; } public int Size { get; set; } } - [Fact] - public void Query_Where_With_Parameter() + [Fact(Timeout = 30000)] + public async Task Query_Where_With_Parameter() { - using var db = new PersonQueryData(); - var (collection, local) = db.GetData(); + var testName = nameof(Query_Where_With_Parameter); + + _output.WriteLine($"starting {testName}"); - var r0 = local - .Where(x => x.Address.State == "FL") - .ToArray(); + try + { + using var db = new PersonQueryData(); + var (collection, local) = db.GetData(); - var r1 = collection.Query() - .Where(x => x.Address.State == "FL") - .ToArray(); + var r0 = local + .Where(x => x.Address.State == "FL") + .ToArray(); - AssertEx.ArrayEqual(r0, r1, true); + var r1 = collection.Query() + .Where(x => x.Address.State == "FL") + .ToArray(); + + AssertEx.ArrayEqual(r0, r1, true); + } + finally + { + _output.WriteLine($"{testName} completed"); + } + + await Task.CompletedTask; } - [Fact] - public void Query_Multi_Where_With_Like() + [Fact(Timeout = 30000)] + public async Task Query_Multi_Where_With_Like() { - using var db = new PersonQueryData(); - var (collection, local) = db.GetData(); + var testName = nameof(Query_Multi_Where_With_Like); + + _output.WriteLine($"starting {testName}"); + + try + { + using var db = new PersonQueryData(); + var (collection, local) = db.GetData(); + + var r0 = local + .Where(x => x.Age >= 10 && x.Age <= 40) + .Where(x => x.Name.StartsWith("Ge")) + .ToArray(); - var r0 = local - .Where(x => x.Age >= 10 && x.Age <= 40) - .Where(x => x.Name.StartsWith("Ge")) - .ToArray(); + var r1 = collection.Query() + .Where(x => x.Age >= 10 && x.Age <= 40) + .Where(x => x.Name.StartsWith("Ge")) + .ToArray(); - var r1 = collection.Query() - .Where(x => x.Age >= 10 && x.Age <= 40) - .Where(x => x.Name.StartsWith("Ge")) - .ToArray(); + AssertEx.ArrayEqual(r0, r1, true); + } + finally + { + _output.WriteLine($"{testName} completed"); + } - AssertEx.ArrayEqual(r0, r1, true); + await Task.CompletedTask; } - [Fact] - public void Query_Single_Where_With_And() + [Fact(Timeout = 30000)] + public async Task Query_Single_Where_With_And() { - using var db = new PersonQueryData(); - var (collection, local) = db.GetData(); + var testName = nameof(Query_Single_Where_With_And); - var r0 = local - .Where(x => x.Age == 25 && x.Active) - .ToArray(); + _output.WriteLine($"starting {testName}"); - var r1 = collection.Query() - .Where("age = 25 AND active = true") - .ToArray(); + try + { + using var db = new PersonQueryData(); + var (collection, local) = db.GetData(); - AssertEx.ArrayEqual(r0, r1, true); + var r0 = local + .Where(x => x.Age == 25 && x.Active) + .ToArray(); + + var r1 = collection.Query() + .Where("age = 25 AND active = true") + .ToArray(); + + AssertEx.ArrayEqual(r0, r1, true); + } + finally + { + _output.WriteLine($"{testName} completed"); + } + + await Task.CompletedTask; } - [Fact] - public void Query_Single_Where_With_Or_And_In() + [Fact(Timeout = 30000)] + public async Task Query_Single_Where_With_Or_And_In() { - using var db = new PersonQueryData(); - var (collection, local) = db.GetData(); + var testName = nameof(Query_Single_Where_With_Or_And_In); + + _output.WriteLine($"starting {testName}"); + + try + { + using var db = new PersonQueryData(); + var (collection, local) = db.GetData(); - var r0 = local - .Where(x => x.Age == 25 || x.Age == 26 || x.Age == 27) - .ToArray(); + var r0 = local + .Where(x => x.Age == 25 || x.Age == 26 || x.Age == 27) + .ToArray(); - var r1 = collection.Query() - .Where("age = 25 OR age = 26 OR age = 27") - .ToArray(); + var r1 = collection.Query() + .Where("age = 25 OR age = 26 OR age = 27") + .ToArray(); - var r2 = collection.Query() - .Where("age IN [25, 26, 27]") - .ToArray(); + var r2 = collection.Query() + .Where("age IN [25, 26, 27]") + .ToArray(); - AssertEx.ArrayEqual(r0, r1, true); - AssertEx.ArrayEqual(r1, r2, true); + AssertEx.ArrayEqual(r0, r1, true); + AssertEx.ArrayEqual(r1, r2, true); + } + finally + { + _output.WriteLine($"{testName} completed"); + } + + await Task.CompletedTask; } - [Fact] - public void Query_With_Array_Ids() + [Fact(Timeout = 30000)] + public async Task Query_With_Array_Ids() { - using var db = new PersonQueryData(); - var (collection, local) = db.GetData(); + var testName = nameof(Query_With_Array_Ids); + + _output.WriteLine($"starting {testName}"); + + try + { + using var db = new PersonQueryData(); + var (collection, local) = db.GetData(); + + var ids = new int[] { 1, 2, 3 }; - var ids = new int[] { 1, 2, 3 }; + var r0 = local + .Where(x => ids.Contains(x.Id)) + .ToArray(); - var r0 = local - .Where(x => ids.Contains(x.Id)) - .ToArray(); + var r1 = collection.Query() + .Where(x => ids.Contains(x.Id)) + .ToArray(); - var r1 = collection.Query() - .Where(x => ids.Contains(x.Id)) - .ToArray(); + AssertEx.ArrayEqual(r0, r1, true); + } + finally + { + _output.WriteLine($"{testName} completed"); + } - AssertEx.ArrayEqual(r0, r1, true); + await Task.CompletedTask; } } -} \ No newline at end of file +} diff --git a/LiteDB.Tests/Resources/ingest-20250922-234735.json b/LiteDB.Tests/Resources/ingest-20250922-234735.json new file mode 100644 index 000000000..1bf07fb4b --- /dev/null +++ b/LiteDB.Tests/Resources/ingest-20250922-234735.json @@ -0,0 +1,3090 @@ +{ + "Id": { + "Timestamp": 0, + "Machine": 0, + "Pid": 0, + "Increment": 0, + "CreationTime": "1970-01-01T00:00:00Z" + }, + "Path": "C:\\Users\\W31rd0\\source\\repos\\External\\LiteDB\\docs-md\\bsondocument\\index.md", + "Title": "index.md", + "Preview": "BsonDocument - LiteDB :: A .NET embedded NoSQL database [Fork me on GitHub](https://github.com/mbdavid/litedb) * [HOME](/) * [DOCS](/docs/) * [API](/api/) * [DOWNLOAD](https://www.nuget.org/packages/LiteDB/) [![Logo](/images/logo_litedb.svg", + "Embedding": [ + -0.022403426, + -0.029359642, + -0.00046093663, + -0.059005667, + 0.0069930996, + 0.016273458, + -0.016602583, + -0.01131023, + 0.014870317, + -0.021842895, + -0.029844096, + 0.014618867, + 0.027098, + 0.017678386, + 0.1401081, + -0.007594465, + -0.010675366, + 0.013672935, + -0.019306185, + -0.013990937, + -0.010655935, + 0.012747789, + -0.009680307, + -0.0003725233, + -0.012974445, + -0.013457543, + -0.02004069, + -0.010407569, + 0.038830165, + 0.014161669, + -0.0070986836, + -0.019631581, + 0.006761844, + 0.013459354, + -0.0056088576, + 0.019640232, + -0.009875782, + -0.005763959, + -0.019631783, + -0.015914338, + 0.005236226, + -0.012504074, + -0.015525924, + 0.00023541797, + 0.005625254, + 0.033247076, + -0.011530345, + 0.039344907, + -0.025501683, + 0.028230812, + 0.008927498, + 0.0035762815, + -0.0031235365, + -0.16083123, + 0.025102789, + -0.0060775676, + -0.010795695, + -0.010582045, + -0.01895601, + 0.00030876783, + -0.030082349, + 0.018458271, + -0.023089295, + -0.01873548, + 0.008966433, + -0.011343133, + 0.007159017, + 0.044083297, + 0.007298392, + -0.01800921, + -0.012547977, + -0.017322049, + -0.012034455, + -0.030201742, + 0.000170388, + -0.033381708, + 0.018013721, + -0.016894994, + -0.0040284162, + 0.03271521, + -0.01723094, + -0.018345203, + -0.014204077, + -0.0012019876, + 0.006193583, + 0.02139132, + 0.010533943, + -0.020427756, + 0.0019250304, + -0.0049338606, + 0.0016191548, + -0.022113353, + 0.043970168, + -0.014303418, + -0.020823121, + -0.008389568, + 0.02995531, + 0.020378115, + 0.006410817, + 0.007303294, + 0.0004188727, + -0.0019098456, + -0.018554227, + -0.0071257097, + 0.01487092, + 0.023285754, + 0.03082619, + 0.024463473, + -0.0024837677, + 0.012460391, + 0.038928714, + -0.010430601, + -0.0019072209, + -0.017237872, + 0.00040964218, + -0.16451721, + 0.009652725, + 0.013252799, + -0.002995977, + -0.028014783, + -0.0129057625, + 4.781136E-05, + -0.04364987, + 0.0035441294, + -0.0010485147, + -0.026462981, + 0.0007883033, + 0.024853852, + 0.012571348, + -0.009706335, + 0.0045404267, + 0.010477795, + -0.028705193, + -0.002271904, + -0.005680185, + -0.014551858, + 0.0075496472, + 0.003543154, + -0.0026087153, + 0.012349665, + -0.005752813, + 0.015127715, + -0.004125628, + 0.005739068, + -0.0091581885, + 0.01948451, + -0.013440935, + 0.013362978, + -0.020010797, + 0.007836782, + 0.014539941, + 0.0016704247, + -0.00043680094, + 0.012970707, + 0.026974145, + -0.023815792, + 0.03286925, + -0.012004105, + -0.016490698, + 0.00996865, + -0.014870556, + 0.0075192875, + 0.006357703, + 0.014883676, + 0.009150702, + 0.025643738, + -0.016747843, + -0.020711513, + 0.0028553, + -0.011741321, + -0.01983534, + 0.048821017, + 0.018790346, + 0.002333851, + -0.029739859, + -0.0077868677, + 0.01775358, + 0.01937985, + -0.0075945724, + 0.008384411, + 0.0027218945, + 0.0045479177, + 0.008860463, + 0.022131134, + -0.006308928, + -0.0059358985, + 0.0062858174, + -0.009938677, + 0.019687274, + -0.034574825, + 0.006099144, + -0.030357888, + 0.020885527, + -0.034125403, + -0.009179564, + -0.046524983, + 0.0071112425, + 0.004672993, + 0.01613161, + 0.039354037, + 0.0109938085, + 0.018880246, + -0.019914784, + -0.013887132, + 0.005261161, + 0.008005545, + -0.0051253596, + -0.033223182, + 0.010891959, + 0.0049805283, + 0.027632454, + -0.0035193423, + 0.028158905, + -0.0066806963, + 0.015886184, + -0.013160046, + 0.000425512, + -0.040521357, + -0.011585348, + -0.023668295, + 0.011333932, + -0.023659935, + 0.021757647, + 0.01771396, + -0.004675648, + -0.011869437, + 0.004460686, + -0.01891154, + -0.015691718, + -0.0047947476, + 0.019440105, + -0.0017463372, + 0.002931811, + -0.007390744, + 0.016800331, + -0.015000693, + -0.009002369, + -0.012018377, + 0.0050052283, + -0.012813123, + 0.009262901, + 0.0028900623, + -0.014743892, + 0.014309646, + 0.011422838, + -0.020850217, + -0.036729638, + -0.008145993, + 0.0023908825, + -0.002916819, + 0.0077269156, + 0.009115261, + -0.010952424, + -0.01739091, + 0.025924731, + 0.03183637, + -0.018783912, + 0.0029010065, + -0.0034771783, + -0.030588815, + 0.0033863182, + -0.017428339, + -0.000392883, + -0.034609813, + -0.0073661203, + 0.0046446323, + -0.008215339, + 0.0019364683, + -0.01290026, + -0.006092591, + 0.020428695, + -0.0111066345, + -0.011413782, + 0.029344479, + 0.013024779, + 0.01664524, + -0.035635605, + 0.0040397733, + -0.011791769, + -0.004657131, + -0.018260598, + -0.0017811331, + -0.006985467, + 0.009833211, + -0.005217589, + 0.014181848, + 0.0044849385, + 0.0012807052, + 0.012097408, + 0.0019277302, + 0.022266183, + -0.0055065863, + -0.011344616, + -0.013782056, + 0.0025386587, + -0.038948778, + -0.013953417, + -0.015070791, + -0.0035632655, + -0.015594945, + -0.013525266, + -0.0064704563, + -0.013779526, + 0.022395598, + 0.036651853, + -0.010250017, + -0.026273921, + 0.01719722, + -0.008974516, + 0.0075204824, + -0.009500355, + 0.005451769, + 0.0034252924, + -0.019310236, + -0.008066025, + 0.010718738, + 0.014653356, + -0.017613143, + -0.0016982113, + -0.015599402, + 0.0052340436, + -0.00091375614, + -0.02551274, + -0.0015617283, + -0.027988646, + -0.01658576, + -0.008463128, + 0.0110607445, + 0.050118092, + -0.012678958, + 0.007808872, + -0.021110876, + 0.009279971, + 0.0034684334, + -0.018151732, + 0.01145809, + 0.0062579643, + -0.008938457, + 0.013874842, + -0.0112013975, + 0.0009349033, + -0.015584958, + -0.018339615, + -0.018574417, + 0.0070009166, + -0.010387486, + 0.006115832, + -0.006258328, + -0.004195821, + 0.005444728, + -0.019064141, + 0.02018605, + 0.0016972887, + -0.001831184, + -0.027230285, + -0.013996396, + -0.016179409, + 0.0073284195, + 0.013820795, + 0.008493814, + -0.01365775, + -0.006140682, + 0.00034712878, + -0.026017336, + -0.005849598, + -0.015940739, + 0.027835889, + -0.0018492535, + -0.034484934, + -0.021779928, + -0.022607213, + -0.009553178, + -0.0119207, + -0.006829833, + -0.014911653, + 0.013614235, + 0.0006104121, + -0.015625784, + -0.001959747, + -0.013153557, + 0.012142541, + -0.002873048, + -0.022675933, + -0.008006445, + -0.0006810303, + 0.030733828, + 0.0130840605, + 0.0010474996, + -0.013813878, + -0.016588597, + -0.013086397, + 0.009064899, + 0.02293584, + 0.0012730457, + -0.013586395, + -0.009338724, + -0.019171966, + -0.012923315, + -0.009867184, + -0.009523822, + -0.0030542219, + 0.03269194, + -0.0027963559, + -0.0036135472, + -0.0035010143, + 0.0019733212, + 0.031656876, + -0.017178593, + -0.013005595, + 0.0002524583, + 0.0022101142, + 0.00467781, + 0.024113836, + -0.022404559, + 0.039458413, + 0.01097727, + -0.03244478, + 0.003397581, + -0.0034327041, + -0.010886031, + -0.0026915057, + -0.0031378202, + -0.0016870847, + -0.0068102838, + -0.010067266, + 0.002168535, + -0.030142225, + 0.011143216, + -0.003951502, + -0.012750695, + -0.015970359, + 0.0010807647, + -0.0077205193, + 0.0032559074, + -0.002868423, + -0.033807516, + -0.011061472, + 0.0026036685, + -0.027909262, + -0.007967789, + 0.021227775, + 0.023293804, + 0.009211581, + 0.0070890537, + 0.004000701, + 0.0064108707, + -0.0052853953, + -0.010673632, + -0.009399295, + -0.0050287414, + 0.027811114, + 0.0071800603, + 0.011476593, + -0.00795184, + 0.02306178, + 0.020515239, + -0.009111394, + 0.0055374526, + -0.031698085, + 0.0025499826, + -0.002754352, + -0.0043336507, + 0.011706446, + 0.0239228, + -0.023223782, + -0.010566733, + 0.009001384, + 0.036829617, + -0.03785355, + 0.0008528181, + -0.024422845, + 0.04685594, + 0.020786878, + 0.003910511, + 0.002976195, + -0.00014327833, + 0.010719232, + 0.0051642423, + 0.024394913, + -0.013672723, + -0.016430967, + -0.019116169, + 0.0009433035, + -0.00856265, + 0.003652576, + -7.39887E-05, + 0.040278137, + 0.009346578, + 0.013431628, + 0.0061533884, + -0.008417977, + 0.018369822, + 0.012031584, + -0.012147189, + -0.016410058, + -0.007519508, + 0.0054452866, + 0.011563157, + 0.0075719524, + -0.007048046, + -0.0068441452, + -0.006706628, + 0.0069890325, + -0.022902997, + -0.016602585, + -0.004388637, + 0.012122343, + -0.024496613, + 0.025846984, + 0.02755195, + -0.0034658143, + 0.0042013624, + 0.009003572, + -0.0051420303, + -0.02228143, + 0.027584864, + -0.0031827393, + -0.0124035515, + -0.012124466, + -0.013270807, + -0.009593365, + -0.020928567, + 0.010119675, + 0.013628045, + -0.03411174, + -0.003024132, + -0.0060310843, + -0.0010840794, + -0.014609187, + -0.0077570057, + -0.0027848904, + 0.016531928, + 0.0075998195, + -0.03174341, + 0.02211748, + -0.0031138104, + 0.0040433942, + 0.003844227, + 0.018291535, + -0.011323092, + -0.013766128, + -0.005808253, + -0.0077797025, + -0.011547211, + 0.003824252, + -0.078102104, + -0.0015570117, + -0.008590909, + -0.013789212, + -0.025533104, + -0.008527448, + -0.007795849, + -0.013372288, + -0.013732528, + 0.024658939, + 0.014171842, + 0.016304497, + 0.02645101, + -0.006039021, + -0.0074556507, + -0.008077784, + -0.015222476, + 0.023978785, + 0.019822367, + 0.0014989033, + -0.002013983, + 0.013636462, + -0.014814738, + -0.00025208152, + -0.019090965, + 0.011560428, + 0.021495236, + 0.010152527, + 0.018160332, + 0.022929028, + -0.031701952, + -0.0082598105, + 0.01928083, + -0.004332563, + -0.011753731, + 0.036390845, + 0.012432513, + -0.035916287, + -0.0083868215, + 0.006663444, + 0.005296124, + 0.011707916, + 0.015266973, + -0.021903545, + 0.008203896, + 0.015312614, + 0.0129980305, + -0.03306198, + -0.013049136, + 0.026203483, + -0.016491877, + -0.020277446, + 0.005367826, + -0.009134419, + -0.017075308, + 0.00038927514, + -0.0014361677, + -0.006433339, + -0.023002146, + 0.0249403, + -0.01012994, + 0.010449045, + -0.012741689, + 0.027932866, + 0.010738298, + 0.026318755, + 0.02294105, + -0.009826604, + -0.011871204, + 0.018868709, + -0.033784658, + 0.009507048, + -0.005687965, + 0.008155046, + -0.0046076993, + 0.010968884, + 0.00974308, + 0.022978095, + 0.017266527, + 0.014776249, + -0.014547683, + 0.008939246, + -0.08159173, + -0.03463518, + 0.03514535, + -0.010358899, + 0.017536126, + 0.014454773, + -0.001627316, + -0.03324442, + 0.0011652842, + -0.03183434, + 0.001111537, + -0.0041838847, + -0.0032984258, + -0.036241427, + -0.012570242, + 0.0026739545, + 0.0023263765, + -0.0037495561, + -0.020421939, + -0.02144136, + 0.013654157, + -0.032001544, + -0.010693061, + -0.0068003493, + -0.018219352, + 0.024157746, + -0.015744325, + 0.0075330785, + -0.023628606, + 0.00062292354, + 0.00047410352, + -0.12649885, + -0.010910126, + 0.008995855, + -0.015693046, + -0.0027200172, + 0.01977412, + 0.012120607, + -0.035438597, + 0.02374227, + -0.020763472, + -0.02686843, + -0.017260265, + -0.03262318, + 0.016277162, + 0.010031226, + 0.106760435, + -0.0077569163, + 0.0006890077, + 0.0064676898, + 0.0031779467, + -0.027710581, + -0.023537492, + 0.009551793, + -0.023765434, + -0.011283847, + 0.0013757058, + -0.006923834, + -0.015028035, + -0.0006454949, + 0.0058666677, + -0.007090307, + -0.00993452, + 0.0006385806, + 0.028003342, + 0.0074981055, + -0.0053397142, + -0.0087134475, + -0.0032118564, + 0.027803402, + 0.014910258, + 0.00034493007, + 0.03389886, + 0.018581955, + -0.027009197, + -0.00983733, + 0.01044897, + 0.0031726898, + -0.0066847457, + -0.003864188, + 0.00602833, + 0.0049895267, + -0.08008352, + 0.018167095, + 0.0094885975, + 0.049853675, + -0.020266522, + -0.013743679, + 0.013598024, + 0.04878666, + 0.0013419393, + 0.001127964, + -0.005819026, + 0.0026221436, + 0.0052104658, + -0.016922308, + -0.025739143, + 0.009519062, + 0.023665804, + 0.0028368826, + -0.0072783665, + 0.017145472, + 0.010514104, + 0.0037155843, + 4.6225086E-05, + -0.006698918, + -0.02804745, + -0.025354587, + -0.019836614, + 0.01084292, + 0.0073829168, + 0.008961453, + 0.007560511, + -0.0054335776, + -0.017135251, + 0.010809551, + -0.03485094, + 0.018528178, + 0.032770816, + 0.016732847, + -0.038354445, + 0.013157594, + 0.041931957, + 0.0036351935, + 0.0077127195, + -0.012258655, + 0.0156083, + 0.017892826, + 0.022659648, + -0.014511685, + -0.015630797, + -0.018078338, + 0.00041883165, + 0.028109275, + -0.008190806, + -0.034542065, + -0.00034415396, + 0.02180523, + 0.008063441, + 0.04667555, + -0.0022326105, + -0.017396484, + -0.0050876914, + -0.012789886, + 0.0009428939, + 0.00088623323, + 0.02232849, + 0.007919229, + -0.0076296986, + -0.005407187, + 0.0034279812, + 0.0075282245, + 0.011273183, + -0.0051901806, + 0.0030135452, + 0.0051557124, + 0.004969021, + -0.00024476353, + 0.025037019, + -0.001902621, + -0.02987793, + -0.0121832555, + 0.0026140544, + -0.0062692906, + 0.0009812334, + 0.013797912, + 0.00310298, + 0.006958881, + -0.014745036, + -0.025971573, + -0.011986872, + 0.019106124, + 0.0068664583, + 0.012334328, + -0.015964035, + 0.023813663, + -0.0015877802, + -0.001377511, + -0.00047856814, + -0.01207296, + -0.029436998, + 0.008924944, + 0.00086105807, + 0.0017695171, + 0.017819542, + -0.00022060973, + 0.0048594265, + -0.013109865, + 0.011035071, + -0.0011105031, + -0.012653495, + 0.0019760493, + 0.000530523, + -0.005567533, + -0.010171221, + -0.0005226935, + -0.0058339722, + 0.009662666, + -0.011289659, + -0.009150747, + -0.0004328751, + 0.010829806, + -0.0014539463, + -0.012057369, + -0.011943067, + 0.0077421186, + -0.0059487564, + -0.008530657, + 0.011675373, + -0.0042761406, + 0.018299935, + 0.013196051, + 0.0014760679, + 0.010491785, + 0.016590657, + 0.011916241, + 0.0004802312, + -0.011157987, + 0.010765805, + -0.014462595, + -0.008190835, + -0.00094058935, + 0.005274542, + 0.0075358767, + -0.015831959, + -0.006446908, + -0.013262572, + -0.017463302, + -0.0011763169, + -0.0025349394, + 0.008405723, + -0.0037472495, + -0.0057604667, + 0.004786326, + 0.00076421554, + 0.003074852, + -0.0063720755, + -0.004565334, + -0.010751763, + -0.0074365833, + 0.0013923798, + -0.00013741392, + 0.02150081, + 0.016540403, + 0.0012492732, + -0.0024438782, + -0.021804811, + 0.0022783962, + 0.026360417, + -0.005713466, + 0.0017834831, + -0.0019549597, + 0.0072752037, + -0.004926768, + 0.013349844, + 0.019007785, + 0.010730276, + 0.006946448, + 0.011081868, + -0.0044664396, + 0.0065287217, + 0.023307659, + 0.0019957144, + 0.019575158, + 0.004931743, + 0.00710419, + 0.006090774, + 0.006268074, + -0.01527183, + 0.0062349336, + 0.0036580411, + -0.004297376, + -0.005302215, + 0.005874469, + 0.0054443968, + 0.009540783, + 0.012502216, + 0.013976434, + 0.011273794, + 0.0023419864, + 0.0049912413, + -0.016430214, + -0.00065225514, + 0.024998872, + 0.0076766177, + -0.015379432, + 0.009843317, + -0.011276321, + -0.006213273, + -0.018192504, + -0.015914155, + 0.00591143, + -0.008424492, + -0.0017707477, + -0.013842965, + 0.024825696, + -0.0073274453, + -0.013979986, + 0.012305418, + 0.007876769, + 0.011879324, + 0.021914901, + -0.012786321, + 0.0063343095, + 0.00045215344, + -0.0055197757, + -0.0009348646, + -0.0055024964, + 0.008673907, + -0.021608308, + 0.003792011, + 0.014024904, + 0.002974364, + -0.011505025, + 0.0010879757, + 0.00014588356, + 0.008107588, + 0.014413345, + -0.0067233406, + 0.0065462277, + -0.015878595, + -0.0016072295, + 0.008702469, + 0.010145347, + 0.007140455, + -0.005311145, + 0.005303868, + 0.005667304, + 0.00670874, + 0.014773951, + 0.008748865, + -0.0011739944, + -0.00092761766, + -0.010325828, + -0.003928573, + 0.003607135, + 0.005393052, + 0.0071756938, + 0.010256883, + 0.028422255, + 0.003381697, + 0.1263441, + 0.008741394, + -0.00080573675, + -0.008343679, + -0.018503975, + -0.004153603, + -0.0204835, + -0.01627261, + -0.013768354, + 0.0043000323, + 0.0055107083, + 0.00061074423, + 0.020405216, + 0.0027704667, + 0.01727016, + -0.0013215333, + -0.01258108, + -0.01018823, + 0.017535536, + 0.012677422, + -0.021991441, + 0.011160045, + -0.013324494, + -0.0048012566, + -0.004719847, + 0.012311401, + 0.01723689, + -0.006481369, + -0.007060782, + 0.008597284, + -0.0053196303, + -0.015361636, + 0.00075633236, + 0.013377137, + -0.027337, + -0.004500213, + -0.00872394, + 0.013821406, + 0.0063662706, + 0.019433996, + -0.0065054474, + -0.013503373, + 0.006084507, + -0.010072519, + -0.016407043, + 0.002832738, + -0.006717647, + 0.009724987, + 0.0014942755, + -0.010678373, + 0.012008322, + -0.006619366, + -0.005084279, + 0.0014388277, + -0.010176289, + -0.0055753533, + -0.013038575, + -0.016222632, + 0.016835261, + -0.015263472, + 0.0008645661, + 0.001295511, + -0.00026129774, + -0.011759816, + 0.00091820373, + -0.009304012, + 0.010321136, + 0.010878747, + -0.0052248384, + -0.007602058, + 0.00030591976, + 0.005054866, + -0.0070585003, + 0.006434543, + 0.029360786, + -0.013923718, + -0.018923609, + 0.0003857755, + 0.0023698388, + -0.0052679842, + -0.0027138293, + 0.0075761783, + 0.0055495803, + -0.0029426378, + 0.0059879185, + 0.0076191504, + -0.009869376, + -0.011196346, + 0.030623361, + 0.006172501, + -0.02525901, + -0.0043444913, + -0.0022055025, + -0.0012884306, + -0.016687125, + 0.007942332, + 0.051323887, + 0.0026077947, + -0.019360093, + 0.0017541646, + -0.0033439596, + -0.0213014, + -0.0058389576, + -0.007502055, + 0.02000536, + 0.01206386, + 0.024201715, + 0.00264272, + -0.01870395, + 0.020702185, + -0.019997414, + -0.0024404768, + 0.0011769983, + 0.01190267, + 0.015364816, + -0.022472668, + 0.00028256892, + 0.006251515, + 0.016799033, + 0.0058718417, + 0.034608185, + 0.00043993315, + -0.0080579445, + 0.0037516034, + 0.0076072123, + 0.0056914645, + -0.017179202, + -0.018264053, + 0.015701333, + -0.013586785, + 0.0009744583, + -0.01655356, + 0.007182411, + -0.011493531, + 0.013163043, + 0.0036099597, + 0.0030518884, + -0.008411569, + -0.007130063, + -0.0053318506, + -0.019845085, + -0.0032917124, + 0.021036124, + 0.0101882145, + -0.008656253, + 0.002317305, + -0.015252207, + 0.004897933, + 0.015121773, + -0.011605336, + 0.008038118, + -0.029585008, + 0.012828104, + 0.005008612, + 0.0020089047, + -0.0035546592, + 0.013504256, + -0.00029786865, + -0.020036787, + 0.0074441843, + 0.0031164845, + -0.0041071605, + 0.0027996853, + 0.0027062935, + 0.005812475, + -0.010379557, + 0.002562867, + 0.008600626, + 0.0006929746, + -0.0013505784, + -0.013901627, + 0.0064392886, + -0.013525709, + -0.002405837, + 0.00034825763, + 0.007866464, + 0.014475913, + -0.008371657, + -0.006230476, + 0.009810819, + -0.008321432, + 0.0141213015, + 0.006372089, + -0.004671006, + -0.014671973, + -0.009150884, + 0.0076055713, + 0.0062476015, + 0.013346353, + 0.0029518115, + -0.0060684485, + 0.011122555, + -0.002581921, + -0.014934149, + 0.0054629734, + -0.011766331, + -0.0111630475, + 0.0018732797, + 0.010010633, + -0.0061014057, + 0.017706187, + -0.019164316, + 0.028630191, + -0.008193841, + 0.002141769, + 0.0040491647, + -0.010385297, + 0.01115, + 0.0019539369, + -0.0053933, + -0.019689282, + -0.015107434, + 0.02016279, + 0.018938288, + 0.0064440123, + 0.0024991767, + 0.003062599, + 0.0014320267, + 0.0038975312, + -0.002968092, + 0.006186391, + -0.01772198, + -0.010638225, + 0.0053245714, + 0.015409638, + 0.0101654325, + 0.018193655, + -0.006299927, + 0.0022456802, + 0.0019648105, + 0.0068175653, + 0.00036282997, + -0.0017757169, + 0.0073594525, + -0.010851468, + 0.010879217, + 0.0013796586, + 0.0048216498, + -0.0049242275, + -0.012197847, + 0.00328047, + -0.0026194786, + -0.0077501987, + 0.005607967, + -0.00687081, + -0.006051933, + 0.005529366, + -0.008795625, + 0.0021690018, + 0.0048948317, + 0.010866955, + 0.0022002805, + 0.009563478, + -0.008557941, + 0.014509304, + -0.0077684224, + -0.0032681497, + -0.0530963, + 0.007805546, + 0.015976945, + 0.0050839353, + 0.0019480818, + 0.005957345, + 0.003771156, + 0.0010133768, + 0.025053868, + -0.004534793, + 0.010574314, + -0.003073328, + -0.0077332226, + 0.004451011, + -0.00072178955, + 0.01171368, + 0.0061754826, + 0.0050637443, + -0.008064818, + -0.0069785956, + 0.0036324281, + 0.004420638, + -0.014198406, + 0.013037858, + -0.004683345, + -0.011972224, + -0.005468102, + 0.006813291, + -0.014931917, + 0.0023561227, + -0.0004987337, + -0.0068944907, + -0.0027524135, + -0.015376024, + 0.0016228976, + 0.018003942, + -0.01118482, + -0.009397608, + 0.0017242164, + -0.010885798, + 0.015809473, + 0.003341188, + -0.012195951, + 0.0024027792, + -0.017697657, + -0.015505318, + 0.0004136584, + -0.00068228826, + 0.014781206, + -0.0109744035, + -0.011791309, + 0.013331616, + -0.034926414, + 0.012626135, + 0.012969697, + -0.0072565894, + 0.024529815, + 0.019245943, + -0.011937941, + 0.0023310375, + 0.020617995, + -0.014887795, + 0.012476784, + -0.0051012193, + 0.0111463675, + -0.0022975386, + 0.018756688, + 0.0145178735, + -0.016024016, + 0.0030910957, + 0.0056914785, + 0.018116176, + -0.013680671, + 0.0155054685, + -0.014127937, + 0.024496699, + 0.0065128263, + 0.01785291, + 0.0075107017, + 0.0046252054, + -0.0014514105, + 0.022361826, + -0.0074819042, + -0.009047077, + -0.0110061765, + -0.0049529285, + -0.016844323, + -0.011856573, + -0.017411485, + -0.0015362019, + 0.010072468, + -0.0021098044, + 0.0021358654, + -0.0066477424, + -0.0007583362, + 0.020409571, + -0.020498965, + 0.00086744153, + -0.0005864157, + 0.0031840997, + 0.0035462773, + -0.02187009, + 0.010319466, + 0.006114995, + 0.0066021904, + 0.008463728, + -0.0019553457, + -0.00024574794, + -0.006343625, + -0.0014450888, + 0.004921786, + 0.01611277, + 0.015325633, + -0.009468702, + 0.027850527, + -0.0036389565, + 0.006198279, + 0.0049907514, + 0.001811113, + -0.013492741, + -0.00775541, + -0.006343879, + -0.012596628, + -0.0054396465, + 0.0016635455, + 0.0057523483, + -0.0039318665, + 0.0023677459, + 0.0066465037, + 0.0026523552, + -0.0013364201, + 0.007217081, + 0.009725687, + 0.017493865, + -0.0059177373, + 0.009506606, + 0.017675813, + 0.009544032, + 0.00017188524, + -0.0028138047, + 0.012507577, + 0.00860561, + 0.00075085077, + 0.0125791775, + -0.0018191249, + 0.0009709518, + 0.010165586, + 0.015283586, + 0.0030456665, + -0.0029762024, + 0.017285604, + 0.0039997636, + 0.0061502843, + 0.007959619, + -0.0063300743, + 0.009908685, + -0.0012322901, + -0.011381605, + 0.006956112, + 0.014550747, + 0.012774769, + 0.0005305543, + 0.009057106, + 0.0020901235, + -0.003110093, + -0.0048880004, + -0.011444917, + 0.0037264735, + -0.0056063714, + -0.0085616885, + -0.0015814349, + -0.0022217243, + 0.00028351598, + 0.015148983, + 0.020803196, + 0.005401527, + 0.0064127883, + -0.011043859, + -0.017961424, + -0.004358304, + 0.0031581128, + 0.010835771, + 0.009455179, + 0.014070414, + 0.004028968, + 0.003024169, + 0.0008448565, + 0.013698257, + -0.0075923605, + -0.021146623, + -0.0022490337, + -0.008081525, + 0.0029291175, + -0.0023516268, + 0.011807142, + 0.0011784676, + 0.00039981605, + 0.004181234, + 0.024691602, + -0.009152487, + -0.0039128317, + 0.008490058, + 0.018238572, + -0.016004356, + 0.003555961, + -0.08192597, + -0.000863234, + -0.009006875, + 0.009619861, + -0.0027179252, + 0.020066164, + -0.0017795301, + 0.00018886555, + -0.005783898, + -0.0025056556, + -0.011164263, + -0.008511682, + -0.001284353, + -0.0031162365, + 0.033287182, + 0.010377032, + 0.004939006, + 0.0005506017, + -0.00868646, + -0.009470227, + -0.0038310713, + -0.011481106, + 0.004479436, + 0.009711419, + 0.01086303, + -0.009318759, + 0.009689785, + -0.00855838, + 0.007702924, + 0.00069479295, + 0.00090084446, + 0.0046790233, + 0.0034036764, + -0.011460945, + 0.026092483, + -0.01201649, + 0.0033445498, + -5.279864E-05, + -0.1332034, + 2.4253413E-05, + -0.009195211, + -0.0039253123, + -0.016108748, + -0.0134321805, + 0.012766984, + -0.00497531, + -0.0035076768, + 0.016052656, + -0.008132208, + -0.018928176, + -0.010116235, + -0.0063502905, + 0.016779248, + -0.0038339645, + -0.0070627555, + 0.010353457, + -0.0024090025, + -0.0023844168, + -0.003770092, + 0.009614418, + 0.0038176698, + 0.017323503, + -0.020593967, + 0.01166955, + 0.009518925, + 0.0019805203, + 0.012822677, + 0.006199852, + -0.00822887, + 0.015212161, + -0.0061634607, + 0.013435695, + -0.00043850671, + -0.0077619934, + -0.000549924, + 0.0032285484, + 0.00010197213, + 0.0018408968, + 0.0017096817, + 0.0005666017, + 0.0009807557, + -0.0053420123, + 0.0027655438, + 0.004059865, + 0.007891844, + -0.01496384, + 0.008258201, + -0.011087228, + -0.021488156, + 0.017932236, + -0.010556557, + -0.0010807072, + -0.007134259, + 0.005757239, + -0.0104712555, + 0.013452886, + -0.013961413, + -0.008747015, + -0.0077976305, + -0.013141693, + -0.000901419, + 0.0019266738, + 0.025519153, + 0.015874857, + -0.010957191, + -0.0035435683, + 0.016529012, + 0.013191712, + -0.012319121, + 0.0101846, + 0.0016823183, + 0.006239723, + 0.012725802, + 0.0009711219, + -0.011343138, + 0.029735828, + -0.0029868619, + 0.0044226116, + 0.025464853, + 0.019222565, + -0.021993918, + 0.006611412, + 0.002881808, + -0.015335298, + -0.014671643, + -0.008629292, + -0.0077357665, + -0.04420551, + -0.007518021, + -0.007319374, + -0.0026523278, + 0.027924443, + -0.009434613, + -0.029308885, + 0.028939899, + 0.0023533911, + -0.009086755, + 0.0005440956, + -0.021904282, + 0.0146433, + -0.0038466826, + -0.006441162, + -0.00219391, + -0.032377932, + -0.004796184, + -0.0064379363, + -0.0073602367, + 0.012153201, + 0.005263595, + -0.01572805, + 0.005822134, + 0.005642668, + -0.0012214105, + -0.0033807494, + -0.009600336, + -0.011106704, + -0.029986322, + 0.008659499, + -0.0024494545, + 0.01704612, + -0.0005304793, + 0.0026042426, + 0.010731772, + 0.023387307, + 0.009707397, + 0.001508715, + -0.012373334, + 0.015140698, + -0.0028596863, + -0.010611514, + -0.023112742, + 0.0226843, + 0.0061762403, + -0.013474421, + -0.008386322, + 0.0031241225, + 0.0041317195, + -0.0010159205, + 0.0105968015, + 0.00043799172, + -0.005834594, + 0.013283638, + 0.0130304005, + 0.013847057, + -0.006041051, + -0.00963778, + -0.0011120961, + 0.022147268, + 0.0121100675, + -0.01734395, + 0.027094914, + 0.015339725, + 0.0002971211, + 0.006276745, + 0.014072737, + -0.004659107, + -0.009125492, + 0.007622575, + 0.018450402, + -0.0066224076, + 0.0002648744, + 0.021975562, + -0.004693732, + -0.012976548, + 0.01124325, + -0.006541572, + 0.0009263419, + 0.007190011, + 0.021140736, + -0.0081197545, + -0.004932599, + 0.015437145, + 0.0066260593, + 0.027118249, + 0.014593077, + -0.0032088377, + -0.0033292398, + 0.013349708, + -0.029363278, + -0.012508933, + -0.016097343, + 0.004759683, + -0.005843096, + 0.03748417, + -0.013641163, + -0.0055177324, + 0.02009207, + -0.010123171, + 0.023073364, + 0.0009887561, + -0.008047879, + 0.0132854525, + -0.0009275946, + 0.007528736, + -0.0034811634, + 0.010450366, + 0.0043061757, + 0.012476671, + -0.007572428, + 0.02145975, + -0.011115575, + -0.15586345, + -0.0049847644, + 0.010136536, + 0.01986175, + 0.0061132647, + -0.0052734595, + 0.028080694, + -0.006290384, + 0.015220576, + -0.032403175, + 0.012423833, + -0.013029465, + -0.0019253892, + 0.007280121, + 0.033705268, + 0.016577652, + 0.018399742, + -0.0068255365, + -0.03220923, + -0.004543452, + -0.018134546, + -0.01379701, + 0.0033623872, + 0.010253348, + -0.026692977, + 0.011411196, + 0.019158361, + 0.0059175715, + -0.0055471477, + 0.008524524, + -0.012932432, + -0.0044529443, + 0.0093733175, + 0.0033805605, + -0.016445197, + -0.0037921593, + -0.007254421, + 0.0138421515, + 0.016535528, + 0.002874398, + -0.010915042, + -0.02461902, + -0.0062816157, + 0.003728902, + -0.007805286, + 0.005006127, + -0.0015536208, + -0.004422211, + -0.0069445013, + -0.021799522, + 0.018444693, + -0.015989006, + 0.013417899, + -0.004434593, + -0.0059754606, + 0.011930365, + 0.009818013, + 0.004953622, + 0.007834619, + -0.019800426, + 0.015144467, + -0.02773726, + -0.003200348, + 0.0034979049, + 0.017708544, + -0.012051616, + -0.004248018, + 0.15758163, + -0.013601965, + 0.0024778869, + 0.040583454, + 0.008658015, + 0.0049698325, + -0.006107417, + -0.016287915, + -0.028166426, + -0.038496204, + -0.0019679659, + 0.012940593, + 0.009101482, + 0.009416192, + -0.018943729, + 0.010310642, + -0.0008360986, + 0.002723768, + 0.020714246, + 0.010936516, + 0.0006332087, + -0.0056363135, + 0.027396727, + -0.0040611844, + 0.0021783905, + 0.011983824, + 0.021303767, + -0.016198255, + 0.015345251, + -0.021439148, + 0.0128215095, + 0.0064273276, + -0.006650713, + -0.0027358322, + 0.015083691, + 0.021928344, + -0.007976437, + -0.0040155333, + -0.004834134, + 0.01121701, + 0.0016135224, + 0.001221014, + -0.005271285, + 0.0015285418, + -0.005426412, + -0.002742984, + -0.008080793, + 0.012594901, + -0.017165923, + -0.011754605, + 0.0139910355, + -0.017369218, + -0.0059719537, + 0.00072849845, + 0.008328111, + 0.001176375, + -0.01689741, + 0.03200672, + 0.009531328, + 0.008912266, + -0.015127736, + -0.012316762, + -0.010398042, + 0.009441018, + 0.02092796, + 0.0053705047, + 0.024156641, + 0.009700522, + 0.00084629207, + -0.11559938, + 0.00275736, + -0.029196778, + -0.01443438, + -0.00039392058, + 0.008371421, + -0.008150486, + -0.007904897, + 0.008033398, + 0.010925204, + -0.0022246155, + -0.019246133, + -0.017359078, + 0.0007854474, + 0.0014308429, + 0.007410896, + -0.027189344, + 0.0014928242, + 0.013243331, + -0.008023301, + -0.006079037, + 0.007806054, + -0.0024808284, + -0.013120234, + 0.014490909, + 0.009221225, + -0.028634522, + -0.0036730708, + 0.0058757165, + 0.010898348, + -0.007601707, + -0.021969028, + -0.007292248, + -0.010118908, + -0.011705717, + -0.0025182986, + -0.029284723, + 0.036782682, + 0.005697732, + 0.007600543, + 0.01596234, + -0.010729596, + -0.016692292, + -0.0041297004, + 0.0047450406, + 0.00252439, + 0.017346254, + -0.0010321605, + -0.026022563, + -0.024422769, + 0.00595834, + 0.019153813, + 0.016662251, + -0.0008910078, + -0.005986011, + 0.015500279, + -0.005494228, + -0.0074392105, + 0.013190215, + 0.0014564113, + -0.0051085358, + -0.0004247, + -0.014679517, + -0.00045163653, + 0.012187741, + -0.0008657473, + 0.011705954, + -0.01902029, + 0.012910283, + -0.015914895, + -0.009398208, + 0.004045795, + -0.007291905, + -0.013714174, + 0.009425267, + 0.001560041, + -0.0030936464, + -0.016992774, + -0.004160824, + -0.0074075535, + -0.027199024, + 0.0052860924, + 0.0044792704, + -5.140819E-05, + 0.033689942, + -0.0019714935, + 0.0030249376, + -0.023204071, + -0.008435415, + 0.0009143821, + -0.018069347, + -0.0051966943, + -0.009677451, + -0.0023233453, + -0.005756082, + -0.006357199, + 0.0066508837, + -0.01247577, + -0.011788698, + -0.023220489, + 0.0016846994, + 0.0036102626, + 0.0052183666, + -0.003419909, + 0.00018692843, + 0.019014845, + -0.0116044, + 0.00010301605, + 0.009499709, + -0.008905137, + 0.029028643, + 0.0022509694, + -0.021286778, + 0.0127313975, + -0.009010347, + -0.018467795, + -0.0002532008, + -0.017837608, + 0.0002952987, + -0.009473734, + 0.009307852, + 0.0022780187, + 0.010560145, + -0.015144415, + -0.0025940975, + -0.01029859, + -0.006643773, + -0.004919439, + -0.029378813, + -0.008840761, + -0.00024542637, + -0.00063210423, + 0.010503519, + -0.0059797945, + 0.023585966, + 0.0034685468, + 0.02246273, + -0.0017340666, + -0.0025739823, + -0.0005464606, + -0.014694852, + -0.019035056, + 0.0025440769, + 0.008356363, + -0.014339151, + -0.011637996, + 0.02056462, + 0.016978774, + 0.0004496715, + -0.0052826265, + 0.026628677, + 0.008549993, + -0.027672792, + 0.009063215, + 0.02615313, + 0.022252316, + -0.001098672, + 0.010224201, + -0.012870405, + 0.0030232917, + 0.01876105, + 0.00026956986, + -0.009898714, + 0.010110426, + 0.023035102, + 0.010099189, + 0.002456104, + 0.002836496, + -0.009493594, + -0.006613166, + -0.020841612, + -0.010797774, + 0.0108629195, + -0.01301814, + 0.014207206, + 0.008552833, + 0.0054358626, + 0.03021074, + 0.009422332, + -0.06593746, + -0.016496444, + -0.008681595, + -0.0029839089, + -0.006394424, + 0.015907038, + -0.002814864, + -0.0035938509, + 0.0016686489, + 0.0044826115, + 0.008552735, + 0.0056748325, + -0.000982712, + -0.024993734, + 0.005367305, + 0.014882862, + -0.020124028, + -0.0046708053, + -0.0046373256, + -0.016598942, + -0.024889382, + -0.0017903575, + 0.008025278, + -0.02059643, + -0.005422563, + -0.010899686, + -0.004982181, + -0.0053771245, + 0.004520475, + -0.00040341192, + 0.00015535369, + -0.005714153, + 0.0056416374, + -0.009641438, + 0.0036548942, + -0.009368841, + -0.0057911044, + -0.013908818, + -0.0008088097, + -0.045707583, + -0.0064272475, + -0.0035715094, + -0.048675366, + -0.0161071, + 0.006254275, + -0.0043884953, + -0.006453895, + 0.009878317, + 8.516172E-05, + -0.019025242, + 0.0030460514, + 0.0069113355, + -0.012684829, + -0.010764474, + 0.0025734222, + -0.027298296, + -0.003632992, + -0.0054196776, + -0.0068565514, + -0.014003866, + -0.0027226899, + -0.012300868, + -0.023790631, + -0.012171043, + -0.0034829325, + 0.0030637549, + -0.0007952009, + -0.014756004, + -0.000985789, + 0.028810078, + -0.0107897585, + -0.023620881, + -0.0025634195, + -0.023533847, + -0.010639978, + 0.0053601926, + 0.012753791, + 0.003534151, + -0.015340165, + 0.019528694, + -0.010181639, + 0.012922135, + 0.0036734901, + 0.022857867, + -0.015564488, + -0.008783747, + 0.0011775615, + -0.11216972, + -0.0027203541, + 0.0067609944, + -0.011277973, + -0.0069900565, + 0.03595616, + -0.0033048885, + 0.056033783, + 0.001219644, + 0.005770666, + 0.0049321014, + -0.012507909, + 0.0012956675, + 0.012054261, + 0.004174496, + 0.012720575, + 0.012954342, + 0.015011639, + -0.0025167728, + 0.017550852, + 0.017742135, + -0.026390063, + -0.009283738, + 0.026091825, + -0.0006922839, + -0.04266941, + -0.027839998, + -0.032156043, + -0.005504858, + 0.022884041, + 0.031296026, + 0.0002523288, + -0.021701338, + 0.008776762, + -0.011707956, + -0.013613593, + -0.009703498, + -0.004085717, + 0.019851122, + -0.01592772, + 0.011615404, + -0.0231172, + -0.014606867, + 0.00907302, + 0.019766696, + 0.007984248, + -0.019023396, + 0.0153540345, + 0.010922684, + -0.0068544825, + 0.005059119, + 0.0039419536, + 0.0049374257, + -0.012949319, + -0.006035883, + -0.022012172, + 0.016516706, + -0.012822898, + 0.009508104, + 0.006784028, + -0.00067208725, + 0.014975665, + -0.0069140145, + -0.01434424, + -0.015386578, + 0.016125156, + -0.022469273, + 0.0065864674, + -0.023036608, + 0.011012066, + 0.0019745391, + -0.020631885, + 0.006192669, + -0.0070069702, + -0.024393067, + 0.021034745, + -0.0017983561, + 0.020475889, + -0.0155953625, + -0.0001004365, + 0.016387356, + -0.016836476, + 0.0009832527, + 0.017399587, + -0.014886876, + 0.018579787, + 0.017073505, + 0.023394551, + 0.010080648, + -0.013155558, + 0.01502724, + -0.01498636, + 0.00931648, + 0.02230891, + -0.013214503, + -0.019491408, + 0.009339695, + -0.013190446, + 0.0039178524, + -0.01569694, + 0.012435557, + -0.0067359963, + 0.0026547431, + 0.0035489881, + -0.0083039105, + 0.0063172863, + 0.027557822, + -0.018408041, + 0.016015526, + -0.0039545293, + 0.0071599623, + 0.0057647545, + -0.008515855, + -0.017155854, + 0.012309949, + -0.012102689, + 0.00095549866, + -0.019265234, + 0.00697389, + -0.011290654, + -0.010160816, + -0.0066482755, + -0.013998708, + -0.014333915, + -0.0063800546, + -0.0006639693, + 0.0043133763, + 0.012596892, + 0.009537409, + 0.013800159, + -0.01646656, + 0.023684096, + 0.01261267, + -0.012432818, + -0.0018257508, + 0.01753085, + 0.002882447, + 0.01480059, + -0.01329215, + 0.015197865, + -0.015168839, + -0.001819818, + -0.004826855, + 0.0040393616, + 0.03001037, + 0.012443914, + -0.0043872604, + 0.022387978, + -0.015421536, + 0.01659672, + 6.413005E-05, + 0.0049724746, + 0.01159461, + -0.012909163, + 0.0029480793, + -0.0004624088, + -0.0027882445, + -0.011732155, + -0.010168474, + -0.0008787677, + -0.00222159, + 0.0041289898, + -0.0038382432, + 0.008087445, + 0.008933167, + 0.004995718, + 0.012082613, + -0.020807132, + 0.015785443, + 0.0045371926, + 0.038656224, + 0.012598347, + -0.017239263, + -0.015334001, + 0.03450599, + 0.007944306, + -0.0044667223, + -0.0035735099, + -0.009085883, + -0.023060728, + 0.022476526, + -0.031775832, + 0.0024212715, + 0.0040613, + -0.017213263, + 0.020796878, + 0.0048325267, + -0.01750123, + 0.007593233, + 0.013469285, + 0.001271786, + -0.0027662679, + 0.020561693, + -0.0045986846, + 0.007015318, + -0.00064752274, + 0.012384753, + 0.01850767, + 0.016539264, + 0.005409016, + 0.0040572966, + 0.009207506, + -0.0028449276, + 0.01172624, + -0.00381147, + -0.012543614, + -0.012616492, + 0.0041751983, + 0.0034730914, + -0.00041522953, + 0.015531239, + -0.01732566, + -0.0010700376, + -0.0015962155, + 0.019750757, + 0.017033769, + 0.011717385, + 0.006318085, + 0.001509402, + 0.022649935, + 0.014686437, + 0.010540203, + -0.022761311, + -0.0057676933, + 0.0049567176, + -0.007854571, + -0.012059976, + -0.020119168, + 0.019623937, + 0.020643411, + 0.0044748397, + 0.010166404, + -0.0018235943, + 0.014491588, + 0.010115415, + -0.0025891329, + -0.00043530777, + 0.005381494, + -0.017875208, + 0.028895741, + 0.025810504, + 0.00845577, + 0.0048469123, + 0.026600175, + -0.005095451, + -0.02336371, + 0.009342066, + 0.005542642, + 0.015045063, + 0.0014927487, + 0.0068422626, + 0.016094334, + -0.018924525, + -0.025126018, + 0.004543486, + 0.00043484027, + 0.014276456, + 0.011034836, + 0.011658914, + -0.02263875, + -0.023373317, + 0.013153496, + -0.0058448496, + 0.007769567, + -0.031305507, + -0.016739247, + -0.01240104, + 0.019422969, + -0.010515184, + 0.004023082, + 0.0030804551, + 0.022296526, + -0.012320264, + -0.009887619, + 0.0026760784, + -0.004812516, + -0.011617295, + -0.008945629, + 0.010489611, + 0.002678152, + 0.0069983616, + -0.009278736, + 0.0030230426, + 0.0054247566, + 0.015194039, + -0.022275893, + 0.004265837, + -0.012746338, + -0.010225737, + -0.000635282, + 0.022351464, + 0.0035695003, + 0.015066835, + -0.0066285394, + 0.02868317, + 0.0064685894, + 0.018515024, + -0.031006264, + 0.019218592, + 0.0058047054, + -0.014332682, + -0.012076745, + -0.022338344, + -0.007566538, + -0.008684084, + 6.1819037E-07, + 0.0016160603, + -0.0137433335, + 0.0029402592, + -0.0013988572, + 0.0013351865, + -0.0007047328, + -0.015126743, + -0.005401052, + -0.028184254, + 0.009872798, + -0.004238505, + 0.018008523, + -0.00071813463, + 0.0070101595, + 0.009638051, + 0.0066361846, + 0.017159766, + 0.007504244, + -0.004509977, + 0.010056429, + -0.0008211454, + 0.0043329885, + -0.0038109224, + 0.016614132, + -0.011834969, + -0.013945062, + 0.0046977014, + -0.018121071, + -0.015248206, + -0.0025274567, + -0.019140163, + 0.01309932, + 0.002989005, + 0.019784195, + -0.01050451, + 0.034265824, + 0.03656205, + -0.019843332, + 0.019724123, + 0.025351219, + 0.0035336295, + 0.030140147, + 0.029535076, + -0.032819774, + -0.029193938, + 0.008580551, + -0.004025638, + 0.0041727065, + -0.01637916, + -0.0011175651, + 0.024987718, + 0.014133839, + -0.004784661, + -0.0137453955, + -0.009331236, + 0.016334588, + 0.0030128544, + 0.0062519806, + -0.010380752, + 0.0041864025, + -0.009353927, + 0.0035493008, + -0.007700624, + 0.0051783957, + -0.0148645425, + 0.012393441, + -0.020747162, + 0.004673742, + 0.001965675, + -0.014216276, + 0.018210629, + -0.0127483355, + 0.0072607007, + 0.0012407143, + -0.017633315, + -0.019350283, + -0.013963972, + -0.021923847, + -0.027245933, + -0.0010087821, + 0.00982289, + 0.0046877833, + 0.0011583987, + -0.0034069968, + -0.006748866, + 0.0008220014, + 0.029567633, + -0.014782255, + -0.01710484, + 0.003755863, + -0.0127040045, + -0.02449824, + -0.017092042, + 0.013613122, + 0.0018351771, + -0.0099954875, + 0.014911085, + 0.012998064, + 0.023924842, + 0.017364722, + 0.021640945, + 0.0076111117, + -0.0028265011, + 0.0036293447, + 0.0012285439, + -0.0020441746, + 0.0030094509, + -0.0062689306, + -0.014508705, + 0.02065106, + 0.0056757587, + 0.014013548, + -0.009081501, + 0.005353121, + -0.01023637, + 0.025805904, + 0.016467307, + 0.0014356838, + -0.003145771, + 0.010923155, + -0.008257566, + -0.01894764, + 0.0065038637, + -0.006211383, + 0.0016889937, + -0.020440403, + 0.0017612337, + -0.021717018, + 0.016045077, + -0.008985531, + -0.006838999, + -0.004744275, + 0.022833062, + 0.0065866956, + -0.019793756, + -0.019162435, + -0.0037823315, + -0.013943861, + 0.001016121, + -0.0017584384, + -0.008287824, + -0.0018560118, + 0.0028416964, + 0.022081941, + 0.009255234, + -0.005700457, + -0.0122221345, + -0.010320049, + -0.008686555, + -0.016383147, + 0.005054264, + 0.003243462, + 0.012693815, + 0.0017625735, + 0.005942188, + -0.0046482184, + -0.019250624, + -0.022220457, + -0.030070094, + -0.012699951, + -0.012187523, + 0.00020783099, + -0.01392611, + 0.021270981, + -0.013408182, + 0.010740026, + -0.03168554, + -0.023072174, + -0.003540653, + 0.0032966393, + 0.016065564, + -0.00073937967, + 0.017096907, + -0.034808062, + -0.01212092, + -0.014810947, + -0.02785775, + 0.021117523, + 0.003448726, + -0.005815306, + -0.019106612, + -0.020534357, + -0.010136233, + 0.005157424, + -0.003270571, + -0.012121161, + -0.0063488963, + -0.00769908, + -0.024439136, + -0.008738811, + -0.0060625016, + -0.007222123, + 0.0069793183, + 0.0054332716, + -0.013080505, + -0.03159355, + 0.01985001, + 0.00015736208, + -0.008494264, + -0.007556829, + 0.023188673, + -0.0040015737, + -0.023389285, + -0.027201023, + -0.014345848, + -0.00019172889, + 0.014818668, + -0.02303567, + 0.0016359553, + -0.020639544, + -0.0012845631, + 0.029141996, + -0.003186298, + 0.00477663, + -0.020251842, + 0.021202674, + -0.020718494, + 0.0007352107, + -0.024727875, + -0.014750569, + -0.027538653, + 0.025968675, + 0.0067956964, + 0.0013047646, + 0.007937409, + 0.012496054, + -0.022173256, + -0.012366622, + -0.004069293, + -0.004688455, + 0.006731516, + 0.011552375, + -0.0064534904, + 0.012942777, + -0.015180196, + 0.015762918, + -0.00645372, + -0.0054410626, + -0.0089021325, + -0.022529272, + 0.010612397, + 0.019415699, + -0.020852925, + 0.005704136, + -0.013845742, + -0.0054938775, + -0.016722085, + -0.0181522, + 0.008094221, + -0.010352087, + 0.02872723, + -0.0075552072, + 0.009022115, + 0.0008710217, + -0.026821673, + -0.010020514, + 0.015155785, + 0.013941547, + -0.0005814426, + -0.006650114, + 0.018623557, + 0.028279388, + -0.00257744, + 0.01798468, + 0.023192609, + -0.007989251, + 0.012490996, + -0.008559883, + 0.027437251, + -0.021442624, + -0.015116034, + -0.03346703, + 0.0053553893, + -0.024934981, + 0.00216263, + -0.020792361, + -0.004106453, + -0.016886832, + 0.00046246493, + -0.010412172, + -0.00094732345, + -0.008549759, + 0.013162843, + 0.018662676, + -0.0029572183, + -0.0006918239, + -0.017820187, + 0.02758241, + 0.0014139675, + -0.001184387, + -0.01843387, + 0.017657245, + 0.008907427, + 0.0024012472, + 0.015461176, + -0.0024773285, + -0.0058024414, + 0.0010203798, + -0.00093692506, + 0.01638991, + 0.021395635, + -0.01676794, + 0.018988565, + -0.0038439815, + 0.0068803146, + 0.022137286, + -0.008891913, + 0.007719931, + 0.019635445, + -0.002669683, + -0.018266771, + -0.009203526, + -0.009794082, + 0.022149993, + 0.006909571, + 0.0041262996, + 0.004437945, + 0.014429412, + -0.0054067755, + 0.022950528, + 0.012768831, + -0.015394789, + -0.007463684, + -0.021814577, + 0.011983734, + 0.007477243, + 0.008797093, + 0.004260692, + -0.0058581117, + -0.0017297139, + -0.004904297, + -0.00828556, + -0.009895036, + -0.010691348, + 0.016718758, + 0.016699353, + -0.0015871241, + 0.006559735, + -0.0014673267, + 0.021935647, + -0.003249199, + 0.019602994, + -0.0072449464, + -0.007738537, + -0.000656223, + -0.005987146, + -0.0027292965, + -0.025903663, + 0.009596118, + 0.026358845, + -0.0045184307, + -0.0051224153, + 0.029061602, + -0.013175618, + 0.026188191, + 0.0005889799, + 0.01062802, + -0.0152420495, + 0.003911869, + 0.011710072, + 0.002216575, + -0.012823992, + -0.012044093, + -0.010297008, + 0.015953882, + -0.012901612, + -0.009018347, + 0.020598294, + 0.012807931, + 0.01036631, + -0.0103104245, + -0.00195963, + 0.021844557, + 0.015407983, + -0.01887487, + 0.021180663, + 0.02904277, + -0.00900347, + 0.0006245657, + 0.0040328563, + 0.16909553, + 0.1443197, + -0.0079108495, + 0.017732581, + 0.010688205, + -0.008013505, + -0.002445023, + 0.010618324, + -0.0021832986, + -0.02098227, + 0.009283516, + -0.0017315133, + 0.020007687, + 0.0046745194, + -0.008980967, + 0.012435853, + -0.0034490826, + 0.002304364, + -0.012098984, + 0.046062212, + 0.0014867267, + 0.0019975123, + -0.021763306, + -0.013432375, + -0.034067564, + 0.004477718, + 0.023709435, + -0.010755928, + 0.029493708, + -0.0037563841, + -0.025096474, + -0.026028518, + -0.019822055, + -0.013186225, + 0.004846214, + 0.0033635262, + 0.026322003, + 0.029254803, + 0.007526233, + -0.0011341033, + -0.031385213, + -0.005040031, + -0.0018527679, + -0.01605337, + 0.026720213, + 0.011588068, + 0.00816062, + -0.011101225, + -0.033244126, + 0.015559814, + 0.01010725, + -0.023611642, + -0.0040965034, + 0.0105789965, + -0.011880153, + -0.016379012, + 0.0026610175, + -0.0008042192, + -0.008591299, + 0.009483353, + 0.024152417, + 0.011803963, + -0.008456854, + -0.009173776, + -0.008486057, + -0.007356003, + -0.0037539552, + 0.0066462746, + 0.0062097483, + 0.027709985, + 0.017934589, + -0.012268969, + -0.017791219, + 0.007463513, + -0.026872378, + -0.016125986, + -0.009482337, + -0.002126241, + -0.0136377355, + 0.007641745, + -0.018323489, + 0.013052224, + -0.004418404, + -0.00679719, + -0.0077302344, + 0.0008945445, + 0.0023328052, + 0.028289862, + 0.10009175, + 0.0033966585, + -0.009787358, + -0.03271596, + 0.0037409735, + 0.0027685282, + 0.01738122, + 0.027202224, + 0.013014808, + -0.019241922, + -0.007597794, + 0.013644396, + -0.009909854, + -0.028216057, + -0.007555482, + -0.007304877, + 0.011141363, + 0.019022344, + 0.0113722645, + 0.0063154125, + 0.0026695463, + -0.006154702, + -0.0064552743, + 0.003097966, + 0.0029722343, + -0.032258235, + 0.0021976673, + 0.0058431276, + -0.013713599, + 0.0005978899, + -0.11458491, + -0.009858059, + -0.012604525, + -0.011857043, + 0.0020588522, + -0.008327129, + -0.015755074, + 0.0005298635, + -8.755697E-07, + 0.0045324517, + 0.012156644, + 0.018711675, + 0.016488519, + -0.019420847, + -0.020298412, + -0.010155479, + 0.004072806, + -0.01762635, + -0.0011212828, + 0.0023366779, + 0.0095338505, + -0.0043420554, + 0.005607327, + 0.004290778, + 0.0073207053, + 0.008897146, + 0.00069104997, + 0.0041908384, + 0.00027268866, + 0.021229591, + -0.01995938, + 0.0008797666, + -0.0092747165, + 0.014104673, + 0.008030241, + 0.0028806778, + 0.0020439965, + 0.0037753154, + 0.021072106, + -0.007671592, + 0.022629702, + -0.04614186, + -0.0029086648, + 0.0007899806, + -0.0033691295, + 0.002604023, + 0.0051616393, + 0.008372272, + -0.0003544448, + 0.0023181862, + 0.00409971, + -0.011935123, + -0.0014922045, + -0.003294035, + -0.02520281, + -0.0033809247, + 0.00672384, + -0.020725748, + -0.023924578, + -0.011249866, + -0.0006656545, + 0.014977598, + 0.017138913, + -0.004826765, + -0.00019335799, + -0.032448776, + 0.012845708, + 0.0047560544, + 0.015104432, + 0.0047637452, + -0.0064079007, + 0.010834491, + -0.010786316, + -0.012596886, + -0.0062859515, + 0.04165462, + -0.01577823, + -0.001114999, + -0.0010543247, + 0.014313129, + -0.013121319, + 0.014169299, + -0.019007605, + 0.10939872, + 0.0051119323, + 0.0060034576, + -0.018779397, + -0.00057756645, + 0.010958136, + 0.016039172, + -0.03796148, + -0.008777769, + 0.019504657, + -0.024865804, + 0.022152022, + -0.0020063939, + -0.0058530523, + 0.00039091732, + 0.0021795654, + 0.012687185, + -0.009697326, + 0.009073556, + -0.009586887, + -0.0072185504, + 0.003469578, + -0.0048057805, + 0.027144434, + -0.005123206, + 0.01018817, + 0.0068952404, + -0.011356372, + -0.016462887, + 0.013012662, + 0.0060690837, + -0.018362023, + 0.00757209, + 0.0022669423, + -0.021601684, + -0.001577404, + 0.024271192, + -0.005838263, + -0.004518065, + -0.013756045, + 0.0025233144, + 0.0016318993, + 0.0040812655, + 0.016012315, + -0.026945297, + 0.20551158, + 0.0019059828, + -0.010217774, + 0.0052372473, + -0.017155742, + 0.011283257, + -0.0052146968, + 0.002614786, + -0.007152995, + 0.016891226, + -0.020955643, + 0.012887667, + -0.0031369291, + 0.0021766236, + 0.00668259, + 0.0051846523, + 0.0006279038, + 0.010779525, + 0.019239811, + 0.02684775, + -0.0017372426, + -0.0005050051, + 0.016148783, + -0.014399273, + -0.0050834394, + 0.025324365, + -0.014691299, + 0.014196108, + 0.020383976, + 0.011910045, + 0.0112336995, + 0.002919187, + -0.0066369083, + -0.03181099, + 0.0058744154, + 0.021922996, + 0.0075478763, + 0.003867872, + 0.029595869, + -0.0064815525, + -0.00129301, + 0.009675881, + -0.021551557, + 0.009722577, + 0.0077534597, + 0.006348233, + -0.0070638866, + -0.007026143, + -0.010967787, + 0.007442958, + 0.006098182, + 0.021202536, + 0.0036393213, + 0.0005835712, + -0.009036124, + 0.006218135, + -0.016838545, + 0.0018550553, + 0.0009816539, + -0.026443722, + 0.014209604, + -0.0111517925, + 0.018385211, + -0.014733362, + 0.008191925, + -0.012552624, + 0.0011367081 + ], + "LastModifiedUtc": "2025-09-22T21:27:03.5881028Z", + "SizeBytes": 3054, + "ContentHash": "D98FACFC326FB49305A6F439E03E04D7A66A3FD67DB06B42DDB39C8054018306", + "IngestedUtc": "2025-09-22T21:47:35.357489Z" +} \ No newline at end of file diff --git a/LiteDB.Tests/Shared/SharedDemoDatabaseCollection.cs b/LiteDB.Tests/Shared/SharedDemoDatabaseCollection.cs new file mode 100644 index 000000000..1112ce922 --- /dev/null +++ b/LiteDB.Tests/Shared/SharedDemoDatabaseCollection.cs @@ -0,0 +1,43 @@ +namespace LiteDB.Tests; + +using System; +using System.IO; +using Xunit; + +[CollectionDefinition("SharedDemoDatabase", DisableParallelization = true)] +public sealed class SharedDemoDatabaseCollection : ICollectionFixture +{ +} + +public sealed class SharedDemoDatabaseFixture : IDisposable +{ + private readonly string _filename; + + public SharedDemoDatabaseFixture() + { + _filename = Path.GetFullPath("Demo.db"); + TryDeleteFile(); + } + + public void Dispose() + { + TryDeleteFile(); + } + + private void TryDeleteFile() + { + try + { + if (File.Exists(_filename)) + { + File.Delete(_filename); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } +} diff --git a/LiteDB.Tests/Utils/CpuBoundFactAttribute.cs b/LiteDB.Tests/Utils/CpuBoundFactAttribute.cs new file mode 100644 index 000000000..50c4a26bb --- /dev/null +++ b/LiteDB.Tests/Utils/CpuBoundFactAttribute.cs @@ -0,0 +1,15 @@ +using System; +using Xunit; + +namespace LiteDB.Tests.Utils; + +class CpuBoundFactAttribute : FactAttribute +{ + public CpuBoundFactAttribute(int minCpuCount = 1) + { + if (minCpuCount > Environment.ProcessorCount) + { + Skip = $"This test requires at least {minCpuCount} processors to run properly."; + } + } +} \ No newline at end of file diff --git a/LiteDB.Tests/Utils/DatabaseFactory.cs b/LiteDB.Tests/Utils/DatabaseFactory.cs new file mode 100644 index 000000000..2932e2133 --- /dev/null +++ b/LiteDB.Tests/Utils/DatabaseFactory.cs @@ -0,0 +1,40 @@ +using System; +using LiteDB; + +namespace LiteDB.Tests.Utils +{ + public enum TestDatabaseType + { + Default, + InMemory, + Disk + } + + public static class DatabaseFactory + { + public static LiteDatabase Create(TestDatabaseType type = TestDatabaseType.Default, string connectionString = null, BsonMapper mapper = null) + { + switch (type) + { + case TestDatabaseType.Default: + case TestDatabaseType.InMemory: + return mapper is null + ? new LiteDatabase(connectionString ?? ":memory:") + : new LiteDatabase(connectionString ?? ":memory:", mapper); + + case TestDatabaseType.Disk: + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentException("Disk databases require a connection string.", nameof(connectionString)); + } + + return mapper is null + ? new LiteDatabase(connectionString) + : new LiteDatabase(connectionString, mapper); + + default: + throw new ArgumentOutOfRangeException(nameof(type), type, null); + } + } + } +} diff --git a/LiteDB.Tests/Utils/Faker.cs b/LiteDB.Tests/Utils/Faker.cs index 8f4a0a534..1e2976d97 100644 --- a/LiteDB.Tests/Utils/Faker.cs +++ b/LiteDB.Tests/Utils/Faker.cs @@ -69,10 +69,7 @@ public static long NextLong(this Random random, long min, long max) return (long)(ulongRand % uRange) + min; } - public static bool NextBool(this Random random) - { - return random.NextSingle() >= 0.5; - } + public static bool NextBool(this Random random) => random.NextDouble() >= 0.5; public static string Departments() => _departments[_random.Next(0, _departments.Length - 1)]; diff --git a/LiteDB.Tests/Utils/IsExternalInit.cs b/LiteDB.Tests/Utils/IsExternalInit.cs new file mode 100644 index 000000000..1c6d736c3 --- /dev/null +++ b/LiteDB.Tests/Utils/IsExternalInit.cs @@ -0,0 +1,10 @@ +#if !NETCOREAPP + +namespace System.Runtime.CompilerServices +{ + internal static class IsExternalInit + { + } +} + +#endif diff --git a/LiteDB.Tests/Utils/Models/Zip.cs b/LiteDB.Tests/Utils/Models/Zip.cs index 8f8a01960..3fddc7d2a 100644 --- a/LiteDB.Tests/Utils/Models/Zip.cs +++ b/LiteDB.Tests/Utils/Models/Zip.cs @@ -16,6 +16,7 @@ public class Zip : IEqualityComparer, IComparable public string City { get; set; } public double[] Loc { get; set; } public string State { get; set; } + public byte[] Payload { get; set; } public int CompareTo(Zip other) { diff --git a/LiteDB.sln b/LiteDB.sln index 848dbb661..7f0051124 100644 --- a/LiteDB.sln +++ b/LiteDB.sln @@ -13,32 +13,151 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LiteDB.Benchmarks", "LiteDB EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LiteDB.Stress", "LiteDB.Stress\LiteDB.Stress.csproj", "{FFBC5669-DA32-4907-8793-7B414279DA3B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LiteDB.Demo.Tools.VectorSearch", "LiteDB.Demo.Tools.VectorSearch\LiteDB.Demo.Tools.VectorSearch.csproj", "{64EEF08C-CE83-4929-B5E4-583BBC332941}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp1", "ConsoleApp1\ConsoleApp1.csproj", "{E8763934-E46A-4AAF-A2B5-E812016DAF84}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Issue_2586_RollbackTransaction", "LiteDB.ReproRunner\Repros\Issue_2586_RollbackTransaction\Issue_2586_RollbackTransaction.csproj", "{BE1D6CA2-134A-404A-8F1A-C48E4E240159}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{D455AC29-7847-4DF4-AD06-69042F8B8885}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LiteDB.ReproRunner", "LiteDB.ReproRunner", "{C172DFBD-9BFC-41A4-82B9-5B9BBC90850D}" + ProjectSection(SolutionItems) = preProject + LiteDB.ReproRunner\README.md = LiteDB.ReproRunner\README.md + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LiteDB.ReproRunner.Cli", "LiteDB.ReproRunner\LiteDB.ReproRunner.Cli\LiteDB.ReproRunner.Cli.csproj", "{CDCF5EED-50F3-4790-B180-10B203EE6B4B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repros", "Repros", "{B0BD59D3-0D10-42BF-A744-533473577C8C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Issue_2561_TransactionMonitor", "LiteDB.ReproRunner\Repros\Issue_2561_TransactionMonitor\Issue_2561_TransactionMonitor.csproj", "{B5BF3DFE-5F26-447A-AF5A-60C6E3D341AC}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{3B786621-2B82-4C69-8FE9-3889ECB36E75}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LiteDB.ReproRunner.Tests", "LiteDB.ReproRunner.Tests\LiteDB.ReproRunner.Tests.csproj", "{3EF8E506-B57B-4A98-AD09-E687F9DC515D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LiteDB.ReproRunner.Shared", "LiteDB.ReproRunner\LiteDB.ReproRunner.Shared\LiteDB.ReproRunner.Shared.csproj", "{CE109129-4017-46E7-BE84-17D4D83296F4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedMutexHarness", "LiteDB.Tests.SharedMutexHarness\SharedMutexHarness.csproj", "{F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Debug|x64.ActiveCfg = Debug|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Debug|x64.Build.0 = Debug|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Debug|x86.ActiveCfg = Debug|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Debug|x86.Build.0 = Debug|Any CPU {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Release|Any CPU.ActiveCfg = Release|Any CPU {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Release|Any CPU.Build.0 = Release|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Release|x64.ActiveCfg = Release|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Release|x64.Build.0 = Release|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Release|x86.ActiveCfg = Release|Any CPU + {9497DA19-1FCA-4C2E-A1AB-8DFAACBC76E1}.Release|x86.Build.0 = Release|Any CPU {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Debug|x64.ActiveCfg = Debug|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Debug|x64.Build.0 = Debug|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Debug|x86.ActiveCfg = Debug|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Debug|x86.Build.0 = Debug|Any CPU {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Release|Any CPU.ActiveCfg = Release|Any CPU {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Release|Any CPU.Build.0 = Release|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Release|x64.ActiveCfg = Release|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Release|x64.Build.0 = Release|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Release|x86.ActiveCfg = Release|Any CPU + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA}.Release|x86.Build.0 = Release|Any CPU {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Debug|Any CPU.Build.0 = Debug|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Debug|x64.ActiveCfg = Debug|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Debug|x64.Build.0 = Debug|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Debug|x86.ActiveCfg = Debug|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Debug|x86.Build.0 = Debug|Any CPU {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Release|Any CPU.ActiveCfg = Release|Any CPU {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Release|Any CPU.Build.0 = Release|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Release|x64.ActiveCfg = Release|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Release|x64.Build.0 = Release|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Release|x86.ActiveCfg = Release|Any CPU + {99887C89-CAE4-4A8D-AC4B-87E28B9B1F87}.Release|x86.Build.0 = Release|Any CPU {DF9C82C1-446F-458A-AA50-78E58BA17273}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DF9C82C1-446F-458A-AA50-78E58BA17273}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Debug|x64.ActiveCfg = Debug|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Debug|x64.Build.0 = Debug|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Debug|x86.ActiveCfg = Debug|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Debug|x86.Build.0 = Debug|Any CPU {DF9C82C1-446F-458A-AA50-78E58BA17273}.Release|Any CPU.ActiveCfg = Release|Any CPU {DF9C82C1-446F-458A-AA50-78E58BA17273}.Release|Any CPU.Build.0 = Release|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Release|x64.ActiveCfg = Release|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Release|x64.Build.0 = Release|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Release|x86.ActiveCfg = Release|Any CPU + {DF9C82C1-446F-458A-AA50-78E58BA17273}.Release|x86.Build.0 = Release|Any CPU {FFBC5669-DA32-4907-8793-7B414279DA3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FFBC5669-DA32-4907-8793-7B414279DA3B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Debug|x64.ActiveCfg = Debug|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Debug|x64.Build.0 = Debug|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Debug|x86.ActiveCfg = Debug|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Debug|x86.Build.0 = Debug|Any CPU {FFBC5669-DA32-4907-8793-7B414279DA3B}.Release|Any CPU.ActiveCfg = Release|Any CPU {FFBC5669-DA32-4907-8793-7B414279DA3B}.Release|Any CPU.Build.0 = Release|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Release|x64.ActiveCfg = Release|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Release|x64.Build.0 = Release|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Release|x86.ActiveCfg = Release|Any CPU + {FFBC5669-DA32-4907-8793-7B414279DA3B}.Release|x86.Build.0 = Release|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Debug|Any CPU.Build.0 = Debug|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Debug|x64.ActiveCfg = Debug|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Debug|x64.Build.0 = Debug|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Debug|x86.ActiveCfg = Debug|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Debug|x86.Build.0 = Debug|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Release|Any CPU.ActiveCfg = Release|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Release|Any CPU.Build.0 = Release|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Release|x64.ActiveCfg = Release|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Release|x64.Build.0 = Release|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Release|x86.ActiveCfg = Release|Any CPU + {64EEF08C-CE83-4929-B5E4-583BBC332941}.Release|x86.Build.0 = Release|Any CPU + {E8763934-E46A-4AAF-A2B5-E812016DAF84}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E8763934-E46A-4AAF-A2B5-E812016DAF84}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E8763934-E46A-4AAF-A2B5-E812016DAF84}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E8763934-E46A-4AAF-A2B5-E812016DAF84}.Release|Any CPU.Build.0 = Release|Any CPU + {BE1D6CA2-134A-404A-8F1A-C48E4E240159}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BE1D6CA2-134A-404A-8F1A-C48E4E240159}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BE1D6CA2-134A-404A-8F1A-C48E4E240159}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BE1D6CA2-134A-404A-8F1A-C48E4E240159}.Release|Any CPU.Build.0 = Release|Any CPU + {CDCF5EED-50F3-4790-B180-10B203EE6B4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDCF5EED-50F3-4790-B180-10B203EE6B4B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDCF5EED-50F3-4790-B180-10B203EE6B4B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDCF5EED-50F3-4790-B180-10B203EE6B4B}.Release|Any CPU.Build.0 = Release|Any CPU + {B5BF3DFE-5F26-447A-AF5A-60C6E3D341AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B5BF3DFE-5F26-447A-AF5A-60C6E3D341AC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B5BF3DFE-5F26-447A-AF5A-60C6E3D341AC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B5BF3DFE-5F26-447A-AF5A-60C6E3D341AC}.Release|Any CPU.Build.0 = Release|Any CPU + {3EF8E506-B57B-4A98-AD09-E687F9DC515D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3EF8E506-B57B-4A98-AD09-E687F9DC515D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3EF8E506-B57B-4A98-AD09-E687F9DC515D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3EF8E506-B57B-4A98-AD09-E687F9DC515D}.Release|Any CPU.Build.0 = Release|Any CPU + {CE109129-4017-46E7-BE84-17D4D83296F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE109129-4017-46E7-BE84-17D4D83296F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE109129-4017-46E7-BE84-17D4D83296F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE109129-4017-46E7-BE84-17D4D83296F4}.Release|Any CPU.Build.0 = Release|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Debug|x64.ActiveCfg = Debug|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Debug|x64.Build.0 = Debug|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Debug|x86.ActiveCfg = Debug|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Debug|x86.Build.0 = Debug|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Release|Any CPU.Build.0 = Release|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Release|x64.ActiveCfg = Release|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Release|x64.Build.0 = Release|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Release|x86.ActiveCfg = Release|Any CPU + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -49,4 +168,15 @@ Global GlobalSection(Performance) = preSolution HasPerformanceSessions = true EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {E8763934-E46A-4AAF-A2B5-E812016DAF84} = {D455AC29-7847-4DF4-AD06-69042F8B8885} + {CDCF5EED-50F3-4790-B180-10B203EE6B4B} = {C172DFBD-9BFC-41A4-82B9-5B9BBC90850D} + {B0BD59D3-0D10-42BF-A744-533473577C8C} = {C172DFBD-9BFC-41A4-82B9-5B9BBC90850D} + {B5BF3DFE-5F26-447A-AF5A-60C6E3D341AC} = {B0BD59D3-0D10-42BF-A744-533473577C8C} + {3EF8E506-B57B-4A98-AD09-E687F9DC515D} = {3B786621-2B82-4C69-8FE9-3889ECB36E75} + {CE109129-4017-46E7-BE84-17D4D83296F4} = {C172DFBD-9BFC-41A4-82B9-5B9BBC90850D} + {BE1D6CA2-134A-404A-8F1A-C48E4E240159} = {B0BD59D3-0D10-42BF-A744-533473577C8C} + {74E32E43-2A57-4A38-BD8C-9108B0DCAEAA} = {3B786621-2B82-4C69-8FE9-3889ECB36E75} + {F7E423B9-B90B-4F4D-B02A-F0101BBA26E6} = {3B786621-2B82-4C69-8FE9-3889ECB36E75} + EndGlobalSection EndGlobal diff --git a/LiteDB/Client/Database/Collections/Index.cs b/LiteDB/Client/Database/Collections/Index.cs index aac550567..0db9c9900 100644 --- a/LiteDB/Client/Database/Collections/Index.cs +++ b/LiteDB/Client/Database/Collections/Index.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Linq.Expressions; using System.Text.RegularExpressions; +using LiteDB.Vector; using static LiteDB.Constants; namespace LiteDB @@ -23,6 +24,21 @@ public bool EnsureIndex(string name, BsonExpression expression, bool unique = fa return _engine.EnsureIndex(_collection, name, expression, unique); } + internal bool EnsureVectorIndex(string name, BsonExpression expression, VectorIndexOptions options) + { + if (string.IsNullOrEmpty(name)) throw new ArgumentNullException(nameof(name)); + if (expression == null) throw new ArgumentNullException(nameof(expression)); + if (options == null) throw new ArgumentNullException(nameof(options)); + + return _engine.EnsureVectorIndex(_collection, name, expression, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteCollectionVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(string name, BsonExpression expression, VectorIndexOptions options) + { + return this.EnsureVectorIndex(name, expression, options); + } + /// /// Create a new permanent index in all documents inside this collections if index not exists already. Returns true if index was created or false if already exits /// @@ -37,6 +53,22 @@ public bool EnsureIndex(BsonExpression expression, bool unique = false) return this.EnsureIndex(name, expression, unique); } + internal bool EnsureVectorIndex(BsonExpression expression, VectorIndexOptions options) + { + if (expression == null) throw new ArgumentNullException(nameof(expression)); + if (options == null) throw new ArgumentNullException(nameof(options)); + + var name = Regex.Replace(expression.Source, @"[^a-z0-9]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + return this.EnsureVectorIndex(name, expression, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteCollectionVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(BsonExpression expression, VectorIndexOptions options) + { + return this.EnsureVectorIndex(expression, options); + } + /// /// Create a new permanent index in all documents inside this collections if index not exists already. /// @@ -49,6 +81,21 @@ public bool EnsureIndex(Expression> keySelector, bool unique = fal return this.EnsureIndex(expression, unique); } + internal bool EnsureVectorIndex(Expression> keySelector, VectorIndexOptions options) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + + var expression = this.GetIndexExpression(keySelector, convertEnumerableToMultiKey: false); + + return this.EnsureVectorIndex(expression, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteCollectionVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(Expression> keySelector, VectorIndexOptions options) + { + return this.EnsureVectorIndex(keySelector, options); + } + /// /// Create a new permanent index in all documents inside this collections if index not exists already. /// @@ -62,14 +109,29 @@ public bool EnsureIndex(string name, Expression> keySelector, bool return this.EnsureIndex(name, expression, unique); } + internal bool EnsureVectorIndex(string name, Expression> keySelector, VectorIndexOptions options) + { + if (options == null) throw new ArgumentNullException(nameof(options)); + + var expression = this.GetIndexExpression(keySelector, convertEnumerableToMultiKey: false); + + return this.EnsureVectorIndex(name, expression, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteCollectionVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(string name, Expression> keySelector, VectorIndexOptions options) + { + return this.EnsureVectorIndex(name, keySelector, options); + } + /// /// Get index expression based on LINQ expression. Convert IEnumerable in MultiKey indexes /// - private BsonExpression GetIndexExpression(Expression> keySelector) + private BsonExpression GetIndexExpression(Expression> keySelector, bool convertEnumerableToMultiKey = true) { var expression = _mapper.GetIndexExpression(keySelector); - if (typeof(K).IsEnumerable() && expression.IsScalar == true) + if (convertEnumerableToMultiKey && typeof(K).IsEnumerable() && expression.IsScalar == true) { if (expression.Type == BsonExpressionType.Path) { diff --git a/LiteDB/Client/Database/ILiteQueryable.cs b/LiteDB/Client/Database/ILiteQueryable.cs index 09d8150ce..e2cd538b7 100644 --- a/LiteDB/Client/Database/ILiteQueryable.cs +++ b/LiteDB/Client/Database/ILiteQueryable.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; @@ -20,12 +20,17 @@ public interface ILiteQueryable : ILiteQueryableResult ILiteQueryable OrderBy(Expression> keySelector, int order = 1); ILiteQueryable OrderByDescending(BsonExpression keySelector); ILiteQueryable OrderByDescending(Expression> keySelector); + ILiteQueryable ThenBy(BsonExpression keySelector); + ILiteQueryable ThenBy(Expression> keySelector); + ILiteQueryable ThenByDescending(BsonExpression keySelector); + ILiteQueryable ThenByDescending(Expression> keySelector); + ILiteQueryable> GroupBy(Expression> keySelector); ILiteQueryable GroupBy(BsonExpression keySelector); ILiteQueryable Having(BsonExpression predicate); - ILiteQueryableResult Select(BsonExpression selector); - ILiteQueryableResult Select(Expression> selector); + ILiteQueryable Select(BsonExpression selector); + ILiteQueryable Select(Expression> selector); } public interface ILiteQueryableResult diff --git a/LiteDB/Client/Database/LiteDatabase.cs b/LiteDB/Client/Database/LiteDatabase.cs index 812ae629c..d301d54da 100644 --- a/LiteDB/Client/Database/LiteDatabase.cs +++ b/LiteDB/Client/Database/LiteDatabase.cs @@ -19,6 +19,7 @@ public partial class LiteDatabase : ILiteDatabase private readonly ILiteEngine _engine; private readonly BsonMapper _mapper; private readonly bool _disposeOnClose; + private readonly int? _checkpointOverride; /// /// Get current instance of BsonMapper used in this database instance (can be BsonMapper.Global) @@ -66,6 +67,27 @@ public LiteDatabase(Stream stream, BsonMapper mapper = null, Stream logStream = _engine = new LiteEngine(settings); _mapper = mapper ?? BsonMapper.Global; _disposeOnClose = true; + + if (logStream == null && stream is not MemoryStream) + { + if (!stream.CanWrite) + { + // Read-only streams cannot participate in eager checkpointing because the process + // writes pages back to the underlying data stream immediately. + } + else + { + // Without a dedicated log stream the WAL lives purely in memory; force + // checkpointing to ensure commits reach the underlying data stream. + var originalCheckpointSize = _engine.Pragma(Pragmas.CHECKPOINT); + + if (originalCheckpointSize != 1) + { + _engine.Pragma(Pragmas.CHECKPOINT, 1); + _checkpointOverride = originalCheckpointSize; + } + } + } } /// @@ -373,6 +395,11 @@ protected virtual void Dispose(bool disposing) { if (disposing && _disposeOnClose) { + if (_checkpointOverride.HasValue) + { + _engine.Pragma(Pragmas.CHECKPOINT, _checkpointOverride.Value); + } + _engine.Dispose(); } } diff --git a/LiteDB/Client/Database/LiteGrouping.cs b/LiteDB/Client/Database/LiteGrouping.cs new file mode 100644 index 000000000..de9dc71f4 --- /dev/null +++ b/LiteDB/Client/Database/LiteGrouping.cs @@ -0,0 +1,36 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace LiteDB +{ + internal static class LiteGroupingFieldNames + { + public const string Key = "key"; + public const string Items = "items"; + } + + /// + /// Concrete implementation used to materialize + /// grouping results coming from query execution. + /// + internal sealed class LiteGrouping : IGrouping + { + internal const string KeyField = LiteGroupingFieldNames.Key; + internal const string ItemsField = LiteGroupingFieldNames.Items; + + private readonly IReadOnlyList _items; + + public LiteGrouping(TKey key, IReadOnlyList items) + { + Key = key; + _items = items; + } + + public TKey Key { get; } + + public IEnumerator GetEnumerator() => _items.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator(); + } +} diff --git a/LiteDB/Client/Database/LiteQueryable.cs b/LiteDB/Client/Database/LiteQueryable.cs index 9ec5a2b02..d5deeecc1 100644 --- a/LiteDB/Client/Database/LiteQueryable.cs +++ b/LiteDB/Client/Database/LiteQueryable.cs @@ -103,14 +103,13 @@ public ILiteQueryable Where(Expression> predicate) #region OrderBy /// - /// Sort the documents of resultset in ascending (or descending) order according to a key (support only one OrderBy) + /// Sort the documents of resultset in ascending (or descending) order according to a key. /// public ILiteQueryable OrderBy(BsonExpression keySelector, int order = Query.Ascending) { - if (_query.OrderBy != null) throw new ArgumentException("ORDER BY already defined in this query builder"); + if (_query.OrderBy.Count > 0) throw new ArgumentException("Multiple OrderBy calls are not supported. Use ThenBy for additional sort keys."); - _query.OrderBy = keySelector; - _query.Order = order; + _query.OrderBy.Add(new QueryOrder(keySelector, order)); return this; } @@ -123,19 +122,71 @@ public ILiteQueryable OrderBy(Expression> keySelector, int orde } /// - /// Sort the documents of resultset in descending order according to a key (support only one OrderBy) + /// Sort the documents of resultset in descending order according to a key. /// public ILiteQueryable OrderByDescending(BsonExpression keySelector) => this.OrderBy(keySelector, Query.Descending); /// - /// Sort the documents of resultset in descending order according to a key (support only one OrderBy) + /// Sort the documents of resultset in descending order according to a key. /// public ILiteQueryable OrderByDescending(Expression> keySelector) => this.OrderBy(keySelector, Query.Descending); + /// + /// Appends an ascending sort expression that is applied when previous keys are equal. + /// + public ILiteQueryable ThenBy(BsonExpression keySelector) + { + if (_query.OrderBy.Count == 0) return this.OrderBy(keySelector, Query.Ascending); + + _query.OrderBy.Add(new QueryOrder(keySelector, Query.Ascending)); + return this; + } + + /// + /// Appends an ascending sort expression that is applied when previous keys are equal. + /// + public ILiteQueryable ThenBy(Expression> keySelector) + { + return this.ThenBy(_mapper.GetExpression(keySelector)); + } + + /// + /// Appends a descending sort expression that is applied when previous keys are equal. + /// + public ILiteQueryable ThenByDescending(BsonExpression keySelector) + { + if (_query.OrderBy.Count == 0) return this.OrderBy(keySelector, Query.Descending); + + _query.OrderBy.Add(new QueryOrder(keySelector, Query.Descending)); + return this; + } + + /// + /// Appends a descending sort expression that is applied when previous keys are equal. + /// + public ILiteQueryable ThenByDescending(Expression> keySelector) + { + return this.ThenByDescending(_mapper.GetExpression(keySelector)); + } + #endregion #region GroupBy + /// + /// Groups the documents of resultset according to a specified key selector expression (support only one GroupBy) + /// + public ILiteQueryable> GroupBy(Expression> keySelector) + { + var expression = _mapper.GetExpression(keySelector); + + this.GroupBy(expression); + + _mapper.RegisterGroupingType(); + + return new LiteQueryable>(_engine, _mapper, _collection, _query); + } + /// /// Groups the documents of resultset according to a specified key selector expression (support only one GroupBy) /// @@ -169,7 +220,7 @@ public ILiteQueryable Having(BsonExpression predicate) /// /// Transform input document into a new output document. Can be used with each document, group by or all source /// - public ILiteQueryableResult Select(BsonExpression selector) + public ILiteQueryable Select(BsonExpression selector) { _query.Select = selector; @@ -179,15 +230,133 @@ public ILiteQueryableResult Select(BsonExpression selector) /// /// Project each document of resultset into a new document/value based on selector expression /// - public ILiteQueryableResult Select(Expression> selector) + public ILiteQueryable Select(Expression> selector) { - if (_query.GroupBy != null) throw new ArgumentException("Use Select(BsonExpression selector) when using GroupBy query"); - _query.Select = _mapper.GetExpression(selector); return new LiteQueryable(_engine, _mapper, _collection, _query); } + private static void ValidateVectorArguments(float[] target, double maxDistance) + { + if (target == null || target.Length == 0) throw new ArgumentException("Target vector must be provided.", nameof(target)); + // Dot-product queries interpret "maxDistance" as a minimum similarity score and may therefore pass negative values. + if (double.IsNaN(maxDistance)) throw new ArgumentOutOfRangeException(nameof(maxDistance), "Similarity threshold must be a valid number."); + } + + private static BsonExpression CreateVectorSimilarityFilter(BsonExpression fieldExpr, float[] target, double maxDistance) + { + if (fieldExpr == null) throw new ArgumentNullException(nameof(fieldExpr)); + + ValidateVectorArguments(target, maxDistance); + + var targetArray = new BsonArray(target.Select(v => new BsonValue(v))); + return BsonExpression.Create($"{fieldExpr.Source} VECTOR_SIM @0 <= @1", targetArray, new BsonValue(maxDistance)); + } + + internal ILiteQueryable VectorWhereNear(string vectorField, float[] target, double maxDistance) + { + if (string.IsNullOrWhiteSpace(vectorField)) throw new ArgumentNullException(nameof(vectorField)); + + var fieldExpr = BsonExpression.Create($"$.{vectorField}"); + return this.VectorWhereNear(fieldExpr, target, maxDistance); + } + + internal ILiteQueryable VectorWhereNear(BsonExpression fieldExpr, float[] target, double maxDistance) + { + var filter = CreateVectorSimilarityFilter(fieldExpr, target, maxDistance); + + _query.Where.Add(filter); + + _query.VectorField = fieldExpr.Source; + _query.VectorTarget = target?.ToArray(); + _query.VectorMaxDistance = maxDistance; + + return this; + } + + internal ILiteQueryable VectorWhereNear(Expression> field, float[] target, double maxDistance) + { + if (field == null) throw new ArgumentNullException(nameof(field)); + + var fieldExpr = _mapper.GetExpression(field); + return this.VectorWhereNear(fieldExpr, target, maxDistance); + } + + internal ILiteQueryableResult VectorTopKNear(Expression> field, float[] target, int k) + { + var fieldExpr = _mapper.GetExpression(field); + return this.VectorTopKNear(fieldExpr, target, k); + } + + internal ILiteQueryableResult VectorTopKNear(string field, float[] target, int k) + { + var fieldExpr = BsonExpression.Create($"$.{field}"); + return this.VectorTopKNear(fieldExpr, target, k); + } + + internal ILiteQueryableResult VectorTopKNear(BsonExpression fieldExpr, float[] target, int k) + { + if (fieldExpr == null) throw new ArgumentNullException(nameof(fieldExpr)); + if (target == null || target.Length == 0) throw new ArgumentException("Target vector must be provided.", nameof(target)); + if (k <= 0) throw new ArgumentOutOfRangeException(nameof(k), "Top-K must be greater than zero."); + + var targetArray = new BsonArray(target.Select(v => new BsonValue(v))); + + // Build VECTOR_SIM as order clause + var simExpr = BsonExpression.Create($"VECTOR_SIM({fieldExpr.Source}, @0)", targetArray); + + _query.VectorField = fieldExpr.Source; + _query.VectorTarget = target?.ToArray(); + _query.VectorMaxDistance = double.MaxValue; + + return this + .OrderBy(simExpr, Query.Ascending) + .Limit(k); + } + + [Obsolete("Add `using LiteDB.Vector;` and call the LiteQueryableVectorExtensions.WhereNear extension instead.")] + public ILiteQueryable WhereNear(string vectorField, float[] target, double maxDistance) + { + return this.VectorWhereNear(vectorField, target, maxDistance); + } + + [Obsolete("Add `using LiteDB.Vector;` and call the LiteQueryableVectorExtensions.WhereNear extension instead.")] + public ILiteQueryable WhereNear(BsonExpression fieldExpr, float[] target, double maxDistance) + { + return this.VectorWhereNear(fieldExpr, target, maxDistance); + } + + [Obsolete("Add `using LiteDB.Vector;` and call the LiteQueryableVectorExtensions.WhereNear extension instead.")] + public ILiteQueryable WhereNear(Expression> field, float[] target, double maxDistance) + { + return this.VectorWhereNear(field, target, maxDistance); + } + + [Obsolete("Add `using LiteDB.Vector;` and call the LiteQueryableVectorExtensions.FindNearest extension instead.")] + public IEnumerable FindNearest(string vectorField, float[] target, double maxDistance) + { + return this.VectorWhereNear(vectorField, target, maxDistance).ToEnumerable(); + } + + [Obsolete("Add `using LiteDB.Vector;` and call the LiteQueryableVectorExtensions.TopKNear extension instead.")] + public ILiteQueryableResult TopKNear(Expression> field, float[] target, int k) + { + return this.VectorTopKNear(field, target, k); + } + + [Obsolete("Add `using LiteDB.Vector;` and call the LiteQueryableVectorExtensions.TopKNear extension instead.")] + public ILiteQueryableResult TopKNear(string field, float[] target, int k) + { + return this.VectorTopKNear(field, target, k); + } + + [Obsolete("Add `using LiteDB.Vector;` and call the LiteQueryableVectorExtensions.TopKNear extension instead.")] + public ILiteQueryableResult TopKNear(BsonExpression fieldExpr, float[] target, int k) + { + return this.VectorTopKNear(fieldExpr, target, k); + } + #endregion #region Offset/Limit/ForUpdate diff --git a/LiteDB/Client/Database/LiteRepository.cs b/LiteDB/Client/Database/LiteRepository.cs index 91a659bef..4d5008ae7 100644 --- a/LiteDB/Client/Database/LiteRepository.cs +++ b/LiteDB/Client/Database/LiteRepository.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Linq.Expressions; +using LiteDB.Vector; using static LiteDB.Constants; namespace LiteDB @@ -16,6 +17,18 @@ public class LiteRepository : ILiteRepository private readonly ILiteDatabase _db = null; + private LiteCollection GetLiteCollection(string collectionName) + { + var collection = _db.GetCollection(collectionName); + + if (collection is LiteCollection liteCollection) + { + return liteCollection; + } + + throw new InvalidOperationException("The current collection implementation does not support vector operations."); + } + /// /// Get database instance /// @@ -173,6 +186,17 @@ public bool EnsureIndex(string name, BsonExpression expression, bool unique = return _db.GetCollection(collectionName).EnsureIndex(name, expression, unique); } + internal bool EnsureVectorIndex(string name, BsonExpression expression, VectorIndexOptions options, string collectionName = null) + { + return this.GetLiteCollection(collectionName).EnsureVectorIndex(name, expression, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteRepositoryVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(string name, BsonExpression expression, VectorIndexOptions options, string collectionName = null) + { + return this.EnsureVectorIndex(name, expression, options, collectionName); + } + /// /// Create a new permanent index in all documents inside this collections if index not exists already. Returns true if index was created or false if already exits /// @@ -184,6 +208,17 @@ public bool EnsureIndex(BsonExpression expression, bool unique = false, strin return _db.GetCollection(collectionName).EnsureIndex(expression, unique); } + internal bool EnsureVectorIndex(BsonExpression expression, VectorIndexOptions options, string collectionName = null) + { + return this.GetLiteCollection(collectionName).EnsureVectorIndex(expression, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteRepositoryVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(BsonExpression expression, VectorIndexOptions options, string collectionName = null) + { + return this.EnsureVectorIndex(expression, options, collectionName); + } + /// /// Create a new permanent index in all documents inside this collections if index not exists already. /// @@ -195,6 +230,17 @@ public bool EnsureIndex(Expression> keySelector, bool unique = return _db.GetCollection(collectionName).EnsureIndex(keySelector, unique); } + internal bool EnsureVectorIndex(Expression> keySelector, VectorIndexOptions options, string collectionName = null) + { + return this.GetLiteCollection(collectionName).EnsureVectorIndex(keySelector, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteRepositoryVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(Expression> keySelector, VectorIndexOptions options, string collectionName = null) + { + return this.EnsureVectorIndex(keySelector, options, collectionName); + } + /// /// Create a new permanent index in all documents inside this collections if index not exists already. /// @@ -207,6 +253,17 @@ public bool EnsureIndex(string name, Expression> keySelector, b return _db.GetCollection(collectionName).EnsureIndex(name, keySelector, unique); } + internal bool EnsureVectorIndex(string name, Expression> keySelector, VectorIndexOptions options, string collectionName = null) + { + return this.GetLiteCollection(collectionName).EnsureVectorIndex(name, keySelector, options); + } + + [Obsolete("Add `using LiteDB.Vector;` and call LiteRepositoryVectorExtensions.EnsureIndex instead.")] + public bool EnsureIndex(string name, Expression> keySelector, VectorIndexOptions options, string collectionName = null) + { + return this.EnsureVectorIndex(name, keySelector, options, collectionName); + } + #endregion #region Shortcuts diff --git a/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs b/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs index 214a36479..46851119d 100644 --- a/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs +++ b/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs @@ -3,7 +3,7 @@ using System.Collections; using System.Collections.Generic; using System.Reflection; -using static LiteDB.Constants; +using System.ComponentModel; namespace LiteDB { @@ -216,35 +216,33 @@ public object Deserialize(Type type, BsonValue value) entity.WaitForInitialization(); // initialize CreateInstance - if (entity.CreateInstance == null) - { - entity.CreateInstance = - this.GetTypeCtor(entity) ?? - ((BsonDocument v) => Reflection.CreateInstance(entity.ForType)); - } + entity.CreateInstance = entity.CreateInstance + ?? GetTypeCtor(entity) + ?? ((BsonDocument _) => Reflection.CreateInstance(entity.ForType)); - var o = _typeInstantiator(type) ?? entity.CreateInstance(doc); + object instance = _typeInstantiator(type) + ?? entity.CreateInstance(doc); - if (o is IDictionary dict) + if (instance is IDictionary dict) { - if (o.GetType().GetTypeInfo().IsGenericType) - { - var k = type.GetGenericArguments()[0]; - var t = type.GetGenericArguments()[1]; + Type keyType = typeof(object); + Type valueType = typeof(object); - this.DeserializeDictionary(k, t, dict, value.AsDocument); - } - else + if (instance.GetType().GetTypeInfo().IsGenericType) { - this.DeserializeDictionary(typeof(object), typeof(object), dict, value.AsDocument); + Type[] generics = type.GetGenericArguments(); + keyType = generics[0]; + valueType = generics[1]; } + + DeserializeDictionary(keyType, valueType, dict, value.AsDocument); } else { - this.DeserializeObject(entity, o, doc); + DeserializeObject(entity, instance, doc); } - return o; + return instance; } // in last case, return value as-is - can cause "cast error" @@ -290,15 +288,29 @@ private object DeserializeList(Type type, BsonArray value) return enumerable; } - private void DeserializeDictionary(Type K, Type T, IDictionary dict, BsonDocument value) + private void DeserializeDictionary(Type keyType, Type valueType, IDictionary dict, BsonDocument value) { - var isKEnum = K.GetTypeInfo().IsEnum; - foreach (var el in value.GetElements()) + foreach (KeyValuePair element in value.GetElements()) { - var k = isKEnum ? Enum.Parse(K, el.Key) : K == typeof(Uri) ? new Uri(el.Key) : Convert.ChangeType(el.Key, K); - var v = this.Deserialize(T, el.Value); + object dictKey; + TypeConverter keyConverter = TypeDescriptor.GetConverter(keyType); + if (keyConverter.CanConvertFrom(typeof(string))) + { + // Here, we deserialize the key based on its type, even though it's a string. This is because + // BsonDocuments only support string keys (not BsonValue). + // However, if we deserialize the string representation, we can have pseudo-support for key types like GUID. + // See https://github.com/litedb-org/LiteDB/issues/546 + dictKey = keyConverter.ConvertFromInvariantString(element.Key); + } + else + { + // Some types (e.g. System.Collections.Hashtable) can't be converted using TypeDescriptor + dictKey = Convert.ChangeType(element.Key, keyType); + } + + object dictValue = Deserialize(valueType, element.Value); - dict.Add(k, v); + dict[dictKey] = dictValue; } } diff --git a/LiteDB/Client/Mapper/BsonMapper.Grouping.cs b/LiteDB/Client/Mapper/BsonMapper.Grouping.cs new file mode 100644 index 000000000..c452a9b49 --- /dev/null +++ b/LiteDB/Client/Mapper/BsonMapper.Grouping.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using System.Linq; + +namespace LiteDB +{ + public partial class BsonMapper + { + internal void RegisterGroupingType() + { + var interfaceType = typeof(IGrouping); + var concreteType = typeof(LiteGrouping); + + if (_customDeserializer.ContainsKey(interfaceType)) + { + return; + } + + BsonValue SerializeGrouping(object value) + { + var grouping = (IGrouping)value; + var items = new BsonArray(); + + foreach (var item in grouping) + { + items.Add(this.Serialize(typeof(TElement), item)); + } + + return new BsonDocument + { + [LiteGroupingFieldNames.Key] = this.Serialize(typeof(TKey), grouping.Key), + [LiteGroupingFieldNames.Items] = items + }; + } + + object DeserializeGrouping(BsonValue value) + { + var document = value.AsDocument; + + var key = (TKey)this.Deserialize(typeof(TKey), document[LiteGroupingFieldNames.Key]); + + var itemsArray = document[LiteGroupingFieldNames.Items].AsArray; + var items = new List(itemsArray.Count); + + foreach (var item in itemsArray) + { + items.Add((TElement)this.Deserialize(typeof(TElement), item)); + } + + return new LiteGrouping(key, items); + } + + this.RegisterType(interfaceType, SerializeGrouping, DeserializeGrouping); + + if (!_customDeserializer.ContainsKey(concreteType)) + { + this.RegisterType(concreteType, SerializeGrouping, DeserializeGrouping); + } + } + } +} diff --git a/LiteDB/Client/Mapper/BsonMapper.Serialize.cs b/LiteDB/Client/Mapper/BsonMapper.Serialize.cs index 93efbe7d3..6156ae5a4 100644 --- a/LiteDB/Client/Mapper/BsonMapper.Serialize.cs +++ b/LiteDB/Client/Mapper/BsonMapper.Serialize.cs @@ -1,9 +1,7 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Linq; using System.Reflection; -using static LiteDB.Constants; namespace LiteDB { @@ -110,7 +108,7 @@ internal BsonValue Serialize(Type type, object obj, int depth) } else if (obj is Enum) { - if (this.EnumAsInteger) + if (EnumAsInteger) { return new BsonValue((int)obj); } @@ -128,52 +126,56 @@ internal BsonValue Serialize(Type type, object obj, int depth) type = obj.GetType(); } - var itemType = type.GetTypeInfo().IsGenericType ? type.GetGenericArguments()[1] : typeof(object); + Type valueType = typeof(object); - return this.SerializeDictionary(itemType, dict, depth); + if (type.GetTypeInfo().IsGenericType) { + Type[] generics = type.GetGenericArguments(); + valueType = generics[1]; + } + + return SerializeDictionary(valueType, dict, depth); } // check if is a list or array else if (obj is IEnumerable) { - return this.SerializeArray(Reflection.GetListItemType(type), obj as IEnumerable, depth); + return SerializeArray(Reflection.GetListItemType(type), obj as IEnumerable, depth); } // otherwise serialize as a plain object else { - return this.SerializeObject(type, obj, depth); + return SerializeObject(type, obj, depth); } } private BsonArray SerializeArray(Type type, IEnumerable array, int depth) { - var arr = new BsonArray(); + BsonArray bsonArray = []; - foreach (var item in array) + foreach (object item in array) { - arr.Add(this.Serialize(type, item, depth)); + bsonArray.Add(Serialize(type, item, depth)); } - return arr; + return bsonArray; } - private BsonDocument SerializeDictionary(Type type, IDictionary dict, int depth) + private BsonDocument SerializeDictionary(Type valueType, IDictionary dict, int depth) { - var o = new BsonDocument(); + BsonDocument bsonDocument = []; - foreach (var key in dict.Keys) + foreach (object key in dict.Keys) { - var value = dict[key]; - var skey = key.ToString(); - - if (key is DateTime dateKey) - { - skey = dateKey.ToString("o"); - } - - o[skey] = this.Serialize(type, value, depth); + object value = dict[key]; + + var stringKey = key is DateTime dateKey + ? dateKey.ToString("o") ?? string.Empty + : key.ToString() ?? string.Empty; + + BsonValue bsonValue = Serialize(valueType, value, depth); + bsonDocument[stringKey] = bsonValue; } - return o; + return bsonDocument; } private BsonDocument SerializeObject(Type type, object obj, int depth) diff --git a/LiteDB/Client/Mapper/Linq/LinqExpressionVisitor.cs b/LiteDB/Client/Mapper/Linq/LinqExpressionVisitor.cs index fe006012f..7ee0d69e9 100644 --- a/LiteDB/Client/Mapper/Linq/LinqExpressionVisitor.cs +++ b/LiteDB/Client/Mapper/Linq/LinqExpressionVisitor.cs @@ -24,7 +24,9 @@ internal class LinqExpressionVisitor : ExpressionVisitor [typeof(Decimal)] = new NumberResolver("DECIMAL"), [typeof(Double)] = new NumberResolver("DOUBLE"), [typeof(ICollection)] = new ICollectionResolver(), + [typeof(IGrouping<,>)] = new GroupingResolver(), [typeof(Enumerable)] = new EnumerableResolver(), + [typeof(MemoryExtensions)] = new MemoryExtensionsResolver(), [typeof(Guid)] = new GuidResolver(), [typeof(Math)] = new MathResolver(), [typeof(Regex)] = new RegexResolver(), @@ -181,6 +183,13 @@ protected override Expression VisitMember(MemberExpression node) /// protected override Expression VisitMethodCall(MethodCallExpression node) { + if (this.IsSpanImplicitConversion(node.Method)) + { + this.Visit(node.Arguments[0]); + + return node; + } + // if special method for index access, eval index value (do not use parameters) if (this.IsMethodIndexEval(node, out var obj, out var idx)) { @@ -202,7 +211,20 @@ protected override Expression VisitMethodCall(MethodCallExpression node) } // if not found in resolver, try run method - if (!this.TryGetResolver(node.Method.DeclaringType, out var type)) + var hasResolver = this.TryGetResolver(node.Method.DeclaringType, out var type); + + if (node.Method.DeclaringType == typeof(Enumerable) && node.Arguments.Count > 0) + { + var first = node.Arguments[0].Type; + + if (first.IsGenericType && first.GetGenericTypeDefinition() == typeof(IGrouping<,>)) + { + type = _resolver[typeof(IGrouping<,>)]; + hasResolver = true; + } + } + + if (!hasResolver) { // if method are called by parameter expression and it's not exists, throw error var isParam = ParameterExpressionVisitor.Test(node); @@ -729,11 +751,13 @@ private object Evaluate(Expression expr, params Type[] validTypes) private bool TryGetResolver(Type declaringType, out ITypeResolver typeResolver) { // get method declaring type - if is from any kind of list, read as Enumerable + var isGrouping = declaringType?.IsGenericType == true && declaringType.GetGenericTypeDefinition() == typeof(IGrouping<,>); var isCollection = Reflection.IsCollection(declaringType); var isEnumerable = Reflection.IsEnumerable(declaringType); var isNullable = Reflection.IsNullable(declaringType); var type = + isGrouping ? typeof(IGrouping<,>) : isCollection ? typeof(ICollection) : isEnumerable ? typeof(Enumerable) : isNullable ? typeof(Nullable) : @@ -741,5 +765,30 @@ private bool TryGetResolver(Type declaringType, out ITypeResolver typeResolver) return _resolver.TryGetValue(type, out typeResolver); } + + private bool IsSpanImplicitConversion(MethodInfo method) + { + if (method == null || method.Name != "op_Implicit" || method.GetParameters().Length != 1) + { + return false; + } + + var returnType = method.ReturnType; + + return this.IsSpanLike(returnType); + } + + private bool IsSpanLike(Type type) + { + if (type == null || !type.IsGenericType) + { + return false; + } + + var definition = type.GetGenericTypeDefinition(); + var name = definition.FullName; + + return name == "System.Span`1" || name == "System.ReadOnlySpan`1"; + } } -} \ No newline at end of file +} diff --git a/LiteDB/Client/Mapper/Linq/TypeResolver/GroupingResolver.cs b/LiteDB/Client/Mapper/Linq/TypeResolver/GroupingResolver.cs new file mode 100644 index 000000000..2470b247e --- /dev/null +++ b/LiteDB/Client/Mapper/Linq/TypeResolver/GroupingResolver.cs @@ -0,0 +1,50 @@ +using System.Collections; +using System.Linq; +using System.Reflection; + +namespace LiteDB +{ + internal class GroupingResolver : EnumerableResolver + { + public override string ResolveMethod(MethodInfo method) + { + var name = Reflection.MethodName(method, 1); + switch (name) + { + case "AsEnumerable()": return "*"; + case "Where(Func)": return "FILTER(* => @1)"; + case "Select(Func)": return "MAP(* => @1)"; + case "Count()": return "COUNT(*)"; + case "Count(Func)": return "COUNT(FILTER(* => @1))"; + case "Any()": return "COUNT(*) > 0"; + case "Any(Func)": return "FILTER(* => @1) ANY %"; + case "All(Func)": return "FILTER(* => @1) ALL %"; + case "Sum()": return "SUM(*)"; + case "Sum(Func)": return "SUM(MAP(* => @1))"; + case "Average()": return "AVG(*)"; + case "Average(Func)": return "AVG(MAP(* => @1))"; + case "Max()": return "MAX(*)"; + case "Max(Func)": return "MAX(MAP(* => @1))"; + case "Min()": return "MIN(*)"; + case "Min(Func)": return "MIN(MAP(* => @1))"; + } + + return base.ResolveMethod(method); + } + + public override string ResolveMember(MemberInfo member) + { + if (member.Name == nameof(IGrouping.Key)) + { + return "@key"; + } + + if (member.Name == nameof(ICollection.Count)) + { + return "COUNT(*)"; + } + + return base.ResolveMember(member); + } + } +} diff --git a/LiteDB/Client/Mapper/Linq/TypeResolver/MemoryExtensionsResolver.cs b/LiteDB/Client/Mapper/Linq/TypeResolver/MemoryExtensionsResolver.cs new file mode 100644 index 000000000..f147e2fbb --- /dev/null +++ b/LiteDB/Client/Mapper/Linq/TypeResolver/MemoryExtensionsResolver.cs @@ -0,0 +1,36 @@ +using System.Reflection; + +namespace LiteDB +{ + internal class MemoryExtensionsResolver : ITypeResolver + { + public string ResolveMethod(MethodInfo method) + { + if (method.Name != nameof(System.MemoryExtensions.Contains)) + return null; + var parameters = method.GetParameters(); + + if (parameters.Length == 2) + { + return "@0 ANY = @1"; + } + + // Support the 3-parameter overload only when comparer defaults to null. + if (parameters.Length == 3) + { + var third = parameters[2]; + + if (third.HasDefaultValue && third.DefaultValue == null) + { + return "@0 ANY = @1"; + } + } + + return null; + } + + public string ResolveMember(MemberInfo member) => null; + + public string ResolveCtor(ConstructorInfo ctor) => null; + } +} diff --git a/LiteDB/Client/Shared/SharedEngine.cs b/LiteDB/Client/Shared/SharedEngine.cs index c25e7d591..1b7bb0b0c 100644 --- a/LiteDB/Client/Shared/SharedEngine.cs +++ b/LiteDB/Client/Shared/SharedEngine.cs @@ -3,10 +3,8 @@ using System.Collections.Generic; using System.IO; using System.Threading; -#if NETFRAMEWORK -using System.Security.AccessControl; -using System.Security.Principal; -#endif +using LiteDB.Client.Shared; +using LiteDB.Vector; namespace LiteDB { @@ -21,24 +19,19 @@ public SharedEngine(EngineSettings settings) { _settings = settings; - var name = Path.GetFullPath(settings.Filename).ToLower().Sha1(); + var name = SharedMutexNameFactory.Create(settings.Filename, settings.SharedMutexNameStrategy); try { -#if NETFRAMEWORK - var allowEveryoneRule = new MutexAccessRule(new SecurityIdentifier(WellKnownSidType.WorldSid, null), - MutexRights.FullControl, AccessControlType.Allow); - - var securitySettings = new MutexSecurity(); - securitySettings.AddAccessRule(allowEveryoneRule); - - _mutex = new Mutex(false, "Global\\" + name + ".Mutex", out _, securitySettings); -#else - _mutex = new Mutex(false, "Global\\" + name + ".Mutex"); -#endif + _mutex = SharedMutexFactory.Create(name); } catch (NotSupportedException ex) { + if (ex is PlatformNotSupportedException) + { + throw; + } + throw new PlatformNotSupportedException("Shared mode is not supported in platforms that do not implement named mutex.", ex); } } @@ -235,6 +228,11 @@ public bool EnsureIndex(string collection, string name, BsonExpression expressio return QueryDatabase(() => _engine.EnsureIndex(collection, name, expression, unique)); } + public bool EnsureVectorIndex(string collection, string name, BsonExpression expression, VectorIndexOptions options) + { + return QueryDatabase(() => _engine.EnsureVectorIndex(collection, name, expression, options)); + } + #endregion public void Dispose() diff --git a/LiteDB/Client/Shared/SharedMutexFactory.cs b/LiteDB/Client/Shared/SharedMutexFactory.cs new file mode 100644 index 000000000..ff70390aa --- /dev/null +++ b/LiteDB/Client/Shared/SharedMutexFactory.cs @@ -0,0 +1,130 @@ +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Win32.SafeHandles; + +namespace LiteDB +{ + internal static class SharedMutexFactory + { + private const string MutexPrefix = "Global\\"; + private const string MutexSuffix = ".Mutex"; + + public static Mutex Create(string name) + { + var fullName = MutexPrefix + name + MutexSuffix; + + if (!IsWindows()) + { + return new Mutex(false, fullName); + } + + try + { + return WindowsMutex.Create(fullName); + } + catch (Win32Exception ex) + { + throw new PlatformNotSupportedException("Shared mode is not supported because named mutex access control is unavailable on this platform.", ex); + } + catch (EntryPointNotFoundException ex) + { + throw new PlatformNotSupportedException("Shared mode is not supported because named mutex access control is unavailable on this platform.", ex); + } + catch (DllNotFoundException ex) + { + throw new PlatformNotSupportedException("Shared mode is not supported because named mutex access control is unavailable on this platform.", ex); + } + } + +#if NET6_0_OR_GREATER + private static bool IsWindows() + { + return OperatingSystem.IsWindows(); + } +#else + private static bool IsWindows() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + } +#endif + + private static class WindowsMutex + { + private const string WorldAccessSecurityDescriptor = "D:(A;;GA;;;WD)"; + private const uint SddlRevision1 = 1; + + public static Mutex Create(string name) + { + IntPtr descriptor = IntPtr.Zero; + + try + { + if (!NativeMethods.ConvertStringSecurityDescriptorToSecurityDescriptor(WorldAccessSecurityDescriptor, SddlRevision1, out descriptor, out _)) + { + throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to create security descriptor for shared mutex."); + } + + var attributes = new NativeMethods.SECURITY_ATTRIBUTES + { + nLength = (uint)Marshal.SizeOf(), + bInheritHandle = 0, + lpSecurityDescriptor = descriptor + }; + + var handle = NativeMethods.CreateMutexEx(ref attributes, name, 0, NativeMethods.MUTEX_ALL_ACCESS); + + if (handle == IntPtr.Zero || handle == NativeMethods.InvalidHandleValue) + { + throw new Win32Exception(Marshal.GetLastWin32Error(), "Failed to create shared mutex with global access."); + } + + var mutex = new Mutex(); + mutex.SafeWaitHandle = new SafeWaitHandle(handle, ownsHandle: true); + + return mutex; + } + finally + { + if (descriptor != IntPtr.Zero) + { + NativeMethods.LocalFree(descriptor); + } + } + } + } + + private static class NativeMethods + { + public static readonly IntPtr InvalidHandleValue = new IntPtr(-1); + public const uint MUTEX_ALL_ACCESS = 0x001F0001; + + [StructLayout(LayoutKind.Sequential)] + public struct SECURITY_ATTRIBUTES + { + public uint nLength; + public IntPtr lpSecurityDescriptor; + public int bInheritHandle; + } + + [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Unicode)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool ConvertStringSecurityDescriptorToSecurityDescriptor( + string StringSecurityDescriptor, + uint StringSDRevision, + out IntPtr SecurityDescriptor, + out uint SecurityDescriptorSize); + + [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] + public static extern IntPtr CreateMutexEx( + ref SECURITY_ATTRIBUTES lpMutexAttributes, + string lpName, + uint dwFlags, + uint dwDesiredAccess); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern IntPtr LocalFree(IntPtr hMem); + } + } +} diff --git a/LiteDB/Client/Shared/SharedMutexNameFactory.cs b/LiteDB/Client/Shared/SharedMutexNameFactory.cs new file mode 100644 index 000000000..8f0198bb0 --- /dev/null +++ b/LiteDB/Client/Shared/SharedMutexNameFactory.cs @@ -0,0 +1,92 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using LiteDB.Engine; + +namespace LiteDB.Client.Shared; + +internal static class SharedMutexNameFactory +{ + // Effective Windows limit for named mutexes (conservative). + private const int WINDOWS_MUTEX_NAME_MAX = 250; + + // If the caller adds "Global\" (7 chars) + the name + ".Mutex" (7 chars) to the mutex name, + // we account for it conservatively here without baking it into the return value. + // Adjust if your caller prepends something longer. + private const int CONSERVATIVE_EXTERNAL_PREFIX_LENGTH = 13; // e.g., "Global\\" + name + ".Mutex" + + internal static string Create(string fileName, SharedMutexNameStrategy strategy) + { + return strategy switch + { + SharedMutexNameStrategy.Default => CreateUsingUriEncodingWithFallback(fileName), + SharedMutexNameStrategy.UriEscape => CreateUsingUriEncoding(fileName), + SharedMutexNameStrategy.Sha1Hash => CreateUsingSha1(fileName), + _ => throw new ArgumentOutOfRangeException(nameof(strategy), strategy, null) + }; + } + + private static string CreateUsingUriEncodingWithFallback(string fileName) + { + var normalized = Normalize(fileName); + var uri = Uri.EscapeDataString(normalized); + + if (IsWindows() && + uri.Length + CONSERVATIVE_EXTERNAL_PREFIX_LENGTH > WINDOWS_MUTEX_NAME_MAX) + { + // Short, stable fallback well under the limit. + return "sha1-" + ComputeSha1Hex(normalized); + } + + return uri; + } + + private static string CreateUsingUriEncoding(string fileName) + { + var normalized = Normalize(fileName); + var uri = Uri.EscapeDataString(normalized); + + if (IsWindows() && + uri.Length + CONSERVATIVE_EXTERNAL_PREFIX_LENGTH > WINDOWS_MUTEX_NAME_MAX) + { + // Fallback to SHA to avoid ArgumentException on Windows. + return "sha1-" + ComputeSha1Hex(normalized); + } + + return uri; + } + + private static bool IsWindows() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + } + + internal static string CreateUsingSha1(string value) + { + var normalized = Normalize(value); + return ComputeSha1Hex(normalized); + } + + private static string Normalize(string path) + { + // Invariant casing + absolute path yields stable identity. + return Path.GetFullPath(path).ToLowerInvariant(); + } + + private static string ComputeSha1Hex(string input) + { + var data = Encoding.UTF8.GetBytes(input); + using var sha = SHA1.Create(); + var hashData = sha.ComputeHash(data); + + var sb = new StringBuilder(hashData.Length * 2); + foreach (var b in hashData) + { + sb.Append(b.ToString("X2")); + } + + return sb.ToString(); + } +} diff --git a/LiteDB/Client/SqlParser/Commands/Select.cs b/LiteDB/Client/SqlParser/Commands/Select.cs index f0269424b..5e09a1d06 100644 --- a/LiteDB/Client/SqlParser/Commands/Select.cs +++ b/LiteDB/Client/SqlParser/Commands/Select.cs @@ -126,18 +126,30 @@ private IBsonDataReader ParseSelect() _tokenizer.ReadToken(); _tokenizer.ReadToken().Expect("BY"); - var orderBy = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); + while (true) + { + var orderBy = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); - var orderByOrder = Query.Ascending; - var orderByToken = _tokenizer.LookAhead(); + var orderByOrder = Query.Ascending; + var orderByToken = _tokenizer.LookAhead(); - if (orderByToken.Is("ASC") || orderByToken.Is("DESC")) - { - orderByOrder = _tokenizer.ReadToken().Is("ASC") ? Query.Ascending : Query.Descending; - } + if (orderByToken.Is("ASC") || orderByToken.Is("DESC")) + { + orderByOrder = _tokenizer.ReadToken().Is("ASC") ? Query.Ascending : Query.Descending; + } + + query.OrderBy.Add(new QueryOrder(orderBy, orderByOrder)); - query.OrderBy = orderBy; - query.Order = orderByOrder; + var next = _tokenizer.LookAhead(); + + if (next.Type == TokenType.Comma) + { + _tokenizer.ReadToken(); + continue; + } + + break; + } } ahead = _tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon); diff --git a/LiteDB/Client/Structures/Query.cs b/LiteDB/Client/Structures/Query.cs index 9bbd6388e..83e02899f 100644 --- a/LiteDB/Client/Structures/Query.cs +++ b/LiteDB/Client/Structures/Query.cs @@ -36,7 +36,9 @@ public static Query All() /// public static Query All(int order = Ascending) { - return new Query { OrderBy = "_id", Order = order }; + var query = new Query(); + query.OrderBy.Add(new QueryOrder(BsonExpression.Create("_id"), order)); + return query; } /// @@ -44,7 +46,9 @@ public static Query All(int order = Ascending) /// public static Query All(string field, int order = Ascending) { - return new Query { OrderBy = field, Order = order }; + var query = new Query(); + query.OrderBy.Add(new QueryOrder(BsonExpression.Create(field), order)); + return query; } /// diff --git a/LiteDB/Client/Vector/LiteCollectionVectorExtensions.cs b/LiteDB/Client/Vector/LiteCollectionVectorExtensions.cs new file mode 100644 index 000000000..7de12c53c --- /dev/null +++ b/LiteDB/Client/Vector/LiteCollectionVectorExtensions.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq.Expressions; + +namespace LiteDB.Vector +{ + /// + /// Extension methods that expose vector-aware index creation for . + /// + public static class LiteCollectionVectorExtensions + { + public static bool EnsureIndex(this ILiteCollection collection, string name, BsonExpression expression, VectorIndexOptions options) + { + return Unwrap(collection).EnsureVectorIndex(name, expression, options); + } + + public static bool EnsureIndex(this ILiteCollection collection, BsonExpression expression, VectorIndexOptions options) + { + return Unwrap(collection).EnsureVectorIndex(expression, options); + } + + public static bool EnsureIndex(this ILiteCollection collection, Expression> keySelector, VectorIndexOptions options) + { + return Unwrap(collection).EnsureVectorIndex(keySelector, options); + } + + public static bool EnsureIndex(this ILiteCollection collection, string name, Expression> keySelector, VectorIndexOptions options) + { + return Unwrap(collection).EnsureVectorIndex(name, keySelector, options); + } + + private static LiteCollection Unwrap(ILiteCollection collection) + { + if (collection is null) + { + throw new ArgumentNullException(nameof(collection)); + } + + if (collection is LiteCollection concrete) + { + return concrete; + } + + throw new ArgumentException("Vector index operations require LiteDB's default collection implementation.", nameof(collection)); + } + } +} diff --git a/LiteDB/Client/Vector/LiteQueryableVectorExtensions.cs b/LiteDB/Client/Vector/LiteQueryableVectorExtensions.cs new file mode 100644 index 000000000..9f89f78a2 --- /dev/null +++ b/LiteDB/Client/Vector/LiteQueryableVectorExtensions.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Linq.Expressions; + +namespace LiteDB.Vector +{ + /// + /// Extension methods that surface vector-aware query capabilities for . + /// + public static class LiteQueryableVectorExtensions + { + public static ILiteQueryable WhereNear(this ILiteQueryable source, string vectorField, float[] target, double maxDistance) + { + return Unwrap(source).VectorWhereNear(vectorField, target, maxDistance); + } + + public static ILiteQueryable WhereNear(this ILiteQueryable source, BsonExpression fieldExpr, float[] target, double maxDistance) + { + return Unwrap(source).VectorWhereNear(fieldExpr, target, maxDistance); + } + + public static ILiteQueryable WhereNear(this ILiteQueryable source, Expression> field, float[] target, double maxDistance) + { + return Unwrap(source).VectorWhereNear(field, target, maxDistance); + } + + public static IEnumerable FindNearest(this ILiteQueryable source, string vectorField, float[] target, double maxDistance) + { + var queryable = Unwrap(source); + return queryable.VectorWhereNear(vectorField, target, maxDistance).ToEnumerable(); + } + + public static ILiteQueryableResult TopKNear(this ILiteQueryable source, Expression> field, float[] target, int k) + { + return Unwrap(source).VectorTopKNear(field, target, k); + } + + public static ILiteQueryableResult TopKNear(this ILiteQueryable source, string field, float[] target, int k) + { + return Unwrap(source).VectorTopKNear(field, target, k); + } + + public static ILiteQueryableResult TopKNear(this ILiteQueryable source, BsonExpression fieldExpr, float[] target, int k) + { + return Unwrap(source).VectorTopKNear(fieldExpr, target, k); + } + + private static LiteQueryable Unwrap(ILiteQueryable source) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + if (source is LiteQueryable liteQueryable) + { + return liteQueryable; + } + + throw new ArgumentException("Vector operations require LiteDB's default queryable implementation.", nameof(source)); + } + } +} diff --git a/LiteDB/Client/Vector/LiteRepositoryVectorExtensions.cs b/LiteDB/Client/Vector/LiteRepositoryVectorExtensions.cs new file mode 100644 index 000000000..0ebfa6a3d --- /dev/null +++ b/LiteDB/Client/Vector/LiteRepositoryVectorExtensions.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq.Expressions; + +namespace LiteDB.Vector +{ + /// + /// Extension methods that expose vector-aware index creation for . + /// + public static class LiteRepositoryVectorExtensions + { + public static bool EnsureIndex(this ILiteRepository repository, string name, BsonExpression expression, VectorIndexOptions options, string collectionName = null) + { + return Unwrap(repository).EnsureVectorIndex(name, expression, options, collectionName); + } + + public static bool EnsureIndex(this ILiteRepository repository, BsonExpression expression, VectorIndexOptions options, string collectionName = null) + { + return Unwrap(repository).EnsureVectorIndex(expression, options, collectionName); + } + + public static bool EnsureIndex(this ILiteRepository repository, Expression> keySelector, VectorIndexOptions options, string collectionName = null) + { + return Unwrap(repository).EnsureVectorIndex(keySelector, options, collectionName); + } + + public static bool EnsureIndex(this ILiteRepository repository, string name, Expression> keySelector, VectorIndexOptions options, string collectionName = null) + { + return Unwrap(repository).EnsureVectorIndex(name, keySelector, options, collectionName); + } + + private static LiteRepository Unwrap(ILiteRepository repository) + { + if (repository is null) + { + throw new ArgumentNullException(nameof(repository)); + } + + if (repository is LiteRepository liteRepository) + { + return liteRepository; + } + + throw new ArgumentException("Vector index operations require LiteDB's default repository implementation.", nameof(repository)); + } + } +} diff --git a/LiteDB/Client/Vector/VectorDistanceMetric.cs b/LiteDB/Client/Vector/VectorDistanceMetric.cs new file mode 100644 index 000000000..4aa7f69d7 --- /dev/null +++ b/LiteDB/Client/Vector/VectorDistanceMetric.cs @@ -0,0 +1,12 @@ +namespace LiteDB.Vector +{ + /// + /// Supported metrics for vector similarity operations. + /// + public enum VectorDistanceMetric : byte + { + Euclidean = 0, + Cosine = 1, + DotProduct = 2 + } +} diff --git a/LiteDB/Client/Vector/VectorIndexOptions.cs b/LiteDB/Client/Vector/VectorIndexOptions.cs new file mode 100644 index 000000000..248f6c9a4 --- /dev/null +++ b/LiteDB/Client/Vector/VectorIndexOptions.cs @@ -0,0 +1,31 @@ +using System; + +namespace LiteDB.Vector +{ + /// + /// Options used when creating a vector-aware index. + /// + public sealed class VectorIndexOptions + { + /// + /// Gets the expected dimensionality of the indexed vectors. + /// + public ushort Dimensions { get; } + + /// + /// Gets the distance metric used when comparing vectors. + /// + public VectorDistanceMetric Metric { get; } + + public VectorIndexOptions(ushort dimensions, VectorDistanceMetric metric = VectorDistanceMetric.Cosine) + { + if (dimensions == 0) + { + throw new ArgumentOutOfRangeException(nameof(dimensions), dimensions, "Dimensions must be greater than zero."); + } + + this.Dimensions = dimensions; + this.Metric = metric; + } + } +} diff --git a/LiteDB/Document/BsonArray.cs b/LiteDB/Document/BsonArray.cs index d911e927f..076d0114c 100644 --- a/LiteDB/Document/BsonArray.cs +++ b/LiteDB/Document/BsonArray.cs @@ -36,6 +36,14 @@ public BsonArray(IEnumerable items) this.AddRange(items); } + + public BsonArray(BsonArray items) + : this() + { + if (items == null) throw new ArgumentNullException(nameof(items)); + + this.AddRange(items); + } public new IList RawValue => (IList)base.RawValue; @@ -73,9 +81,8 @@ public void AddRange(TCollection collection) foreach (var bsonValue in collection) { - list.Add(bsonValue ?? Null); + list.Add(bsonValue ?? Null); } - } public void AddRange(IEnumerable items) diff --git a/LiteDB/Document/BsonType.cs b/LiteDB/Document/BsonType.cs index 1b00cba41..133361927 100644 --- a/LiteDB/Document/BsonType.cs +++ b/LiteDB/Document/BsonType.cs @@ -1,4 +1,4 @@ -namespace LiteDB +namespace LiteDB { /// /// All supported BsonTypes in sort order @@ -26,6 +26,7 @@ public enum BsonType : byte Boolean = 12, DateTime = 13, - MaxValue = 14 + MaxValue = 14, + Vector = 100, } } \ No newline at end of file diff --git a/LiteDB/Document/BsonValue.cs b/LiteDB/Document/BsonValue.cs index 62d62ecac..80c83422b 100644 --- a/LiteDB/Document/BsonValue.cs +++ b/LiteDB/Document/BsonValue.cs @@ -119,6 +119,12 @@ protected BsonValue(BsonType type, object rawValue) this.RawValue = rawValue; } + protected BsonValue(float[] value) + { + this.Type = BsonType.Vector; + this.RawValue = value; + } + public BsonValue(object value) { this.RawValue = value; @@ -135,6 +141,7 @@ public BsonValue(object value) else if (value is ObjectId) this.Type = BsonType.ObjectId; else if (value is Guid) this.Type = BsonType.Guid; else if (value is Boolean) this.Type = BsonType.Boolean; + else if (value is float[]) this.Type = BsonType.Vector; else if (value is DateTime) { this.Type = BsonType.DateTime; @@ -246,6 +253,10 @@ public virtual BsonValue this[int index] [DebuggerBrowsable(DebuggerBrowsableState.Never)] public Guid AsGuid => (Guid)this.RawValue; + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public float[] AsVector => (float[])this.RawValue; + + #endregion #region IsTypes @@ -289,6 +300,9 @@ public virtual BsonValue this[int index] [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsGuid => this.Type == BsonType.Guid; + [DebuggerBrowsable(DebuggerBrowsableState.Never)] + public bool IsVector => this.Type == BsonType.Vector; + [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsDateTime => this.Type == BsonType.DateTime; @@ -410,6 +424,18 @@ public static implicit operator BsonValue(Guid value) return new BsonValue(value); } + // Vector + public static implicit operator float[](BsonValue value) + { + return value.AsVector; + } + + // Vector + public static implicit operator BsonValue(float[] value) + { + return new BsonVector(value); + } + // Boolean public static implicit operator Boolean(BsonValue value) { @@ -567,6 +593,25 @@ public virtual int CompareTo(BsonValue other, Collation collation) case BsonType.Guid: return this.AsGuid.CompareTo(other.AsGuid); case BsonType.Boolean: return this.AsBoolean.CompareTo(other.AsBoolean); + case BsonType.Vector: + { + var left = this.AsVector; + var right = other.AsVector; + var length = Math.Min(left.Length, right.Length); + + for (var i = 0; i < length; i++) + { + var result = left[i].CompareTo(right[i]); + if (result != 0) + { + return result; + } + } + + if (left.Length == right.Length) return 0; + + return left.Length < right.Length ? -1 : 1; + } case BsonType.DateTime: var d0 = this.AsDateTime; var d1 = other.AsDateTime; @@ -667,6 +712,7 @@ internal virtual int GetBytesCount(bool recalc) case BsonType.Boolean: return 1; case BsonType.DateTime: return 8; + case BsonType.Vector: return 2 + (4 * this.AsVector.Length); case BsonType.Document: return this.AsDocument.GetBytesCount(recalc); case BsonType.Array: return this.AsArray.GetBytesCount(recalc); diff --git a/LiteDB/Document/BsonVector.cs b/LiteDB/Document/BsonVector.cs new file mode 100644 index 000000000..826db2d26 --- /dev/null +++ b/LiteDB/Document/BsonVector.cs @@ -0,0 +1,28 @@ +using System.Linq; + +namespace LiteDB; + +public class BsonVector(float[] values) : BsonValue(values) +{ + public float[] Values => AsVector; + + public BsonValue Clone() + { + return new BsonVector((float[])Values.Clone()); + } + + public override bool Equals(object? obj) + { + return obj is BsonVector other && Values.SequenceEqual(other.Values); + } + + public override int GetHashCode() + { + return Values.Aggregate(17, (acc, f) => acc * 31 + f.GetHashCode()); + } + + public override string ToString() + { + return $"[{string.Join(", ", Values.Select(v => v.ToString("0.###")))}]"; + } +} \ No newline at end of file diff --git a/LiteDB/Document/Expression/Methods/Vector.cs b/LiteDB/Document/Expression/Methods/Vector.cs new file mode 100644 index 000000000..a8189be31 --- /dev/null +++ b/LiteDB/Document/Expression/Methods/Vector.cs @@ -0,0 +1,55 @@ +using System; +using System.Linq; + +namespace LiteDB; + +internal partial class BsonExpressionMethods +{ + public static BsonValue VECTOR_SIM(BsonValue left, BsonValue right) + { + if (!(left.IsArray || left.Type == BsonType.Vector) || !(right.IsArray || right.Type == BsonType.Vector)) + return BsonValue.Null; + + var query = left.IsArray + ? left.AsArray + : new BsonArray(left.AsVector.Select(x => (BsonValue)x)); + + var candidate = right.IsVector + ? right.AsVector + : right.AsArray.Select(x => + { + try { return (float)x.AsDouble; } + catch { return float.NaN; } + }).ToArray(); + + if (query.Count != candidate.Length) return BsonValue.Null; + + double dot = 0, magQ = 0, magC = 0; + + for (int i = 0; i < candidate.Length; i++) + { + double q; + try + { + q = query[i].AsDouble; + } + catch + { + return BsonValue.Null; + } + + var c = (double)candidate[i]; + + if (double.IsNaN(c)) return BsonValue.Null; + + dot += q * c; + magQ += q * q; + magC += c * c; + } + + if (magQ == 0 || magC == 0) return BsonValue.Null; + + var cosine = 1.0 - (dot / (Math.Sqrt(magQ) * Math.Sqrt(magC))); + return double.IsNaN(cosine) ? BsonValue.Null : cosine; + } +} \ No newline at end of file diff --git a/LiteDB/Document/Expression/Parser/BsonExpressionFunctions.cs b/LiteDB/Document/Expression/Parser/BsonExpressionFunctions.cs index 3c8796d69..9c33cbe1b 100644 --- a/LiteDB/Document/Expression/Parser/BsonExpressionFunctions.cs +++ b/LiteDB/Document/Expression/Parser/BsonExpressionFunctions.cs @@ -59,5 +59,10 @@ public static IEnumerable SORT(BsonDocument root, Collation collation { return SORT(root, collation, parameters, input, sortExpr, order: 1); } + + public static BsonValue VECTOR_SIM(BsonDocument root, Collation collation, BsonDocument parameters, BsonValue left, BsonValue right) + { + return BsonExpressionMethods.VECTOR_SIM(left, right); + } } } diff --git a/LiteDB/Document/Expression/Parser/BsonExpressionOperators.cs b/LiteDB/Document/Expression/Parser/BsonExpressionOperators.cs index 61cee8ce8..6007e0db3 100644 --- a/LiteDB/Document/Expression/Parser/BsonExpressionOperators.cs +++ b/LiteDB/Document/Expression/Parser/BsonExpressionOperators.cs @@ -202,10 +202,16 @@ public static BsonValue IN(Collation collation, BsonValue left, BsonValue right) } else { - return left == right; + return collation.Equals(left, right); } } + /// + /// Compute the cosine distance between two vectors (or arrays that can be interpreted as vectors). + /// Returns null when the arguments cannot be converted into vectors of matching lengths. + /// + public static BsonValue VECTOR_SIM(BsonValue left, BsonValue right) => BsonExpressionMethods.VECTOR_SIM(left, right); + public static BsonValue IN_ANY(Collation collation, IEnumerable left, BsonValue right) => left.Any(x => IN(collation, x, right)); public static BsonValue IN_ALL(Collation collation, IEnumerable left, BsonValue right) => left.All(x => IN(collation, x, right)); diff --git a/LiteDB/Document/Expression/Parser/BsonExpressionParser.cs b/LiteDB/Document/Expression/Parser/BsonExpressionParser.cs index 77ca4afe5..a1736ad9f 100644 --- a/LiteDB/Document/Expression/Parser/BsonExpressionParser.cs +++ b/LiteDB/Document/Expression/Parser/BsonExpressionParser.cs @@ -35,6 +35,9 @@ internal class BsonExpressionParser ["+"] = Tuple.Create("+", M("ADD"), BsonExpressionType.Add), ["-"] = Tuple.Create("-", M("MINUS"), BsonExpressionType.Subtract), + // vector similarity operator returns the cosine distance between two vectors + ["VECTOR_SIM"] = Tuple.Create(" VECTOR_SIM ", M("VECTOR_SIM"), BsonExpressionType.VectorSim), + // predicate ["LIKE"] = Tuple.Create(" LIKE ", M("LIKE"), BsonExpressionType.Like), ["BETWEEN"] = Tuple.Create(" BETWEEN ", M("BETWEEN"), BsonExpressionType.Between), @@ -74,7 +77,7 @@ internal class BsonExpressionParser // logic (will use Expression.AndAlso|OrElse) ["AND"] = Tuple.Create(" AND ", (MethodInfo)null, BsonExpressionType.And), - ["OR"] = Tuple.Create(" OR ", (MethodInfo)null, BsonExpressionType.Or) + ["OR"] = Tuple.Create(" OR ", (MethodInfo)null, BsonExpressionType.Or), }; private static readonly MethodInfo _parameterPathMethod = M("PARAMETER_PATH"); @@ -1177,6 +1180,9 @@ private static BsonExpression TryParseFunction(Tokenizer tokenizer, ExpressionCo case "MAP": return ParseFunction(token, BsonExpressionType.Map, tokenizer, context, parameters, scope); case "FILTER": return ParseFunction(token, BsonExpressionType.Filter, tokenizer, context, parameters, scope); case "SORT": return ParseFunction(token, BsonExpressionType.Sort, tokenizer, context, parameters, scope); + case "VECTOR_SIM": + return ParseFunction(token, BsonExpressionType.VectorSim, tokenizer, context, parameters, scope, + convertScalarLeftToEnumerable: false, isScalarResult: true); } return null; @@ -1186,7 +1192,7 @@ private static BsonExpression TryParseFunction(Tokenizer tokenizer, ExpressionCo /// Parse expression functions, like MAP, FILTER or SORT. /// MAP(items[*] => @.Name) /// - private static BsonExpression ParseFunction(string functionName, BsonExpressionType type, Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) + private static BsonExpression ParseFunction(string functionName, BsonExpressionType type, Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope, bool convertScalarLeftToEnumerable = true, bool isScalarResult = false) { // check if next token are ( otherwise returns null (is not a function) if (tokenizer.LookAhead().Type != TokenType.OpenParenthesis) return null; @@ -1197,7 +1203,7 @@ private static BsonExpression ParseFunction(string functionName, BsonExpressionT var left = ParseSingleExpression(tokenizer, context, parameters, scope); // if left is a scalar expression, convert into enumerable expression (avoid to use [*] all the time) - if (left.IsScalar) + if (convertScalarLeftToEnumerable && left.IsScalar) { left = ConvertToEnumerable(left); } @@ -1269,7 +1275,7 @@ private static BsonExpression ParseFunction(string functionName, BsonExpressionT Parameters = parameters, IsImmutable = isImmutable, UseSource = useSource, - IsScalar = false, + IsScalar = isScalarResult, Fields = fields, Expression = Expression.Call(method, args.ToArray()), Source = src.ToString() diff --git a/LiteDB/Document/Expression/Parser/BsonExpressionType.cs b/LiteDB/Document/Expression/Parser/BsonExpressionType.cs index 605ae9075..9c4bd84f3 100644 --- a/LiteDB/Document/Expression/Parser/BsonExpressionType.cs +++ b/LiteDB/Document/Expression/Parser/BsonExpressionType.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Linq; @@ -44,6 +44,7 @@ public enum BsonExpressionType : byte Map = 27, Filter = 28, Sort = 29, - Source = 30 + Source = 30, + VectorSim = 50 } } diff --git a/LiteDB/Document/Json/JsonWriter.cs b/LiteDB/Document/Json/JsonWriter.cs index eaa7ad85c..0b420cede 100644 --- a/LiteDB/Document/Json/JsonWriter.cs +++ b/LiteDB/Document/Json/JsonWriter.cs @@ -116,6 +116,12 @@ private void WriteValue(BsonValue value) case BsonType.MaxValue: this.WriteExtendDataType("$maxValue", "1"); break; + + case BsonType.Vector: + var vector = value.AsVector; + var array = new BsonArray(vector.Select(x => (BsonValue)x)); + this.WriteArray(array); + break; } } diff --git a/LiteDB/Engine/Disk/DiskReader.cs b/LiteDB/Engine/Disk/DiskReader.cs index c419f16b4..adf0e4313 100644 --- a/LiteDB/Engine/Disk/DiskReader.cs +++ b/LiteDB/Engine/Disk/DiskReader.cs @@ -50,7 +50,7 @@ public PageBuffer ReadPage(long position, bool writable, FileOrigin origin) _cache.GetWritablePage(position, origin, (pos, buf) => this.ReadStream(stream, pos, buf)) : _cache.GetReadablePage(position, origin, (pos, buf) => this.ReadStream(stream, pos, buf)); -#if DEBUG +#if DEBUG || TESTING _state.SimulateDiskReadFail?.Invoke(page); #endif diff --git a/LiteDB/Engine/Disk/DiskService.cs b/LiteDB/Engine/Disk/DiskService.cs index 73e7910b5..ca2e8d15c 100644 --- a/LiteDB/Engine/Disk/DiskService.cs +++ b/LiteDB/Engine/Disk/DiskService.cs @@ -190,7 +190,7 @@ public int WriteLogDisk(IEnumerable pages) // set log stream position to page stream.Position = page.Position; -#if DEBUG +#if DEBUG || TESTING _state.SimulateDiskWriteFail?.Invoke(page); #endif @@ -202,6 +202,7 @@ public int WriteLogDisk(IEnumerable pages) count++; } + stream.Flush(); } return count; diff --git a/LiteDB/Engine/Disk/Serializer/BufferReader.NetCore.cs b/LiteDB/Engine/Disk/Serializer/BufferReader.NetCore.cs new file mode 100644 index 000000000..3458326b9 --- /dev/null +++ b/LiteDB/Engine/Disk/Serializer/BufferReader.NetCore.cs @@ -0,0 +1,167 @@ +using System; +using static LiteDB.Constants; + +namespace LiteDB.Engine; + +internal partial class BufferReader +{ + /// + /// Read string with fixed size + /// + public string ReadString(int count) + { + if (count == 0) + { + return string.Empty; + } + + // if fits in current segment, use inner array - otherwise copy from multiples segments + if (_currentPosition + count <= _current.Count) + { + var span = new ReadOnlySpan(_current.Array, _current.Offset + _currentPosition, count); + var value = StringEncoding.UTF8.GetString(span); + + this.MoveForward(count); + + return value; + } + + const int stackLimit = 256; + + if (count <= stackLimit) + { + Span stackBuffer = stackalloc byte[stackLimit]; + var destination = stackBuffer.Slice(0, count); + + this.ReadIntoSpan(destination); + + return StringEncoding.UTF8.GetString(destination); + } + + var rented = _bufferPool.Rent(count); + + try + { + var destination = rented.AsSpan(0, count); + + this.ReadIntoSpan(destination); + + return StringEncoding.UTF8.GetString(destination); + } + finally + { + _bufferPool.Return(rented, true); + } + } + + private void ReadIntoSpan(Span destination) + { + var offset = 0; + + while (offset < destination.Length) + { + if (_currentPosition == _current.Count) + { + this.MoveForward(0); + + if (_isEOF) + { + break; + } + } + + var available = _current.Count - _currentPosition; + var toCopy = Math.Min(destination.Length - offset, available); + + var source = new ReadOnlySpan(_current.Array, _current.Offset + _currentPosition, toCopy); + source.CopyTo(destination.Slice(offset, toCopy)); + + this.MoveForward(toCopy); + offset += toCopy; + } + + ENSURE(offset == destination.Length, "current value must fit inside defined buffer"); + } + + /// + /// Reading string until find \0 at end + /// + public string ReadCString() + { + // first try read CString in current segment + if (this.TryReadCStringCurrentSegment(out var value)) + { + return value; + } + else + { + const int stackLimit = 256; + + Span stackBuffer = stackalloc byte[stackLimit]; + Span destination = stackBuffer; + byte[] rented = null; + var total = 0; + + while (true) + { + if (_currentPosition == _current.Count) + { + this.MoveForward(0); + + if (_isEOF) + { + ENSURE(false, "missing null terminator for CString"); + } + + continue; + } + + var available = _current.Count - _currentPosition; + + var span = new ReadOnlySpan(_current.Array, _current.Offset + _currentPosition, available); + var terminator = span.IndexOf((byte)0x00); + var take = terminator >= 0 ? terminator : span.Length; + + var required = total + take; + + if (required > destination.Length) + { + var newLength = Math.Max(required, Math.Max(destination.Length * 2, stackLimit * 2)); + var buffer = _bufferPool.Rent(newLength); + + destination.Slice(0, total).CopyTo(buffer.AsSpan(0, total)); + + if (rented != null) + { + _bufferPool.Return(rented, true); + } + + rented = buffer; + destination = rented.AsSpan(); + } + + if (take > 0) + { + span.Slice(0, take).CopyTo(destination.Slice(total)); + total += take; + this.MoveForward(take); + } + + if (terminator >= 0) + { + this.MoveForward(1); // +1 to '\0' + break; + } + } + + var result = StringEncoding.UTF8.GetString(destination.Slice(0, total)); + + if (rented != null) + { + _bufferPool.Return(rented, true); + } + + return result; + } + } +} \ No newline at end of file diff --git a/LiteDB/Engine/Disk/Serializer/BufferReader.NetStd.cs b/LiteDB/Engine/Disk/Serializer/BufferReader.NetStd.cs new file mode 100644 index 000000000..de41a2742 --- /dev/null +++ b/LiteDB/Engine/Disk/Serializer/BufferReader.NetStd.cs @@ -0,0 +1,72 @@ +using System.IO; +using static LiteDB.Constants; + +namespace LiteDB.Engine; + +internal partial class BufferReader +{ + /// + /// Read string with fixed size + /// + public string ReadString(int count) + { + string value; + + // if fits in current segment, use inner array - otherwise copy from multiples segments + if (_currentPosition + count <= _current.Count) + { + value = StringEncoding.UTF8.GetString(_current.Array, _current.Offset + _currentPosition, count); + + this.MoveForward(count); + } + else + { + // rent a buffer to be re-usable + var buffer = _bufferPool.Rent(count); + + this.Read(buffer, 0, count); + + value = StringEncoding.UTF8.GetString(buffer, 0, count); + + _bufferPool.Return(buffer, true); + } + + return value; + } + + /// + /// Reading string until find \0 at end + /// + public string ReadCString() + { + // first try read CString in current segment + if (this.TryReadCStringCurrentSegment(out var value)) + { + return value; + } + else + { + using (var mem = new MemoryStream()) + { + // copy all first segment + var initialCount = _current.Count - _currentPosition; + + mem.Write(_current.Array, _current.Offset + _currentPosition, initialCount); + + this.MoveForward(initialCount); + + // and go to next segment + while (_current[_currentPosition] != 0x00 && _isEOF == false) + { + mem.WriteByte(_current[_currentPosition]); + + this.MoveForward(1); + } + + this.MoveForward(1); // +1 to '\0' + + return StringEncoding.UTF8.GetString(mem.ToArray()); + } + } + } +} \ No newline at end of file diff --git a/LiteDB/Engine/Disk/Serializer/BufferReader.cs b/LiteDB/Engine/Disk/Serializer/BufferReader.cs index 98618124f..980ff2542 100644 --- a/LiteDB/Engine/Disk/Serializer/BufferReader.cs +++ b/LiteDB/Engine/Disk/Serializer/BufferReader.cs @@ -9,7 +9,7 @@ namespace LiteDB.Engine /// /// Read multiple array segment as a single linear segment - Forward Only /// - internal class BufferReader : IDisposable + internal partial class BufferReader : IDisposable { private readonly IEnumerator _source; private readonly bool _utcDate; @@ -145,72 +145,7 @@ public void Consume() #endregion #region Read String - - /// - /// Read string with fixed size - /// - public string ReadString(int count) - { - string value; - - // if fits in current segment, use inner array - otherwise copy from multiples segments - if (_currentPosition + count <= _current.Count) - { - value = StringEncoding.UTF8.GetString(_current.Array, _current.Offset + _currentPosition, count); - - this.MoveForward(count); - } - else - { - // rent a buffer to be re-usable - var buffer = _bufferPool.Rent(count); - - this.Read(buffer, 0, count); - - value = StringEncoding.UTF8.GetString(buffer, 0, count); - - _bufferPool.Return(buffer, true); - } - - return value; - } - - /// - /// Reading string until find \0 at end - /// - public string ReadCString() - { - // first try read CString in current segment - if (this.TryReadCStringCurrentSegment(out var value)) - { - return value; - } - else - { - using (var mem = new MemoryStream()) - { - // copy all first segment - var initialCount = _current.Count - _currentPosition; - - mem.Write(_current.Array, _current.Offset + _currentPosition, initialCount); - - this.MoveForward(initialCount); - - // and go to next segment - while (_current[_currentPosition] != 0x00 && _isEOF == false) - { - mem.WriteByte(_current[_currentPosition]); - - this.MoveForward(1); - } - - this.MoveForward(1); // +1 to '\0' - - return StringEncoding.UTF8.GetString(mem.ToArray()); - } - } - } - + /// /// Try read CString in current segment avoind read byte-to-byte over segments /// @@ -239,7 +174,7 @@ private bool TryReadCStringCurrentSegment(out string value) #endregion #region Read Numbers - + private T ReadNumber(Func convert, int size) { T value; @@ -267,7 +202,9 @@ private T ReadNumber(Func convert, int size) public Int32 ReadInt32() => this.ReadNumber(BitConverter.ToInt32, 4); public Int64 ReadInt64() => this.ReadNumber(BitConverter.ToInt64, 8); + public UInt16 ReadUInt16() => this.ReadNumber(BitConverter.ToUInt16, 2); public UInt32 ReadUInt32() => this.ReadNumber(BitConverter.ToUInt32, 4); + public Single ReadSingle() => this.ReadNumber(BitConverter.ToSingle, 4); public Double ReadDouble() => this.ReadNumber(BitConverter.ToDouble, 8); public Decimal ReadDecimal() @@ -352,6 +289,20 @@ public bool ReadBoolean() return value; } + private BsonValue ReadVector() + { + var length = this.ReadUInt16(); + var values = new float[length]; + + for (var i = 0; i < length; i++) + { + values[i] = this.ReadSingle(); + } + + return new BsonVector(values); + } + + /// /// Write single byte /// @@ -413,10 +364,14 @@ public BsonValue ReadIndexKey() case BsonType.MinValue: return BsonValue.MinValue; case BsonType.MaxValue: return BsonValue.MaxValue; + case BsonType.Vector: return this.ReadVector(); + default: throw new NotImplementedException(); } } + + #endregion #region BsonDocument as SPECS @@ -592,8 +547,12 @@ private BsonValue ReadElement(HashSet remaining, out string name) { return BsonValue.MaxValue; } + else if (type == 0x64) // Vector + { + return this.ReadVector(); + } - throw new NotSupportedException("BSON type not supported"); + throw new NotSupportedException("BSON type not supported"); } #endregion diff --git a/LiteDB/Engine/Disk/Serializer/BufferWriter.NetCore.cs b/LiteDB/Engine/Disk/Serializer/BufferWriter.NetCore.cs new file mode 100644 index 000000000..c82861aca --- /dev/null +++ b/LiteDB/Engine/Disk/Serializer/BufferWriter.NetCore.cs @@ -0,0 +1,156 @@ +using System; +using static LiteDB.Constants; + +namespace LiteDB.Engine; + +internal partial class BufferWriter +{ + private const int StackAllocationThreshold = 256; + + /// + /// Write String with \0 at end + /// + public partial void WriteCString(string value) + { + if (value.IndexOf('\0') > -1) throw LiteException.InvalidNullCharInString(); + + var bytesCount = StringEncoding.UTF8.GetByteCount(value); + + if (this.TryWriteInline(value.AsSpan(), bytesCount, 1)) + { + this.Write((byte)0x00); + return; + } + + if (bytesCount <= StackAllocationThreshold) + { + Span stackBuffer = stackalloc byte[StackAllocationThreshold]; + var buffer = stackBuffer.Slice(0, bytesCount); + + StringEncoding.UTF8.GetBytes(value.AsSpan(), buffer); + + this.WriteSpan(buffer); + } + else + { + var rented = _bufferPool.Rent(bytesCount); + + try + { + var buffer = rented.AsSpan(0, bytesCount); + + StringEncoding.UTF8.GetBytes(value.AsSpan(), buffer); + + this.WriteSpan(buffer); + } + finally + { + _bufferPool.Return(rented, true); + } + } + + this.Write((byte)0x00); + } + + /// + /// Write string into output buffer. + /// Support direct string (with no length information) or BSON specs: with (legnth + 1) [4 bytes] before and '\0' at end = 5 extra bytes + /// + public partial void WriteString(string value, bool specs) + { + var count = StringEncoding.UTF8.GetByteCount(value); + + if (specs) + { + this.Write(count + 1); // write Length + 1 (for \0) + } + + if (this.TryWriteInline(value.AsSpan(), count, specs ? 1 : 0)) + { + if (specs) + { + this.Write((byte)0x00); + } + + return; + } + + if (count <= StackAllocationThreshold) + { + Span stackBuffer = stackalloc byte[StackAllocationThreshold]; + var buffer = stackBuffer.Slice(0, count); + + StringEncoding.UTF8.GetBytes(value.AsSpan(), buffer); + + this.WriteSpan(buffer); + } + else + { + var rented = _bufferPool.Rent(count); + + try + { + var buffer = rented.AsSpan(0, count); + + StringEncoding.UTF8.GetBytes(value.AsSpan(), buffer); + + this.WriteSpan(buffer); + } + finally + { + _bufferPool.Return(rented, true); + } + } + + if (specs) + { + this.Write((byte)0x00); + } + } + + private bool TryWriteInline(ReadOnlySpan chars, int byteCount, int extraBytes) + { + var required = byteCount + extraBytes; + + if (required > _current.Count - _currentPosition) + { + return false; + } + + if (byteCount > 0) + { + var destination = new Span(_current.Array, _current.Offset + _currentPosition, byteCount); + var written = StringEncoding.UTF8.GetBytes(chars, destination); + ENSURE(written == byteCount, "encoded byte count mismatch"); + + this.MoveForward(byteCount); + } + + return true; + } + + private void WriteSpan(ReadOnlySpan source) + { + var offset = 0; + + while (offset < source.Length) + { + if (_currentPosition == _current.Count) + { + this.MoveForward(0); + + ENSURE(_isEOF == false, "current value must fit inside defined buffer"); + } + + var available = _current.Count - _currentPosition; + var toCopy = Math.Min(source.Length - offset, available); + + var target = new Span(_current.Array, _current.Offset + _currentPosition, toCopy); + source.Slice(offset, toCopy).CopyTo(target); + + this.MoveForward(toCopy); + offset += toCopy; + } + } +} + diff --git a/LiteDB/Engine/Disk/Serializer/BufferWriter.NetStd.cs b/LiteDB/Engine/Disk/Serializer/BufferWriter.NetStd.cs new file mode 100644 index 000000000..ab38a6809 --- /dev/null +++ b/LiteDB/Engine/Disk/Serializer/BufferWriter.NetStd.cs @@ -0,0 +1,79 @@ +using static LiteDB.Constants; + +namespace LiteDB.Engine; + +internal partial class BufferWriter +{ + /// + /// Write String with \0 at end + /// + public partial void WriteCString(string value) + { + if (value.IndexOf('\0') > -1) throw LiteException.InvalidNullCharInString(); + + var bytesCount = StringEncoding.UTF8.GetByteCount(value); + var available = _current.Count - _currentPosition; // avaiable in current segment + + // can write direct in current segment (use < because need +1 \0) + if (bytesCount < available) + { + StringEncoding.UTF8.GetBytes(value, 0, value.Length, _current.Array, _current.Offset + _currentPosition); + + _current[_currentPosition + bytesCount] = 0x00; + + this.MoveForward(bytesCount + 1); // +1 to '\0' + } + else + { + var buffer = _bufferPool.Rent(bytesCount); + + StringEncoding.UTF8.GetBytes(value, 0, value.Length, buffer, 0); + + this.Write(buffer, 0, bytesCount); + + _current[_currentPosition] = 0x00; + + this.MoveForward(1); + + _bufferPool.Return(buffer, true); + } + } + + /// + /// Write string into output buffer. + /// Support direct string (with no length information) or BSON specs: with (legnth + 1) [4 bytes] before and '\0' at end = 5 extra bytes + /// + public partial void WriteString(string value, bool specs) + { + var count = StringEncoding.UTF8.GetByteCount(value); + + if (specs) + { + this.Write(count + 1); // write Length + 1 (for \0) + } + + if (count <= _current.Count - _currentPosition) + { + StringEncoding.UTF8.GetBytes(value, 0, value.Length, _current.Array, _current.Offset + _currentPosition); + + this.MoveForward(count); + } + else + { + // rent a buffer to be re-usable + var buffer = _bufferPool.Rent(count); + + StringEncoding.UTF8.GetBytes(value, 0, value.Length, buffer, 0); + + this.Write(buffer, 0, count); + + _bufferPool.Return(buffer, true); + } + + if (specs) + { + this.Write((byte)0x00); + } + } +} + diff --git a/LiteDB/Engine/Disk/Serializer/BufferWriter.cs b/LiteDB/Engine/Disk/Serializer/BufferWriter.cs index 11a0fb71e..83c6c26f8 100644 --- a/LiteDB/Engine/Disk/Serializer/BufferWriter.cs +++ b/LiteDB/Engine/Disk/Serializer/BufferWriter.cs @@ -1,7 +1,6 @@ using System; using System.Buffers; using System.Collections.Generic; -using System.Text; using static LiteDB.Constants; namespace LiteDB.Engine @@ -9,7 +8,7 @@ namespace LiteDB.Engine /// /// Write data types/BSON data into byte[]. It's forward only and support multi buffer slice as source /// - internal class BufferWriter : IDisposable + internal partial class BufferWriter : IDisposable { private readonly IEnumerator _source; @@ -145,81 +144,20 @@ public void Consume() } #endregion - + #region String - + /// /// Write String with \0 at end /// - public void WriteCString(string value) - { - if (value.IndexOf('\0') > -1) throw LiteException.InvalidNullCharInString(); - - var bytesCount = StringEncoding.UTF8.GetByteCount(value); - var available = _current.Count - _currentPosition; // avaiable in current segment - - // can write direct in current segment (use < because need +1 \0) - if (bytesCount < available) - { - StringEncoding.UTF8.GetBytes(value, 0, value.Length, _current.Array, _current.Offset + _currentPosition); - - _current[_currentPosition + bytesCount] = 0x00; - - this.MoveForward(bytesCount + 1); // +1 to '\0' - } - else - { - var buffer = _bufferPool.Rent(bytesCount); - - StringEncoding.UTF8.GetBytes(value, 0, value.Length, buffer, 0); - - this.Write(buffer, 0, bytesCount); - - _current[_currentPosition] = 0x00; - - this.MoveForward(1); - - _bufferPool.Return(buffer, true); - } - } - + public partial void WriteCString(string value); + /// - /// Write string into output buffer. + /// Write string into output buffer. /// Support direct string (with no length information) or BSON specs: with (legnth + 1) [4 bytes] before and '\0' at end = 5 extra bytes /// - public void WriteString(string value, bool specs) - { - var count = StringEncoding.UTF8.GetByteCount(value); - - if (specs) - { - this.Write(count + 1); // write Length + 1 (for \0) - } - - if (count <= _current.Count - _currentPosition) - { - StringEncoding.UTF8.GetBytes(value, 0, value.Length, _current.Array, _current.Offset + _currentPosition); - - this.MoveForward(count); - } - else - { - // rent a buffer to be re-usable - var buffer = _bufferPool.Rent(count); - - StringEncoding.UTF8.GetBytes(value, 0, value.Length, buffer, 0); - - this.Write(buffer, 0, count); - - _bufferPool.Return(buffer, true); - } - - if (specs) - { - this.Write((byte)0x00); - } - } - + public partial void WriteString(string value, bool specs); + #endregion #region Numbers @@ -246,7 +184,9 @@ private void WriteNumber(T value, Action toBytes, int size) public void Write(Int32 value) => this.WriteNumber(value, BufferExtensions.ToBytes, 4); public void Write(Int64 value) => this.WriteNumber(value, BufferExtensions.ToBytes, 8); + public void Write(UInt16 value) => this.WriteNumber(value, BufferExtensions.ToBytes, 2); public void Write(UInt32 value) => this.WriteNumber(value, BufferExtensions.ToBytes, 4); + public void Write(Single value) => this.WriteNumber(value, BufferExtensions.ToBytes, 4); public void Write(Double value) => this.WriteNumber(value, BufferExtensions.ToBytes, 8); public void Write(Decimal value) @@ -333,6 +273,18 @@ internal void Write(PageAddress address) this.Write(address.Index); } + public void Write(float[] vector) + { + ENSURE(vector.Length <= ushort.MaxValue, "Vector length must fit into UInt16"); + + this.Write((ushort)vector.Length); + + for (var i = 0; i < vector.Length; i++) + { + this.Write(vector[i]); + } + } + #endregion #region BsonDocument as SPECS @@ -476,6 +428,11 @@ private void WriteElement(string key, BsonValue value) this.Write((byte)0x7F); this.WriteCString(key); break; + case BsonType.Vector: + this.Write((byte)0x64); // ✅ 0x64 = 100 + this.WriteCString(key); + this.Write(value.AsVector); // ✅ This should exist + break; } } @@ -486,4 +443,4 @@ public void Dispose() _source?.Dispose(); } } -} \ No newline at end of file +} diff --git a/LiteDB/Engine/Engine/Delete.cs b/LiteDB/Engine/Engine/Delete.cs index 1f8a5471d..5ce01db69 100644 --- a/LiteDB/Engine/Engine/Delete.cs +++ b/LiteDB/Engine/Engine/Delete.cs @@ -21,6 +21,7 @@ public int Delete(string collection, IEnumerable ids) var collectionPage = snapshot.CollectionPage; var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); + var vectorService = new VectorIndexService(snapshot, _header.Pragmas.Collation); if (collectionPage == null) return 0; @@ -38,6 +39,11 @@ public int Delete(string collection, IEnumerable ids) _state.Validate(); + foreach (var (_, metadata) in collectionPage.GetVectorIndexes()) + { + vectorService.Delete(metadata, pkNode.DataBlock); + } + // remove object data data.Delete(pkNode.DataBlock); diff --git a/LiteDB/Engine/Engine/Index.cs b/LiteDB/Engine/Engine/Index.cs index 157542848..139d4e9b2 100644 --- a/LiteDB/Engine/Engine/Index.cs +++ b/LiteDB/Engine/Engine/Index.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; +using LiteDB.Vector; using static LiteDB.Constants; namespace LiteDB.Engine @@ -94,6 +95,73 @@ public bool EnsureIndex(string collection, string name, BsonExpression expressio }); } + /// + /// Create a new vector index (or do nothing if already exists) for a collection/field. + /// + public bool EnsureVectorIndex(string collection, string name, BsonExpression expression, VectorIndexOptions options) + { + if (collection.IsNullOrWhiteSpace()) throw new ArgumentNullException(nameof(collection)); + if (name.IsNullOrWhiteSpace()) throw new ArgumentNullException(nameof(name)); + if (expression == null) throw new ArgumentNullException(nameof(expression)); + if (options == null) throw new ArgumentNullException(nameof(options)); + if (expression.Fields.Count == 0) throw new ArgumentException("Vector index expressions must reference a document field.", nameof(expression)); + + if (name.Length > INDEX_NAME_MAX_LENGTH) throw LiteException.InvalidIndexName(name, collection, "MaxLength = " + INDEX_NAME_MAX_LENGTH); + if (!name.IsWord()) throw LiteException.InvalidIndexName(name, collection, "Use only [a-Z$_]"); + if (name.StartsWith("$")) throw LiteException.InvalidIndexName(name, collection, "Index name can't start with `$`"); + + return this.AutoTransaction(transaction => + { + var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true); + var collectionPage = snapshot.CollectionPage; + var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); + var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); + var vectorService = new VectorIndexService(snapshot, _header.Pragmas.Collation); + + var existing = collectionPage.GetCollectionIndex(name); + var existingMetadata = collectionPage.GetVectorIndexMetadata(name); + + if (existing != null && existing.IndexType != 1) + { + throw LiteException.IndexAlreadyExist(name); + } + + if (existing != null && existingMetadata != null) + { + if (existing.Expression != expression.Source) + { + throw LiteException.IndexAlreadyExist(name); + } + + if (existingMetadata.Dimensions != options.Dimensions || existingMetadata.Metric != options.Metric) + { + throw new LiteException(0, $"Vector index '{name}' already exists with different options."); + } + + return false; + } + + LOG($"create vector index `{collection}.{name}`", "COMMAND"); + + var tuple = collectionPage.InsertVectorIndex(name, expression.Source, options.Dimensions, options.Metric); + + foreach (var pkNode in new IndexAll("_id", LiteDB.Query.Ascending).Run(collectionPage, indexer)) + { + _state.Validate(); + + using (var reader = new BufferReader(data.Read(pkNode.DataBlock))) + { + var doc = reader.ReadDocument(expression.Fields).GetValue(); + vectorService.Upsert(tuple.Index, tuple.Metadata, doc, pkNode.DataBlock); + } + + transaction.Safepoint(); + } + + return true; + }); + } + /// /// Drop an index from a collection /// @@ -119,12 +187,25 @@ public bool DropIndex(string collection, string name) // no index, no drop if (index == null) return false; + if (index.IndexType == 1) + { + var metadata = col.GetVectorIndexMetadata(name); + if (metadata != null) + { + var vectorService = new VectorIndexService(snapshot, _header.Pragmas.Collation); + vectorService.Drop(metadata); + } + + snapshot.CollectionPage.DeleteCollectionIndex(name); + return true; + } + // delete all data pages + indexes pages indexer.DropIndex(index); // remove index entry in collection page snapshot.CollectionPage.DeleteCollectionIndex(name); - + return true; }); } diff --git a/LiteDB/Engine/Engine/Insert.cs b/LiteDB/Engine/Engine/Insert.cs index 7f3097df0..bc94b7c95 100644 --- a/LiteDB/Engine/Engine/Insert.cs +++ b/LiteDB/Engine/Engine/Insert.cs @@ -23,6 +23,7 @@ public int Insert(string collection, IEnumerable docs, BsonAutoId var count = 0; var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); + var vectorService = new VectorIndexService(snapshot, _header.Pragmas.Collation); LOG($"insert `{collection}`", "COMMAND"); @@ -32,7 +33,7 @@ public int Insert(string collection, IEnumerable docs, BsonAutoId transaction.Safepoint(); - this.InsertDocument(snapshot, doc, autoId, indexer, data); + this.InsertDocument(snapshot, doc, autoId, indexer, data, vectorService); count++; } @@ -44,7 +45,7 @@ public int Insert(string collection, IEnumerable docs, BsonAutoId /// /// Internal implementation of insert a document /// - private void InsertDocument(Snapshot snapshot, BsonDocument doc, BsonAutoId autoId, IndexService indexer, DataService data) + private void InsertDocument(Snapshot snapshot, BsonDocument doc, BsonAutoId autoId, IndexService indexer, DataService data, VectorIndexService vectorService) { // if no _id, use AutoId if (!doc.TryGetValue("_id", out var id)) @@ -72,7 +73,7 @@ private void InsertDocument(Snapshot snapshot, BsonDocument doc, BsonAutoId auto IndexNode last = null; // for each index, insert new IndexNode - foreach (var index in snapshot.CollectionPage.GetCollectionIndexes()) + foreach (var index in snapshot.CollectionPage.GetCollectionIndexes().Where(x => x.IndexType == 0)) { // for each index, get all keys (supports multi-key) - gets distinct values only // if index are unique, get single key only @@ -87,6 +88,11 @@ private void InsertDocument(Snapshot snapshot, BsonDocument doc, BsonAutoId auto last = node; } } + + foreach (var (vectorIndex, metadata) in snapshot.CollectionPage.GetVectorIndexes()) + { + vectorService.Upsert(vectorIndex, metadata, doc, dataBlock); + } } } } \ No newline at end of file diff --git a/LiteDB/Engine/Engine/Rebuild.cs b/LiteDB/Engine/Engine/Rebuild.cs index 37036bad6..b85fe6f6f 100644 --- a/LiteDB/Engine/Engine/Rebuild.cs +++ b/LiteDB/Engine/Engine/Rebuild.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Text; +using LiteDB.Vector; using static LiteDB.Constants; @@ -62,6 +63,7 @@ internal void RebuildContent(IFileReader reader) var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true); var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); + var vectorService = new VectorIndexService(snapshot, _header.Pragmas.Collation); // get all documents from current collection var docs = reader.GetDocuments(collection); @@ -71,16 +73,28 @@ internal void RebuildContent(IFileReader reader) { transaction.Safepoint(); - this.InsertDocument(snapshot, doc, BsonAutoId.ObjectId, indexer, data); + this.InsertDocument(snapshot, doc, BsonAutoId.ObjectId, indexer, data, vectorService); } // first create all user indexes (exclude _id index) foreach (var index in reader.GetIndexes(collection)) { - this.EnsureIndex(collection, - index.Name, - BsonExpression.Create(index.Expression), - index.Unique); + if (index.IndexType == 1 && index.VectorMetadata != null) + { + this.EnsureVectorIndex( + collection, + index.Name, + BsonExpression.Create(index.Expression), + new VectorIndexOptions(index.VectorMetadata.Dimensions, index.VectorMetadata.Metric)); + } + else + { + this.EnsureIndex( + collection, + index.Name, + BsonExpression.Create(index.Expression), + index.Unique); + } } } diff --git a/LiteDB/Engine/Engine/Transaction.cs b/LiteDB/Engine/Engine/Transaction.cs index 4db9f57b3..b88977069 100644 --- a/LiteDB/Engine/Engine/Transaction.cs +++ b/LiteDB/Engine/Engine/Transaction.cs @@ -112,8 +112,8 @@ private void CommitAndReleaseTransaction(TransactionService transaction) _monitor.ReleaseTransaction(transaction); // try checkpoint when finish transaction and log file are bigger than checkpoint pragma value (in pages) - if (_header.Pragmas.Checkpoint > 0 && - _disk.GetFileLength(FileOrigin.Log) > (_header.Pragmas.Checkpoint * PAGE_SIZE)) + if (_header.Pragmas.Checkpoint > 0 && + _disk.GetFileLength(FileOrigin.Log) >= (_header.Pragmas.Checkpoint * PAGE_SIZE)) { _walIndex.TryCheckpoint(); } diff --git a/LiteDB/Engine/Engine/Update.cs b/LiteDB/Engine/Engine/Update.cs index 0d825beea..b8ac36e97 100644 --- a/LiteDB/Engine/Engine/Update.cs +++ b/LiteDB/Engine/Engine/Update.cs @@ -21,6 +21,7 @@ public int Update(string collection, IEnumerable docs) var collectionPage = snapshot.CollectionPage; var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); + var vectorService = new VectorIndexService(snapshot, _header.Pragmas.Collation); var count = 0; if (collectionPage == null) return 0; @@ -33,7 +34,7 @@ public int Update(string collection, IEnumerable docs) transaction.Safepoint(); - if (this.UpdateDocument(snapshot, collectionPage, doc, indexer, data)) + if (this.UpdateDocument(snapshot, collectionPage, doc, indexer, data, vectorService)) { count++; } @@ -94,7 +95,7 @@ IEnumerable transformDocs() /// /// Implement internal update document /// - private bool UpdateDocument(Snapshot snapshot, CollectionPage col, BsonDocument doc, IndexService indexer, DataService data) + private bool UpdateDocument(Snapshot snapshot, CollectionPage col, BsonDocument doc, IndexService indexer, DataService data, VectorIndexService vectorService) { // normalize id before find var id = doc["_id"]; @@ -113,6 +114,10 @@ private bool UpdateDocument(Snapshot snapshot, CollectionPage col, BsonDocument // update data storage data.Update(col, pkNode.DataBlock, doc); + foreach (var (vectorIndex, metadata) in col.GetVectorIndexes()) + { + vectorService.Upsert(vectorIndex, metadata, doc, pkNode.DataBlock); + } // get all current non-pk index nodes from this data block (slot, key, nodePosition) var oldKeys = indexer.GetNodeList(pkNode.NextNode) @@ -122,7 +127,7 @@ private bool UpdateDocument(Snapshot snapshot, CollectionPage col, BsonDocument // build a list of all new key index keys var newKeys = new List>(); - foreach (var index in col.GetCollectionIndexes().Where(x => x.Name != "_id")) + foreach (var index in col.GetCollectionIndexes().Where(x => x.Name != "_id" && x.IndexType == 0)) { // getting all keys from expression over document var keys = index.BsonExpr.GetIndexKeys(doc, _header.Pragmas.Collation); diff --git a/LiteDB/Engine/Engine/Upsert.cs b/LiteDB/Engine/Engine/Upsert.cs index e9545c0ea..077c4f9aa 100644 --- a/LiteDB/Engine/Engine/Upsert.cs +++ b/LiteDB/Engine/Engine/Upsert.cs @@ -22,6 +22,7 @@ public int Upsert(string collection, IEnumerable docs, BsonAutoId var collectionPage = snapshot.CollectionPage; var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); + var vectorService = new VectorIndexService(snapshot, _header.Pragmas.Collation); var count = 0; LOG($"upsert `{collection}`", "COMMAND"); @@ -33,9 +34,9 @@ public int Upsert(string collection, IEnumerable docs, BsonAutoId transaction.Safepoint(); // first try update document (if exists _id), if not found, do insert - if (doc["_id"] == BsonValue.Null || this.UpdateDocument(snapshot, collectionPage, doc, indexer, data) == false) + if (doc["_id"] == BsonValue.Null || this.UpdateDocument(snapshot, collectionPage, doc, indexer, data, vectorService) == false) { - this.InsertDocument(snapshot, doc, autoId, indexer, data); + this.InsertDocument(snapshot, doc, autoId, indexer, data, vectorService); count++; } } diff --git a/LiteDB/Engine/EngineSettings.cs b/LiteDB/Engine/EngineSettings.cs index e78034ab2..8400c7f63 100644 --- a/LiteDB/Engine/EngineSettings.cs +++ b/LiteDB/Engine/EngineSettings.cs @@ -71,6 +71,11 @@ public class EngineSettings /// Is used to transform a from the database on read. This can be used to upgrade data from older versions. /// public Func ReadTransform { get; set; } + + /// + /// Determines how the mutex name is generated. + /// + public SharedMutexNameStrategy SharedMutexNameStrategy { get; set; } /// /// Create new IStreamFactory for datafile diff --git a/LiteDB/Engine/EngineState.cs b/LiteDB/Engine/EngineState.cs index bd3dafac7..dfce2e984 100644 --- a/LiteDB/Engine/EngineState.cs +++ b/LiteDB/Engine/EngineState.cs @@ -18,7 +18,7 @@ internal class EngineState private readonly LiteEngine _engine; // can be null for unit tests private readonly EngineSettings _settings; -#if DEBUG +#if DEBUG || TESTING public Action SimulateDiskReadFail = null; public Action SimulateDiskWriteFail = null; #endif diff --git a/LiteDB/Engine/FileReader/FileReaderV8.cs b/LiteDB/Engine/FileReader/FileReaderV8.cs index 3230c9d57..74ca8e8c2 100644 --- a/LiteDB/Engine/FileReader/FileReaderV8.cs +++ b/LiteDB/Engine/FileReader/FileReaderV8.cs @@ -371,61 +371,38 @@ private void LoadIndexes() continue; } - var page = result.Value; - var buffer = page.Buffer; - - var count = buffer.ReadByte(CollectionPage.P_INDEXES); // 1 byte - var position = CollectionPage.P_INDEXES + 1; - - // handle error per collection try { - for (var i = 0; i < count; i++) - { - position += 2; // skip: slot (1 byte) + indexType (1 byte) - - var name = buffer.ReadCString(position, out var nameLength); - - position += nameLength; - - var expr = buffer.ReadCString(position, out var exprLength); - - position += exprLength; - - var unique = buffer.ReadBool(position); - - position++; - - position += 15; // head 5 bytes, tail 5 bytes, reserved 1 byte, freeIndexPageList 4 bytes + var page = result.Value; + var collectionPage = new CollectionPage(page.Buffer); - ENSURE(!string.IsNullOrEmpty(name), "Index name can't be empty (collection {0} - index: {1})", collection.Key, i); - ENSURE(!string.IsNullOrEmpty(expr), "Index expression can't be empty (collection {0} - index: {1})", collection.Key, i); + foreach (var index in collectionPage.GetCollectionIndexes()) + { + if (index.Name == "_id") continue; - var indexInfo = new IndexInfo + var info = new IndexInfo { Collection = collection.Key, - Name = name, - Expression = expr, - Unique = unique + Name = index.Name, + Expression = index.Expression, + Unique = index.Unique, + IndexType = index.IndexType, + VectorMetadata = index.IndexType == 1 ? collectionPage.GetVectorIndexMetadata(index.Name) : null }; - // ignore _id index - if (name == "_id") continue; - if (_indexes.TryGetValue(collection.Key, out var indexInfos)) { - indexInfos.Add(indexInfo); + indexInfos.Add(info); } else { - _indexes[collection.Key] = new List { indexInfo }; + _indexes[collection.Key] = new List { info }; } } } catch (Exception ex) { this.HandleError(ex, pageInfo); - continue; } } } diff --git a/LiteDB/Engine/FileReader/IndexInfo.cs b/LiteDB/Engine/FileReader/IndexInfo.cs index 04b26682d..5dd68c05c 100644 --- a/LiteDB/Engine/FileReader/IndexInfo.cs +++ b/LiteDB/Engine/FileReader/IndexInfo.cs @@ -13,5 +13,7 @@ internal class IndexInfo public string Name { get; set; } public string Expression { get; set; } public bool Unique { get; set; } + public byte IndexType { get; set; } + public VectorIndexMetadata VectorMetadata { get; set; } } } diff --git a/LiteDB/Engine/ILiteEngine.cs b/LiteDB/Engine/ILiteEngine.cs index 067589999..b0a2ab3ae 100644 --- a/LiteDB/Engine/ILiteEngine.cs +++ b/LiteDB/Engine/ILiteEngine.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using LiteDB.Vector; namespace LiteDB.Engine { @@ -25,6 +26,7 @@ public interface ILiteEngine : IDisposable bool RenameCollection(string name, string newName); bool EnsureIndex(string collection, string name, BsonExpression expression, bool unique); + bool EnsureVectorIndex(string collection, string name, BsonExpression expression, VectorIndexOptions options); bool DropIndex(string collection, string name); BsonValue Pragma(string name); diff --git a/LiteDB/Engine/LiteEngine.cs b/LiteDB/Engine/LiteEngine.cs index 59bb84b4c..2e36024c8 100644 --- a/LiteDB/Engine/LiteEngine.cs +++ b/LiteDB/Engine/LiteEngine.cs @@ -243,7 +243,7 @@ internal List Close(Exception ex) #endregion -#if DEBUG +#if DEBUG || TESTING // exposes for unit tests internal TransactionMonitor GetMonitor() => _monitor; internal Action SimulateDiskReadFail { set => _state.SimulateDiskReadFail = value; } diff --git a/LiteDB/Engine/Pages/BasePage.cs b/LiteDB/Engine/Pages/BasePage.cs index 09f87c851..52cda352c 100644 --- a/LiteDB/Engine/Pages/BasePage.cs +++ b/LiteDB/Engine/Pages/BasePage.cs @@ -7,7 +7,7 @@ namespace LiteDB.Engine { - internal enum PageType { Empty = 0, Header = 1, Collection = 2, Index = 3, Data = 4 } + internal enum PageType { Empty = 0, Header = 1, Collection = 2, Index = 3, Data = 4, VectorIndex = 5 } internal class BasePage { @@ -738,6 +738,7 @@ public static T ReadPage(PageBuffer buffer) if (typeof(T) == typeof(HeaderPage)) return (T)(object)new HeaderPage(buffer); if (typeof(T) == typeof(CollectionPage)) return (T)(object)new CollectionPage(buffer); if (typeof(T) == typeof(IndexPage)) return (T)(object)new IndexPage(buffer); + if (typeof(T) == typeof(VectorIndexPage)) return (T)(object)new VectorIndexPage(buffer); if (typeof(T) == typeof(DataPage)) return (T)(object)new DataPage(buffer); throw new InvalidCastException(); @@ -751,6 +752,7 @@ public static T CreatePage(PageBuffer buffer, uint pageID) { if (typeof(T) == typeof(CollectionPage)) return (T)(object)new CollectionPage(buffer, pageID); if (typeof(T) == typeof(IndexPage)) return (T)(object)new IndexPage(buffer, pageID); + if (typeof(T) == typeof(VectorIndexPage)) return (T)(object)new VectorIndexPage(buffer, pageID); if (typeof(T) == typeof(DataPage)) return (T)(object)new DataPage(buffer, pageID); throw new InvalidCastException(); diff --git a/LiteDB/Engine/Pages/CollectionPage.cs b/LiteDB/Engine/Pages/CollectionPage.cs index 4db992238..2d4ce8d58 100644 --- a/LiteDB/Engine/Pages/CollectionPage.cs +++ b/LiteDB/Engine/Pages/CollectionPage.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Text; +using LiteDB.Vector; using static LiteDB.Constants; namespace LiteDB.Engine @@ -26,6 +27,7 @@ internal class CollectionPage : BasePage /// All indexes references for this collection /// private readonly Dictionary _indexes = new Dictionary(); + private readonly Dictionary _vectorIndexes = new Dictionary(); public CollectionPage(PageBuffer buffer, uint pageID) : base(buffer, pageID, PageType.Collection) @@ -65,6 +67,16 @@ public CollectionPage(PageBuffer buffer) _indexes[index.Name] = index; } + + var vectorCount = r.ReadByte(); + + for (var i = 0; i < vectorCount; i++) + { + var name = r.ReadCString(); + var metadata = new VectorIndexMetadata(r); + + _vectorIndexes[name] = metadata; + } } } @@ -92,6 +104,14 @@ public override PageBuffer UpdateBuffer() { index.UpdateBuffer(w); } + + w.Write((byte)_vectorIndexes.Count); + + foreach (var pair in _vectorIndexes) + { + w.WriteCString(pair.Key); + pair.Value.UpdateBuffer(w); + } } return base.UpdateBuffer(); @@ -138,22 +158,54 @@ public CollectionIndex[] GetCollectionIndexesSlots() return indexes; } + private int GetSerializedLength(int additionalIndexLength, int additionalVectorLength) + { + var length = 1 + _indexes.Sum(x => CollectionIndex.GetLength(x.Value)) + additionalIndexLength; + + length += 1 + _vectorIndexes.Sum(x => GetVectorMetadataLength(x.Key)) + additionalVectorLength; + + return length; + } + + private static int GetVectorMetadataLength(string name) + { + return StringEncoding.UTF8.GetByteCount(name) + 1 + VectorIndexMetadata.GetLength(); + } + + public IEnumerable<(CollectionIndex Index, VectorIndexMetadata Metadata)> GetVectorIndexes() + { + foreach (var pair in _vectorIndexes) + { + if (_indexes.TryGetValue(pair.Key, out var index)) + { + yield return (index, pair.Value); + } + } + } + + public VectorIndexMetadata GetVectorIndexMetadata(string name) + { + return _vectorIndexes.TryGetValue(name, out var metadata) ? metadata : null; + } + /// /// Insert new index inside this collection page /// public CollectionIndex InsertCollectionIndex(string name, string expr, bool unique) { - var totalLength = 1 + - _indexes.Sum(x => CollectionIndex.GetLength(x.Value)) + - CollectionIndex.GetLength(name, expr); + if (_indexes.ContainsKey(name) || _vectorIndexes.ContainsKey(name)) + { + throw LiteException.IndexAlreadyExist(name); + } + + var totalLength = this.GetSerializedLength(CollectionIndex.GetLength(name, expr), 0); - // check if has space avaiable if (_indexes.Count == 255 || totalLength >= P_INDEXES_COUNT) throw new LiteException(0, $"This collection has no more space for new indexes"); var slot = (byte)(_indexes.Count == 0 ? 0 : (_indexes.Max(x => x.Value.Slot) + 1)); var index = new CollectionIndex(slot, 0, name, expr, unique); - + _indexes[name] = index; this.IsDirty = true; @@ -161,6 +213,30 @@ public CollectionIndex InsertCollectionIndex(string name, string expr, bool uniq return index; } + public (CollectionIndex Index, VectorIndexMetadata Metadata) InsertVectorIndex(string name, string expr, ushort dimensions, VectorDistanceMetric metric) + { + if (_indexes.ContainsKey(name) || _vectorIndexes.ContainsKey(name)) + { + throw LiteException.IndexAlreadyExist(name); + } + + var totalLength = this.GetSerializedLength(CollectionIndex.GetLength(name, expr), GetVectorMetadataLength(name)); + + if (_indexes.Count == 255 || totalLength >= P_INDEXES_COUNT) throw new LiteException(0, $"This collection has no more space for new indexes"); + + var slot = (byte)(_indexes.Count == 0 ? 0 : (_indexes.Max(x => x.Value.Slot) + 1)); + + var index = new CollectionIndex(slot, 1, name, expr, false); + var metadata = new VectorIndexMetadata(slot, dimensions, metric); + + _indexes[name] = index; + _vectorIndexes[name] = metadata; + + this.IsDirty = true; + + return (index, metadata); + } + /// /// Return index instance and mark as updatable /// @@ -177,6 +253,7 @@ public CollectionIndex UpdateCollectionIndex(string name) public void DeleteCollectionIndex(string name) { _indexes.Remove(name); + _vectorIndexes.Remove(name); this.IsDirty = true; } diff --git a/LiteDB/Engine/Pages/VectorIndexPage.cs b/LiteDB/Engine/Pages/VectorIndexPage.cs new file mode 100644 index 000000000..504adbbb6 --- /dev/null +++ b/LiteDB/Engine/Pages/VectorIndexPage.cs @@ -0,0 +1,55 @@ +using System.Collections.Generic; +using static LiteDB.Constants; + +namespace LiteDB.Engine +{ + internal sealed class VectorIndexPage : BasePage + { + public VectorIndexPage(PageBuffer buffer) + : base(buffer) + { + ENSURE(this.PageType == PageType.VectorIndex, "page type must be vector index page"); + + if (this.PageType != PageType.VectorIndex) throw LiteException.InvalidPageType(PageType.VectorIndex, this); + } + + public VectorIndexPage(PageBuffer buffer, uint pageID) + : base(buffer, pageID, PageType.VectorIndex) + { + } + + public VectorIndexNode GetNode(byte index) + { + var segment = base.Get(index); + + return new VectorIndexNode(this, index, segment); + } + + public VectorIndexNode InsertNode(PageAddress dataBlock, float[] vector, int bytesLength, byte levelCount, PageAddress externalVector) + { + var segment = base.Insert((ushort)bytesLength, out var index); + + return new VectorIndexNode(this, index, segment, dataBlock, vector, levelCount, externalVector); + } + + public void DeleteNode(byte index) + { + base.Delete(index); + } + + public IEnumerable GetNodes() + { + foreach (var index in base.GetUsedIndexs()) + { + yield return this.GetNode(index); + } + } + + public static byte FreeListSlot(int freeBytes) + { + ENSURE(freeBytes >= 0, "freeBytes must be positive"); + + return freeBytes >= MAX_INDEX_LENGTH ? (byte)0 : (byte)1; + } + } +} diff --git a/LiteDB/Engine/Query/IndexQuery/VectorIndexQuery.cs b/LiteDB/Engine/Query/IndexQuery/VectorIndexQuery.cs new file mode 100644 index 000000000..de4d586b2 --- /dev/null +++ b/LiteDB/Engine/Query/IndexQuery/VectorIndexQuery.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace LiteDB.Engine +{ + internal sealed class VectorIndexQuery : Index, IDocumentLookup + { + private readonly Snapshot _snapshot; + private readonly CollectionIndex _index; + private readonly VectorIndexMetadata _metadata; + private readonly float[] _target; + private readonly double _maxDistance; + private readonly int? _limit; + private readonly Collation _collation; + + private readonly Dictionary _cache = new Dictionary(); + + public string Expression => _index.Expression; + + public VectorIndexQuery( + string name, + Snapshot snapshot, + CollectionIndex index, + VectorIndexMetadata metadata, + float[] target, + double maxDistance, + int? limit, + Collation collation) + : base(name, Query.Ascending) + { + _snapshot = snapshot; + _index = index; + _metadata = metadata; + _target = target; + _maxDistance = maxDistance; + _limit = limit; + _collation = collation; + } + + public override uint GetCost(CollectionIndex index) + { + return 1; + } + + public override IEnumerable Execute(IndexService indexer, CollectionIndex index) + { + throw new NotSupportedException(); + } + + public override IEnumerable Run(CollectionPage col, IndexService indexer) + { + _cache.Clear(); + + var service = new VectorIndexService(_snapshot, _collation); + var results = service.Search(_metadata, _target, _maxDistance, _limit).ToArray(); + + foreach (var result in results) + { + var rawId = result.Document.RawId; + + if (rawId.IsEmpty) + { + continue; + } + + _cache[rawId] = result.Document; + yield return new IndexNode(result.Document); + } + } + + public BsonDocument Load(IndexNode node) + { + return node.Key as BsonDocument; + } + + public BsonDocument Load(PageAddress rawId) + { + return _cache.TryGetValue(rawId, out var document) ? document : null; + } + + public override string ToString() + { + return "VECTOR INDEX SEARCH"; + } + } +} diff --git a/LiteDB/Engine/Query/Pipeline/BasePipe.cs b/LiteDB/Engine/Query/Pipeline/BasePipe.cs index a16bbd953..29ea59b6a 100644 --- a/LiteDB/Engine/Query/Pipeline/BasePipe.cs +++ b/LiteDB/Engine/Query/Pipeline/BasePipe.cs @@ -154,24 +154,63 @@ protected IEnumerable Filter(IEnumerable source, Bso /// /// ORDER BY: Sort documents according orderby expression and order asc/desc /// - protected IEnumerable OrderBy(IEnumerable source, BsonExpression expr, int order, int offset, int limit) + protected IEnumerable OrderBy(IEnumerable source, OrderBy orderBy, int offset, int limit) { - var keyValues = source - .Select(x => new KeyValuePair(expr.ExecuteScalar(x, _pragmas.Collation), x.RawId)); + var segments = orderBy.Segments; - using (var sorter = new SortService(_tempDisk, order, _pragmas)) + if (segments.Count == 1) { - sorter.Insert(keyValues); + var segment = segments[0]; + var keyValues = source + .Select(doc => new KeyValuePair(segment.Expression.ExecuteScalar(doc, _pragmas.Collation), doc.RawId)); - LOG($"sort {sorter.Count} keys in {sorter.Containers.Count} containers", "SORT"); + using (var sorter = new SortService(_tempDisk, new[] { segment.Order }, _pragmas)) + { + sorter.Insert(keyValues); + + LOG($"sort {sorter.Count} keys in {sorter.Containers.Count} containers", "SORT"); + + var result = sorter.Sort().Skip(offset).Take(limit); - var result = sorter.Sort().Skip(offset).Take(limit); + foreach (var keyValue in result) + { + var doc = _lookup.Load(keyValue.Value); + + yield return doc; + } + } + } + else + { + var orders = segments.Select(x => x.Order).ToArray(); + + var keyValues = source + .Select(doc => + { + var values = new BsonValue[segments.Count]; - foreach (var keyValue in result) + for (var i = 0; i < segments.Count; i++) + { + values[i] = segments[i].Expression.ExecuteScalar(doc, _pragmas.Collation); + } + + return new KeyValuePair(SortKey.FromValues(values, orders), doc.RawId); + }); + + using (var sorter = new SortService(_tempDisk, orders, _pragmas)) { - var doc = _lookup.Load(keyValue.Value); + sorter.Insert(keyValues); - yield return doc; + LOG($"sort {sorter.Count} keys in {sorter.Containers.Count} containers", "SORT"); + + var result = sorter.Sort().Skip(offset).Take(limit); + + foreach (var keyValue in result) + { + var doc = _lookup.Load(keyValue.Value); + + yield return doc; + } } } } diff --git a/LiteDB/Engine/Query/Pipeline/GroupByPipe.cs b/LiteDB/Engine/Query/Pipeline/GroupByPipe.cs index 438e5a756..bbe8dbdaa 100644 --- a/LiteDB/Engine/Query/Pipeline/GroupByPipe.cs +++ b/LiteDB/Engine/Query/Pipeline/GroupByPipe.cs @@ -36,17 +36,23 @@ public override IEnumerable Pipe(IEnumerable nodes, Que source = this.Filter(source, expr); } - // run orderBy used in GroupBy (if not already ordered by index) - if (query.OrderBy != null) + // run orderBy used to prepare data for grouping (if not already ordered by index) + if (query.GroupBy.OrderBy != null) { - source = this.OrderBy(source, query.OrderBy.Expression, query.OrderBy.Order, 0, int.MaxValue); + source = this.OrderBy(source, query.GroupBy.OrderBy, 0, int.MaxValue); } // apply groupby var groups = this.GroupBy(source, query.GroupBy); // apply group filter and transform result - var result = this.SelectGroupBy(groups, query.GroupBy); + var result = this.SelectGroupBy(groups, query.GroupBy, query.OrderBy); + + if (query.OrderBy != null) + { + return this.OrderGroupedResult(result, query.OrderBy, query.Offset, query.Limit) + .Select(x => x.Document); + } // apply offset if (query.Offset > 0) result = result.Skip(query.Offset); @@ -54,13 +60,26 @@ public override IEnumerable Pipe(IEnumerable nodes, Que // apply limit if (query.Limit < int.MaxValue) result = result.Take(query.Limit); - return result; + return result.Select(x => x.Document); } /// /// GROUP BY: Apply groupBy expression and aggregate results in DocumentGroup /// - private IEnumerable GroupBy(IEnumerable source, GroupBy groupBy) + private readonly struct GroupSource + { + public GroupSource(BsonValue key, DocumentCacheEnumerable documents) + { + this.Key = key; + this.Documents = documents; + } + + public BsonValue Key { get; } + + public DocumentCacheEnumerable Documents { get; } + } + + private IEnumerable GroupBy(IEnumerable source, GroupBy groupBy) { using (var enumerator = source.GetEnumerator()) { @@ -70,11 +89,9 @@ private IEnumerable GroupBy(IEnumerable s { var key = groupBy.Expression.ExecuteScalar(enumerator.Current, _pragmas.Collation); - groupBy.Select.Parameters["key"] = key; - var group = YieldDocuments(key, enumerator, groupBy, done); - yield return new DocumentCacheEnumerable(group, _lookup); + yield return new GroupSource(key, new DocumentCacheEnumerable(group, _lookup)); } } } @@ -90,15 +107,13 @@ private IEnumerable YieldDocuments(BsonValue key, IEnumerator YieldDocuments(BsonValue key, IEnumerator - private IEnumerable SelectGroupBy(IEnumerable groups, GroupBy groupBy) + private readonly struct GroupedResult + { + public GroupedResult(BsonValue key, BsonDocument document, BsonValue[] orderValues) + { + this.Key = key; + this.Document = document; + this.OrderValues = orderValues; + } + + public BsonValue Key { get; } + + public BsonDocument Document { get; } + + public BsonValue[] OrderValues { get; } + } + + private static void SetKeyParameter(BsonExpression expression, BsonValue key) + { + if (expression?.Parameters != null) + { + expression.Parameters["key"] = key; + } + } + + private IEnumerable SelectGroupBy(IEnumerable groups, GroupBy groupBy, OrderBy resultOrderBy) { var defaultName = groupBy.Select.DefaultFieldName(); foreach (var group in groups) { - // transfom group result if contains select expression - BsonValue value; + var key = group.Key; + var cache = group.Documents; + + SetKeyParameter(groupBy.Select, key); + SetKeyParameter(groupBy.Having, key); + + BsonDocument document = null; + BsonValue[] orderValues = null; try { if (groupBy.Having != null) { - var filter = groupBy.Having.ExecuteScalar(group, null, null, _pragmas.Collation); + var filter = groupBy.Having.ExecuteScalar(cache, null, null, _pragmas.Collation); - if (!filter.IsBoolean || !filter.AsBoolean) continue; + if (!filter.IsBoolean || !filter.AsBoolean) + { + continue; + } } - value = groupBy.Select.ExecuteScalar(group, null, null, _pragmas.Collation); + BsonValue value; + + if (ReferenceEquals(groupBy.Select, BsonExpression.Root)) + { + var items = new BsonArray(); + + foreach (var groupDocument in cache) + { + items.Add(groupDocument); + } + + value = new BsonDocument + { + [LiteGroupingFieldNames.Key] = key, + [LiteGroupingFieldNames.Items] = items + }; + } + else + { + value = groupBy.Select.ExecuteScalar(cache, null, null, _pragmas.Collation); + } + + if (value.IsDocument) + { + document = value.AsDocument; + } + else + { + document = new BsonDocument { [defaultName] = value }; + } + + if (resultOrderBy != null) + { + var segments = resultOrderBy.Segments; + + orderValues = new BsonValue[segments.Count]; + + for (var i = 0; i < segments.Count; i++) + { + var expression = segments[i].Expression; + + SetKeyParameter(expression, key); + + orderValues[i] = expression.ExecuteScalar(cache, document, null, _pragmas.Collation); + } + } } finally { - group.Dispose(); + cache.Dispose(); } - if (value.IsDocument) + yield return new GroupedResult(key, document, orderValues); + } + } + + /// + /// Apply ORDER BY over grouped projection using in-memory sorting. + /// + private IEnumerable OrderGroupedResult(IEnumerable source, OrderBy orderBy, int offset, int limit) + { + var segments = orderBy.Segments; + var orders = segments.Select(x => x.Order).ToArray(); + var buffer = new List<(SortKey Key, GroupedResult Result)>(); + + foreach (var item in source) + { + var values = item.OrderValues ?? new BsonValue[segments.Count]; + + if (item.OrderValues == null) { - yield return value.AsDocument; + for (var i = 0; i < segments.Count; i++) + { + var expression = segments[i].Expression; + + SetKeyParameter(expression, item.Key); + + values[i] = expression.ExecuteScalar(item.Document, _pragmas.Collation); + } } - else + + var key = SortKey.FromValues(values, orders); + + buffer.Add((key, item)); + } + + buffer.Sort((left, right) => left.Key.CompareTo(right.Key, _pragmas.Collation)); + + var skipped = 0; + var returned = 0; + + foreach (var item in buffer) + { + if (skipped < offset) { - yield return new BsonDocument { [defaultName] = value }; + skipped++; + continue; + } + + yield return item.Result; + + returned++; + + if (limit != int.MaxValue && returned >= limit) + { + yield break; } } } diff --git a/LiteDB/Engine/Query/Pipeline/QueryPipe.cs b/LiteDB/Engine/Query/Pipeline/QueryPipe.cs index f33199fac..0e8bfe0cd 100644 --- a/LiteDB/Engine/Query/Pipeline/QueryPipe.cs +++ b/LiteDB/Engine/Query/Pipeline/QueryPipe.cs @@ -46,7 +46,7 @@ public override IEnumerable Pipe(IEnumerable nodes, Que if (query.OrderBy != null) { // pipe: orderby with offset+limit - source = this.OrderBy(source, query.OrderBy.Expression, query.OrderBy.Order, query.Offset, query.Limit); + source = this.OrderBy(source, query.OrderBy, query.Offset, query.Limit); } else { diff --git a/LiteDB/Engine/Query/Query.cs b/LiteDB/Engine/Query/Query.cs index 86fc07da1..c0056bc23 100644 --- a/LiteDB/Engine/Query/Query.cs +++ b/LiteDB/Engine/Query/Query.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -17,8 +17,7 @@ public partial class Query public List Includes { get; } = new List(); public List Where { get; } = new List(); - public BsonExpression OrderBy { get; set; } = null; - public int Order { get; set; } = Query.Ascending; + public List OrderBy { get; } = new List(); public BsonExpression GroupBy { get; set; } = null; public BsonExpression Having { get; set; } = null; @@ -27,6 +26,11 @@ public partial class Query public int Limit { get; set; } = int.MaxValue; public bool ForUpdate { get; set; } = false; + public string VectorField { get; set; } = null; + public float[] VectorTarget { get; set; } = null; + public double VectorMaxDistance { get; set; } = double.MaxValue; + public bool HasVectorFilter => VectorField != null && VectorTarget != null; + public string Into { get; set; } public BsonAutoId IntoAutoId { get; set; } = BsonAutoId.ObjectId; @@ -69,10 +73,7 @@ public string ToSQL(string collection) sb.AppendLine($"INCLUDE {string.Join(", ", this.Includes.Select(x => x.Source))}"); } - if (this.Where.Count > 0) - { - sb.AppendLine($"WHERE {string.Join(" AND ", this.Where.Select(x => x.Source))}"); - } + if (this.GroupBy != null) { @@ -84,9 +85,12 @@ public string ToSQL(string collection) sb.AppendLine($"HAVING {this.Having.Source}"); } - if (this.OrderBy != null) + if (this.OrderBy.Count > 0) { - sb.AppendLine($"ORDER BY {this.OrderBy.Source} {(this.Order == Query.Ascending ? "ASC" : "DESC")}"); + var orderBy = this.OrderBy + .Select(x => $"{x.Expression.Source} {(x.Order == Query.Ascending ? "ASC" : "DESC")}"); + + sb.AppendLine($"ORDER BY {string.Join(", ", orderBy)}"); } if (this.Limit != int.MaxValue) @@ -104,6 +108,37 @@ public string ToSQL(string collection) sb.AppendLine($"FOR UPDATE"); } + if (this.HasVectorFilter) + { + var field = this.VectorField; + + if (!string.IsNullOrEmpty(field)) + { + field = field.Trim(); + + if (!field.StartsWith("$", StringComparison.Ordinal)) + { + field = field.StartsWith(".", StringComparison.Ordinal) + ? "$" + field + : "$." + field; + } + } + + var vectorExpr = $"VECTOR_SIM({field}, [{string.Join(",", this.VectorTarget)}])"; + if (this.Where.Count > 0) + { + sb.AppendLine($"WHERE ({string.Join(" AND ", this.Where.Select(x => x.Source))}) AND {vectorExpr} <= {this.VectorMaxDistance}"); + } + else + { + sb.AppendLine($"WHERE {vectorExpr} <= {this.VectorMaxDistance}"); + } + } + else if (this.Where.Count > 0) + { + sb.AppendLine($"WHERE {string.Join(" AND ", this.Where.Select(x => x.Source))}"); + } + return sb.ToString().Trim(); } } diff --git a/LiteDB/Engine/Query/QueryOptimization.cs b/LiteDB/Engine/Query/QueryOptimization.cs index 7a34d4de6..2188d96dd 100644 --- a/LiteDB/Engine/Query/QueryOptimization.cs +++ b/LiteDB/Engine/Query/QueryOptimization.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using static LiteDB.Constants; @@ -15,6 +15,7 @@ internal class QueryOptimization private readonly Collation _collation; private readonly QueryPlan _queryPlan; private readonly List _terms = new List(); + private bool _vectorOrderConsumed; public QueryOptimization(Snapshot snapshot, Query query, IEnumerable source, Collation collation) { @@ -150,7 +151,10 @@ private void DefineQueryFields() fields.AddRange(_query.Includes.SelectMany(x => x.Fields)); fields.AddRange(_query.GroupBy?.Fields); fields.AddRange(_query.Having?.Fields); - fields.AddRange(_query.OrderBy?.Fields); + if (_query.OrderBy.Count > 0) + { + fields.AddRange(_query.OrderBy.SelectMany(x => x.Expression.Fields)); + } // if contains $, all fields must be deserialized if (fields.Contains("$")) @@ -173,28 +177,37 @@ private void DefineIndex() // if index are not defined yet, get index if (_queryPlan.Index == null) { - // try select best index (if return null, there is no good choice) - var indexCost = this.ChooseIndex(_queryPlan.Fields); - - // if found an index, use-it - if (indexCost != null) + if (this.TrySelectVectorIndex(out var vectorIndex, out selected)) { - _queryPlan.Index = indexCost.Index; - _queryPlan.IndexCost = indexCost.Cost; - _queryPlan.IndexExpression = indexCost.IndexExpression; + _queryPlan.Index = vectorIndex; + _queryPlan.IndexCost = vectorIndex.GetCost(null); + _queryPlan.IndexExpression = vectorIndex.Expression; } else { - // if has no index to use, use full scan over _id - var pk = _snapshot.CollectionPage.PK; + // try select best index (if return null, there is no good choice) + var indexCost = this.ChooseIndex(_queryPlan.Fields); - _queryPlan.Index = new IndexAll("_id", Query.Ascending); - _queryPlan.IndexCost = _queryPlan.Index.GetCost(pk); - _queryPlan.IndexExpression = "$._id"; - } + // if found an index, use-it + if (indexCost != null) + { + _queryPlan.Index = indexCost.Index; + _queryPlan.IndexCost = indexCost.Cost; + _queryPlan.IndexExpression = indexCost.IndexExpression; + } + else + { + // if has no index to use, use full scan over _id + var pk = _snapshot.CollectionPage.PK; + + _queryPlan.Index = new IndexAll("_id", Query.Ascending); + _queryPlan.IndexCost = _queryPlan.Index.GetCost(pk); + _queryPlan.IndexExpression = "$._id"; + } - // get selected expression used as index - selected = indexCost?.Expression; + // get selected expression used as index + selected = indexCost?.Expression; + } } else { @@ -277,11 +290,12 @@ private IndexCost ChooseIndex(HashSet fields) } // if no index found, try use same index in orderby/groupby/preferred - if (lowest == null && (_query.OrderBy != null || _query.GroupBy != null || preferred != null)) + if (lowest == null && (_query.OrderBy.Count > 0 || _query.GroupBy != null || preferred != null)) { + var orderByExpr = _query.OrderBy.Count > 0 ? _query.OrderBy[0].Expression.Source : null; var index = indexes.FirstOrDefault(x => x.Expression == _query.GroupBy?.Source) ?? - indexes.FirstOrDefault(x => x.Expression == _query.OrderBy?.Source) ?? + indexes.FirstOrDefault(x => x.Expression == orderByExpr) ?? indexes.FirstOrDefault(x => x.Expression == preferred); if (index != null) @@ -293,6 +307,217 @@ private IndexCost ChooseIndex(HashSet fields) return lowest; } + private bool TrySelectVectorIndex(out VectorIndexQuery index, out BsonExpression consumedTerm) + { + index = null; + consumedTerm = null; + + string expression = null; + float[] target = null; + double maxDistance = double.MaxValue; + var matchedFromOrderBy = false; + + foreach (var term in _terms) + { + if (this.TryParseVectorPredicate(term, out expression, out target, out maxDistance)) + { + consumedTerm = term; + break; + } + } + + if (expression == null && _query.OrderBy.Count > 0) + { + foreach (var order in _query.OrderBy) + { + if (this.TryParseVectorExpression(order.Expression, out expression, out target)) + { + matchedFromOrderBy = true; + maxDistance = double.MaxValue; + break; + } + } + } + + if (expression == null && _query.VectorTarget != null && _query.VectorField != null) + { + expression = NormalizeVectorField(_query.VectorField); + target = _query.VectorTarget?.ToArray(); + maxDistance = _query.VectorMaxDistance; + matchedFromOrderBy = matchedFromOrderBy || (_query.OrderBy.Any(order => order.Expression?.Type == BsonExpressionType.VectorSim)); + } + + if (expression == null || target == null) + { + return false; + } + + int? limit = _query.Limit != int.MaxValue ? _query.Limit : (int?)null; + + foreach (var (candidate, metadata) in _snapshot.CollectionPage.GetVectorIndexes()) + { + if (!string.Equals(candidate.Expression, expression, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (metadata.Dimensions != target.Length) + { + continue; + } + + index = new VectorIndexQuery(candidate.Name, _snapshot, candidate, metadata, target, maxDistance, limit, _collation); + + if (matchedFromOrderBy) + { + _vectorOrderConsumed = true; + } + + return true; + } + + return false; + } + + private bool TryParseVectorPredicate(BsonExpression predicate, out string expression, out float[] target, out double maxDistance) + { + expression = null; + target = null; + maxDistance = double.NaN; + + if (predicate == null) + { + return false; + } + + if ((predicate.Type == BsonExpressionType.LessThan || predicate.Type == BsonExpressionType.LessThanOrEqual) && + this.TryParseVectorExpression(predicate.Left, out expression, out target) && + TryConvertToDouble(predicate.Right?.ExecuteScalar(_collation), out maxDistance)) + { + return true; + } + + if ((predicate.Type == BsonExpressionType.GreaterThan || predicate.Type == BsonExpressionType.GreaterThanOrEqual) && + this.TryParseVectorExpression(predicate.Right, out expression, out target) && + TryConvertToDouble(predicate.Left?.ExecuteScalar(_collation), out maxDistance)) + { + return true; + } + + expression = null; + target = null; + maxDistance = double.NaN; + return false; + } + + private bool TryParseVectorExpression(BsonExpression expression, out string fieldExpression, out float[] target) + { + fieldExpression = null; + target = null; + + if (expression == null || expression.Type != BsonExpressionType.VectorSim) + { + return false; + } + + var field = expression.Left; + if (field == null || string.IsNullOrEmpty(field.Source)) + { + return false; + } + + var targetValue = expression.Right?.ExecuteScalar(_collation); + + if (!TryConvertToVector(targetValue, out target)) + { + return false; + } + + fieldExpression = field.Source; + return true; + } + + private static bool TryConvertToVector(BsonValue value, out float[] vector) + { + vector = null; + + if (value == null || value.IsNull) + { + return false; + } + + if (value.Type == BsonType.Vector) + { + vector = value.AsVector.ToArray(); + return true; + } + + if (!value.IsArray) + { + return false; + } + + var array = value.AsArray; + var buffer = new float[array.Count]; + + for (var i = 0; i < array.Count; i++) + { + var item = array[i]; + + if (item.IsNull) + { + return false; + } + + try + { + buffer[i] = (float)item.AsDouble; + } + catch + { + return false; + } + } + + vector = buffer; + return true; + } + + private static bool TryConvertToDouble(BsonValue value, out double number) + { + number = double.NaN; + + if (value == null || value.IsNull || !value.IsNumber) + { + return false; + } + + number = value.AsDouble; + return !double.IsNaN(number); + } + + private static string NormalizeVectorField(string field) + { + if (string.IsNullOrWhiteSpace(field)) + { + return field; + } + + field = field.Trim(); + + if (field.StartsWith("$", StringComparison.Ordinal)) + { + return field; + } + + if (field.StartsWith(".", StringComparison.Ordinal)) + { + field = field.Substring(1); + } + + return "$." + field; + } + #endregion #region OrderBy / GroupBy Definition @@ -303,22 +528,28 @@ private IndexCost ChooseIndex(HashSet fields) private void DefineOrderBy() { // if has no order by, returns null - if (_query.OrderBy == null) return; + if (_query.OrderBy.Count == 0) return; + + if (_vectorOrderConsumed) + { + _queryPlan.OrderBy = null; + return; + } - var orderBy = new OrderBy(_query.OrderBy, _query.Order); + var orderBy = new OrderBy(_query.OrderBy.Select(x => new OrderByItem(x.Expression, x.Order))); - // if index expression are same as orderBy, use index to sort - just update index order - if (orderBy.Expression.Source == _queryPlan.IndexExpression) + // if index expression are same as primary OrderBy segment, use index order configuration + if (orderBy.PrimaryExpression.Source == _queryPlan.IndexExpression) { - // re-use index order and no not run OrderBy - // update index order to be same as required in OrderBy - _queryPlan.Index.Order = orderBy.Order; + _queryPlan.Index.Order = orderBy.PrimaryOrder; - // in this case "query.OrderBy" will be null - orderBy = null; + if (orderBy.Segments.Count == 1) + { + orderBy = null; + } } - // otherwise, query.OrderBy will be setted according user defined + // otherwise, query.OrderBy will be set according user defined _queryPlan.OrderBy = orderBy; } @@ -329,25 +560,25 @@ private void DefineGroupBy() { if (_query.GroupBy == null) return; - if (_query.OrderBy != null) throw new NotSupportedException("GROUP BY expression do not support ORDER BY"); if (_query.Includes.Count > 0) throw new NotSupportedException("GROUP BY expression do not support INCLUDE"); - var groupBy = new GroupBy(_query.GroupBy, _queryPlan.Select.Expression, _query.Having); - var orderBy = (OrderBy)null; + var expression = _query.GroupBy; + var select = _queryPlan.Select.Expression; + var having = _query.Having; + var groupOrderBy = (OrderBy)null; - // if groupBy use same expression in index, set group by order to MaxValue to not run - if (groupBy.Expression.Source == _queryPlan.IndexExpression) + // if groupBy use same expression in index, no additional ordering is required before grouping + if (expression.Source == _queryPlan.IndexExpression) { - // great - group by expression are same used in index - no changes here + // index already provides grouped ordering } else { // create orderBy expression - orderBy = new OrderBy(groupBy.Expression, Query.Ascending); + groupOrderBy = new OrderBy(new[] { new OrderByItem(expression, Query.Ascending) }); } - _queryPlan.GroupBy = groupBy; - _queryPlan.OrderBy = orderBy; + _queryPlan.GroupBy = new GroupBy(expression, select, having, groupOrderBy); } #endregion @@ -364,7 +595,7 @@ private void DefineIncludes() // test if field are using in any filter or orderBy var used = _queryPlan.Filters.Any(x => x.Fields.Contains(field)) || - (_queryPlan.OrderBy?.Expression.Fields.Contains(field) ?? false); + (_queryPlan.OrderBy?.ContainsField(field) ?? false); if (used) { diff --git a/LiteDB/Engine/Query/Structures/GroupBy.cs b/LiteDB/Engine/Query/Structures/GroupBy.cs index 193cc1227..d92a615eb 100644 --- a/LiteDB/Engine/Query/Structures/GroupBy.cs +++ b/LiteDB/Engine/Query/Structures/GroupBy.cs @@ -18,11 +18,14 @@ internal class GroupBy public BsonExpression Having { get; } - public GroupBy(BsonExpression expression, BsonExpression select, BsonExpression having) + public OrderBy OrderBy { get; } + + public GroupBy(BsonExpression expression, BsonExpression select, BsonExpression having, OrderBy orderBy) { this.Expression = expression; this.Select = select; this.Having = having; + this.OrderBy = orderBy; } } } diff --git a/LiteDB/Engine/Query/Structures/OrderBy.cs b/LiteDB/Engine/Query/Structures/OrderBy.cs index 63ae6fd4c..c216520d4 100644 --- a/LiteDB/Engine/Query/Structures/OrderBy.cs +++ b/LiteDB/Engine/Query/Structures/OrderBy.cs @@ -12,14 +12,39 @@ namespace LiteDB.Engine /// internal class OrderBy { - public BsonExpression Expression { get; } + private readonly List _segments; + + public OrderBy(IEnumerable segments) + { + if (segments == null) throw new ArgumentNullException(nameof(segments)); + + _segments = segments.ToList(); + + if (_segments.Count == 0) + { + throw new ArgumentException("OrderBy requires at least one segment", nameof(segments)); + } + } + + public IReadOnlyList Segments => _segments; - public int Order { get; set; } + public BsonExpression PrimaryExpression => _segments[0].Expression; - public OrderBy(BsonExpression expression, int order) + public int PrimaryOrder => _segments[0].Order; + + public bool ContainsField(string field) => _segments.Any(x => x.Expression.Fields.Contains(field)); + } + + internal class OrderByItem + { + public OrderByItem(BsonExpression expression, int order) { - this.Expression = expression; + this.Expression = expression ?? throw new ArgumentNullException(nameof(expression)); this.Order = order; } + + public BsonExpression Expression { get; } + + public int Order { get; } } } diff --git a/LiteDB/Engine/Query/Structures/QueryOrder.cs b/LiteDB/Engine/Query/Structures/QueryOrder.cs new file mode 100644 index 000000000..012561bd0 --- /dev/null +++ b/LiteDB/Engine/Query/Structures/QueryOrder.cs @@ -0,0 +1,18 @@ +namespace LiteDB +{ + /// + /// Represents a single ORDER BY segment containing the expression and direction. + /// + public class QueryOrder + { + public QueryOrder(BsonExpression expression, int order) + { + this.Expression = expression; + this.Order = order; + } + + public BsonExpression Expression { get; } + + public int Order { get; } + } +} diff --git a/LiteDB/Engine/Query/Structures/QueryPlan.cs b/LiteDB/Engine/Query/Structures/QueryPlan.cs index c567a3db2..70f8c56bd 100644 --- a/LiteDB/Engine/Query/Structures/QueryPlan.cs +++ b/LiteDB/Engine/Query/Structures/QueryPlan.cs @@ -179,11 +179,11 @@ public BsonDocument GetExecutionPlan() if (this.OrderBy != null) { - doc["orderBy"] = new BsonDocument + doc["orderBy"] = new BsonArray(this.OrderBy.Segments.Select(x => new BsonDocument { - ["expr"] = this.OrderBy.Expression.Source, - ["order"] = this.OrderBy.Order, - }; + ["expr"] = x.Expression.Source, + ["order"] = x.Order, + })); } if (this.Limit != int.MaxValue) @@ -203,12 +203,23 @@ public BsonDocument GetExecutionPlan() if (this.GroupBy != null) { - doc["groupBy"] = new BsonDocument + var group = new BsonDocument { ["expr"] = this.GroupBy.Expression.Source, ["having"] = this.GroupBy.Having?.Source, ["select"] = this.GroupBy.Select?.Source }; + + if (this.GroupBy.OrderBy != null) + { + group["orderBy"] = new BsonArray(this.GroupBy.OrderBy.Segments.Select(x => new BsonDocument + { + ["expr"] = x.Expression.Source, + ["order"] = x.Order + })); + } + + doc["groupBy"] = group; } else { diff --git a/LiteDB/Engine/Services/SnapShot.cs b/LiteDB/Engine/Services/SnapShot.cs index f34840e2f..083efcc32 100644 --- a/LiteDB/Engine/Services/SnapShot.cs +++ b/LiteDB/Engine/Services/SnapShot.cs @@ -329,6 +329,30 @@ public IndexPage GetFreeIndexPage(int bytesLength, ref uint freeIndexPageList) return page; } + /// + /// Get a vector index page with enough free space for a new node. + /// + public VectorIndexPage GetFreeVectorPage(int bytesLength, ref uint freeVectorPageList) + { + ENSURE(!_disposed, "the snapshot is disposed"); + + VectorIndexPage page; + + if (freeVectorPageList == uint.MaxValue) + { + page = this.NewPage(); + } + else + { + page = this.GetPage(freeVectorPageList); + + ENSURE(page.FreeBytes > bytesLength, "this page shout be space enouth for this new vector node"); + ENSURE(page.PageListSlot == 0, "this page should be in slot #0"); + } + + return page; + } + /// /// Get a new empty page from disk: can be a reused page (from header free list) or file extend /// Never re-use page from same transaction @@ -488,6 +512,42 @@ public void AddOrRemoveFreeIndexList(IndexPage page, ref uint startPageID) } } + /// + /// Add/Remove a vector index page from single free list + /// + public void AddOrRemoveFreeVectorList(VectorIndexPage page, ref uint startPageID) + { + ENSURE(!_disposed, "the snapshot is disposed"); + + var newSlot = VectorIndexPage.FreeListSlot(page.FreeBytes); + var isOnList = page.PageListSlot == 0; + var mustKeep = newSlot == 0; + + if (page.ItemsCount == 0) + { + if (isOnList) + { + this.RemoveFreeList(page, ref startPageID); + } + + this.DeletePage(page); + } + else + { + if (isOnList && !mustKeep) + { + this.RemoveFreeList(page, ref startPageID); + } + else if (!isOnList && mustKeep) + { + this.AddFreeList(page, ref startPageID); + } + + page.PageListSlot = newSlot; + page.IsDirty = true; + } + } + /// /// Add page into double linked-list (always add as first element) /// @@ -507,7 +567,7 @@ private void AddFreeList(T page, ref uint startPageID) where T : BasePage page.NextPageID = startPageID; page.IsDirty = true; - ENSURE(page.PageType == PageType.Data || page.PageType == PageType.Index, "only data/index pages must be first on free stack"); + ENSURE(page.PageType == PageType.Data || page.PageType == PageType.Index || page.PageType == PageType.VectorIndex, "only data/index pages must be first on free stack"); startPageID = page.PageID; @@ -560,9 +620,10 @@ private void DeletePage(T page) { ENSURE(page.PrevPageID == uint.MaxValue && page.NextPageID == uint.MaxValue, "before delete a page, no linked list with any another page"); ENSURE(page.ItemsCount == 0 && page.UsedBytes == 0 && page.HighestIndex == byte.MaxValue && page.FragmentedBytes == 0, "no items on page when delete this page"); - ENSURE(page.PageType == PageType.Data || page.PageType == PageType.Index, "only data/index page can be deleted"); + ENSURE(page.PageType == PageType.Data || page.PageType == PageType.Index || page.PageType == PageType.VectorIndex, "only data/index page can be deleted"); DEBUG(!_collectionPage.FreeDataPageList.Any(x => x == page.PageID), "this page cann't be deleted because free data list page is linked o this page"); DEBUG(!_collectionPage.GetCollectionIndexes().Any(x => x.FreeIndexPageList == page.PageID), "this page cann't be deleted because free index list page is linked o this page"); + DEBUG(!_collectionPage.GetVectorIndexes().Any(x => x.Metadata.Reserved == page.PageID), "this page cann't be deleted because free vector list page is linked o this page"); DEBUG(page.Buffer.Slice(PAGE_HEADER_SIZE, PAGE_SIZE - PAGE_HEADER_SIZE - 1).All(0), "page content shloud be empty"); // mark page as empty and dirty @@ -603,7 +664,8 @@ public void DropCollection(Action safePoint) ENSURE(!_disposed, "the snapshot is disposed"); var indexer = new IndexService(this, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); - + VectorIndexService vectorIndexer = null; + // CollectionPage will be last deleted page (there is no NextPageID from CollectionPage) _transPages.FirstDeletedPageID = _collectionPage.PageID; _transPages.LastDeletedPageID = _collectionPage.PageID; @@ -618,6 +680,11 @@ public void DropCollection(Action safePoint) // getting all indexes pages from all indexes foreach(var index in _collectionPage.GetCollectionIndexes()) { + if (index.IndexType == 1) + { + continue; + } + // add head/tail (same page) to be deleted indexPages.Add(index.Head.PageID); @@ -628,6 +695,15 @@ public void DropCollection(Action safePoint) safePoint(); } } + + + foreach (var (_, metadata) in _collectionPage.GetVectorIndexes()) + { + vectorIndexer ??= new VectorIndexService(this, _header.Pragmas.Collation); + vectorIndexer.Drop(metadata); + + safePoint(); + } // now, mark all pages as deleted foreach (var pageID in indexPages) diff --git a/LiteDB/Engine/Services/TransactionService.cs b/LiteDB/Engine/Services/TransactionService.cs index 7373251da..7e2d3de0a 100644 --- a/LiteDB/Engine/Services/TransactionService.cs +++ b/LiteDB/Engine/Services/TransactionService.cs @@ -296,11 +296,17 @@ public void Rollback() // but first, if writable, discard changes if (snapshot.Mode == LockMode.Write) { - // discard all dirty pages - _disk.DiscardDirtyPages(snapshot.GetWritablePages(true, true).Select(x => x.Buffer)); - - // discard all clean pages - _disk.DiscardCleanPages(snapshot.GetWritablePages(false, true).Select(x => x.Buffer)); + // discard all dirty pages (only buffers still writable) + _disk.DiscardDirtyPages(snapshot + .GetWritablePages(true, true) + .Select(x => x.Buffer) + .Where(x => x.ShareCounter == BUFFER_WRITABLE)); + + // discard all clean pages (only buffers still writable) + _disk.DiscardCleanPages(snapshot + .GetWritablePages(false, true) + .Select(x => x.Buffer) + .Where(x => x.ShareCounter == BUFFER_WRITABLE)); } // now, release pages @@ -406,11 +412,17 @@ protected virtual void Dispose(bool dispose) // release writable snapshots foreach (var snapshot in _snapshots.Values.Where(x => x.Mode == LockMode.Write)) { - // discard all dirty pages - _disk.DiscardDirtyPages(snapshot.GetWritablePages(true, true).Select(x => x.Buffer)); - - // discard all clean pages - _disk.DiscardCleanPages(snapshot.GetWritablePages(false, true).Select(x => x.Buffer)); + // discard all dirty pages (only buffers still writable) + _disk.DiscardDirtyPages(snapshot + .GetWritablePages(true, true) + .Select(x => x.Buffer) + .Where(x => x.ShareCounter == BUFFER_WRITABLE)); + + // discard all clean pages (only buffers still writable) + _disk.DiscardCleanPages(snapshot + .GetWritablePages(false, true) + .Select(x => x.Buffer) + .Where(x => x.ShareCounter == BUFFER_WRITABLE)); } // release buffers in read-only snaphosts diff --git a/LiteDB/Engine/Services/VectorIndexService.cs b/LiteDB/Engine/Services/VectorIndexService.cs new file mode 100644 index 000000000..fce044551 --- /dev/null +++ b/LiteDB/Engine/Services/VectorIndexService.cs @@ -0,0 +1,978 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LiteDB; +using LiteDB.Vector; + +namespace LiteDB.Engine +{ + internal sealed class VectorIndexService + { + private const int EfConstruction = 24; + private const int DefaultEfSearch = 32; + + private readonly Snapshot _snapshot; + private readonly Collation _collation; + private readonly Random _random = new Random(); + + private DataService _vectorData; + + private readonly struct NodeDistance + { + public NodeDistance(PageAddress address, double distance, double similarity) + { + this.Address = address; + this.Distance = double.IsNaN(distance) ? double.PositiveInfinity : distance; + this.Similarity = similarity; + } + + public PageAddress Address { get; } + public double Distance { get; } + public double Similarity { get; } + } + + public int LastVisitedCount { get; private set; } + + public VectorIndexService(Snapshot snapshot, Collation collation) + { + _snapshot = snapshot; + _collation = collation; + } + + public void Upsert(CollectionIndex index, VectorIndexMetadata metadata, BsonDocument document, PageAddress dataBlock) + { + var value = index.BsonExpr.ExecuteScalar(document, _collation); + + if (!TryExtractVector(value, metadata.Dimensions, out var vector)) + { + this.Delete(metadata, dataBlock); + return; + } + + this.Delete(metadata, dataBlock); + this.Insert(metadata, dataBlock, vector); + } + + public void Delete(VectorIndexMetadata metadata, PageAddress dataBlock) + { + if (!this.TryFindNode(metadata, dataBlock, out var address, out var node)) + { + return; + } + + this.RemoveNode(metadata, address, node); + } + + public IEnumerable<(BsonDocument Document, double Distance)> Search( + VectorIndexMetadata metadata, + float[] target, + double maxDistance, + int? limit) + { + if (metadata.Root.IsEmpty) + { + this.LastVisitedCount = 0; + return Enumerable.Empty<(BsonDocument Document, double Distance)>(); + } + + var data = new DataService(_snapshot, uint.MaxValue); + var vectorCache = new Dictionary(); + var visited = new HashSet(); + + this.LastVisitedCount = 0; + + var entryPoint = metadata.Root; + var entryNode = this.GetNode(entryPoint); + var entryTopLevel = entryNode.LevelCount - 1; + var currentEntry = entryPoint; + + for (var level = entryTopLevel; level > 0; level--) + { + currentEntry = this.GreedySearch(metadata, target, currentEntry, level, vectorCache, visited); + } + + var effectiveLimit = limit.HasValue && limit.Value > 0 + ? Math.Max(limit.Value * 4, DefaultEfSearch) + : DefaultEfSearch; + + var candidates = this.SearchLayer( + metadata, + target, + currentEntry, + 0, + effectiveLimit, + effectiveLimit, + visited, + vectorCache); + + var results = new List<(BsonDocument Document, double Distance, double Similarity)>(); + + var pruneDistance = metadata.Metric == VectorDistanceMetric.DotProduct + ? double.PositiveInfinity + : maxDistance; + + var hasExplicitSimilarity = metadata.Metric == VectorDistanceMetric.DotProduct + && !double.IsPositiveInfinity(maxDistance) + && maxDistance < double.MaxValue; + + var baseMinSimilarity = hasExplicitSimilarity ? maxDistance : double.NegativeInfinity; + var minSimilarity = baseMinSimilarity; + + foreach (var candidate in candidates) + { + var compareDistance = candidate.Distance; + var meetsThreshold = metadata.Metric == VectorDistanceMetric.DotProduct + ? !double.IsNaN(candidate.Similarity) && candidate.Similarity >= minSimilarity + : !double.IsNaN(compareDistance) && compareDistance <= pruneDistance; + + if (!meetsThreshold) + { + continue; + } + + var node = this.GetNode(candidate.Address); + using var reader = new BufferReader(data.Read(node.DataBlock)); + var document = reader.ReadDocument().GetValue(); + document.RawId = node.DataBlock; + results.Add((document, candidate.Distance, candidate.Similarity)); + } + + if (metadata.Metric == VectorDistanceMetric.DotProduct) + { + results = results + .OrderByDescending(x => x.Similarity) + .ToList(); + + if (limit.HasValue) + { + results = results.Take(limit.Value).ToList(); + if (results.Count == limit.Value) + { + minSimilarity = Math.Max(baseMinSimilarity, results.Min(x => x.Similarity)); + } + } + + return results.Select(x => (x.Document, x.Similarity)); + } + + results = results + .OrderBy(x => x.Distance) + .ToList(); + + if (limit.HasValue) + { + results = results.Take(limit.Value).ToList(); + if (results.Count == limit.Value) + { + pruneDistance = Math.Min(pruneDistance, results.Max(x => x.Distance)); + } + } + + return results.Select(x => (x.Document, x.Distance)); + } + + public void Drop(VectorIndexMetadata metadata) + { + this.ClearTree(metadata); + + metadata.Root = PageAddress.Empty; + metadata.Reserved = uint.MaxValue; + _snapshot.CollectionPage.IsDirty = true; + } + + public static double ComputeDistance(float[] candidate, float[] target, VectorDistanceMetric metric, out double similarity) + { + similarity = double.NaN; + + if (candidate.Length != target.Length) + { + return double.NaN; + } + + switch (metric) + { + case VectorDistanceMetric.Cosine: + return ComputeCosineDistance(candidate, target); + case VectorDistanceMetric.Euclidean: + return ComputeEuclideanDistance(candidate, target); + case VectorDistanceMetric.DotProduct: + similarity = ComputeDotProduct(candidate, target); + return -similarity; + default: + throw new ArgumentOutOfRangeException(nameof(metric)); + } + } + + private void Insert(VectorIndexMetadata metadata, PageAddress dataBlock, float[] vector) + { + var levelCount = this.SampleLevel(); + var length = VectorIndexNode.GetLength(vector.Length, out var storesInline); + var freeList = metadata.Reserved; + var page = _snapshot.GetFreeVectorPage(length, ref freeList); + metadata.Reserved = freeList; + + PageAddress externalVector = PageAddress.Empty; + VectorIndexNode node; + + try + { + if (!storesInline) + { + externalVector = this.StoreVector(vector); + } + + node = page.InsertNode(dataBlock, vector, length, levelCount, externalVector); + + freeList = metadata.Reserved; + metadata.Reserved = uint.MaxValue; + _snapshot.AddOrRemoveFreeVectorList(page, ref freeList); + metadata.Reserved = freeList; + } + catch + { + if (!storesInline && !externalVector.IsEmpty) + { + this.ReleaseVectorData(externalVector); + } + + metadata.Reserved = freeList; + + throw; + } + + _snapshot.CollectionPage.IsDirty = true; + + var newAddress = node.Position; + + if (metadata.Root.IsEmpty) + { + metadata.Root = newAddress; + _snapshot.CollectionPage.IsDirty = true; + return; + } + + var vectorCache = new Dictionary + { + [newAddress] = vector + }; + + var entryPoint = metadata.Root; + var entryNode = this.GetNode(entryPoint); + var entryTopLevel = entryNode.LevelCount - 1; + var newTopLevel = levelCount - 1; + + if (newTopLevel > entryTopLevel) + { + metadata.Root = newAddress; + _snapshot.CollectionPage.IsDirty = true; + entryTopLevel = newTopLevel; + } + + var currentEntry = entryPoint; + + for (var level = entryTopLevel; level > newTopLevel; level--) + { + currentEntry = this.GreedySearch(metadata, vector, currentEntry, level, vectorCache, null); + } + + var maxLevelToConnect = Math.Min(entryNode.LevelCount - 1, newTopLevel); + + for (var level = maxLevelToConnect; level >= 0; level--) + { + var candidates = this.SearchLayer( + metadata, + vector, + currentEntry, + level, + VectorIndexNode.MaxNeighborsPerLevel, + EfConstruction, + null, + vectorCache); + + var selected = this.SelectNeighbors( + candidates.Where(x => x.Address != newAddress).ToList(), + VectorIndexNode.MaxNeighborsPerLevel); + + node.SetNeighbors(level, selected.Select(x => x.Address).ToList()); + + foreach (var neighbor in selected) + { + this.EnsureBidirectional(metadata, neighbor.Address, newAddress, level, vectorCache); + } + + if (selected.Count > 0) + { + currentEntry = selected[0].Address; + } + } + } + + private PageAddress GreedySearch( + VectorIndexMetadata metadata, + float[] target, + PageAddress start, + int level, + Dictionary vectorCache, + HashSet globalVisited) + { + var current = start; + this.RegisterVisit(globalVisited, current); + + var currentVector = this.GetVector(metadata, current, vectorCache); + var currentDistance = NormalizeDistance(ComputeDistance(currentVector, target, metadata.Metric, out _)); + + var improved = true; + + while (improved) + { + improved = false; + + var node = this.GetNode(current); + foreach (var neighbor in node.GetNeighbors(level)) + { + if (neighbor.IsEmpty) + { + continue; + } + + this.RegisterVisit(globalVisited, neighbor); + + var neighborVector = this.GetVector(metadata, neighbor, vectorCache); + var neighborDistance = NormalizeDistance(ComputeDistance(neighborVector, target, metadata.Metric, out _)); + + if (neighborDistance < currentDistance) + { + current = neighbor; + currentDistance = neighborDistance; + improved = true; + } + } + } + + return current; + } + + private List SearchLayer( + VectorIndexMetadata metadata, + float[] target, + PageAddress entryPoint, + int level, + int maxResults, + int explorationFactor, + HashSet globalVisited, + Dictionary vectorCache) + { + var results = new List(); + var candidates = new List(); + var visited = new HashSet(); + + if (entryPoint.IsEmpty) + { + return results; + } + + var entryVector = this.GetVector(metadata, entryPoint, vectorCache); + var entryDistance = ComputeDistance(entryVector, target, metadata.Metric, out var entrySimilarity); + var entryNode = new NodeDistance(entryPoint, entryDistance, entrySimilarity); + + InsertOrdered(results, entryNode, Math.Max(1, explorationFactor)); + candidates.Add(entryNode); + visited.Add(entryPoint); + this.RegisterVisit(globalVisited, entryPoint); + + while (candidates.Count > 0) + { + var index = GetMinimumIndex(candidates); + var current = candidates[index]; + candidates.RemoveAt(index); + + var worstAllowed = results.Count >= explorationFactor + ? results[results.Count - 1].Distance + : double.PositiveInfinity; + + if (current.Distance > worstAllowed) + { + continue; + } + + var node = this.GetNode(current.Address); + + foreach (var neighbor in node.GetNeighbors(level)) + { + if (neighbor.IsEmpty || !visited.Add(neighbor)) + { + continue; + } + + this.RegisterVisit(globalVisited, neighbor); + + var neighborVector = this.GetVector(metadata, neighbor, vectorCache); + var distance = ComputeDistance(neighborVector, target, metadata.Metric, out var similarity); + var candidate = new NodeDistance(neighbor, distance, similarity); + + if (InsertOrdered(results, candidate, Math.Max(1, explorationFactor))) + { + candidates.Add(candidate); + } + } + } + + return this.SelectNeighbors(results, Math.Max(1, maxResults)); + } + + private void EnsureBidirectional(VectorIndexMetadata metadata, PageAddress source, PageAddress target, int level, Dictionary vectorCache) + { + var node = this.GetNode(source); + var neighbors = node.GetNeighbors(level).ToList(); + + if (!neighbors.Contains(target)) + { + neighbors.Add(target); + } + + var pruned = this.PruneNeighbors(metadata, source, neighbors, vectorCache); + node.SetNeighbors(level, pruned); + } + + private IReadOnlyList PruneNeighbors(VectorIndexMetadata metadata, PageAddress source, List neighbors, Dictionary vectorCache) + { + var unique = new HashSet(neighbors.Where(x => !x.IsEmpty && x != source)); + + if (unique.Count == 0) + { + return Array.Empty(); + } + + var sourceVector = this.GetVector(metadata, source, vectorCache); + var scored = new List(); + + foreach (var neighbor in unique) + { + var neighborVector = this.GetVector(metadata, neighbor, vectorCache); + var distance = ComputeDistance(sourceVector, neighborVector, metadata.Metric, out _); + scored.Add(new NodeDistance(neighbor, distance, double.NaN)); + } + + return this.SelectNeighbors(scored, VectorIndexNode.MaxNeighborsPerLevel) + .Select(x => x.Address) + .ToList(); + } + + private void RemoveNode(VectorIndexMetadata metadata, PageAddress address, VectorIndexNode node) + { + var start = PageAddress.Empty; + + for (var level = 0; level < node.LevelCount && start.IsEmpty; level++) + { + foreach (var neighbor in node.GetNeighbors(level)) + { + if (!neighbor.IsEmpty) + { + start = neighbor; + break; + } + } + } + + for (var level = 0; level < node.LevelCount; level++) + { + foreach (var neighbor in node.GetNeighbors(level)) + { + if (neighbor.IsEmpty) + { + continue; + } + + var neighborNode = this.GetNode(neighbor); + neighborNode.RemoveNeighbor(level, address); + } + } + + if (metadata.Root == address) + { + metadata.Root = this.SelectNewRoot(metadata, address, start); + _snapshot.CollectionPage.IsDirty = true; + } + + this.ReleaseNode(metadata, node); + } + + private PageAddress SelectNewRoot(VectorIndexMetadata metadata, PageAddress removed, PageAddress start) + { + if (start.IsEmpty) + { + return PageAddress.Empty; + } + + var best = PageAddress.Empty; + byte bestLevel = 0; + + var visited = new HashSet(); + var queue = new Queue(); + queue.Enqueue(start); + + while (queue.Count > 0) + { + var current = queue.Dequeue(); + if (current == removed || !visited.Add(current)) + { + continue; + } + + var node = this.GetNode(current); + var levelCount = node.LevelCount; + + if (best.IsEmpty || levelCount > bestLevel) + { + best = current; + bestLevel = levelCount; + } + + for (var level = 0; level < levelCount; level++) + { + foreach (var neighbor in node.GetNeighbors(level)) + { + if (!neighbor.IsEmpty && neighbor != removed) + { + queue.Enqueue(neighbor); + } + } + } + } + + return best; + } + + private bool TryFindNode(VectorIndexMetadata metadata, PageAddress dataBlock, out PageAddress address, out VectorIndexNode node) + { + address = PageAddress.Empty; + node = null; + + if (metadata.Root.IsEmpty) + { + return false; + } + + var visited = new HashSet(); + var queue = new Queue(); + queue.Enqueue(metadata.Root); + + while (queue.Count > 0) + { + var current = queue.Dequeue(); + if (!visited.Add(current)) + { + continue; + } + + var candidate = this.GetNode(current); + + if (candidate.DataBlock == dataBlock) + { + address = current; + node = candidate; + return true; + } + + for (var level = 0; level < candidate.LevelCount; level++) + { + foreach (var neighbor in candidate.GetNeighbors(level)) + { + if (!neighbor.IsEmpty) + { + queue.Enqueue(neighbor); + } + } + } + } + + return false; + } + + private void ClearTree(VectorIndexMetadata metadata) + { + if (metadata.Root.IsEmpty) + { + return; + } + + var visited = new HashSet(); + var stack = new Stack(); + stack.Push(metadata.Root); + + while (stack.Count > 0) + { + var address = stack.Pop(); + if (!visited.Add(address)) + { + continue; + } + + var node = this.GetNode(address); + + for (var level = 0; level < node.LevelCount; level++) + { + foreach (var neighbor in node.GetNeighbors(level)) + { + if (!neighbor.IsEmpty && !visited.Contains(neighbor)) + { + stack.Push(neighbor); + } + } + } + + this.ReleaseNode(metadata, node); + } + } + + private void ReleaseNode(VectorIndexMetadata metadata, VectorIndexNode node) + { + this.ReleaseVectorData(node); + + var page = node.Page; + page.DeleteNode(node.Position.Index); + var freeList = metadata.Reserved; + metadata.Reserved = uint.MaxValue; + _snapshot.AddOrRemoveFreeVectorList(page, ref freeList); + metadata.Reserved = freeList; + } + + private VectorIndexNode GetNode(PageAddress address) + { + var page = _snapshot.GetPage(address.PageID); + + return page.GetNode(address.Index); + } + + private float[] GetVector(VectorIndexMetadata metadata, PageAddress address, Dictionary cache) + { + if (!cache.TryGetValue(address, out var vector)) + { + var node = this.GetNode(address); + vector = node.HasInlineVector + ? node.ReadVector() + : this.ReadExternalVector(node, metadata); + cache[address] = vector; + } + + return vector; + } + + private float[] ReadExternalVector(VectorIndexNode node, VectorIndexMetadata metadata) + { + if (node.ExternalVector.IsEmpty) + { + return Array.Empty(); + } + + var dimensions = metadata.Dimensions; + if (dimensions == 0) + { + return Array.Empty(); + } + + var totalBytes = dimensions * sizeof(float); + var vector = new float[dimensions]; + var bytesCopied = 0; + + foreach (var slice in this.GetVectorDataService().Read(node.ExternalVector)) + { + if (bytesCopied >= totalBytes) + { + break; + } + + var available = Math.Min(slice.Count, totalBytes - bytesCopied); + + if ((available & 3) != 0) + { + throw new LiteException(0, "Vector data block is corrupted."); + } + + Buffer.BlockCopy(slice.Array, slice.Offset, vector, bytesCopied, available); + bytesCopied += available; + } + + if (bytesCopied != totalBytes) + { + throw new LiteException(0, "Vector data block is incomplete."); + } + + return vector; + } + + private PageAddress StoreVector(float[] vector) + { + if (vector.Length == 0) + { + return PageAddress.Empty; + } + + var totalBytes = vector.Length * sizeof(float); + var bytesWritten = 0; + var firstBlock = PageAddress.Empty; + DataBlock lastBlock = null; + + while (bytesWritten < totalBytes) + { + var remaining = totalBytes - bytesWritten; + var chunk = Math.Min(remaining, DataService.MAX_DATA_BYTES_PER_PAGE); + + if ((chunk & 3) != 0) + { + chunk -= chunk & 3; + } + + if (chunk <= 0) + { + chunk = remaining; + } + + var dataPage = _snapshot.GetFreeDataPage(chunk + DataBlock.DATA_BLOCK_FIXED_SIZE); + var block = dataPage.InsertBlock(chunk, bytesWritten > 0); + + if (lastBlock != null) + { + lastBlock.SetNextBlock(block.Position); + } + else + { + firstBlock = block.Position; + } + + Buffer.BlockCopy(vector, bytesWritten, block.Buffer.Array, block.Buffer.Offset, chunk); + + _snapshot.AddOrRemoveFreeDataList(dataPage); + + lastBlock = block; + bytesWritten += chunk; + } + + return firstBlock; + } + + private void ReleaseVectorData(VectorIndexNode node) + { + if (node.HasInlineVector) + { + return; + } + + this.ReleaseVectorData(node.ExternalVector); + } + + private void ReleaseVectorData(PageAddress address) + { + if (address.IsEmpty) + { + return; + } + + this.GetVectorDataService().Delete(address); + } + + private DataService GetVectorDataService() + { + return _vectorData ??= new DataService(_snapshot, uint.MaxValue); + } + + private byte SampleLevel() + { + var level = 1; + + lock (_random) + { + while (level < VectorIndexNode.MaxLevels && _random.NextDouble() < 0.5d) + { + level++; + } + } + + return (byte)level; + } + + private void RegisterVisit(HashSet visited, PageAddress address) + { + if (address.IsEmpty) + { + return; + } + + if (visited != null) + { + if (!visited.Add(address)) + { + return; + } + } + + this.LastVisitedCount++; + } + + private static int GetMinimumIndex(List candidates) + { + var index = 0; + var best = candidates[0].Distance; + + for (var i = 1; i < candidates.Count; i++) + { + if (candidates[i].Distance < best) + { + best = candidates[i].Distance; + index = i; + } + } + + return index; + } + + private static bool InsertOrdered(List list, NodeDistance item, int maxSize) + { + if (maxSize <= 0) + { + return false; + } + + var inserted = false; + + var index = list.FindIndex(x => item.Distance < x.Distance); + + if (index >= 0) + { + list.Insert(index, item); + inserted = true; + } + else if (list.Count < maxSize) + { + list.Add(item); + inserted = true; + } + + if (list.Count > maxSize) + { + list.RemoveAt(list.Count - 1); + } + + return inserted; + } + + private List SelectNeighbors(List candidates, int maxNeighbors) + { + if (candidates.Count == 0 || maxNeighbors <= 0) + { + return new List(); + } + + var seen = new HashSet(); + + return candidates + .OrderBy(x => x.Distance) + .Where(x => seen.Add(x.Address)) + .Take(maxNeighbors) + .ToList(); + } + + private static double NormalizeDistance(double distance) + { + return double.IsNaN(distance) ? double.PositiveInfinity : distance; + } + + private static double ComputeCosineDistance(float[] candidate, float[] target) + { + double dot = 0d; + double magCandidate = 0d; + double magTarget = 0d; + + for (var i = 0; i < candidate.Length; i++) + { + var c = candidate[i]; + var t = target[i]; + + dot += c * t; + magCandidate += c * c; + magTarget += t * t; + } + + if (magCandidate == 0 || magTarget == 0) + { + return double.NaN; + } + + var cosine = dot / (Math.Sqrt(magCandidate) * Math.Sqrt(magTarget)); + return 1d - cosine; + } + + private static double ComputeEuclideanDistance(float[] candidate, float[] target) + { + double sum = 0d; + + for (var i = 0; i < candidate.Length; i++) + { + var diff = candidate[i] - target[i]; + sum += diff * diff; + } + + return Math.Sqrt(sum); + } + + private static double ComputeDotProduct(float[] candidate, float[] target) + { + double sum = 0d; + + for (var i = 0; i < candidate.Length; i++) + { + sum += candidate[i] * target[i]; + } + + return sum; + } + + private static bool TryExtractVector(BsonValue value, ushort expectedDimensions, out float[] vector) + { + vector = null; + + if (value.IsNull) + { + return false; + } + + float[] buffer; + + if (value.Type == BsonType.Vector) + { + buffer = value.AsVector.ToArray(); + } + else if (value.IsArray) + { + buffer = new float[value.AsArray.Count]; + + for (var i = 0; i < buffer.Length; i++) + { + var item = value.AsArray[i]; + + try + { + buffer[i] = (float)item.AsDouble; + } + catch + { + return false; + } + } + } + else + { + return false; + } + + if (buffer.Length != expectedDimensions) + { + return false; + } + + vector = buffer; + return true; + } + } +} + diff --git a/LiteDB/Engine/SharedMutexNameStrategy.cs b/LiteDB/Engine/SharedMutexNameStrategy.cs new file mode 100644 index 000000000..4683264ec --- /dev/null +++ b/LiteDB/Engine/SharedMutexNameStrategy.cs @@ -0,0 +1,9 @@ + +namespace LiteDB.Engine; + +public enum SharedMutexNameStrategy +{ + Default, + UriEscape, + Sha1Hash +} \ No newline at end of file diff --git a/LiteDB/Engine/Sort/SortContainer.cs b/LiteDB/Engine/Sort/SortContainer.cs index 9705f5afa..60ffd9318 100644 --- a/LiteDB/Engine/Sort/SortContainer.cs +++ b/LiteDB/Engine/Sort/SortContainer.cs @@ -18,6 +18,7 @@ internal class SortContainer : IDisposable { private readonly Collation _collation; private readonly int _size; + private readonly int[] _orders; private int _remaining = 0; private int _count = 0; @@ -49,10 +50,11 @@ internal class SortContainer : IDisposable /// public int Count => _count; - public SortContainer(Collation collation, int size) + public SortContainer(Collation collation, int size, IReadOnlyList orders) { _collation = collation; _size = size; + _orders = orders as int[] ?? orders.ToArray(); } public void Insert(IEnumerable> items, int order, BufferSlice buffer) @@ -108,6 +110,11 @@ public bool MoveNext() } var key = _reader.ReadIndexKey(); + + if (_orders.Length > 1) + { + key = SortKey.FromBsonValue(key, _orders); + } var value = _reader.ReadPageAddress(); this.Current = new KeyValuePair(key, value); diff --git a/LiteDB/Engine/Sort/SortKey.cs b/LiteDB/Engine/Sort/SortKey.cs new file mode 100644 index 000000000..6b1ba2f89 --- /dev/null +++ b/LiteDB/Engine/Sort/SortKey.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using LiteDB; + +namespace LiteDB.Engine +{ + internal class SortKey : BsonArray + { + private readonly int[] _orders; + + private SortKey(IEnumerable values, IEnumerable orders) + : base(values?.Select(x => x ?? BsonValue.Null) ?? throw new ArgumentNullException(nameof(values))) + { + if (orders == null) throw new ArgumentNullException(nameof(orders)); + + _orders = orders as int[] ?? orders.ToArray(); + + if (_orders.Length != this.Count) + { + throw new ArgumentException("Orders length must match values length", nameof(orders)); + } + } + + private SortKey(BsonArray array, IEnumerable orders) + : base(array) + { + if (orders == null) throw new ArgumentNullException(nameof(orders)); + + _orders = orders as int[] ?? orders.ToArray(); + + if (_orders.Length != this.Count) + { + throw new ArgumentException("Orders length must match values length", nameof(orders)); + } + } + + public override int CompareTo(BsonValue other) + { + return this.CompareTo(other, Collation.Binary); + } + + public override int CompareTo(BsonValue other, Collation collation) + { + if (other is SortKey sortKey) + { + var length = Math.Min(this.Count, sortKey.Count); + + for (var i = 0; i < length; i++) + { + var result = this[i].CompareTo(sortKey[i], collation); + + if (result == 0) continue; + + return _orders[i] == Query.Descending ? -result : result; + } + + if (this.Count == sortKey.Count) return 0; + + return this.Count < sortKey.Count ? -1 : 1; + } + + if (other is BsonArray array) + { + return this.CompareTo(new SortKey(array, Enumerable.Repeat(Query.Ascending, array.Count)), collation); + } + + return base.CompareTo(other, collation); + } + + public static SortKey FromValues(IReadOnlyList values, IReadOnlyList orders) + { + if (values == null) throw new ArgumentNullException(nameof(values)); + if (orders == null) throw new ArgumentNullException(nameof(orders)); + + return new SortKey(values, orders); + } + + public static SortKey FromBsonValue(BsonValue value, IReadOnlyList orders) + { + if (value is SortKey sortKey) return sortKey; + + if (value is BsonArray array) + { + return new SortKey(array.ToArray(), orders); + } + + return new SortKey(new[] { value }, orders); + } + } +} diff --git a/LiteDB/Engine/Sort/SortService.cs b/LiteDB/Engine/Sort/SortService.cs index 2ec04a0c2..208c8473e 100644 --- a/LiteDB/Engine/Sort/SortService.cs +++ b/LiteDB/Engine/Sort/SortService.cs @@ -26,7 +26,7 @@ internal class SortService : IDisposable private readonly int _containerSize; private readonly Done _done = new Done { Running = true }; - private readonly int _order; + private readonly int[] _orders; private readonly EnginePragmas _pragmas; private readonly BufferSlice _buffer; private readonly Lazy _reader; @@ -43,10 +43,13 @@ internal class SortService : IDisposable /// public IReadOnlyCollection Containers => _containers; - public SortService(SortDisk disk, int order, EnginePragmas pragmas) + public SortService(SortDisk disk, IReadOnlyList orders, EnginePragmas pragmas) { _disk = disk; - _order = order; + if (orders == null) throw new ArgumentNullException(nameof(orders)); + if (orders.Count == 0) throw new ArgumentException("Orders must contain at least one segment", nameof(orders)); + + _orders = orders as int[] ?? orders.ToArray(); _pragmas = pragmas; _containerSize = disk.ContainerSize; @@ -86,10 +89,12 @@ public void Insert(IEnumerable> items) // slit all items in sorted containers foreach (var containerItems in this.SliptValues(items, _done)) { - var container = new SortContainer(_pragmas.Collation, _containerSize); + var container = new SortContainer(_pragmas.Collation, _containerSize, _orders); // insert segmented items inside a container - reuse same buffer slice - container.Insert(containerItems, _order, _buffer); + var order = _orders.Length == 1 ? _orders[0] : Query.Ascending; + + container.Insert(containerItems, order, _buffer); _containers.Add(container); @@ -133,7 +138,7 @@ public IEnumerable> Sort() } else { - var diffOrder = _order * -1; + var diffOrder = _orders.Length == 1 ? _orders[0] * -1 : -1; // merge sort with all containers while (_containers.Any(x => !x.IsEOF)) diff --git a/LiteDB/Engine/Structures/PageBuffer.cs b/LiteDB/Engine/Structures/PageBuffer.cs index d4b95c18f..7892c24d6 100644 --- a/LiteDB/Engine/Structures/PageBuffer.cs +++ b/LiteDB/Engine/Structures/PageBuffer.cs @@ -62,7 +62,7 @@ public void Release() Interlocked.Decrement(ref this.ShareCounter); } -#if DEBUG +#if DEBUG || TESTING ~PageBuffer() { ENSURE(this.ShareCounter == 0, $"share count must be 0 in destroy PageBuffer (current: {this.ShareCounter})"); diff --git a/LiteDB/Engine/Structures/VectorIndexMetadata.cs b/LiteDB/Engine/Structures/VectorIndexMetadata.cs new file mode 100644 index 000000000..9f62d0366 --- /dev/null +++ b/LiteDB/Engine/Structures/VectorIndexMetadata.cs @@ -0,0 +1,79 @@ +using LiteDB.Vector; +using System; + +namespace LiteDB.Engine +{ + /// + /// Metadata persisted for a vector-aware index. + /// + internal sealed class VectorIndexMetadata + { + /// + /// Slot index [0-255] reserved in the collection page. + /// + public byte Slot { get; } + + /// + /// Number of components expected in vector payloads. + /// + public ushort Dimensions { get; } + + /// + /// Distance metric applied during nearest-neighbour evaluation. + /// + public VectorDistanceMetric Metric { get; } + + /// + /// Head pointer to the persisted vector index structure. + /// + public PageAddress Root { get; set; } + + /// + /// Additional metadata for engine specific bookkeeping. + /// + public uint Reserved { get; set; } + + public VectorIndexMetadata(byte slot, ushort dimensions, VectorDistanceMetric metric) + { + if (dimensions == 0) + { + throw new ArgumentOutOfRangeException(nameof(dimensions), dimensions, "Dimensions must be greater than zero"); + } + + this.Slot = slot; + this.Dimensions = dimensions; + this.Metric = metric; + this.Root = PageAddress.Empty; + this.Reserved = uint.MaxValue; + } + + public VectorIndexMetadata(BufferReader reader) + { + this.Slot = reader.ReadByte(); + this.Dimensions = reader.ReadUInt16(); + this.Metric = (VectorDistanceMetric)reader.ReadByte(); + this.Root = reader.ReadPageAddress(); + this.Reserved = reader.ReadUInt32(); + } + + public void UpdateBuffer(BufferWriter writer) + { + writer.Write(this.Slot); + writer.Write(this.Dimensions); + writer.Write((byte)this.Metric); + writer.Write(this.Root); + writer.Write(this.Reserved); + } + + public static int GetLength() + { + return + 1 + // Slot + 2 + // Dimensions + 1 + // Metric + PageAddress.SIZE + // Root + 4; // Reserved + } + } + +} diff --git a/LiteDB/Engine/Structures/VectorIndexNode.cs b/LiteDB/Engine/Structures/VectorIndexNode.cs new file mode 100644 index 000000000..f0b090f30 --- /dev/null +++ b/LiteDB/Engine/Structures/VectorIndexNode.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using static LiteDB.Constants; + +namespace LiteDB.Engine +{ + internal sealed class VectorIndexNode + { + private const int P_DATA_BLOCK = 0; + private const int P_LEVEL_COUNT = P_DATA_BLOCK + PageAddress.SIZE; + private const int P_LEVELS = P_LEVEL_COUNT + 1; + + public const int MaxLevels = 4; + public const int MaxNeighborsPerLevel = 8; + + private const int LEVEL_STRIDE = 1 + (MaxNeighborsPerLevel * PageAddress.SIZE); + private const int P_VECTOR = P_LEVELS + (MaxLevels * LEVEL_STRIDE); + private const int P_VECTOR_POINTER = P_VECTOR + 2; + + private readonly VectorIndexPage _page; + private readonly BufferSlice _segment; + + public PageAddress Position { get; } + + public PageAddress DataBlock { get; private set; } + + public VectorIndexPage Page => _page; + + public byte LevelCount { get; private set; } + + public int Dimensions { get; } + + public bool HasInlineVector { get; } + + public PageAddress ExternalVector { get; } + + public VectorIndexNode(VectorIndexPage page, byte index, BufferSlice segment) + { + _page = page; + _segment = segment; + + this.Position = new PageAddress(page.PageID, index); + this.DataBlock = segment.ReadPageAddress(P_DATA_BLOCK); + this.LevelCount = segment.ReadByte(P_LEVEL_COUNT); + + var length = segment.ReadUInt16(P_VECTOR); + + if (length == 0) + { + this.HasInlineVector = false; + this.ExternalVector = segment.ReadPageAddress(P_VECTOR_POINTER); + this.Dimensions = 0; + } + else + { + this.HasInlineVector = true; + this.ExternalVector = PageAddress.Empty; + this.Dimensions = length; + } + } + + public VectorIndexNode(VectorIndexPage page, byte index, BufferSlice segment, PageAddress dataBlock, float[] vector, byte levelCount, PageAddress externalVector) + { + if (vector == null) throw new ArgumentNullException(nameof(vector)); + if (levelCount == 0 || levelCount > MaxLevels) throw new ArgumentOutOfRangeException(nameof(levelCount)); + + _page = page; + _segment = segment; + + this.Position = new PageAddress(page.PageID, index); + this.DataBlock = dataBlock; + this.LevelCount = levelCount; + this.Dimensions = vector.Length; + this.HasInlineVector = externalVector.IsEmpty; + this.ExternalVector = externalVector; + + segment.Write(dataBlock, P_DATA_BLOCK); + segment.Write(levelCount, P_LEVEL_COUNT); + + for (var level = 0; level < MaxLevels; level++) + { + var offset = GetLevelOffset(level); + segment.Write((byte)0, offset); + + var position = offset + 1; + + for (var i = 0; i < MaxNeighborsPerLevel; i++) + { + segment.Write(PageAddress.Empty, position); + position += PageAddress.SIZE; + } + } + + if (this.HasInlineVector) + { + segment.Write(vector, P_VECTOR); + } + else + { + if (externalVector.IsEmpty) + { + throw new ArgumentException("External vector address must be provided when vector is stored out of page.", nameof(externalVector)); + } + + segment.Write((ushort)0, P_VECTOR); + segment.Write(externalVector, P_VECTOR_POINTER); + } + + page.IsDirty = true; + } + + public static int GetLength(int dimensions, out bool storesInline) + { + var inlineLength = + PageAddress.SIZE + // DataBlock + 1 + // Level count + (MaxLevels * LEVEL_STRIDE) + + 2 + // vector length prefix + (dimensions * sizeof(float)); + + var maxNodeLength = PAGE_SIZE - PAGE_HEADER_SIZE - BasePage.SLOT_SIZE; + + if (inlineLength <= maxNodeLength) + { + storesInline = true; + return inlineLength; + } + + storesInline = false; + + return + PageAddress.SIZE + // DataBlock + 1 + // Level count + (MaxLevels * LEVEL_STRIDE) + + 2 + // sentinel prefix + PageAddress.SIZE; // pointer to external vector + } + + public IReadOnlyList GetNeighbors(int level) + { + if (level < 0 || level >= MaxLevels) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + var offset = GetLevelOffset(level); + var count = _segment.ReadByte(offset); + var neighbors = new List(count); + var position = offset + 1; + + for (var i = 0; i < count; i++) + { + neighbors.Add(_segment.ReadPageAddress(position)); + position += PageAddress.SIZE; + } + + return neighbors; + } + + public void SetNeighbors(int level, IReadOnlyList neighbors) + { + if (level < 0 || level >= MaxLevels) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + if (neighbors == null) + { + throw new ArgumentNullException(nameof(neighbors)); + } + + var offset = GetLevelOffset(level); + var count = Math.Min(neighbors.Count, MaxNeighborsPerLevel); + + _segment.Write((byte)count, offset); + + var position = offset + 1; + + var i = 0; + + for (; i < count; i++) + { + _segment.Write(neighbors[i], position); + position += PageAddress.SIZE; + } + + for (; i < MaxNeighborsPerLevel; i++) + { + _segment.Write(PageAddress.Empty, position); + position += PageAddress.SIZE; + } + + _page.IsDirty = true; + } + + public bool TryAddNeighbor(int level, PageAddress address) + { + if (level < 0 || level >= MaxLevels) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + var current = this.GetNeighbors(level); + + if (current.Contains(address)) + { + return false; + } + + if (current.Count >= MaxNeighborsPerLevel) + { + return false; + } + + var expanded = new List(current.Count + 1); + expanded.AddRange(current); + expanded.Add(address); + + this.SetNeighbors(level, expanded); + + return true; + } + + public bool RemoveNeighbor(int level, PageAddress address) + { + if (level < 0 || level >= MaxLevels) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + var current = this.GetNeighbors(level); + + if (!current.Contains(address)) + { + return false; + } + + var reduced = current + .Where(x => x != address) + .ToList(); + + this.SetNeighbors(level, reduced); + + return true; + } + + public void SetLevelCount(byte levelCount) + { + if (levelCount == 0 || levelCount > MaxLevels) + { + throw new ArgumentOutOfRangeException(nameof(levelCount)); + } + + this.LevelCount = levelCount; + _segment.Write(levelCount, P_LEVEL_COUNT); + _page.IsDirty = true; + } + + private static int GetLevelOffset(int level) + { + return P_LEVELS + (level * LEVEL_STRIDE); + } + + public void UpdateVector(float[] vector) + { + if (vector == null) throw new ArgumentNullException(nameof(vector)); + + if (this.HasInlineVector == false) + { + throw new InvalidOperationException("Inline vector update is not supported for externally stored vectors."); + } + + if (vector.Length != this.Dimensions) + { + throw new ArgumentException("Vector length must match node dimensions.", nameof(vector)); + } + + _segment.Write(vector, P_VECTOR); + _page.IsDirty = true; + } + + public float[] ReadVector() + { + if (!this.HasInlineVector) + { + throw new InvalidOperationException("Vector is stored externally and must be loaded from the data pages."); + } + + return _segment.ReadVector(P_VECTOR); + } + } +} diff --git a/LiteDB/LiteDB.csproj b/LiteDB/LiteDB.csproj index 67eeb45f1..467816e98 100644 --- a/LiteDB/LiteDB.csproj +++ b/LiteDB/LiteDB.csproj @@ -1,76 +1,76 @@ - + - - net4.5;netstandard1.3;netstandard2.0 - 5.0.21 - 5.0.21 - 5.0.21 - 5.0.21 - Maurício David - LiteDB - LiteDB - A lightweight embedded .NET NoSQL document store in a single datafile - MIT - en-US - LiteDB - LiteDB - database nosql embedded - icon_64x64.png - MIT - https://www.litedb.org - https://github.com/mbdavid/LiteDB - git - LiteDB - LiteDB - true - 1.6.1 - 1701;1702;1705;1591;0618 - bin\$(Configuration)\$(TargetFramework)\LiteDB.xml - true - LiteDB.snk - true - latest - - - + + netstandard2.0;net8.0 + Maurício David + LiteDB + LiteDB - A lightweight embedded .NET NoSQL document store in a single datafile + MIT + en-US + LiteDB + LiteDB + database nosql embedded + icon_64x64.png + MIT + https://www.litedb.org + https://github.com/litedb-org/LiteDB + git + LiteDB + LiteDB + true + 1701;1702;1705;1591;0618 + bin\$(Configuration)\$(TargetFramework)\LiteDB.xml + true + latest + - - TRACE;DEBUG - + - - HAVE_SHA1_MANAGED;HAVE_APP_DOMAIN;HAVE_PROCESS;HAVE_ENVIRONMENT - + + $(DefineConstants);TRACE;DEBUG + - - HAVE_SHA1_MANAGED - + + $(DefineConstants);HAVE_SHA1_MANAGED + - - - - - + + $(DefineConstants);HAVE_SHA1_MANAGED;HAVE_APP_DOMAIN;HAVE_PROCESS;HAVE_ENVIRONMENT + - - - - + + $(DefineConstants);TESTING + + - - - - + + + + - - - - - + + + + + + + + + + + + + + + + + + diff --git a/LiteDB/LiteDB.snk b/LiteDB/LiteDB.snk deleted file mode 100644 index baaa34055..000000000 Binary files a/LiteDB/LiteDB.snk and /dev/null differ diff --git a/LiteDB/Utils/Constants.cs b/LiteDB/Utils/Constants.cs index 93458193d..8f0d0eeb4 100644 --- a/LiteDB/Utils/Constants.cs +++ b/LiteDB/Utils/Constants.cs @@ -5,8 +5,8 @@ using System.Runtime.CompilerServices; using System.Threading; -[assembly: InternalsVisibleTo("LiteDB.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010029e66990e22110ce40a7197e37f8f82df3332c399e696df7f27d09e14ee590ac2dda735d4777fe554c427540bde93b14d3d26c04731c963383dcaa18859c8cbcd4a1a9c394d1204f474c2ab6f23a2eaadf81eb8a7a3d3cc73658868b0302163b92a2614ca050ab703be33c3e1d76f55b11f4f87cb73558f3aa69c1ce726d9ee8")] -#if DEBUG +[assembly: InternalsVisibleTo("LiteDB.Tests")] +#if DEBUG || TESTING [assembly: InternalsVisibleTo("ConsoleApp1")] #endif @@ -102,7 +102,7 @@ internal class Constants /// /// Initial seed for Random /// -#if DEBUG +#if DEBUG || TESTING public const int RANDOMIZER_SEED = 3131; #else public const int RANDOMIZER_SEED = 0; diff --git a/LiteDB/Utils/Extensions/BufferSliceExtensions.cs b/LiteDB/Utils/Extensions/BufferSliceExtensions.cs index 3e7422d01..049ce931f 100644 --- a/LiteDB/Utils/Extensions/BufferSliceExtensions.cs +++ b/LiteDB/Utils/Extensions/BufferSliceExtensions.cs @@ -40,6 +40,11 @@ public static UInt32 ReadUInt32(this BufferSlice buffer, int offset) return BitConverter.ToUInt32(buffer.Array, buffer.Offset + offset); } + public static float ReadSingle(this BufferSlice buffer, int offset) + { + return BitConverter.ToSingle(buffer.Array, buffer.Offset + offset); + } + public static Int64 ReadInt64(this BufferSlice buffer, int offset) { return BitConverter.ToInt64(buffer.Array, buffer.Offset + offset); @@ -88,6 +93,18 @@ public static DateTime ReadDateTime(this BufferSlice buffer, int offset) return new DateTime(ticks, DateTimeKind.Utc); } + public static float[] ReadVector(this BufferSlice buffer, int offset) + { + var count = buffer.ReadUInt16(offset); + offset += 2; // move offset to first float + var vector = new float[count]; + for (var i = 0; i < count; i++) + { + vector[i] = BitConverter.ToSingle(buffer.Array, buffer.Offset + offset + (i * 4)); + } + return vector; + } + public static PageAddress ReadPageAddress(this BufferSlice buffer, int offset) { return new PageAddress(buffer.ReadUInt32(offset), buffer[offset + 4]); @@ -162,6 +179,7 @@ public static BsonValue ReadIndexKey(this BufferSlice buffer, int offset) case BsonType.MinValue: return BsonValue.MinValue; case BsonType.MaxValue: return BsonValue.MaxValue; + case BsonType.Vector: return buffer.ReadVector(offset); default: throw new NotImplementedException(); } @@ -201,6 +219,11 @@ public static void Write(this BufferSlice buffer, UInt32 value, int offset) value.ToBytes(buffer.Array, buffer.Offset + offset); } + public static void Write(this BufferSlice buffer, float value, int offset) + { + BitConverter.GetBytes(value).CopyTo(buffer.Array, buffer.Offset + offset); + } + public static void Write(this BufferSlice buffer, Int64 value, int offset) { value.ToBytes(buffer.Array, buffer.Offset + offset); @@ -236,6 +259,17 @@ public static void Write(this BufferSlice buffer, Guid value, int offset) buffer.Write(value.ToByteArray(), offset); } + public static void Write(this BufferSlice buffer, float[] value, int offset) + { + buffer.Write((ushort)value.Length, offset); + offset += 2; + foreach (var v in value) + { + BitConverter.GetBytes(v).CopyTo(buffer.Array, buffer.Offset + offset); + offset += 4; + } + } + public static void Write(this BufferSlice buffer, ObjectId value, int offset) { value.ToByteArray(buffer.Array, buffer.Offset + offset); @@ -316,6 +350,7 @@ public static void WriteIndexKey(this BufferSlice buffer, BsonValue value, int o case BsonType.Boolean: buffer[offset] = (value.AsBoolean) ? (byte)1 : (byte)0; break; case BsonType.DateTime: buffer.Write(value.AsDateTime, offset); break; + case BsonType.Vector: buffer.Write(value.AsVector, offset); break; default: throw new NotImplementedException(); } diff --git a/LiteDB/Utils/Extensions/StringExtensions.cs b/LiteDB/Utils/Extensions/StringExtensions.cs index 00ed9b54c..c34afebf8 100644 --- a/LiteDB/Utils/Extensions/StringExtensions.cs +++ b/LiteDB/Utils/Extensions/StringExtensions.cs @@ -37,24 +37,6 @@ public static bool IsWord(this string str) return true; } - public static string Sha1(this string value) - { - var data = Encoding.UTF8.GetBytes(value); - - using (var sha = SHA1.Create()) - { - var hashData = sha.ComputeHash(data); - var hash = new StringBuilder(); - - foreach (var b in hashData) - { - hash.Append(b.ToString("X2")); - } - - return hash.ToString(); - } - } - /// /// Implement SqlLike in C# string - based on /// https://stackoverflow.com/a/8583383/3286260 diff --git a/LiteDB/Utils/Tokenizer.cs b/LiteDB/Utils/Tokenizer.cs index 4cbdf169d..99d2bde68 100644 --- a/LiteDB/Utils/Tokenizer.cs +++ b/LiteDB/Utils/Tokenizer.cs @@ -98,7 +98,8 @@ internal class Token "LIKE", "IN", "AND", - "OR" + "OR", + "VECTOR_SIM" }; public Token(TokenType tokenType, string value, long position) diff --git a/README.md b/README.md index 2859e3b1d..f3f780e82 100644 --- a/README.md +++ b/README.md @@ -168,8 +168,4 @@ LiteDB is digitally signed courtesy of [SignPath](https://www.signpath.io) - - -## License - -[MIT](http://opensource.org/licenses/MIT) + \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index ef38c21d7..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 5.0.{build} -branches: - only: - - master -image: Visual Studio 2022 -configuration: - - Debug - - Release -before_build: - - cmd: nuget restore LiteDB.sln -build: - project: LiteDB.sln - publish_nuget: true - verbosity: minimal -for: -- - matrix: - only: - - configuration: Release - artifacts: - - path: LiteDB\bin\Release\LiteDB*.nupkg - deploy: - - provider: Webhook - url: https://app.signpath.io/API/v1/f5b329b8-705f-4d6c-928a-19465b83716b/Integrations/AppVeyor?ProjectKey=LiteDB.git&SigningPolicyKey=release-signing - authorization: - secure: 3eLjGkpQC1wg1s5GIEqs7yk/V8OZNnpKmpwdsaloGExc5jMspM4nA7u/UlG5ugraEyXRC05ZxLU4FIfH2V2BEg== diff --git a/docs/reprorunner.md b/docs/reprorunner.md new file mode 100644 index 000000000..bd3725804 --- /dev/null +++ b/docs/reprorunner.md @@ -0,0 +1,140 @@ +# ReproRunner CI and JSON Contract + +The ReproRunner CLI discovers, validates, and executes LiteDB reproduction projects. This document +captures the machine-readable schema emitted by `list --json`, the OS constraint syntax consumed by +CI, and the knobs available to run repros locally or from GitHub Actions. + +## JSON inventory contract + +Running `reprorunner list --json` (or `dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- list --json`) +produces a stable payload with one entry per repro: + +```json +{ + "repros": [ + { + "name": "AnyRepro", + "supports": ["any"] + }, + { + "name": "WindowsOnly", + "supports": ["windows"] + }, + { + "name": "PinnedUbuntu", + "os": { + "includeLabels": ["ubuntu-22.04"] + } + } + ] +} +``` + +The top level includes: + +- `repros` – array of repro descriptors. +- Each repro has a unique `name` (matching the manifest id). +- `supports` – optional list describing the broad platform family. Accepted values are `windows`, + `linux`, and `any`. Omitted or empty means `any`. +- `os` – optional advanced constraints that refine the supported runner labels. + +### Advanced OS constraints + +The `os` object supports four optional arrays. Each entry is compared in a case-insensitive manner +against the repository's OS matrix. + +```json +"os": { + "includePlatforms": ["linux"], + "includeLabels": ["ubuntu-22.04"], + "excludePlatforms": ["windows"], + "excludeLabels": ["ubuntu-24.04"] +} +``` + +Resolution rules: + +1. Start with the labels implied by `supports` (`any` => all labels). +2. Intersect with `includePlatforms` (if present) and `includeLabels` (if present). +3. Remove any labels present in `excludePlatforms` and `excludeLabels`. +4. The final set is intersected with the repo-level label inventory. If the result is empty, the repro + is skipped and the CI generator prints a warning. + +Unknown platforms or labels are ignored for the purposes of scheduling but are reported in the matrix +summary so the manifest can be corrected. + +## Centralised OS label inventory + +Supported GitHub runner labels live in `.github/os-matrix.json` and are shared across workflows: + +```json +{ + "linux": ["ubuntu-22.04", "ubuntu-24.04"], + "windows": ["windows-2022"] +} +``` + +When a new runner label is added to the repository, update this file and every workflow (including +ReproRunner) picks up the change automatically. + +## New GitHub Actions workflow + +`.github/workflows/reprorunner.yml` drives ReproRunner executions on CI. It offers two entry points: + +- Manual triggers via `workflow_dispatch`. +- Automatic execution via `workflow_call` from the main `ci.yml` workflow. +- Optional inputs: + - `filter` - regular expression to narrow the repro list. + - `ref` - commit, branch, or tag to check out. + +### Job layout + +1. **generate-matrix** + - Checks out the requested ref. + - Restores/builds the CLI and captures the JSON inventory: `reprorunner list --json [--filter ]`. + - Loads `.github/os-matrix.json`, applies each repro's constraints, and emits a matrix of `{ os, repro }` pairs. + - Writes a summary of scheduled/skipped repros (with reasons) to `$GITHUB_STEP_SUMMARY`. + - Uploads `repros.json` for debugging. + +2. **repro** + - Runs once per matrix entry using `runs-on: ${{ matrix.os }}`. + - Builds the CLI in Release mode. + - Executes `reprorunner run --ci --target-os ""`. + - Uploads `logs--` artifacts (`artifacts/` plus the CLI `runs/` folder when present). + - Appends a per-job summary snippet (status + artifact hint). + +The `repro` job is skipped automatically when no repro qualifies after constraint evaluation. + +## Running repros locally + +Most local workflows mirror CI: + +- List repros (optionally filtered): + + ```bash + dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- list --json --filter Fast + ``` + +- Execute a repro under CI settings (for example, on Windows): + + ```bash + dotnet run --project LiteDB.ReproRunner/LiteDB.ReproRunner.Cli -- \ + run Issue_2561_TransactionMonitor --ci --target-os windows-2022 + ``` + +- View generated artifacts under `LiteDB.ReproRunner/LiteDB.ReproRunner.Cli/bin///runs/...` + or in the CI job artifacts prefixed with `logs-`. + +When crafting new repro manifests, prefer `supports` for broad platform gating and the `os` block for +precise runner pinning. + +## Troubleshooting matrix expansion + +- **Repro skipped unexpectedly** – run `reprorunner show ` to confirm the declared OS metadata. + Verify the values match the keys in `.github/os-matrix.json`. +- **Unknown platform/label warnings** – the manifest references a runner that is not present in the OS + matrix. Update the manifest or add the missing label to `.github/os-matrix.json`. +- **Empty workflow after filtering** – double-check the `filter` regex and ensure the CLI discovers at + least one repro whose name matches the expression. + + diff --git a/docs/versioning.md b/docs/versioning.md new file mode 100644 index 000000000..661c1b600 --- /dev/null +++ b/docs/versioning.md @@ -0,0 +1,57 @@ +# Versioning + +LiteDB uses GitVersion for semantic versioning across local builds and CI. The configuration lives in `GitVersion.yml` and is consumed by both MSBuild (via `GitVersion.MsBuild`) and the GitHub workflows. + +> [!NOTE] +> Environments that expose the repository as a detached worktree or otherwise hide the `.git/HEAD` sentinel (for example, some test harnesses) automatically fall back to a static `0.0.0-detached` version so builds can proceed without GitVersion. + +## Branch semantics + +- `master` is the mainline branch. Each direct commit or merge increments the patch number unless an annotated `v*` tag (or `+semver:` directive) requests a larger bump. +- `dev` tracks the next patch version and produces prerelease builds like `6.0.1-prerelease.0003`. The numeric suffix is zero-padded for predictable ordering. +- Feature branches (`feature/*`, `bugfix/*`, `chore/*`, `refactor/*`, `pr/*`) inherit their base version but do not publish artifacts. They exist purely for validation. + +The first prerelease that precedes the 6.0.0 release (commit `a0298891ddcaf7ba48c679f1052a6f442f6c094f`) remains the baseline for the prerelease numbering history. + +## GitHub workflows + +- `publish-prerelease.yml` runs on every push to `dev`. It resolves the semantic version with GitVersion, runs the full test suite, packs the library, and pushes the resulting prerelease package to NuGet. GitHub releases are intentionally skipped for now. +- `publish-release.yml` is manual (`workflow_dispatch`). It computes the release version and can optionally push to NuGet and/or create a GitHub release via boolean inputs. GitHub releases use a zero-padded prerelease counter for predictable sorting in the UI, while NuGet publishing keeps the standard GitVersion output. By default it performs a dry run (build + pack only) so we keep the publishing path disabled until explicitly requested. +- `tag-version.yml` lets you start a manual major/minor/patch bump. It tags the specified ref (defaults to `master`) with the next `v*` version so future builds pick up the new baseline. Use this after validating a release candidate. + +## Dry-running versions + +GitVersion is registered as a local dotnet tool. Restore the tool once (`dotnet tool restore`) and use one of the helpers: + +```powershell +# PowerShell (Windows, macOS, Linux) +./scripts/gitver/gitversion.ps1 # show version for HEAD +./scripts/gitver/gitversion.ps1 dev~3 # inspect an arbitrary commit +./scripts/gitver/gitversion.ps1 -Json # emit raw JSON +``` + +```bash +# Bash (macOS, Linux, Git Bash on Windows) +./scripts/gitver/gitversion.sh # show version for HEAD +./scripts/gitver/gitversion.sh dev~3 # inspect an arbitrary commit +./scripts/gitver/gitversion.sh --json # emit raw JSON +``` + +Both scripts resolve the git ref to a SHA, execute GitVersion with the repository configuration, and echo the key fields (FullSemVer, NuGetVersion, InformationalVersion, BranchName). + +## Manual bumps + +1. Merge the desired changes into `master`. +2. Run the **Tag version** workflow from the Actions tab, pick `master`, and choose `patch`, `minor`, or `major`. +3. The workflow creates and pushes the annotated `v*` tag. The next prerelease build from `dev` will increment accordingly, and the next stable run from `master` will match the tagged version. + +`+semver:` commit messages are still honoured. For example, including `+semver: minor` in a commit on `master` advances the minor version even without a tag. + +## Working locally + +- `dotnet build` / `dotnet pack` automatically consume the GitVersion-generated values; no extra parameters are required. +- To bypass GitVersion temporarily (e.g., for experiments), set `GitVersion_NoFetch=false` in the build command. Reverting the property restores normal behaviour. +- When you are ready to publish a prerelease, push to `dev` and let the workflow take care of packing and nuget push. + +For historical reference, the `v6.0.0-prerelease.0001` tag remains anchored to commit `a0298891ddcaf7ba48c679f1052a6f442f6c094f`, ensuring version ordering continues correctly from the original timeline. + diff --git a/scripts/gitver/create-github-release.ps1 b/scripts/gitver/create-github-release.ps1 new file mode 100644 index 000000000..df18664c7 --- /dev/null +++ b/scripts/gitver/create-github-release.ps1 @@ -0,0 +1,82 @@ +[CmdletBinding()] +param( + [Parameter(Position = 0)] + [string]$Ref = 'HEAD', + [switch]$Push +) + +$ErrorActionPreference = 'Stop' + +$repoRoot = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..' '..') + +# Restore tools if needed +if (-not (Get-Command dotnet-gitversion -ErrorAction SilentlyContinue)) { + Write-Host "Restoring .NET tools..." + dotnet tool restore +} + +# Get version info +$jsonText = & "$PSScriptRoot/gitversion.ps1" -Ref $Ref -Json +$data = $jsonText | ConvertFrom-Json + +$majorMinorPatch = $data.MajorMinorPatch +$preLabel = $data.PreReleaseLabel +$preNumber = [int]$data.PreReleaseNumber +$sha = $data.Sha + +if ([string]::IsNullOrEmpty($preLabel) -or $preLabel -eq 'null') { + throw "Commit $sha is not a prerelease version. Use this script only for prerelease tags." +} + +# Format with zero-padded prerelease number for proper GitHub sorting +$paddedVersion = '{0}-{1}.{2:0000}' -f $majorMinorPatch, $preLabel, $preNumber +$tagName = "v$paddedVersion" + +Write-Host "" +Write-Host "Creating prerelease tag:" -ForegroundColor Cyan +Write-Host " Commit: $sha" +Write-Host " Tag: $tagName" +Write-Host " Version: $paddedVersion" +Write-Host "" + +# Check if tag already exists +$existingTag = git -C $repoRoot rev-parse --verify --quiet "refs/tags/$tagName" 2>$null +if ($existingTag) { + # we dont abort because we want to allow to separate creation and pushing of tags + Write-Host "Tag $tagName already exists." -ForegroundColor Yellow +} else{ + # Create annotated tag + $message = "Prerelease $paddedVersion" + git -C $repoRoot tag -a $tagName -m $message $sha + + if ($LASTEXITCODE -ne 0) { + throw "Failed to create tag" + } + + Write-Host "Tag created successfully!" -ForegroundColor Green + +} + + +if ($Push) { + Write-Host "" + Write-Host "Pushing tag to https://github.com/litedb-org/LiteDB..." -ForegroundColor Cyan + git -C $repoRoot push https://github.com/litedb-org/LiteDB $tagName + + if ($LASTEXITCODE -ne 0) { + throw "Failed to push tag" + } + + Write-Host "Tag pushed successfully!" -ForegroundColor Green + Write-Host "" + Write-Host "You can now create a GitHub release at:" -ForegroundColor Yellow + Write-Host " https://github.com/litedb-org/LiteDB/releases/new?tag=$tagName" +} +else { + Write-Host "" + Write-Host "Tag created locally. To push it, run:" -ForegroundColor Yellow + Write-Host " git push https://github.com/litedb-org/LiteDB $tagName" + Write-Host "" + Write-Host "Or re-run this script with -Push:" + Write-Host " ./scripts/gitver/tag-prerelease.ps1 -Push" +} \ No newline at end of file diff --git a/scripts/gitver/gitversion.ps1 b/scripts/gitver/gitversion.ps1 new file mode 100644 index 000000000..c28317b6d --- /dev/null +++ b/scripts/gitver/gitversion.ps1 @@ -0,0 +1,101 @@ +[CmdletBinding()] +param( + [Parameter(Position = 0)] + [string]$Ref = 'HEAD', + [string]$Branch, + [switch]$Json, + [switch]$NoRestore +) + +$ErrorActionPreference = 'Stop' + +$repoRoot = Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..' '..') +$manifestPath = Join-Path $repoRoot '.config/dotnet-tools.json' +$gitVersionConfig = Join-Path $repoRoot 'GitVersion.yml' + +if (-not (Test-Path $manifestPath)) { + throw "Tool manifest not found at $manifestPath" +} + +if (-not (Test-Path $gitVersionConfig)) { + throw "GitVersion configuration not found at $gitVersionConfig" +} + +$sha = ((& git -C $repoRoot rev-parse --verify --quiet "$Ref").Trim()) +if (-not $sha) { + throw "Unable to resolve git ref '$Ref'" +} + +if (-not $Branch) { + $candidates = (& git -C $repoRoot branch --contains $sha) | ForEach-Object { + ($_ -replace '^[*+\s]+', '').Trim() + } | Where-Object { $_ } + + foreach ($candidate in @('dev', 'develop', 'master', 'main')) { + if ($candidates -contains $candidate) { + $Branch = $candidate + break + } + } + + if (-not $Branch -and $candidates) { + $Branch = $candidates[0] + } +} + +if (-not $Branch) { + $Branch = "gv-temp-" + $sha.Substring(0, 7) +} + +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("litedb-gv-" + [Guid]::NewGuid().ToString('N')) + +try { + git clone --quiet --local --no-hardlinks "$repoRoot" "$tempRoot" + + Push-Location -LiteralPath $tempRoot + + git checkout -B "$Branch" "$sha" | Out-Null + + if (-not (Test-Path '.config')) { + New-Item -ItemType Directory -Path '.config' | Out-Null + } + + Copy-Item -Path $manifestPath -Destination '.config/dotnet-tools.json' -Force + Copy-Item -Path $gitVersionConfig -Destination 'GitVersion.yml' -Force + + if (-not $NoRestore) { + dotnet tool restore | Out-Null + } + + $jsonText = dotnet tool run dotnet-gitversion /output json + + if ($LASTEXITCODE -ne 0) { + throw 'dotnet-gitversion failed' + } + + if ($Json) { + $jsonText + return + } + + $data = $jsonText | ConvertFrom-Json + + $semVer = $data.MajorMinorPatch + if ([string]::IsNullOrEmpty($data.PreReleaseLabel) -eq $false) { + $preNumber = [int]$data.PreReleaseNumber + $semVer = '{0}-{1}.{2:0000}' -f $data.MajorMinorPatch, $data.PreReleaseLabel, $preNumber + } + + $line = '{0,-22} {1}' + Write-Host ($line -f 'Resolved SHA:', $sha) + Write-Host ($line -f 'FullSemVer:', $semVer) + Write-Host ($line -f 'NuGetVersion:', $semVer) + Write-Host ($line -f 'Informational:', "$semVer+$($data.ShortSha)") + Write-Host ($line -f 'BranchName:', $data.BranchName) +} +finally { + Pop-Location -ErrorAction SilentlyContinue + if (Test-Path $tempRoot) { + Remove-Item -Recurse -Force $tempRoot + } +} diff --git a/scripts/gitver/gitversion.sh b/scripts/gitver/gitversion.sh new file mode 100644 index 000000000..4b434a73e --- /dev/null +++ b/scripts/gitver/gitversion.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +REF="HEAD" +JSON_OUTPUT=false +NO_RESTORE=false +WORKTREE="" +TARGET="$ROOT" + +while [[ $# -gt 0 ]]; do + case "$1" in + --json) + JSON_OUTPUT=true + shift + ;; + --no-restore) + NO_RESTORE=true + shift + ;; + *) + REF="$1" + shift + ;; + esac +done + +cleanup() { + if [[ -n "$WORKTREE" && -d "$WORKTREE" ]]; then + git -C "$ROOT" worktree remove --force "$WORKTREE" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +if [[ "$REF" != "HEAD" ]]; then + SHA=$(git -C "$ROOT" rev-parse --verify --quiet "$REF") + if [[ -z "$SHA" ]]; then + echo "Unable to resolve git ref '$REF'" >&2 + exit 1 + fi + WORKTREE="$(mktemp -d -t litedb-gv-XXXXXX)" + git -C "$ROOT" worktree add --detach "$WORKTREE" "$SHA" >/dev/null + TARGET="$WORKTREE" +else + SHA=$(git -C "$ROOT" rev-parse HEAD) +fi + +cd "$TARGET" + +if [[ "$NO_RESTORE" != "true" ]]; then + dotnet tool restore >/dev/null +fi + +JSON=$(dotnet tool run dotnet-gitversion /output json) + +if [[ "$JSON_OUTPUT" == "true" ]]; then + printf '%s\n' "$JSON" + exit 0 +fi + +MAJOR_MINOR_PATCH=$(jq -r '.MajorMinorPatch' <<<"$JSON") +PRE_LABEL=$(jq -r '.PreReleaseLabel' <<<"$JSON") +PRE_NUMBER=$(jq -r '.PreReleaseNumber' <<<"$JSON") +SHORT_SHA=$(jq -r '.ShortSha' <<<"$JSON") +BRANCH=$(jq -r '.BranchName' <<<"$JSON") + +SEMVER="$MAJOR_MINOR_PATCH" +if [[ "$PRE_LABEL" != "" && "$PRE_LABEL" != "null" ]]; then + printf -v PRE_PADDED '%04d' "$PRE_NUMBER" + SEMVER="${MAJOR_MINOR_PATCH}-${PRE_LABEL}.${PRE_PADDED}" +fi + +printf '%-22s %s\n' "Resolved SHA:" "$SHA" +printf '%-22s %s\n' "FullSemVer:" "$SEMVER" +printf '%-22s %s\n' "NuGetVersion:" "$SEMVER" +printf '%-22s %s\n' "Informational:" "${SEMVER}+${SHORT_SHA}" +printf '%-22s %s\n' "BranchName:" "$BRANCH" \ No newline at end of file diff --git a/tests.ci.runsettings b/tests.ci.runsettings new file mode 100644 index 000000000..b730669db --- /dev/null +++ b/tests.ci.runsettings @@ -0,0 +1,18 @@ + + + + + + 180000 + 60000 + + + 1 + true + + + + + false + + \ No newline at end of file diff --git a/tests.runsettings b/tests.runsettings new file mode 100644 index 000000000..c97856d4c --- /dev/null +++ b/tests.runsettings @@ -0,0 +1,7 @@ + + + + 300000 + 30000 + +