src: Add dogfooding integ tests

Author: antonsynd

.claude/settings.local.json

@@ -25,7 +25,8 @@
       "Bash(for:*)",
       "Bash(do dotnet test --no-build)",
       "Bash(dotnet run:*)",
-      "Bash(dotnet restore:*)"
+      "Bash(dotnet restore:*)",
+      "mcp__Desktop_Commander__create_directory"
     ]
   }
 }

src/Sharpy.Compiler.Tests/Integration/FileBasedIntegrationTests.cs

@@ -0,0 +1,123 @@
+using System.Reflection;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Sharpy.Compiler.Tests.Integration;
+
+/// <summary>
+/// File-based integration tests that discover .spy test files and their expected outputs.
+///
+/// Test files are organized in the TestFixtures directory:
+///   TestFixtures/
+///     feature_name/
+///       test_name.spy          - Sharpy source code
+///       test_name.expected     - Expected stdout output
+///       test_name.error        - (optional) Expected compilation error substring
+///
+/// A test passes if:
+///   - The .spy file compiles and executes successfully
+///   - The stdout matches the contents of .expected exactly
+///
+/// Error tests (when .error file exists):
+///   - The .spy file should fail to compile
+///   - The error message should contain the text in .error
+/// </summary>
+public class FileBasedIntegrationTests : IntegrationTestBase
+{
+    private static readonly string FixturesPath;
+
+    static FileBasedIntegrationTests()
+    {
+        // Find the TestFixtures directory relative to the test assembly
+        var assemblyLocation = Assembly.GetExecutingAssembly().Location;
+        var assemblyDir = Path.GetDirectoryName(assemblyLocation)!;
+
+        // Navigate from bin/Debug/net10.0 to the source directory
+        FixturesPath = Path.GetFullPath(Path.Combine(
+            assemblyDir, "..", "..", "..", "Integration", "TestFixtures"));
+    }
+
+    public FileBasedIntegrationTests(ITestOutputHelper output) : base(output)
+    {
+    }
+
+    /// <summary>
+    /// Discovers all test fixtures by scanning the TestFixtures directory.
+    /// </summary>
+    public static IEnumerable<object[]> GetTestFixtures()
+    {
+        if (!Directory.Exists(FixturesPath))
+        {
+            yield break;
+        }
+
+        // Find all .spy files recursively
+        foreach (var spyFile in Directory.EnumerateFiles(FixturesPath, "*.spy", SearchOption.AllDirectories))
+        {
+            var relativePath = Path.GetRelativePath(FixturesPath, spyFile);
+            var testName = Path.ChangeExtension(relativePath, null).Replace(Path.DirectorySeparatorChar, '/');
+            yield return new object[] { testName, spyFile };
+        }
+    }
+
+    [Theory]
+    [MemberData(nameof(GetTestFixtures))]
+    public void RunTestFixture(string testName, string spyFilePath)
+    {
+        Output.WriteLine($"Running test: {testName}");
+        Output.WriteLine($"Source file: {spyFilePath}");
+
+        // Read the source file
+        var source = File.ReadAllText(spyFilePath);
+        Output.WriteLine("=== Sharpy Source ===");
+        Output.WriteLine(source);
+        Output.WriteLine("=====================");
+
+        // Check if this is an error test
+        var errorFilePath = Path.ChangeExtension(spyFilePath, ".error");
+        var isErrorTest = File.Exists(errorFilePath);
+
+        // Execute the test
+        var result = CompileAndExecute(source, Path.GetFileName(spyFilePath));
+
+        if (isErrorTest)
+        {
+            // Error test: compilation should fail
+            var expectedError = File.ReadAllText(errorFilePath).Trim();
+            Output.WriteLine($"Expected error: {expectedError}");
+
+            Assert.False(result.Success,
+                $"Expected compilation to fail, but it succeeded. Output: {result.StandardOutput}");
+
+            var actualErrors = string.Join("\n", result.CompilationErrors);
+            Assert.Contains(expectedError, actualErrors, StringComparison.OrdinalIgnoreCase);
+        }
+        else
+        {
+            // Success test: compilation should succeed and output should match
+            var expectedFilePath = Path.ChangeExtension(spyFilePath, ".expected");
+
+            Assert.True(File.Exists(expectedFilePath),
+                $"Missing expected output file: {expectedFilePath}");
+
+            var expectedOutput = File.ReadAllText(expectedFilePath);
+
+            Assert.True(result.Success,
+                $"Compilation failed: {string.Join("\n", result.CompilationErrors)}");
+
+            Assert.Equal(expectedOutput, result.StandardOutput);
+        }
+    }
+
+    /// <summary>
+    /// Verifies that the test fixtures directory exists and contains at least one test.
+    /// This is a sanity check to ensure the test discovery is working.
+    /// </summary>
+    [Fact]
+    public void TestFixturesDirectory_Exists()
+    {
+        Output.WriteLine($"Looking for fixtures in: {FixturesPath}");
+        Assert.True(Directory.Exists(FixturesPath),
+            $"TestFixtures directory not found at: {FixturesPath}");
+    }
+}

src/Sharpy.Compiler.Tests/Integration/TestFixtures/README.md

@@ -0,0 +1,70 @@
+# Test Fixtures
+
+This directory contains file-based integration tests for the Sharpy compiler.
+
+## Structure
+
+```
+TestFixtures/
+├── basics/              # Basic language features
+│   ├── hello_world.spy
+│   ├── hello_world.expected
+│   ├── arithmetic.spy
+│   └── arithmetic.expected
+├── functions/           # Function-related tests
+├── control_flow/        # Control flow tests (if/elif/else, loops)
+├── errors/              # Tests that expect compilation errors
+└── README.md
+```
+
+## Adding a New Test
+
+### Success Test (compilation should succeed)
+
+1. Create a `.spy` file with your Sharpy source code
+2. Create a matching `.expected` file with the exact expected stdout output
+
+Example:
+```
+my_feature/
+├── test_name.spy        # Your Sharpy code
+└── test_name.expected   # Expected stdout output (including newlines)
+```
+
+### Error Test (compilation should fail)
+
+1. Create a `.spy` file with Sharpy code that should fail to compile
+2. Create a matching `.error` file with a substring that should appear in the error message
+
+Example:
+```
+errors/
+├── undefined_var.spy    # Code with an undefined variable
+└── undefined_var.error  # Contains: "undefined_var" (substring to match)
+```
+
+## Running Tests
+
+```bash
+# Run all file-based tests
+dotnet test --filter "FullyQualifiedName~FileBasedIntegrationTests"
+
+# Run a specific test (by name)
+dotnet test --filter "FullyQualifiedName~FileBasedIntegrationTests&DisplayName~hello_world"
+```
+
+## Converting Dogfood Outputs
+
+To convert a successful dogfood output to a test fixture:
+
+1. Copy the `source.spy` file to an appropriate subdirectory
+2. Rename it to a descriptive name (e.g., `abstract_class_basic.spy`)
+3. Create the matching `.expected` file with the expected output
+4. Run the test to verify
+
+## Notes
+
+- Expected output files should match stdout exactly, including trailing newlines
+- Error files use case-insensitive substring matching
+- Tests are discovered automatically at runtime - no need to register them
+- Test names in the runner are based on the file path relative to TestFixtures/

src/Sharpy.Compiler.Tests/Integration/TestFixtures/basics/arithmetic.expected

@@ -0,0 +1,5 @@
+15
+5
+50
+2
+0

src/Sharpy.Compiler.Tests/Integration/TestFixtures/basics/arithmetic.spy

@@ -0,0 +1,8 @@
+x: int = 10
+y: int = 5
+
+print(x + y)
+print(x - y)
+print(x * y)
+print(x // y)
+print(x % y)

src/Sharpy.Compiler.Tests/Integration/TestFixtures/basics/hello_world.expected

@@ -0,0 +1 @@
+Hello, World!

src/Sharpy.Compiler.Tests/Integration/TestFixtures/basics/hello_world.spy

@@ -0,0 +1 @@
+print("Hello, World!")

src/Sharpy.Compiler.Tests/Integration/TestFixtures/basics/type_inference.expected

@@ -0,0 +1,4 @@
+42
+3.14
+hello
+True

src/Sharpy.Compiler.Tests/Integration/TestFixtures/basics/type_inference.spy

@@ -0,0 +1,9 @@
+x = 42
+y = 3.14
+z = "hello"
+flag = True
+
+print(x)
+print(y)
+print(z)
+print(flag)

src/Sharpy.Compiler.Tests/Integration/TestFixtures/control_flow/for_loop_range.expected

@@ -0,0 +1,5 @@
+0
+1
+2
+3
+4