First stab at implementing an indexer

Author: olafurpg

.github/workflows/test.yml

@@ -30,4 +30,3 @@ jobs:
       with:
         dotnet-version: 6.0.x
     - run: dotnet test
-

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store
+.idea
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 ##
@@ -348,3 +350,4 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
+

NOTICE

@@ -0,0 +1,10 @@
+# tcz717/LsifDotnet
+
+scip-dotnet contains contains parts which are derived from the
+tcz717/LsifDotnet project. The original license is Apache 2 and can be
+found here: https://github.com/tcz717/LsifDotnet/blob/main/LICENSE
+
+The parts that are derived from LsifDotnet are mostly related to parsing
+command-line options and loading the MSBuild Workspace, not the indexing logic
+itself.
+

ScipDotnet.Tests/Position.cs

@@ -0,0 +1,22 @@
+using Scip;
+
+namespace ScipDotnet.Tests;
+
+public record Position(int Line, int Character) : IComparable<Position>
+{
+    public int CompareTo(Position? other)
+    {
+        if (other == null)
+        {
+            return 1;
+        }
+
+        if (other.Line != Line)
+        {
+            return Line - other.Line;
+        }
+
+        return Character - other.Character;
+    }
+
+}

ScipDotnet.Tests/Range.cs

@@ -0,0 +1,33 @@
+using Scip;
+
+namespace ScipDotnet.Tests;
+
+public record Range(Position Start, Position End) : IComparable<Range>
+{
+    public int CompareTo(Range? other)
+    {
+        if (other == null)
+        {
+            return 1;
+        }
+
+        var byStart = Start.CompareTo(other.Start);
+        if (byStart != 0)
+        {
+            return byStart;
+        }
+
+        return End.CompareTo(other.End);
+    }
+
+    public static Range FromOccurrence(Occurrence occ)
+    {
+        if (occ.Range.Count == 3)
+            return new Range(new Position(occ.Range[0], occ.Range[1]), new Position(occ.Range[0], occ.Range[2]));
+
+        if (occ.Range.Count == 4)
+            return new Range(new Position(occ.Range[0], occ.Range[1]), new Position(occ.Range[2], occ.Range[3]));
+
+        throw new ArgumentException($"occurrence.range must have length 3 or length 4, obtained {occ.Range}");
+    }
+}

ScipDotnet.Tests/ScipDotnet.Tests.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="DiffPlex" Version="1.7.1" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+        <PackageReference Include="NUnit" Version="3.13.3" />
+        <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
+        <PackageReference Include="NUnit.Analyzers" Version="3.3.0" />
+        <PackageReference Include="coverlet.collector" Version="3.1.2" />
+        <PackageReference Include="TextCopy" Version="6.2.0" />
+        <PackageReference Include="Verify.NUnit" Version="19.3.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\ScipDotnet\ScipDotnet.csproj" />
+    </ItemGroup>
+
+</Project>

ScipDotnet.Tests/SnapshotTests.cs

@@ -0,0 +1,256 @@
+using System.Diagnostics;
+using System.Text;
+using DiffPlex.DiffBuilder;
+using DiffPlex.DiffBuilder.Model;
+using Scip;
+using Index = Scip.Index;
+
+namespace ScipDotnet.Tests;
+
+[TestFixture]
+public class SnapshotTests
+{
+    [Test, TestCaseSource(nameof(ListSnapshotInputDirectories))]
+    public void Snapshot(string inputDirectory)
+    {
+        var indexFile = IndexDirectory(inputDirectory);
+        var indexBytes = File.ReadAllBytes(indexFile);
+        var index = Index.Parser.ParseFrom(indexBytes);
+        var snapshots = index.Documents.ToDictionary(document => document.RelativePath,
+            document => FormatDocument(index, document));
+        var outputDirectory =
+            Path.GetFullPath(Path.Join(inputDirectory, "../../", "output", Path.GetFileName(inputDirectory)));
+        var isUpdateSnapshots = Environment.GetEnvironmentVariable("SCIP_UPDATE_SNAPSHOTS") != null;
+        if (isUpdateSnapshots)
+        {
+            if (Directory.Exists(outputDirectory))
+            {
+                Directory.Delete(outputDirectory, true);
+            }
+
+            foreach (var (relativePath, snapshot) in snapshots)
+            {
+                var documentPath = Path.Join(outputDirectory, relativePath);
+                var directory = Path.GetDirectoryName(documentPath);
+                if (directory == null) continue;
+                Directory.CreateDirectory(directory);
+                File.WriteAllText(documentPath, snapshot);
+            }
+        }
+        else
+        {
+            var files = new List<string>();
+            RecursivelyListFiles(outputDirectory, files);
+            Assert.Multiple(() =>
+            {
+                foreach (var file in files)
+                {
+                    var relativePath = Path.GetRelativePath(outputDirectory, file);
+                    var obtained = snapshots.GetValueOrDefault(relativePath, "");
+                    var expected = File.ReadAllText(file);
+                    var diff = DiffStrings(obtained, expected);
+                    if (diff.Length > 0)
+                    {
+                        Assert.Fail("(+ expected, - obtained). To update the expected output to match the obtained behavior, run: " +
+                                    "SCIP_UPDATE_SNAPSHOTS=true dotnet test\n\n" + diff, file);
+                    }
+                }
+
+                var filesSet = new HashSet<string>(files);
+                foreach (var (relativePath, _) in snapshots)
+                {
+                    var outputPath = Path.Join(outputDirectory, relativePath);
+                    if (filesSet.Contains(outputPath))
+                    {
+                        continue;
+                    }
+
+                    Assert.Fail(
+                        $"relative path '{relativePath}' missing an output file. To fix this problem, run the following command: SCIP_UPDATE_SNAPSHOTS=true dotnet test");
+                }
+            });
+        }
+    }
+
+    private static string DiffStrings(string obtained, string expected)
+    {
+        var diff = InlineDiffBuilder.Diff(expected, obtained);
+        var sb = new StringBuilder();
+        for (var i = 0; i < diff.Lines.Count; i++)
+        {
+            var line = diff.Lines[i];
+            if (line.Type == ChangeType.Unchanged)
+            {
+                continue;
+            }
+
+            if (i > 0 && diff.Lines[i - 1].Type == ChangeType.Unchanged)
+            {
+                sb.Append("  ").AppendLine(diff.Lines[i - 1].Text);
+            }
+
+            switch (line.Type)
+            {
+                case ChangeType.Inserted:
+                    sb.Append("+ ");
+                    break;
+                case ChangeType.Deleted:
+                    sb.Append("- ");
+                    break;
+            }
+
+            sb.AppendLine(line.Text);
+        }
+
+        return sb.ToString();
+    }
+
+    private static string[] ListSnapshotInputDirectories()
+    {
+        var inputs = Path.Join(RootDirectory(), "snapshots", "input");
+        return Directory.GetDirectories(inputs);
+    }
+
+    private static void RecursivelyListFiles(string path, List<string> result)
+    {
+        if (!Directory.Exists(path)) return;
+        result.AddRange(Directory.GetFiles(path));
+        foreach (var directory in Directory.GetDirectories(path))
+        {
+            RecursivelyListFiles(directory, result);
+        }
+    }
+
+    private static string IndexDirectory(string directory)
+    {
+        var include = Environment.GetEnvironmentVariable("SCIP_INCLUDE");
+        var includeOption = include != null ? $" --include {include}" : "";
+        var arguments = $"run --project ScipDotnet -- index --working-directory {directory}{includeOption}";
+        var process = new Process()
+        {
+            StartInfo = new ProcessStartInfo()
+            {
+                FileName = "dotnet",
+                Arguments = arguments,
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                WorkingDirectory = RootDirectory(),
+                RedirectStandardError = true
+            }
+        };
+        process.Start();
+        process.WaitForExit();
+        if (process.ExitCode != 0)
+        {
+            var stdout = process.StandardOutput.ReadToEnd();
+            var stderr = process.StandardError.ReadToEnd();
+            Assert.Fail(
+                $"non-zero exit code {process.ExitCode} indexing {directory}\ndotnet {arguments}\n{stdout}{stderr}");
+        }
+
+        return Path.Join(directory, "index.scip");
+    }
+
+    private static string RootDirectory()
+    {
+        var process = new Process()
+        {
+            StartInfo = new ProcessStartInfo()
+            {
+                // The working directory of `dotnet test` is not the root directory of the project
+                // so we infer it by invoking `git rev-parse --show-toplevel`. It would be cleaner
+                // to get the root directory from MSBuild but I wasn't able to figure out how to do it
+                // after searching for ~20 minutes. This works for now and unblocks writing tests.
+                FileName = "git",
+                Arguments = "rev-parse --show-toplevel",
+                UseShellExecute = false,
+                RedirectStandardOutput = true
+            }
+        };
+        process.Start();
+        return process.StandardOutput.ReadToEnd().Trim();
+    }
+
+
+    private Dictionary<String, SymbolInformation> SymbolTable(Document document)
+    {
+        var result = new Dictionary<String, SymbolInformation>();
+        foreach (var info in document.Symbols)
+        {
+            // Intentionally crash the test if we emit conflicting SymbolInformation
+            if (!result.TryAdd(info.Symbol, info))
+            {
+                Assert.Fail("duplicate SymbolInformation '{0}'", info.Symbol);
+            }
+        }
+
+        return result;
+    }
+
+    private string FormatDocument(Index index, Document document)
+    {
+        var sb = new StringBuilder();
+        var inputPath = Path.Join(
+            index.Metadata.ProjectRoot.Replace("file://", String.Empty),
+            document.RelativePath);
+        var occurrences = document.Occurrences.ToList();
+        occurrences.Sort(CompareOccurrences);
+        var symtab = SymbolTable(document);
+        var occurrenceIndex = 0;
+        var lines = File.ReadAllLines(inputPath);
+        for (var lineNumber = 0; lineNumber < lines.Length; lineNumber++)
+        {
+            var line = lines[lineNumber].Replace("\t", " ");
+            sb.Append("  ").AppendLine(line);
+            while (occurrenceIndex < occurrences.Count)
+            {
+                var occurrence = occurrences[occurrenceIndex];
+                var range = Range.FromOccurrence(occurrence);
+                if (range.Start.Line != range.End.Line)
+                    continue; // TODO: support in the future
+                if (range.Start.Line != lineNumber)
+                    break;
+                var isDefinition = (occurrence.SymbolRoles & (int)SymbolRole.Definition) != 0;
+                var role = isDefinition ? "definition" : "reference";
+                var length = range.End.Character - range.Start.Character;
+                var indent = new String(' ', range.Start.Character);
+                sb.Append("//")
+                    .Append(indent)
+                    .Append(new String('^', length))
+                    .Append(' ')
+                    .Append(role)
+                    .Append(' ')
+                    .AppendLine(occurrence.Symbol);
+
+                if (isDefinition)
+                {
+                    var info = symtab.GetValueOrDefault(occurrence.Symbol, new SymbolInformation());
+                    var prefix = "//" + indent + new String(' ', length + 1);
+                    foreach (var documentation in info.Documentation)
+                    {
+                        sb.Append(prefix).Append("documentation ").AppendLine(documentation.Replace("\n", "\\n"));
+                    }
+
+                    foreach (var relationship in info.Relationships)
+                    {
+                        sb.Append(prefix).Append("relationship ");
+                        if (relationship.IsDefinition) sb.Append("definition ");
+                        if (relationship.IsImplementation) sb.Append("implementation ");
+                        if (relationship.IsReference) sb.Append("reference ");
+                        if (relationship.IsTypeDefinition) sb.Append("type_definition ");
+                        sb.AppendLine(relationship.Symbol);
+                    }
+                }
+
+                occurrenceIndex++;
+            }
+        }
+
+        return sb.ToString();
+    }
+
+    private static int CompareOccurrences(Occurrence a, Occurrence b)
+    {
+        return Range.FromOccurrence(a).CompareTo(Range.FromOccurrence(b));
+    }
+}

ScipDotnet.Tests/Usings.cs

@@ -0,0 +1 @@
+global using NUnit.Framework;