Merge pull request #81 from microsoft/open-source-powershell-extractor

PS: Open source the powershell extractor

Author: dilanbhalla

powershell/README.md

@@ -0,0 +1,12 @@
+# Powershell Extractor
+
+## Directories:
+- `extractor`
+  - Powershell extractor source code
+- `ql`
+  - QL libraries and queries for Powershell (to be written)
+- `tools`
+  - Directory containing files that must be copied to powershell/tools in the directory containing the CodeQL CLI. This will be done automatically by `build.ps1` (see below).
+
+## How to build the Powershell:
+- Run `build.ps1 path-to-codeql-cli-folder` where `path-to-codeql-cli-folder` is the path to the folder containing the CodeQL CLI (i.e., `codeql.exe`).

powershell/build.ps1

@@ -0,0 +1,17 @@
+param (
+    [Parameter(Mandatory=$true)][string]$cliFolder
+)
+
+$toolsWin64Folder = Join-Path (Join-Path (Join-Path $cliFolder "powershell") "tools") "win64"
+dotnet publish (Join-Path "extractor" "powershell.sln") -o $toolsWin64Folder
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "Build failed"
+    exit 1
+}
+
+$powershellFolder = Join-Path -Path $cliFolder -ChildPath "powershell"
+Copy-Item -Path codeql-extractor.yml -Destination $powershellFolder -Force
+$qlLibFolder = Join-Path -Path "ql" -ChildPath "lib"
+Copy-Item -Path (Join-Path $qlLibFolder "semmlecode.powershell.dbscheme") -Destination $powershellFolder -Force
+Copy-Item -Path (Join-Path $qlLibFolder "semmlecode.powershell.dbscheme.stats") -Destination $powershellFolder -Force
+Copy-Item -Path "tools" -Destination $powershellFolder -Recurse -Force

powershell/codeql-extractor.yml

@@ -0,0 +1,10 @@
+name: "powershell"
+display_name: "powershell"
+version: 0.0.1
+column_kind: "utf16"
+legacy_qltest_extraction: true
+file_types:
+  - name: powershell
+    display_name: powershellscripts
+    extensions:
+      - .ps1

powershell/extractor/Microsoft.Extractor.Tests/Microsoft.Extractor.Tests.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Semmle.Extraction.PowerShell.Standalone\Semmle.Extraction.PowerShell.Standalone.csproj" />
+  </ItemGroup>
+
+</Project>

powershell/extractor/Microsoft.Extractor.Tests/TrapSanitizer.cs

@@ -0,0 +1,88 @@
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace Microsoft.Extraction.Tests;
+
+/// <summary>
+/// This class provides a method for sanitizing a trap files in tests so they can be validated. 
+///     The resulting trap file will not be valid for making a codeqldb due to missing file metadata,
+///     which is removed to ensure the test case can match the expected trap.
+/// </summary>
+internal class TrapSanitizer
+{
+    // Regex to match the IDs in the file (# followed by digits)
+    private static readonly Regex CaptureId = new Regex($"#([0-9]+)");
+
+    /// <summary>
+    /// Sanitize a Trap file to check equality by removing run specific things like file names and squashing ids
+    ///     to a consistent range
+    /// </summary>
+    /// <param name="TrapContents">The lines of the trap file to sanitize</param>
+    /// <returns>A string containing the sanitized contents</returns>
+    public static string SanitizeTrap(string[] TrapContents)
+    {
+        StringBuilder sb = new();
+        int largestId = 0;
+        int startingLineActual = -1;
+        for (int i = 0; i < TrapContents.Length; i++)
+        {
+            // The first line with actual extracted contents will be after the numlines line
+            if (TrapContents[i].StartsWith("numlines"))
+            {
+                startingLineActual = i + 1;
+                break;
+            }
+            // If a line before numlines has an ID it is a candidate for largest id found
+            if (CaptureId.IsMatch(TrapContents[i]))
+            {
+                largestId = int.Max(largestId, int.Parse(CaptureId.Matches(TrapContents[i])[0].Groups[1].Captures[0].Value));
+            }
+        }
+
+        // Starting from the line after numlines declaration
+        for (int i = startingLineActual; i < TrapContents.Length; i++)
+        {
+            // Replace IDs in each line based on the largest previous ID found
+            // Reserve #1 for the File
+            sb.Append(SanitizeLine(TrapContents[i], largestId - 1));
+            sb.Append(Environment.NewLine);
+        }
+
+        return sb.ToString();
+    }
+
+    /// <summary>
+    /// Sanitize a single line of trap content given the largest previously used id number to ignore,
+    ///     subtracting the offset from those IDs.
+    /// </summary>
+    /// <param name="trapContent">A single line of trap content</param>
+    /// <param name="offset">The offset to apply</param>
+    /// <returns>A sanitized line</returns>
+    private static string SanitizeLine(string trapContent, int offset)
+    {
+        var matches = CaptureId.Matches(trapContent);
+        if (!matches.Any())
+        {
+            return trapContent;
+        }
+        var sb = new StringBuilder();
+        int lastIndex = 0;
+        foreach (Match match in matches)
+        {
+            var capture = match.Groups[1].Captures[0];
+            sb.Append(trapContent[lastIndex..capture.Index]);
+            lastIndex = capture.Index + capture.Length;
+            int newInt = int.Parse(capture.Value);
+            if (newInt > 1)
+            {
+                sb.Append(newInt - offset);
+            }
+            else
+            {
+                sb.Append(newInt);
+            }
+        }
+        sb.Append(trapContent[lastIndex..]);
+        return sb.ToString();
+    }
+}

powershell/extractor/Microsoft.Extractor.Tests/Traps.cs

@@ -0,0 +1,212 @@
+using System.Reflection;
+using System.Text.RegularExpressions;
+using Microsoft.Extraction.Tests;
+using Semmle.Extraction;
+using Semmle.Extraction.PowerShell.Standalone;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+using Semmle.Extraction.PowerShell;
+
+namespace Microsoft.Extractor.Tests;
+
+internal static class PathHolder
+{
+    internal static string powershellSource = Path.Join("..", "..", "..", "..", "..", "samples", "code");
+    internal static string expectedTraps = Path.Join("..", "..", "..", "..", "..", "samples", "traps");
+    internal static string schemaPath = Path.Join("..", "..", "..", "..", "..", "config", "semmlecode.powershell.dbscheme");
+    internal static string generatedTraps = Path.Join(".", Path.GetFullPath(powershellSource).Replace(":", "_"));
+}
+public class TrapTestFixture : IDisposable
+{
+    public TrapTestFixture()
+    {
+        // Setup here
+    }
+
+    public void Dispose()
+    {
+        // Delete the generated traps
+        Directory.Delete(PathHolder.generatedTraps, true);
+    }
+}
+
+public class Traps : IClassFixture<TrapTestFixture>
+{
+    private readonly ITestOutputHelper _output;
+    public Traps(ITestOutputHelper output)
+    {
+        _output = output;
+    }
+
+    private static Regex schemaDeclStart = new("([a-zA-Z_]+)\\(");
+    private static Regex schemaEnd = new("^\\)");
+    private static Regex commentEnd = new("\\*/");
+
+    /// <summary>
+    /// Naiively parse the schema and try to determine how many parameters each table expects
+    /// </summary>
+    /// <param name="schemaContents"></param>
+    /// <returns>Dictionary mapping table name to number of parameters</returns>
+    private static Dictionary<string, int> ParseSchema(string[] schemaContents)
+    {
+        bool isParsingTable = false;
+        int expectedNumEntries = 0;
+        string targetName = string.Empty;
+        Dictionary<string, int> output = new();
+        for (int index = 0; index < schemaContents.Length; index++)
+        {
+            if (!isParsingTable)
+            {
+                if (schemaDeclStart.IsMatch(schemaContents[index]))
+                {
+                    targetName = schemaDeclStart.Matches(schemaContents[index])[0].Groups[1].Captures[0].Value;
+                    isParsingTable = true;
+                    expectedNumEntries = 0;
+                }
+            }
+            else
+            {
+                if (commentEnd.IsMatch(schemaContents[index]))
+                {
+                    isParsingTable = false;
+                    expectedNumEntries = 0;
+                }
+                if (schemaEnd.IsMatch(schemaContents[index]))
+                {
+                    output.Add(targetName, expectedNumEntries);
+                    isParsingTable = false;
+                    expectedNumEntries++;
+                }
+                else
+                {
+                    expectedNumEntries++;
+                }
+            }
+        }
+
+        return output;
+    }
+
+    /// <summary>
+    /// Check that the Schema entries match the implemented methods in Tuples.cs
+    /// </summary>
+    [Fact]
+    public void Schema_Matches_Tuples()
+    {
+        string[] schemaContents = File.ReadLines(PathHolder.schemaPath).ToArray();
+        Dictionary<string, int> expected = ParseSchema(schemaContents);
+        // Get all the nonpublic static methods from the Tuples classes
+        var methods = typeof(Semmle.Extraction.PowerShell.Tuples)
+            .GetMethods(BindingFlags.Static | BindingFlags.NonPublic)
+            .Union(typeof(Semmle.Extraction.Tuples).GetMethods(BindingFlags.Static | BindingFlags.NonPublic))
+            // Select a tuple of the method, its parameters
+            .Select(method => (method, method.GetParameters(),
+                // the expected number of parameters - one fewer than actual if the first is a TextWriter, and the name of the method
+                method.GetParameters()[0].ParameterType.Name.Equals("TextWriter") ? method.GetParameters().Length - 1 : method.GetParameters().Length , method.Name));
+        List<string> errors = new();
+        List<string> warnings = new();
+        // If a tuple method exists and doesn't have a matching schema entry that is an error, as the produce traps won't be match
+        foreach (var method in methods)
+        {
+            if (expected.Any(entry => method.Name == entry.Key && (method.Item3) == entry.Value))
+            {
+                continue;
+            }
+            errors.Add($"Tuple {method.Name} does not match any schema entry, expected {method.Item3} parameters.");
+        }
+        // If the schema has a superfluous entity that is a warning, as the extractor simply cannot product those things
+        foreach (var entry in expected)
+        {
+            if (methods.Any(method => method.Name == entry.Key && (method.Item3) == entry.Value))
+            {
+                continue;
+            }
+            warnings.Add($"Schema entry {entry.Key} does not match any implemented Tuple, expected {entry.Value} parameters.");
+        }
+
+        foreach (var warning in warnings)
+        {
+            _output.WriteLine($"Warning: {warning}");
+        }
+        foreach (var error in errors)
+        {
+            _output.WriteLine($"Error: {error}");
+        }
+        Assert.Empty(errors);
+    }
+    
+    [Fact]
+    public void Verify_Sample_Traps()
+    {
+        string[] expectedTrapsFiles = Directory.GetFiles(PathHolder.expectedTraps);
+        int numFailures = 0;
+        foreach (string expected in expectedTrapsFiles)
+        {
+            if (File.ReadAllText(expected).Contains("extractor_messages"))
+            {
+                numFailures++;
+                _output.WriteLine($"Expected sample trap {expected} has extractor error messages.");
+            }
+        }
+
+        if (numFailures > 0)
+        {
+            _output.WriteLine($"{numFailures} errors were detected.");
+        }
+        Assert.Equal(0, numFailures);
+    }
+
+
+    [Fact]
+    public void Compare_Generated_Traps()
+    {
+        string[] args = new string[] { PathHolder.powershellSource };
+        int exitcode = Program.Main(args);
+        Assert.Equal(0, exitcode);
+        string[] generatedTrapsFiles = Directory.GetFiles(PathHolder.generatedTraps);
+        string[] expectedTrapsFiles = Directory.GetFiles(PathHolder.expectedTraps);
+
+        Assert.NotEmpty(generatedTrapsFiles);
+        int numFailures = 0;
+        var generatedFileNames = generatedTrapsFiles.Select(x => (Path.GetFileName(x), x)).ToList();
+        var expectedFileNames = expectedTrapsFiles.Select(x => (Path.GetFileName(x), x)).ToList();
+        foreach (var expectedTrapFile in expectedFileNames)
+        {
+            if (generatedFileNames.Any(x => x.Item1 == expectedTrapFile.Item1)) continue;
+            numFailures++;
+            _output.WriteLine($"{expectedTrapFile} has no matching filename in generated.");
+        }
+        foreach (var generated in generatedFileNames)
+        {
+            var expected = expectedFileNames.FirstOrDefault(filePath => filePath.Item1.Equals(generated.Item1));
+            if (expected.Item1 is null || expected.x is null)
+            {
+                numFailures++;
+                _output.WriteLine($"{generated.Item1} has no matching filename in expected.");
+            }
+            else
+            {
+                if (File.ReadAllText(generated.x).Contains("extractor_messages"))
+                {
+                    _output.WriteLine($"Test generated trap {generated} has extractor error messages.");
+                    numFailures++;
+                    continue;
+                }
+                string generatedFileSanitized = TrapSanitizer.SanitizeTrap(File.ReadAllLines(generated.x));
+                string expectedFileSanitized = TrapSanitizer.SanitizeTrap(File.ReadAllLines(expected.x));
+                if (!generatedFileSanitized.Equals(expectedFileSanitized))
+                {
+                    numFailures++;
+                    _output.WriteLine($"{generated} does not match {expected}");
+                }
+            }
+        }
+
+        if (numFailures > 0)
+        {
+            _output.WriteLine($"{numFailures} errors were detected.");
+        }
+        Assert.Equal(expectedTrapsFiles.Length, generatedTrapsFiles.Length);
+        Assert.Equal(0, numFailures);
+    }
+}

powershell/extractor/Microsoft.Extractor.Tests/Usings.cs

@@ -0,0 +1 @@
+global using Xunit;