Adding Azure.Functions.Cli.TestFramework.csproj to have E2E tests run on

Azure.Functions.Cli.sln

@@ -21,6 +21,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Functions.Cli.Abstrac
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CoreToolsHost", "src\CoreToolsHost\CoreToolsHost.csproj", "{0333D5B6-B628-4605-A51E-D0AEE4C3F1FC}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Functions.Cli.TestFramework", "test\Cli\TestFramework\Azure.Functions.Cli.TestFramework.csproj", "{3A8E1907-E3A2-1CE0-BA8B-805B655FAF09}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -51,6 +53,10 @@ Global
 		{0333D5B6-B628-4605-A51E-D0AEE4C3F1FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0333D5B6-B628-4605-A51E-D0AEE4C3F1FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0333D5B6-B628-4605-A51E-D0AEE4C3F1FC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3A8E1907-E3A2-1CE0-BA8B-805B655FAF09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3A8E1907-E3A2-1CE0-BA8B-805B655FAF09}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3A8E1907-E3A2-1CE0-BA8B-805B655FAF09}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3A8E1907-E3A2-1CE0-BA8B-805B655FAF09}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -63,6 +69,7 @@ Global
 		{78231B55-D243-46F1-9C7F-7831B40ED2D8} = {154FDAF2-0E86-450E-BE57-4E3D410B0FAC}
 		{BC78165E-CE5B-4303-BB8E-BC172E5B86E0} = {154FDAF2-0E86-450E-BE57-4E3D410B0FAC}
 		{0333D5B6-B628-4605-A51E-D0AEE4C3F1FC} = {5F51C958-39C0-4E0C-9165-71D0BCE647BC}
+		{3A8E1907-E3A2-1CE0-BA8B-805B655FAF09} = {6EE1D011-2334-44F2-9D41-608B969DAE6D}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {FA1E01D6-A57B-4061-A333-EDC511D283C0}

test/Cli/TestFramework/Assertions/CommandResultAssertions.cs

@@ -0,0 +1,43 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+// Based off of: https://github.com/dotnet/sdk/blob/e793aa4709d28cd783712df40413448250e26fea/test/Microsoft.NET.TestFramework/Assertions/CommandResultAssertions.cs
+using Azure.Functions.Cli.Abstractions.Command;
+using FluentAssertions;
+using FluentAssertions.Execution;
+
+namespace Azure.Functions.Cli.TestFramework.Assertions
+{
+    public class CommandResultAssertions(CommandResult commandResult)
+    {
+        private readonly CommandResult _commandResult = commandResult;
+
+        public CommandResultAssertions ExitWith(int expectedExitCode)
+        {
+            Execute.Assertion.ForCondition(_commandResult.ExitCode == expectedExitCode)
+                .FailWith($"Expected command to exit with {expectedExitCode} but it did not. Error message: {_commandResult.StdErr}");
+            return this;
+        }
+
+        public AndConstraint<CommandResultAssertions> HaveStdOutContaining(string pattern)
+        {
+            Execute.Assertion.ForCondition(_commandResult.StdOut is not null && _commandResult.StdOut.Contains(pattern))
+                .FailWith($"The command output did not contain expected result: {pattern}{Environment.NewLine}");
+            return new AndConstraint<CommandResultAssertions>(this);
+        }
+
+        public AndConstraint<CommandResultAssertions> HaveStdErrContaining(string pattern)
+        {
+            Execute.Assertion.ForCondition(_commandResult.StdErr is not null && _commandResult.StdErr.Contains(pattern))
+                .FailWith($"The command output did not contain expected result: {pattern}{Environment.NewLine}");
+            return new AndConstraint<CommandResultAssertions>(this);
+        }
+
+        public AndConstraint<CommandResultAssertions> NotHaveStdOutContaining(string pattern)
+        {
+            Execute.Assertion.ForCondition(_commandResult.StdOut is not null && !_commandResult.StdOut.Contains(pattern))
+                .FailWith($"The command output did contain expected result: {pattern}{Environment.NewLine}");
+            return new AndConstraint<CommandResultAssertions>(this);
+        }
+    }
+}

test/Cli/TestFramework/Assertions/CommandResultExtensions.cs

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+// Copied from: https://github.com/dotnet/sdk/blob/e793aa4709d28cd783712df40413448250e26fea/test/Microsoft.NET.TestFramework/Assertions/CommandResultExtensions.cs
+using Azure.Functions.Cli.Abstractions.Command;
+
+namespace Azure.Functions.Cli.TestFramework.Assertions
+{
+    public static class CommandResultExtensions
+    {
+        public static CommandResultAssertions Should(this CommandResult commandResult) => new(commandResult);
+    }
+}

test/Cli/TestFramework/Azure.Functions.Cli.TestFramework.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectCapability Remove="TestContainer" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Storage.Queues" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="FluentAssertions"/>
+    <PackageReference Include="StyleCop.Analyzers" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="$(RepoSrcRoot)Cli\Abstractions\Azure.Functions.Cli.Abstractions.csproj" />
+  </ItemGroup>
+
+</Project>

test/Cli/TestFramework/CommandInfo.cs

@@ -0,0 +1,61 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+// Based off of: https://github.com/dotnet/sdk/blob/e793aa4709d28cd783712df40413448250e26fea/test/Microsoft.NET.TestFramework/Commands/SdkCommandSpec.cs
+using System.Diagnostics;
+using Azure.Functions.Cli.Abstractions.Command;
+
+namespace Azure.Functions.Cli.TestFramework
+{
+    public class CommandInfo
+    {
+        public required string FileName { get; set; }
+
+        public List<string> Arguments { get; set; } = [];
+
+        public Dictionary<string, string> Environment { get; set; } = [];
+
+        public List<string> EnvironmentToRemove { get; } = [];
+
+        public required string WorkingDirectory { get; set; }
+
+        public string? TestName { get; set; }
+
+        public Command ToCommand()
+        {
+            var process = new Process()
+            {
+                StartInfo = ToProcessStartInfo()
+            };
+
+            return new Command(process, trimTrailingNewlines: true);
+        }
+
+        public ProcessStartInfo ToProcessStartInfo()
+        {
+            var psi = new ProcessStartInfo
+            {
+                FileName = FileName,
+                Arguments = string.Join(" ", Arguments),
+                UseShellExecute = false
+            };
+
+            foreach (KeyValuePair<string, string> kvp in Environment)
+            {
+                psi.Environment[kvp.Key] = kvp.Value;
+            }
+
+            foreach (string envToRemove in EnvironmentToRemove)
+            {
+                psi.Environment.Remove(envToRemove);
+            }
+
+            if (WorkingDirectory is not null)
+            {
+                psi.WorkingDirectory = WorkingDirectory;
+            }
+
+            return psi;
+        }
+    }
+}

test/Cli/TestFramework/Commands/FuncCommand.cs

@@ -0,0 +1,217 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+// Based off of: https://github.com/dotnet/sdk/blob/e793aa4709d28cd783712df40413448250e26fea/test/Microsoft.NET.TestFramework/Commands/TestCommand.cs
+using System.Diagnostics;
+using Azure.Functions.Cli.Abstractions.Command;
+using Xunit.Abstractions;
+
+namespace Azure.Functions.Cli.TestFramework.Commands
+{
+    public abstract class FuncCommand(ITestOutputHelper log)
+    {
+        private readonly Dictionary<string, string> _environment = [];
+
+        public ITestOutputHelper Log { get; } = log;
+
+        public string? WorkingDirectory { get; set; }
+
+        public List<string> Arguments { get; set; } = [];
+
+        public List<string> EnvironmentToRemove { get; } = [];
+
+        // These only work via Execute(), not when using GetProcessStartInfo()
+        public Action<string>? CommandOutputHandler { get; set; }
+
+        public Func<Process, Task>? ProcessStartedHandler { get; set; }
+
+        public StreamWriter? FileWriter { get; private set; } = null;
+
+        public string? LogFilePath { get; private set; }
+
+        protected abstract CommandInfo CreateCommand(IEnumerable<string> args);
+
+        public FuncCommand WithEnvironmentVariable(string name, string value)
+        {
+            _environment[name] = value;
+            return this;
+        }
+
+        public FuncCommand WithWorkingDirectory(string workingDirectory)
+        {
+            WorkingDirectory = workingDirectory;
+            return this;
+        }
+
+        private CommandInfo CreateCommandInfo(IEnumerable<string> args)
+        {
+            CommandInfo commandInfo = CreateCommand(args);
+            foreach (KeyValuePair<string, string> kvp in _environment)
+            {
+                commandInfo.Environment[kvp.Key] = kvp.Value;
+            }
+
+            foreach (string envToRemove in EnvironmentToRemove)
+            {
+                commandInfo.EnvironmentToRemove.Add(envToRemove);
+            }
+
+            if (WorkingDirectory is not null)
+            {
+                commandInfo.WorkingDirectory = WorkingDirectory;
+            }
+
+            if (Arguments.Count != 0)
+            {
+                commandInfo.Arguments = [.. Arguments, .. commandInfo.Arguments];
+            }
+
+            return commandInfo;
+        }
+
+        public ProcessStartInfo GetProcessStartInfo(params string[] args)
+        {
+            CommandInfo commandSpec = CreateCommandInfo(args);
+            return commandSpec.ToProcessStartInfo();
+        }
+
+        public virtual CommandResult Execute(IEnumerable<string> args)
+        {
+            CommandInfo spec = CreateCommandInfo(args);
+            ICommand command = spec
+                .ToCommand()
+                .CaptureStdOut()
+                .CaptureStdErr();
+
+            string? funcExeDirectory = Path.GetDirectoryName(spec.FileName);
+
+            if (!string.IsNullOrEmpty(funcExeDirectory))
+            {
+                Directory.SetCurrentDirectory(funcExeDirectory);
+            }
+
+            string? directoryToLogTo = Environment.GetEnvironmentVariable("DirectoryToLogTo");
+            if (string.IsNullOrEmpty(directoryToLogTo))
+            {
+                directoryToLogTo = Directory.GetCurrentDirectory();
+            }
+
+            // Ensure directory exists
+            Directory.CreateDirectory(directoryToLogTo);
+
+            // Create a more unique filename to avoid conflicts
+            string uniqueId = Guid.NewGuid().ToString("N")[..8];
+            LogFilePath = Path.Combine(
+                directoryToLogTo,
+                $"func_{spec.Arguments.First()}_{spec.TestName}_{DateTime.Now:yyyyMMdd_HHmmss}_{uniqueId}.log");
+
+            // Make sure we're only opening the file once
+            try
+            {
+                // Open with FileShare.Read to allow others to read but not write
+                var fileStream = new FileStream(LogFilePath, FileMode.Create, FileAccess.Write, FileShare.Read);
+                FileWriter = new StreamWriter(fileStream)
+                {
+                    AutoFlush = true
+                };
+
+                // Write initial information
+                FileWriter.WriteLine($"=== Test started at {DateTime.Now} ===");
+                FileWriter.WriteLine($"Test Name: {spec.TestName}");
+                string? display = $"func {string.Join(" ", spec.Arguments)}";
+                FileWriter.WriteLine($"Command: {display}");
+                FileWriter.WriteLine($"Working Directory: {spec.WorkingDirectory ?? "not specified"}");
+                FileWriter.WriteLine("====================================");
+
+                command.OnOutputLine(line =>
+                {
+                    try
+                    {
+                        // Write to the file if it's still open
+                        if (FileWriter is not null && FileWriter.BaseStream is not null)
+                        {
+                            FileWriter.WriteLine($"[STDOUT] {line}");
+                            FileWriter.Flush();
+                        }
+
+                        Log.WriteLine($"》   {line}");
+                        CommandOutputHandler?.Invoke(line);
+                    }
+                    catch (Exception ex)
+                    {
+                        Log.WriteLine($"Error writing to log file: {ex.Message}");
+                    }
+                });
+
+                command.OnErrorLine(line =>
+                {
+                    try
+                    {
+                        // Write to the file if it's still open
+                        if (FileWriter is not null && FileWriter.BaseStream is not null)
+                        {
+                            FileWriter.WriteLine($"[STDERR] {line}");
+                            FileWriter.Flush();
+                        }
+
+                        if (!string.IsNullOrEmpty(line))
+                        {
+                            Log.WriteLine($"❌   {line}");
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        Log.WriteLine($"Error writing to log file: {ex.Message}");
+                    }
+                });
+
+                Log.WriteLine($"Executing '{display}':");
+                Log.WriteLine($"Output being captured to: {LogFilePath}");
+
+                CommandResult result = ((Command)command).Execute(ProcessStartedHandler, FileWriter);
+
+                FileWriter.WriteLine("====================================");
+                FileWriter.WriteLine($"Command exited with code: {result.ExitCode}");
+                FileWriter.WriteLine($"=== Test ended at {DateTime.Now} ===");
+
+                Log.WriteLine($"Command '{display}' exited with exit code {result.ExitCode}.");
+
+                return result;
+            }
+            finally
+            {
+                // Make sure to close and dispose the writer
+                if (FileWriter is not null)
+                {
+                    try
+                    {
+                        FileWriter.Close();
+                        FileWriter.Dispose();
+                    }
+                    catch (Exception ex)
+                    {
+                        Log.WriteLine($"Error closing log file: {ex.Message}");
+                    }
+                }
+            }
+        }
+
+        public static void LogCommandResult(ITestOutputHelper log, CommandResult result)
+        {
+            log.WriteLine($"> {result.StartInfo.FileName} {result.StartInfo.Arguments}");
+            log.WriteLine(result.StdOut);
+
+            if (!string.IsNullOrEmpty(result.StdErr))
+            {
+                log.WriteLine(string.Empty);
+                log.WriteLine("StdErr:");
+                log.WriteLine(result.StdErr);
+            }
+
+            if (result.ExitCode != 0)
+            {
+                log.WriteLine($"Exit Code: {result.ExitCode}");
+            }
+        }
+    }
+}