diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..e721531 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,30 @@ +# This workflow will build a .NET project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net + +name: Main + +on: + push: + branches: [ "main" ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + dotnet-version: [ '8.0', '9.0' ] + + steps: + - uses: actions/checkout@v4 + - name: Setup .NET ${{ matrix.dotnet-version }} + uses: actions/setup-dotnet@v3 + with: + dotnet-version: ${{ matrix.dotnet-version }}.x + - name: Restore + run: dotnet restore ./Ocelot.Discovery.Nacos.sln -p:TargetFramework=net${{ matrix.dotnet-version }} + - name: Build + run: dotnet build --no-restore ./Ocelot.Discovery.Nacos.sln --framework net${{ matrix.dotnet-version }} + - name: Unit Tests + run: dotnet test --no-restore --no-build --verbosity minimal --framework net${{ matrix.dotnet-version }} ./unit/Ocelot.Discovery.Nacos.UnitTests.csproj + # - name: Acceptance Tests + # run: dotnet test --no-restore --no-build --verbosity minimal --framework net${{ matrix.dotnet-version }} ./acceptance/Ocelot.Discovery.Nacos.AcceptanceTests.csproj diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..d1cee2e --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,24 @@ +# This workflow will build a .NET project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net + +name: PR + +on: pull_request + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup .NET 8.0 + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 8.0.x + - name: Restore + run: dotnet restore ./Ocelot.Discovery.Nacos.sln -p:TargetFramework=net8.0 + - name: Build + run: dotnet build --no-restore ./Ocelot.Discovery.Nacos.sln --framework net8.0 + - name: Unit Tests + run: dotnet test --no-restore --no-build --verbosity normal --framework net8.0 ./unit/Ocelot.Discovery.Nacos.UnitTests.csproj + # - name: Acceptance Tests + # run: dotnet test --no-restore --no-build --verbosity normal --framework net8.0 ./acceptance/Ocelot.Discovery.Nacos.AcceptanceTests.csproj diff --git a/.gitignore b/.gitignore index a4fe18b..1384b31 100644 --- a/.gitignore +++ b/.gitignore @@ -398,3 +398,5 @@ FodyWeavers.xsd # JetBrains Rider *.sln.iml +.idea/* +/Ocelot.Discovery.Nacos/.idea diff --git a/Ocelot.Discovery.Nacos.sln b/Ocelot.Discovery.Nacos.sln new file mode 100644 index 0000000..f264491 --- /dev/null +++ b/Ocelot.Discovery.Nacos.sln @@ -0,0 +1,43 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35527.113 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ocelot.Discovery.Nacos", "src\Ocelot.Discovery.Nacos.csproj", "{CEF24699-3E41-D971-ACCA-FEF5CCB2011C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ocelot.Discovery.Nacos.UnitTests", "unit\Ocelot.Discovery.Nacos.UnitTests.csproj", "{FAD17C0B-4F8F-99A6-1419-9665E4210346}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ocelot.Discovery.Nacos.AcceptanceTests", "acceptance\Ocelot.Discovery.Nacos.AcceptanceTests.csproj", "{D14C0D65-EDF4-42DA-8028-948E6A15F41B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}" + ProjectSection(SolutionItems) = preProject + .github\workflows\main.yml = .github\workflows\main.yml + .github\workflows\pr.yml = .github\workflows\pr.yml + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CEF24699-3E41-D971-ACCA-FEF5CCB2011C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CEF24699-3E41-D971-ACCA-FEF5CCB2011C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CEF24699-3E41-D971-ACCA-FEF5CCB2011C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CEF24699-3E41-D971-ACCA-FEF5CCB2011C}.Release|Any CPU.Build.0 = Release|Any CPU + {FAD17C0B-4F8F-99A6-1419-9665E4210346}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FAD17C0B-4F8F-99A6-1419-9665E4210346}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FAD17C0B-4F8F-99A6-1419-9665E4210346}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FAD17C0B-4F8F-99A6-1419-9665E4210346}.Release|Any CPU.Build.0 = Release|Any CPU + {D14C0D65-EDF4-42DA-8028-948E6A15F41B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D14C0D65-EDF4-42DA-8028-948E6A15F41B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D14C0D65-EDF4-42DA-8028-948E6A15F41B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D14C0D65-EDF4-42DA-8028-948E6A15F41B}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {0E583663-7152-4575-9AE5-FF3D78AEAA1C} + EndGlobalSection +EndGlobal diff --git a/acceptance/ConcurrentSteps.cs b/acceptance/ConcurrentSteps.cs new file mode 100644 index 0000000..f65586f --- /dev/null +++ b/acceptance/ConcurrentSteps.cs @@ -0,0 +1,273 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using Ocelot.Discovery.Nacos.AcceptanceTests.LoadBalancer; +using Ocelot.LoadBalancer; +using Shouldly; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Net; +using System.Runtime.CompilerServices; +using System.Text; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests; + +public class ConcurrentSteps : Steps, IDisposable +{ + protected Task[] _tasks; + protected ServiceHandler[] _handlers; + protected ConcurrentDictionary _responses; + protected volatile int[] _counters; + + public ConcurrentSteps() + { + _tasks = Array.Empty(); + _handlers = Array.Empty(); + _responses = new(); + _counters = Array.Empty(); + } + + public override void Dispose() + { + foreach (var handler in _handlers) + { + handler?.Dispose(); + } + + foreach (var response in _responses.Values) + { + response?.Dispose(); + } + + foreach (var task in _tasks) + { + task?.Dispose(); + } + + base.Dispose(); + GC.SuppressFinalize(this); + } + + protected void GivenServiceInstanceIsRunning(string url, string response) + => GivenServiceInstanceIsRunning(url, response, HttpStatusCode.OK); + + protected void GivenServiceInstanceIsRunning(string url, string response, HttpStatusCode statusCode) + { + _handlers = new ServiceHandler[1]; // allocate single instance + _counters = new int[1]; // single counter + GivenServiceIsRunning(url, response, 0, statusCode); + _counters[0] = 0; + } + + protected void GivenThereIsAServiceRunningOn(string url, string basePath, string responseBody) + { + var handler = new ServiceHandler(); + _handlers = new ServiceHandler[] { handler }; + handler.GivenThereIsAServiceRunningOn(url, basePath, MapGet(basePath, responseBody)); + } + + protected void GivenMultipleServiceInstancesAreRunning(string[] urls, [CallerMemberName] string serviceName = null) + { + serviceName ??= new Uri(urls[0]).Host; + string[] responses = urls.Select(u => $"{serviceName}|url({u})").ToArray(); + GivenMultipleServiceInstancesAreRunning(urls, responses, HttpStatusCode.OK); + } + + protected void GivenMultipleServiceInstancesAreRunning(string[] urls, string[] responses) + => GivenMultipleServiceInstancesAreRunning(urls, responses, HttpStatusCode.OK); + + protected void GivenMultipleServiceInstancesAreRunning(string[] urls, string[] responses, HttpStatusCode statusCode) + { + Debug.Assert(urls.Length == responses.Length, "Length mismatch!"); + _handlers = new ServiceHandler[urls.Length]; // allocate multiple instances + _counters = new int[urls.Length]; // multiple counters + for (int i = 0; i < urls.Length; i++) + { + GivenServiceIsRunning(urls[i], responses[i], i, statusCode); + _counters[i] = 0; + } + } + + private void GivenServiceIsRunning(string url, string response) + => GivenServiceIsRunning(url, response, 0, HttpStatusCode.OK); + private void GivenServiceIsRunning(string url, string response, int index) + => GivenServiceIsRunning(url, response, index, HttpStatusCode.OK); + + private void GivenServiceIsRunning(string url, string response, int index, HttpStatusCode successCode) + { + response ??= successCode.ToString(); + _handlers[index] ??= new(); + var serviceHandler = _handlers[index]; + serviceHandler.GivenThereIsAServiceRunningOn(url, MapGet(index, response, successCode)); + } + + protected static RequestDelegate MapGet(string path, string responseBody) => MapGet(path, responseBody, HttpStatusCode.OK); + protected static RequestDelegate MapGet(string path, string responseBody, HttpStatusCode statusCode) => async context => + { + var downstreamPath = !string.IsNullOrEmpty(context.Request.PathBase.Value) + ? context.Request.PathBase.Value + : context.Request.Path.Value; + bool isMatch = downstreamPath == path; + context.Response.StatusCode = (int)(isMatch ? statusCode : HttpStatusCode.NotFound); + await context.Response.WriteAsync(isMatch ? responseBody : "Not Found"); + }; + + public static class HeaderNames + { + public const string ServiceIndex = nameof(LeaseEventArgs.ServiceIndex); + public const string Host = nameof(Uri.Host); + public const string Port = nameof(Uri.Port); + public const string Counter = nameof(Counter); + } + + protected RequestDelegate MapGet(int index, string body) => MapGet(index, body, HttpStatusCode.OK); + protected RequestDelegate MapGet(int index, string body, HttpStatusCode successCode) => async context => + { + // Don't delay during the first service call + if (Volatile.Read(ref _counters[index]) > 0) + { + await Task.Delay(Random.Shared.Next(5, 15)); // emulate integration delay up to 15 milliseconds + } + + string responseBody; + var request = context.Request; + var response = context.Response; + try + { + int count = Interlocked.Increment(ref _counters[index]); + responseBody = string.Concat(count, ':', body); + + response.StatusCode = (int)successCode; + response.Headers.Append(HeaderNames.ServiceIndex, new StringValues(index.ToString())); + response.Headers.Append(HeaderNames.Host, new StringValues(request.Host.Host)); + response.Headers.Append(HeaderNames.Port, new StringValues(request.Host.Port.ToString())); + response.Headers.Append(HeaderNames.Counter, new StringValues(count.ToString())); + await response.WriteAsync(responseBody); + } + catch (Exception exception) + { + responseBody = string.Concat(1, ':', exception.StackTrace); + response.StatusCode = (int)HttpStatusCode.InternalServerError; + await response.WriteAsync(responseBody); + } + }; + + public Task[] WhenIGetUrlOnTheApiGatewayConcurrently(string url, int times) + => RunParallelRequests(times, (i) => url); + + public Task[] WhenIGetUrlOnTheApiGatewayConcurrently(int times, params string[] urls) + => RunParallelRequests(times, (i) => urls[i % urls.Length]); + + protected Task[] RunParallelRequests(int times, Func urlFunc) + { + _tasks = new Task[times]; + _responses = new(times, times); + for (var i = 0; i < times; i++) + { + var url = urlFunc(i); + _tasks[i] = GetParallelResponse(url, i); + _responses[i] = null; + } + + Task.WaitAll(_tasks); + return _tasks; + } + + private async Task GetParallelResponse(string url, int threadIndex) + { + var response = await _ocelotClient.GetAsync(url); + var content = await response.Content.ReadAsStringAsync(); + var counterString = content.Contains(':') + ? content.Split(':')[0] // let the first fragment is counter value + : "0"; + int count = int.Parse(counterString); + count.ShouldBeGreaterThan(0); + _responses[threadIndex] = response; + } + + public void ThenAllStatusCodesShouldBe(HttpStatusCode expected) + => _responses.ShouldAllBe(response => response.Value.StatusCode == expected); + public void ThenAllResponseBodiesShouldBe(string expectedBody) + => _responses.ShouldAllBe(response => response.Value.Content.ReadAsStringAsync().Result == expectedBody); + + protected string CalledTimesMessage() + => $"All values are [{string.Join(',', _counters)}]"; + + public void ThenAllServicesShouldHaveBeenCalledTimes(int expected) + => _counters.Sum().ShouldBe(expected, CalledTimesMessage()); + + public void ThenServiceShouldHaveBeenCalledTimes(int index, int expected) + => _counters[index].ShouldBe(expected, CalledTimesMessage()); + + public void ThenServicesShouldHaveBeenCalledTimes(params int[] expected) + { + for (int i = 0; i < expected.Length; i++) + { + _counters[i].ShouldBe(expected[i], CalledTimesMessage()); + } + } + + public static int Bottom(int totalRequests, int totalServices) + => totalRequests / totalServices; + public static int Top(int totalRequests, int totalServices) + { + int bottom = Bottom(totalRequests, totalServices); + return totalRequests - (bottom * totalServices) + bottom; + } + + public void ThenAllServicesCalledRealisticAmountOfTimes(int bottom, int top) + { + var customMessage = new StringBuilder() + .AppendLine($"{nameof(bottom)}: {bottom}") + .AppendLine($" {nameof(top)}: {top}") + .AppendLine($" All values are [{string.Join(',', _counters)}]") + .ToString(); + int sum = 0, totalSum = _counters.Sum(); + + // Last offline services cannot be called at all, thus don't assert zero counters + for (int i = 0; i < _counters.Length && sum < totalSum; i++) + { + int actual = _counters[i]; + actual.ShouldBeInRange(bottom, top, customMessage); + sum += actual; + } + } + + public void ThenAllServicesCalledOptimisticAmountOfTimes(ILoadBalancerAnalyzer analyzer) + { + if (analyzer == null) return; + int bottom = analyzer.BottomOfConnections(), + top = analyzer.TopOfConnections(); + ThenAllServicesCalledRealisticAmountOfTimes(bottom, top); // with unstable checkings + } + + public void ThenServiceCountersShouldMatchLeasingCounters(ILoadBalancerAnalyzer analyzer, int[] ports, int totalRequests) + { + if (analyzer == null || ports == null) + return; + + analyzer.ShouldNotBeNull().Analyze(); + analyzer.Events.Count.ShouldBe(totalRequests, $"{nameof(ILoadBalancerAnalyzer.ServiceName)}: {analyzer.ServiceName}"); + + var leasingCounters = analyzer?.GetHostCounters() ?? new(); + var sortedLeasingCountersByPort = ports.Select(port => leasingCounters.FirstOrDefault(kv => kv.Key.DownstreamPort == port).Value).ToArray(); + for (int i = 0; i < ports.Length; i++) + { + var host = leasingCounters.Keys.FirstOrDefault(k => k.DownstreamPort == ports[i]); + + // Leasing info/counters can be absent because of offline service instance with exact port in unstable scenario + if (host != null) + { + var customMessage = new StringBuilder() + .AppendLine($"{nameof(ILoadBalancerAnalyzer.ServiceName)}: {analyzer.ServiceName}") + .AppendLine($" Port: {ports[i]}") + .AppendLine($" Host: {host}") + .AppendLine($" Service counters: [{string.Join(',', _counters)}]") + .AppendLine($" Leasing counters: [{string.Join(',', sortedLeasingCountersByPort)}]") // should have order of _counters + .ToString(); + int counter1 = _counters[i]; + int counter2 = leasingCounters[host]; + counter1.ShouldBe(counter2, customMessage); + } + } + } +} diff --git a/acceptance/LoadBalancer/ILoadBalancerAnalyzer.cs b/acceptance/LoadBalancer/ILoadBalancerAnalyzer.cs new file mode 100644 index 0000000..0e46741 --- /dev/null +++ b/acceptance/LoadBalancer/ILoadBalancerAnalyzer.cs @@ -0,0 +1,18 @@ +using Ocelot.LoadBalancer; +using Ocelot.Values; +using System.Collections.Concurrent; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests.LoadBalancer; + +public interface ILoadBalancerAnalyzer +{ + string ServiceName { get; } + string GenerationPrefix { get; } + ConcurrentBag Events { get; } + object Analyze(); + Dictionary GetHostCounters(); + Dictionary ToHostCountersDictionary(IEnumerable> grouping); + bool HasManyServiceGenerations(int maxGeneration); + int BottomOfConnections(); + int TopOfConnections(); +} diff --git a/acceptance/LoadBalancer/LeastConnectionAnalyzer.cs b/acceptance/LoadBalancer/LeastConnectionAnalyzer.cs new file mode 100644 index 0000000..788010c --- /dev/null +++ b/acceptance/LoadBalancer/LeastConnectionAnalyzer.cs @@ -0,0 +1,28 @@ +using Microsoft.AspNetCore.Http; +using Ocelot.LoadBalancer; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.Values; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests.LoadBalancer; + +internal sealed class LeastConnectionAnalyzer : LoadBalancerAnalyzer, ILoadBalancer +{ + private readonly LeastConnection loadBalancer; + + public LeastConnectionAnalyzer(Func>> services, string serviceName) + : base(serviceName) + { + loadBalancer = new(services, serviceName); + loadBalancer.Leased += Me_Leased; + } + + private void Me_Leased(object sender, LeaseEventArgs args) => Events.Add(args); + + public override string Type => nameof(LeastConnectionAnalyzer); + public override Task> LeaseAsync(HttpContext httpContext) => loadBalancer.LeaseAsync(httpContext); + public override void Release(ServiceHostAndPort hostAndPort) => loadBalancer.Release(hostAndPort); + + public override Dictionary ToHostCountersDictionary(IEnumerable> grouping) + => grouping.ToDictionary(g => g.Key, g => g.Count(e => e.Lease == g.Key)); +} diff --git a/acceptance/LoadBalancer/LeastConnectionAnalyzerCreator.cs b/acceptance/LoadBalancer/LeastConnectionAnalyzerCreator.cs new file mode 100644 index 0000000..3819dc2 --- /dev/null +++ b/acceptance/LoadBalancer/LeastConnectionAnalyzerCreator.cs @@ -0,0 +1,22 @@ +using Ocelot.Configuration; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.ServiceDiscovery.Providers; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests.LoadBalancer; + +internal sealed class LeastConnectionAnalyzerCreator : ILoadBalancerCreator +{ + // We need to adhere to the same implementations of RoundRobinCreator, which results in a significant design overhead, (until redesigned) + public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) + { + var loadBalancer = new LeastConnectionAnalyzer( + serviceProvider.GetAsync, + !string.IsNullOrEmpty(route.ServiceName) // if service discovery mode then use service name; otherwise use balancer key + ? route.ServiceName + : route.LoadBalancerKey); + return new OkResponse(loadBalancer); + } + + public string Type => nameof(LeastConnectionAnalyzer); +} diff --git a/acceptance/LoadBalancer/LoadBalancerAnalyzer.cs b/acceptance/LoadBalancer/LoadBalancerAnalyzer.cs new file mode 100644 index 0000000..51da1bf --- /dev/null +++ b/acceptance/LoadBalancer/LoadBalancerAnalyzer.cs @@ -0,0 +1,118 @@ +using Microsoft.AspNetCore.Http; +using Ocelot.LoadBalancer; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.Values; +using System.Collections.Concurrent; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests.LoadBalancer; + +internal class LoadBalancerAnalyzer : ILoadBalancerAnalyzer, ILoadBalancer +{ + protected readonly string _serviceName; + protected LoadBalancerAnalyzer(string serviceName) => _serviceName = serviceName; + + public string ServiceName => _serviceName; + public virtual string GenerationPrefix => "Gen:"; + public ConcurrentBag Events { get; } = new(); + + public virtual object Analyze() + { + var allGenerations = Events + .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) + .Where(generation => !string.IsNullOrEmpty(generation)) + .Distinct().ToArray(); + var allIndices = Events.Select(e => e.ServiceIndex) + .Distinct().OrderBy(index => index).ToArray(); + + Dictionary> eventsPerGeneration = new(); + foreach (var generation in allGenerations) + { + var l = Events.Where(e => e.Service.Tags.Contains(generation)).ToList(); + eventsPerGeneration.Add(generation, l); + } + + Dictionary> generationIndices = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.ServiceIndex).Distinct().ToList(); + generationIndices.Add(generation, l); + } + + Dictionary> generationLeases = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.Lease).ToList(); + generationLeases.Add(generation, l); + } + + Dictionary> generationHosts = new(); + foreach (var generation in allGenerations) + { + var l = eventsPerGeneration[generation].Select(e => e.Lease.HostAndPort).Distinct().ToList(); + generationHosts.Add(generation, l); + } + + Dictionary> generationLeasesWithMaxConnections = new(); + foreach (var generation in allGenerations) + { + List leases = new(); + var uniqueHosts = generationHosts[generation]; + foreach (var host in uniqueHosts) + { + int max = generationLeases[generation].Where(l => l == host).Max(l => l.Connections); + Lease wanted = generationLeases[generation].Find(l => l == host && l.Connections == max); + leases.Add(wanted); + } + + leases = leases.OrderBy(l => l.HostAndPort.DownstreamPort).ToList(); + generationLeasesWithMaxConnections.Add(generation, leases); + } + + return generationLeasesWithMaxConnections; + } + + public virtual bool HasManyServiceGenerations(int maxGeneration) + { + int[] generations = new int[maxGeneration + 1]; + string[] tags = new string[maxGeneration + 1]; + for (int i = 0; i < generations.Length; i++) + { + generations[i] = i; + tags[i] = GenerationPrefix + i; + } + + var all = Events + .Select(e => e.Service.Tags.FirstOrDefault(t => t.StartsWith(GenerationPrefix))) + .Distinct().ToArray(); + return all.All(tags.Contains); + } + + public virtual Dictionary GetHostCounters() + { + var hosts = Events.Select(e => e.Lease.HostAndPort).Distinct().ToList(); + var grouping = Events + .GroupBy(e => e.Lease.HostAndPort) + .OrderBy(g => g.Key.DownstreamPort); + return ToHostCountersDictionary(grouping); + } + + public virtual Dictionary ToHostCountersDictionary(IEnumerable> grouping) + => grouping.ToDictionary(g => g.Key, g => g.Count(e => e.Lease == g.Key)); + + public virtual int BottomOfConnections() + { + var hostCounters = GetHostCounters(); + return hostCounters.Min(_ => _.Value); + } + + public virtual int TopOfConnections() + { + var hostCounters = GetHostCounters(); + return hostCounters.Max(_ => _.Value); + } + + public virtual string Type => nameof(LoadBalancerAnalyzer); + public virtual Task> LeaseAsync(HttpContext httpContext) => Task.FromResult>(new ErrorResponse(new UnableToFindLoadBalancerError(GetType().Name))); + public virtual void Release(ServiceHostAndPort hostAndPort) { } +} diff --git a/acceptance/LoadBalancer/LoadBalancerTests.cs b/acceptance/LoadBalancer/LoadBalancerTests.cs new file mode 100644 index 0000000..300303b --- /dev/null +++ b/acceptance/LoadBalancer/LoadBalancerTests.cs @@ -0,0 +1,127 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Ocelot.Configuration; +using Ocelot.Configuration.File; +using Ocelot.DependencyInjection; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.ServiceDiscovery.Providers; +using Ocelot.Values; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests; +using LoadBalancer; + +public sealed class LoadBalancerTests : ConcurrentSteps, IDisposable +{ + [Theory] + [Trait("Feat", "211")] + [InlineData(false)] // original scenario, clean config + [InlineData(true)] // extended scenario using analyzer + public void ShouldLoadBalanceRequestWithLeastConnection(bool withAnalyzer) + { + var ports = PortFinder.GetPorts(2); + var route = GivenRoute(withAnalyzer ? nameof(LeastConnectionAnalyzer) : nameof(LeastConnection), ports); + var configuration = GivenConfiguration(route); + var downstreamServiceUrls = ports.Select(DownstreamUrl).ToArray(); + LeastConnectionAnalyzer lbAnalyzer = null; + LeastConnectionAnalyzer getAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) + { + //lock (LoadBalancerHouse.SyncRoot) // Note, synch locking is implemented in LoadBalancerHouse + return lbAnalyzer ??= new LeastConnectionAnalyzerCreator().Create(route, provider)?.Data as LeastConnectionAnalyzer; + } + Action withLeastConnectionAnalyzer = (s) + => s.AddOcelot().AddCustomLoadBalancer(getAnalyzer); + GivenMultipleServiceInstancesAreRunning(downstreamServiceUrls); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(withAnalyzer ? withLeastConnectionAnalyzer : WithAddOcelot); + WhenIGetUrlOnTheApiGatewayConcurrently("/", 99); + ThenAllServicesShouldHaveBeenCalledTimes(99); + ThenAllServicesCalledOptimisticAmountOfTimes(lbAnalyzer); + ThenServiceCountersShouldMatchLeasingCounters(lbAnalyzer, ports, 99); + ThenAllServicesCalledRealisticAmountOfTimes(Bottom(99, ports.Length), Top(99, ports.Length)); + ThenServicesShouldHaveBeenCalledTimes(50, 49); // strict assertion + } + + [Theory] + [Trait("Bug", "365")] + [InlineData(false)] // original scenario, clean config + [InlineData(true)] // extended scenario using analyzer + public void ShouldLoadBalanceRequestWithRoundRobin(bool withAnalyzer) + { + var ports = PortFinder.GetPorts(2); + var route = GivenRoute(withAnalyzer ? nameof(RoundRobinAnalyzer) : nameof(RoundRobin), ports); + var configuration = GivenConfiguration(route); + var downstreamServiceUrls = ports.Select(DownstreamUrl).ToArray(); + RoundRobinAnalyzer lbAnalyzer = null; + RoundRobinAnalyzer getAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) + { + //lock (LoadBalancerHouse.SyncRoot) // Note, synch locking is implemented in LoadBalancerHouse + return lbAnalyzer ??= new RoundRobinAnalyzerCreator().Create(route, provider)?.Data as RoundRobinAnalyzer; + } + Action withRoundRobinAnalyzer = (s) + => s.AddOcelot().AddCustomLoadBalancer(getAnalyzer); + GivenMultipleServiceInstancesAreRunning(downstreamServiceUrls); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(withAnalyzer ? withRoundRobinAnalyzer : WithAddOcelot); + WhenIGetUrlOnTheApiGatewayConcurrently("/", 99); + ThenAllServicesShouldHaveBeenCalledTimes(99); + ThenAllServicesCalledOptimisticAmountOfTimes(lbAnalyzer); + ThenServiceCountersShouldMatchLeasingCounters(lbAnalyzer, ports, 99); + ThenAllServicesCalledRealisticAmountOfTimes(Bottom(99, ports.Length), Top(99, ports.Length)); + ThenServicesShouldHaveBeenCalledTimes(50, 49); // strict assertion + } + + [Fact] + [Trait("Feat", "961")] + public void ShouldLoadBalanceRequestWithCustomLoadBalancer() + { + Func loadBalancerFactoryFunc = + (serviceProvider, route, discoveryProvider) => new CustomLoadBalancer(discoveryProvider.GetAsync); + var ports = PortFinder.GetPorts(2); + var route = GivenRoute(nameof(CustomLoadBalancer), ports); + var configuration = GivenConfiguration(route); + var downstreamServiceUrls = ports.Select(DownstreamUrl).ToArray(); + Action withCustomLoadBalancer = (s) + => s.AddOcelot().AddCustomLoadBalancer(loadBalancerFactoryFunc); + GivenMultipleServiceInstancesAreRunning(downstreamServiceUrls); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(withCustomLoadBalancer); + WhenIGetUrlOnTheApiGatewayConcurrently("/", 50); + ThenAllServicesShouldHaveBeenCalledTimes(50); + ThenAllServicesCalledRealisticAmountOfTimes(Bottom(50, ports.Length), Top(50, ports.Length)); + ThenServicesShouldHaveBeenCalledTimes(25, 25); // strict assertion + } + + private sealed class CustomLoadBalancer : ILoadBalancer + { + private readonly Func>> _services; + private static readonly object _lock = new(); + private int _last; + + public string Type => nameof(CustomLoadBalancer); + public CustomLoadBalancer(Func>> services) => _services = services; + + public async Task> LeaseAsync(HttpContext httpContext) + { + var services = await _services(); + lock (_lock) + { + if (_last >= services.Count) _last = 0; + var next = services[_last++]; + return new OkResponse(next.HostAndPort); + } + } + + public void Release(ServiceHostAndPort hostAndPort) { } + } + + private FileRoute GivenRoute(string loadBalancer, params int[] ports) => new() + { + DownstreamPathTemplate = "/", + DownstreamScheme = Uri.UriSchemeHttp, + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new() { HttpMethods.Get }, + LoadBalancerOptions = new() { Type = loadBalancer ?? nameof(LeastConnection) }, + DownstreamHostAndPorts = ports.Select(Localhost).ToList(), + }; +} diff --git a/acceptance/LoadBalancer/RoundRobinAnalyzer.cs b/acceptance/LoadBalancer/RoundRobinAnalyzer.cs new file mode 100644 index 0000000..7100466 --- /dev/null +++ b/acceptance/LoadBalancer/RoundRobinAnalyzer.cs @@ -0,0 +1,30 @@ +using Microsoft.AspNetCore.Http; +using Ocelot.LoadBalancer; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.Values; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests.LoadBalancer; + +internal sealed class RoundRobinAnalyzer : LoadBalancerAnalyzer, ILoadBalancer +{ + private readonly RoundRobin loadBalancer; + + public RoundRobinAnalyzer(Func>> services, string serviceName) + : base(serviceName) + { + loadBalancer = new(services, serviceName); + loadBalancer.Leased += Me_Leased; + } + + private void Me_Leased(object sender, LeaseEventArgs args) => Events.Add(args); + + public override string Type => nameof(RoundRobinAnalyzer); + public override Task> LeaseAsync(HttpContext httpContext) => loadBalancer.LeaseAsync(httpContext); + public override void Release(ServiceHostAndPort hostAndPort) => loadBalancer.Release(hostAndPort); + + public override string GenerationPrefix => nameof(EndpointsV1.Metadata.Generation) + ":"; + + public override Dictionary ToHostCountersDictionary(IEnumerable> grouping) + => grouping.ToDictionary(g => g.Key, g => g.Max(e => e.Lease.Connections)); +} diff --git a/acceptance/LoadBalancer/RoundRobinAnalyzerCreator.cs b/acceptance/LoadBalancer/RoundRobinAnalyzerCreator.cs new file mode 100644 index 0000000..cc04db4 --- /dev/null +++ b/acceptance/LoadBalancer/RoundRobinAnalyzerCreator.cs @@ -0,0 +1,22 @@ +using Ocelot.Configuration; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Responses; +using Ocelot.ServiceDiscovery.Providers; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests.LoadBalancer; + +internal sealed class RoundRobinAnalyzerCreator : ILoadBalancerCreator +{ + // We need to adhere to the same implementations of RoundRobinCreator, which results in a significant design overhead, (until redesigned) + public Response Create(DownstreamRoute route, IServiceDiscoveryProvider serviceProvider) + { + var loadBalancer = new RoundRobinAnalyzer( + serviceProvider.GetAsync, + !string.IsNullOrEmpty(route.ServiceName) // if service discovery mode then use service name; otherwise use balancer key + ? route.ServiceName + : route.LoadBalancerKey); + return new OkResponse(loadBalancer); + } + + public string Type => nameof(RoundRobinAnalyzer); +} diff --git a/acceptance/NacosServiceDiscoveryTests.cs b/acceptance/NacosServiceDiscoveryTests.cs new file mode 100644 index 0000000..59d5224 --- /dev/null +++ b/acceptance/NacosServiceDiscoveryTests.cs @@ -0,0 +1,542 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Ocelot.Configuration; +using Ocelot.Configuration.File; +using Ocelot.DependencyInjection; +using Ocelot.LoadBalancer.LoadBalancers; +using Ocelot.Logging; +using Ocelot.ServiceDiscovery.Providers; +using Ocelot.Values; +using Shouldly; +using System.Net; +using System.Runtime.CompilerServices; +using System.Xml.Serialization; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests; +using LoadBalancer; + +public sealed class NacosServiceDiscoveryTests : ConcurrentSteps, IDisposable +{ + private readonly string _kubernetesUrl; + private readonly ServiceHandler _kubernetesHandler; + private string _receivedToken; + //private readonly Action _kubeClientOptionsConfigure; + + public NacosServiceDiscoveryTests() + { + _kubernetesUrl = DownstreamUrl(PortFinder.GetRandomPort()); + //_kubeClientOptionsConfigure = opts => + //{ + // opts.ApiEndPoint = new Uri(_kubernetesUrl); + // opts.AccessToken = "txpc696iUhbVoudg164r93CxDTrKRVWG"; + // opts.AuthStrategy = KubeAuthStrategy.BearerToken; + // opts.AllowInsecure = true; + //}; + _kubernetesHandler = new(); + } + + public override void Dispose() + { + _kubernetesHandler.Dispose(); + base.Dispose(); + } + + [Fact] + public async Task ShouldReturnServicesFromK8s() + { + const string namespaces = nameof(NacosServiceDiscoveryTests); + const string serviceName = nameof(ShouldReturnServicesFromK8s); + var servicePort = PortFinder.GetRandomPort(); + var downstreamUrl = LoopbackLocalhostUrl(servicePort); + var downstream = new Uri(downstreamUrl); + var subsetV1 = GivenSubsetAddress(downstream); + var endpoints = GivenEndpoints(subsetV1); + var route = GivenRouteWithServiceName(namespaces); + var configuration = GivenKubeConfiguration(namespaces, route); + var downstreamResponse = serviceName; + + GivenServiceInstanceIsRunning(downstreamUrl, downstreamResponse); + GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(WithKubernetes); + await WhenIGetUrlOnTheApiGateway("/"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenTheResponseBodyShouldBe($"1:{downstreamResponse}"); + ThenAllServicesShouldHaveBeenCalledTimes(1); + ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG"); + } + + [Theory] + [Trait("Feat", "1967")] + [InlineData("", HttpStatusCode.BadGateway)] + [InlineData("http", HttpStatusCode.OK)] + public async Task ShouldReturnServicesByPortNameAsDownstreamScheme(string downstreamScheme, HttpStatusCode statusCode) + { + const string serviceName = "example-web"; + const string namespaces = "default"; + var servicePort = PortFinder.GetRandomPort(); + var downstreamUrl = LoopbackLocalhostUrl(servicePort); + var downstream = new Uri(downstreamUrl); + var subsetV1 = GivenSubsetAddress(downstream); + + // Ports[0] -> port(https, 443) + // Ports[1] -> port(http, not 80) + subsetV1.Ports.Insert(0, new() + { + Name = "https", // This service instance is offline -> BadGateway + Port = 443, + }); + var endpoints = GivenEndpoints(subsetV1); + var route = GivenRouteWithServiceName(namespaces); + route.DownstreamPathTemplate = "/{url}"; + route.DownstreamScheme = downstreamScheme; // !!! Warning !!! Select port by name as scheme + route.UpstreamPathTemplate = "/api/example/{url}"; + route.ServiceName = serviceName; // "example-web" + var configuration = GivenKubeConfiguration(namespaces, route); + + GivenServiceInstanceIsRunning(downstreamUrl, nameof(ShouldReturnServicesByPortNameAsDownstreamScheme)); + GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(WithKubernetes); + await WhenIGetUrlOnTheApiGateway("/api/example/1"); + ThenTheStatusCodeShouldBe(statusCode); + ThenTheResponseBodyShouldBe(downstreamScheme == "http" + ? "1:" + nameof(ShouldReturnServicesByPortNameAsDownstreamScheme) + : string.Empty); + ThenAllServicesShouldHaveBeenCalledTimes(downstreamScheme == "http" ? 1 : 0); + ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG"); + } + + [Theory] + [Trait("Bug", "2110")] + [InlineData(1, 30)] + [InlineData(2, 50)] + [InlineData(3, 50)] + [InlineData(4, 50)] + [InlineData(5, 50)] + [InlineData(6, 99)] + [InlineData(7, 99)] + [InlineData(8, 99)] + [InlineData(9, 999)] + [InlineData(10, 999)] + public void ShouldHighlyLoadOnStableKubeProvider_WithRoundRobinLoadBalancing(int totalServices, int totalRequests) + { + const int ZeroGeneration = 0; + var (endpoints, servicePorts) = ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer(totalServices); + GivenThereIsAFakeKubernetesProvider(endpoints); // stable, services will not be removed from the list + + HighlyLoadOnKubeProviderAndRoundRobinBalancer(totalRequests, ZeroGeneration); + + int bottom = totalRequests / totalServices, + top = totalRequests - (bottom * totalServices) + bottom; + ThenAllServicesCalledRealisticAmountOfTimes(bottom, top); + ThenServiceCountersShouldMatchLeasingCounters(_roundRobinAnalyzer, servicePorts, totalRequests); + } + + [Theory] + [Trait("Bug", "2110")] + [InlineData(5, 50, 1)] + [InlineData(5, 50, 2)] + [InlineData(5, 50, 3)] + [InlineData(5, 50, 4)] + public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(int totalServices, int totalRequests, int k8sGeneration) + { + int failPerThreads = (totalRequests / k8sGeneration) - 1; // k8sGeneration means number of offline services + var (endpoints, servicePorts) = ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer(totalServices); + GivenThereIsAFakeKubernetesProvider(endpoints, false, k8sGeneration, failPerThreads); // false means unstable, k8sGeneration services will be removed from the list + + HighlyLoadOnKubeProviderAndRoundRobinBalancer(totalRequests, k8sGeneration); + + ThenAllServicesCalledOptimisticAmountOfTimes(_roundRobinAnalyzer); // with unstable checkings + ThenServiceCountersShouldMatchLeasingCounters(_roundRobinAnalyzer, servicePorts, totalRequests); + } + + [Fact] + [Trait("Feat", "2256")] + public async Task ShouldReturnServicesFromK8s_AddKubernetesWithNullConfigureOptions() + { + const string namespaces = nameof(NacosServiceDiscoveryTests); + const string serviceName = nameof(ShouldReturnServicesFromK8s_AddKubernetesWithNullConfigureOptions); + var servicePort = PortFinder.GetRandomPort(); + var downstreamUrl = LoopbackLocalhostUrl(servicePort); + var downstream = new Uri(downstreamUrl); + var subsetV1 = GivenSubsetAddress(downstream); + var endpoints = GivenEndpoints(subsetV1); + var route = GivenRouteWithServiceName(namespaces); + var configuration = GivenKubeConfiguration(namespaces, route, "txpc696iUhbVoudg164r93CxDTrKRVWG"); + var downstreamResponse = serviceName; + GivenServiceInstanceIsRunning(downstreamUrl, downstreamResponse); + GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(AddKubernetesWithNullConfigureOptions); + await WhenIGetUrlOnTheApiGateway("/"); + ThenTheStatusCodeShouldBe(HttpStatusCode.OK); + ThenTheResponseBodyShouldBe($"1:{downstreamResponse}"); + ThenAllServicesShouldHaveBeenCalledTimes(1); + ThenTheTokenIs("Bearer txpc696iUhbVoudg164r93CxDTrKRVWG"); + } + + private void AddKubernetesWithNullConfigureOptions(IServiceCollection services) + => services.AddOcelot(); //.AddKubernetes(configureOptions: null); + + private (EndpointsV1 Endpoints, int[] ServicePorts) ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer( + int totalServices, + [CallerMemberName] string serviceName = nameof(ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer)) + { + const string namespaces = nameof(NacosServiceDiscoveryTests); + var servicePorts = PortFinder.GetPorts(totalServices); + var downstreamUrls = servicePorts + .Select(port => LoopbackLocalhostUrl(port, Array.IndexOf(servicePorts, port))) + .ToArray(); // based on localhost aka loopback network interface + var downstreams = downstreamUrls.Select(url => new Uri(url)) + .ToList(); + var downstreamResponses = downstreams + .Select(ds => $"{serviceName}:{ds.Host}:{ds.Port}") + .ToArray(); + var subset = new EndpointSubsetV1(); + downstreams.ForEach(ds => GivenSubsetAddress(ds, subset)); + var endpoints = GivenEndpoints(subset, serviceName); // totalServices service instances with different ports + var route = GivenRouteWithServiceName(namespaces, serviceName, nameof(RoundRobinAnalyzer)); // !!! + var configuration = GivenKubeConfiguration(namespaces, route); + GivenMultipleServiceInstancesAreRunning(downstreamUrls, downstreamResponses); + GivenThereIsAConfiguration(configuration); + GivenOcelotIsRunningWithServices(WithKubernetesAndRoundRobin); + return (endpoints, servicePorts); + } + + private void HighlyLoadOnKubeProviderAndRoundRobinBalancer(int totalRequests, int k8sGenerationNo) + { + // Act + WhenIGetUrlOnTheApiGatewayConcurrently("/", totalRequests); // load by X parallel requests + + // Assert + _k8sCounter.ShouldBeGreaterThanOrEqualTo(totalRequests); // integration endpoint called times + _k8sServiceGeneration.ShouldBe(k8sGenerationNo); + ThenAllStatusCodesShouldBe(HttpStatusCode.OK); + ThenAllServicesShouldHaveBeenCalledTimes(totalRequests); + _roundRobinAnalyzer.ShouldNotBeNull().Analyze(); + _roundRobinAnalyzer.Events.Count.ShouldBe(totalRequests); + _roundRobinAnalyzer.HasManyServiceGenerations(k8sGenerationNo).ShouldBeTrue(); + } + + private void ThenTheTokenIs(string token) + { + _receivedToken.ShouldBe(token); + } + + private EndpointsV1 GivenEndpoints(EndpointSubsetV1 subset, [CallerMemberName] string serviceName = "") + { + var e = new EndpointsV1() + { + Kind = "endpoint", + ApiVersion = "1.0", + Metadata = new() + { + Name = serviceName, + Namespace = nameof(NacosServiceDiscoveryTests), + }, + }; + e.Subsets.Add(subset); + return e; + } + + private static EndpointSubsetV1 GivenSubsetAddress(Uri downstream, EndpointSubsetV1 subset = null) + { + subset ??= new(); + subset.Addresses.Add(new() + { + Ip = Dns.GetHostAddresses(downstream.Host).Select(x => x.ToString()).First(a => a.Contains('.')), // 127.0.0.1 + Hostname = downstream.Host, + }); + subset.Ports.Add(new() + { + Name = downstream.Scheme, + Port = downstream.Port, + }); + return subset; + } + + private FileRoute GivenRouteWithServiceName(string serviceNamespace, + [CallerMemberName] string serviceName = null, + string loadBalancerType = nameof(LeastConnection)) => new() + { + DownstreamPathTemplate = "/", + DownstreamScheme = null, // the scheme should not be defined in service discovery scenarios by default, only ServiceName + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new() { HttpMethods.Get }, + ServiceName = serviceName, // !!! + ServiceNamespace = serviceNamespace, + LoadBalancerOptions = new() { Type = loadBalancerType }, + }; + + private FileConfiguration GivenKubeConfiguration(string serviceNamespace, FileRoute route, string token = null) + { + var u = new Uri(_kubernetesUrl); + var configuration = GivenConfiguration(route); + configuration.GlobalConfiguration.ServiceDiscoveryProvider = new() + { + Scheme = u.Scheme, + Host = u.Host, + Port = u.Port, + Type = "", //nameof(Kube), + PollingInterval = 0, + Namespace = serviceNamespace, + Token = token ?? "Test", + }; + return configuration; + } + + private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, + [CallerMemberName] string serviceName = nameof(NacosServiceDiscoveryTests), string namespaces = nameof(NacosServiceDiscoveryTests)) + => GivenThereIsAFakeKubernetesProvider(endpoints, true, 0, 0, serviceName, namespaces); + + private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isStable, int offlineServicesNo, int offlinePerThreads, + [CallerMemberName] string serviceName = nameof(NacosServiceDiscoveryTests), string namespaces = nameof(NacosServiceDiscoveryTests)) + { + _k8sCounter = 0; + _kubernetesHandler.GivenThereIsAServiceRunningOn(_kubernetesUrl, async context => + { + await Task.Delay(Random.Shared.Next(1, 10)); // emulate integration delay up to 10 milliseconds + if (context.Request.Path.Value == $"/api/v1/namespaces/{namespaces}/endpoints/{serviceName}") + { + string json; + lock (K8sCounterLocker) + { + _k8sCounter++; + var subset = endpoints.Subsets[0]; + + // Each offlinePerThreads-th request to integrated K8s endpoint should fail + if (!isStable && _k8sCounter % offlinePerThreads == 0 && _k8sCounter >= offlinePerThreads) + { + while (offlineServicesNo-- > 0) + { + int index = subset.Addresses.Count - 1; // Random.Shared.Next(0, subset.Addresses.Count - 1); + subset.Addresses.RemoveAt(index); + subset.Ports.RemoveAt(index); + } + + _k8sServiceGeneration++; + } + + endpoints.Metadata.Generation = _k8sServiceGeneration; + json = JsonConvert.SerializeObject(endpoints/*,KubeResourceClient.SerializerSettings*/); + } + + if (context.Request.Headers.TryGetValue("Authorization", out var values)) + { + _receivedToken = values.First(); + } + + context.Response.Headers.Append("Content-Type", "application/json"); + await context.Response.WriteAsync(json); + } + }); + } + + private static ServiceDescriptor GetValidateScopesDescriptor() + => ServiceDescriptor.Singleton>( + new DefaultServiceProviderFactory(new() { ValidateScopes = true })); + private IOcelotBuilder AddKubernetes(IServiceCollection services) => services + .Replace(GetValidateScopesDescriptor()) + .AddOcelot();//.AddKubernetes(_kubeClientOptionsConfigure); + + private void WithKubernetes(IServiceCollection services) => AddKubernetes(services); + private void WithKubernetesAndRoundRobin(IServiceCollection services) => AddKubernetes(services) + .AddCustomLoadBalancer(GetRoundRobinAnalyzer); + //.Services.RemoveAll().AddSingleton(); + + private int _k8sCounter, _k8sServiceGeneration; + private static readonly object K8sCounterLocker = new(); + private RoundRobinAnalyzer _roundRobinAnalyzer; + private RoundRobinAnalyzer GetRoundRobinAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) + { + lock (K8sCounterLocker) + { + return _roundRobinAnalyzer ??= new RoundRobinAnalyzerCreator().Create(route, provider)?.Data as RoundRobinAnalyzer; //??= new RoundRobinAnalyzer(provider.GetAsync, route.ServiceName); + } + } +} + +internal class FakeKubeServiceCreator //: KubeServiceCreator +{ + public FakeKubeServiceCreator(IOcelotLoggerFactory factory) { } //: base(factory) { } + + protected /*override*/ ServiceHostAndPort GetServiceHostAndPort(/*KubeRegistryConfiguration configuration,*/ EndpointsV1 endpoint, EndpointSubsetV1 subset, EndpointAddressV1 address) + { + var ports = subset.Ports; + var index = subset.Addresses.IndexOf(address); + var portV1 = ports[index]; + //Logger.LogDebug(() => $"K8s service with key '{configuration.KeyOfServiceInK8s}' and address {address.Ip}; Detected port is {portV1.Name}:{portV1.Port}. Total {ports.Count} ports of [{string.Join(',', ports.Select(p => p.Name))}]."); + return new ServiceHostAndPort(address.Ip, (int)portV1.Port, portV1.Name); + } + + protected /*override*/ IEnumerable GetServiceTags(/*KubeRegistryConfiguration configuration,*/ EndpointsV1 endpoint, EndpointSubsetV1 subset, EndpointAddressV1 address) + { + var tags = new List(); //base.GetServiceTags(configuration, endpoint, subset, address).ToList(); + long gen = endpoint.Metadata.Generation ?? 0L; + tags.Add($"{nameof(endpoint.Metadata.Generation)}:{gen}"); + return tags; + } +} + +public class EndpointsV1 //: KubeResourceV1 +{ + public List Subsets { get; } = new List(); + public bool ShouldSerializeSubsets() => Subsets.Count > 0; + + [JsonProperty("metadata")] + public ObjectMetaV1 Metadata { get; set; } + + [JsonProperty("kind")] + public string Kind { get; set; } + + [JsonProperty("apiVersion")] + public string ApiVersion { get; set; } +} + +public class EndpointSubsetV1 +{ + [JsonProperty("addresses", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + public List Addresses { get; } = new List(); + + + [JsonProperty("notReadyAddresses", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + public List NotReadyAddresses { get; } = new List(); + + + [JsonProperty("ports", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + public List Ports { get; } = new List(); + + + public bool ShouldSerializeAddresses() + { + return Addresses.Count > 0; + } + + public bool ShouldSerializeNotReadyAddresses() + { + return NotReadyAddresses.Count > 0; + } + + public bool ShouldSerializePorts() + { + return Ports.Count > 0; + } +} + +public class EndpointAddressV1 +{ + [JsonProperty("hostname", NullValueHandling = NullValueHandling.Ignore)] + public string Hostname { get; set; } + + [JsonProperty("nodeName", NullValueHandling = NullValueHandling.Ignore)] + public string NodeName { get; set; } + + //[JsonProperty("targetRef", NullValueHandling = NullValueHandling.Ignore)] + //public ObjectReferenceV1 TargetRef { get; set; } + + [JsonProperty("ip", NullValueHandling = NullValueHandling.Include)] + public string Ip { get; set; } +} + +public class EndpointPortV1 +{ + [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] + public string Name { get; set; } + + [JsonProperty("appProtocol", NullValueHandling = NullValueHandling.Ignore)] + public string AppProtocol { get; set; } + + [JsonProperty("protocol", NullValueHandling = NullValueHandling.Ignore)] + public string Protocol { get; set; } + + [JsonProperty("port", NullValueHandling = NullValueHandling.Ignore)] + public int? Port { get; set; } +} + +public class ObjectMetaV1 +{ + [JsonExtensionData] + private readonly Dictionary _extensionData = new Dictionary(); + + [JsonProperty("uid", NullValueHandling = NullValueHandling.Ignore)] + public string Uid { get; set; } + + [JsonProperty("generateName", NullValueHandling = NullValueHandling.Ignore)] + public string GenerateName { get; set; } + + [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] + public string Name { get; set; } + + [JsonProperty("namespace", NullValueHandling = NullValueHandling.Ignore)] + public string Namespace { get; set; } + + [JsonProperty("selfLink", NullValueHandling = NullValueHandling.Ignore)] + public string SelfLink { get; set; } + + [JsonProperty("generation", NullValueHandling = NullValueHandling.Ignore)] + public long? Generation { get; set; } + + [JsonProperty("resourceVersion", NullValueHandling = NullValueHandling.Ignore)] + public string ResourceVersion { get; set; } + + [JsonProperty("creationTimestamp", NullValueHandling = NullValueHandling.Ignore)] + public DateTime? CreationTimestamp { get; set; } + + [JsonProperty("deletionTimestamp", NullValueHandling = NullValueHandling.Ignore)] + public DateTime? DeletionTimestamp { get; set; } + + [JsonProperty("annotations", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + public Dictionary Annotations { get; } = new Dictionary(); + + + [JsonProperty("deletionGracePeriodSeconds", NullValueHandling = NullValueHandling.Ignore)] + public long? DeletionGracePeriodSeconds { get; set; } + + [JsonProperty("finalizers", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + public List Finalizers { get; } = new List(); + + + [JsonProperty("labels", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + public Dictionary Labels { get; } = new Dictionary(); + + + //[JsonProperty("managedFields", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + //public List ManagedFields { get; } = new List(); + + + //[JsonProperty("ownerReferences", ObjectCreationHandling = ObjectCreationHandling.Reuse)] + //public List OwnerReferences { get; } = new List(); + + + [JsonIgnore] + public IDictionary ExtensionData => _extensionData; + + public bool ShouldSerializeAnnotations() + { + return Annotations.Count > 0; + } + + public bool ShouldSerializeFinalizers() + { + return Finalizers.Count > 0; + } + + public bool ShouldSerializeLabels() + { + return Labels.Count > 0; + } + + //public bool ShouldSerializeManagedFields() + //{ + // return ManagedFields.Count > 0; + //} + + //public bool ShouldSerializeOwnerReferences() + //{ + // return OwnerReferences.Count > 0; + //} +} diff --git a/acceptance/Ocelot.Discovery.Nacos.AcceptanceTests.csproj b/acceptance/Ocelot.Discovery.Nacos.AcceptanceTests.csproj new file mode 100644 index 0000000..8d82f15 --- /dev/null +++ b/acceptance/Ocelot.Discovery.Nacos.AcceptanceTests.csproj @@ -0,0 +1,48 @@ + + + + net6.0;net8.0 + latest + enable + enable + + false + true + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/acceptance/PortFinder.cs b/acceptance/PortFinder.cs new file mode 100644 index 0000000..73158d3 --- /dev/null +++ b/acceptance/PortFinder.cs @@ -0,0 +1,68 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests; + +public static class PortFinder +{ + private const int EndPortRange = 45000; + private static int CurrentPort = 20000; + private static readonly object LockObj = new(); + private static readonly ConcurrentBag UsedPorts = new(); + + /// + /// Gets a pseudo-random port from the range [, ] for one testing scenario. + /// + /// New allocated port. + /// Critical situation where available ports range has been exceeded. + public static int GetRandomPort() + { + lock (LockObj) + { + ExceedingPortRangeException.ThrowIf(CurrentPort > EndPortRange); + return UsePort(CurrentPort++); + } + } + + /// + /// Gets the exact number of ports from the range [, ] for one testing scenario. + /// + /// The number of wanted ports. + /// Array of allocated ports. + /// Critical situation where available ports range has been exceeded. + public static int[] GetPorts(int count) + { + var ports = new int[count]; + lock (LockObj) + { + for (int i = 0; i < count; i++, CurrentPort++) + { + ExceedingPortRangeException.ThrowIf(CurrentPort > EndPortRange); + ports[i] = UsePort(CurrentPort); + } + } + return ports; + } + + private static int UsePort(int port) + { + UsedPorts.Add(port); // TODO Review or remove, now useless + + var ipe = new IPEndPoint(IPAddress.Loopback, port); + + using var socket = new Socket(ipe.AddressFamily, SocketType.Stream, ProtocolType.Tcp); + socket.Bind(ipe); + socket.Close(); + return port; + } +} + +public class ExceedingPortRangeException : Exception +{ + public ExceedingPortRangeException() + : base("Cannot find available port to bind to!") { } + + public static void ThrowIf(bool condition) + => _ = condition ? throw new ExceedingPortRangeException() : 0; +} diff --git a/acceptance/ServiceHandler.cs b/acceptance/ServiceHandler.cs new file mode 100644 index 0000000..4c2a4d7 --- /dev/null +++ b/acceptance/ServiceHandler.cs @@ -0,0 +1,128 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System.Net; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests; + +public class ServiceHandler : IDisposable +{ + private IWebHost _builder; + + public void GivenThereIsAServiceRunningOn(string baseUrl, RequestDelegate handler) + { + _builder = TestHostBuilder.Create() + .UseUrls(baseUrl) + .UseKestrel() + .UseContentRoot(Directory.GetCurrentDirectory()) + .UseIISIntegration() + .Configure(app => + { + app.Run(handler); + }) + .Build(); + + _builder.Start(); + } + + public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, RequestDelegate handler) + { + _builder = TestHostBuilder.Create() + .UseUrls(baseUrl) + .UseKestrel() + .UseContentRoot(Directory.GetCurrentDirectory()) + .UseIISIntegration() + .Configure(app => + { + app.UsePathBase(basePath); + app.Run(handler); + }) + .Build(); + + _builder.Start(); + } + + public void GivenThereIsAServiceRunningOnWithKestrelOptions(string baseUrl, string basePath, Action options, RequestDelegate handler) + { + _builder = TestHostBuilder.Create() + .UseUrls(baseUrl) + .UseKestrel() + .ConfigureKestrel(options ?? WithDefaultKestrelServerOptions) // ! + .UseContentRoot(Directory.GetCurrentDirectory()) + .UseIISIntegration() + .Configure(app => + { + app.UsePathBase(basePath); + app.Run(handler); + }) + .Build(); + + _builder.Start(); + } + + internal void WithDefaultKestrelServerOptions(KestrelServerOptions options) + { + } + + public void GivenThereIsAServiceRunningOn(string baseUrl, string basePath, string fileName, string password, int port, RequestDelegate handler) + { + _builder = TestHostBuilder.Create() + .UseUrls(baseUrl) + .UseKestrel(options => + { + options.Listen(IPAddress.Loopback, port, listenOptions => + { + listenOptions.UseHttps(fileName, password); + }); + }) + .UseContentRoot(Directory.GetCurrentDirectory()) + .Configure(app => + { + app.UsePathBase(basePath); + app.Run(handler); + }) + .Build(); + + _builder.Start(); + } + + public async Task StartFakeDownstreamService(string url, Func, Task> middleware) + { + _builder = TestHostBuilder.Create() + .ConfigureServices(s => { }).UseKestrel() + .UseUrls(url) + .UseContentRoot(Directory.GetCurrentDirectory()) + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: false); + config.AddEnvironmentVariables(); + }) + .ConfigureLogging((hostingContext, logging) => + { + logging.AddConfiguration(hostingContext.Configuration.GetSection("Logging")); + logging.AddConsole(); + }) + .Configure(app => + { + app.UseWebSockets(); + app.Use(middleware); + }) + .UseIISIntegration() + .Build(); + + await _builder.StartAsync(); + } + + public void Dispose() + { + _builder?.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/acceptance/Steps.cs b/acceptance/Steps.cs new file mode 100644 index 0000000..d0afda9 --- /dev/null +++ b/acceptance/Steps.cs @@ -0,0 +1,1007 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +//using IdentityServer4.AccessTokenValidation; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Ocelot.Configuration.ChangeTracking; +using Ocelot.Configuration.Creator; +using Ocelot.Configuration.File; +using Ocelot.Configuration.Repository; +using Ocelot.DependencyInjection; +using Ocelot.Logging; +using Ocelot.Middleware; +using System.Diagnostics; +using System.IO.Compression; +using System.Net.Http.Headers; +using System.Text; +//using static Ocelot.AcceptanceTests.HttpDelegatingHandlersTests; +using ConfigurationBuilder = Microsoft.Extensions.Configuration.ConfigurationBuilder; +using CookieHeaderValue = Microsoft.Net.Http.Headers.CookieHeaderValue; +using MediaTypeHeaderValue = System.Net.Http.Headers.MediaTypeHeaderValue; +using Shouldly; +using System.Net; +using Moq; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests; + +public class Steps : IDisposable +{ + protected TestServer _ocelotServer; + protected HttpClient _ocelotClient; + protected HttpResponseMessage _response; + private HttpContent _postContent; + private BearerToken _token; + public string RequestIdKey = "OcRequestId"; + private readonly Random _random; + protected readonly Guid _testId; + protected readonly string _ocelotConfigFileName; + + // TODO Merge both members + protected IWebHostBuilder _webHostBuilder; + protected IWebHostBuilder _ocelotBuilder; + + private IWebHost _ocelotHost; // TODO remove because of one reference + private IOcelotConfigurationChangeTokenSource _changeToken; + + public Steps() + { + _random = new Random(); + _testId = Guid.NewGuid(); + _ocelotConfigFileName = $"{_testId:N}-{ConfigurationBuilderExtensions.PrimaryConfigFile}"; + Files = new() { _ocelotConfigFileName }; + Folders = new(); + } + + protected List Files { get; } + protected List Folders { get; } + protected string TestID { get => _testId.ToString("N"); } + + protected static FileHostAndPort Localhost(int port) => new("localhost", port); + protected static string DownstreamUrl(int port) => $"{Uri.UriSchemeHttp}://localhost:{port}"; + protected static string LoopbackLocalhostUrl(int port, int loopbackIndex = 0) => $"{Uri.UriSchemeHttp}://127.0.0.{++loopbackIndex}:{port}"; + + protected static FileConfiguration GivenConfiguration(params FileRoute[] routes) => new() + { + Routes = new(routes), + }; + + protected static FileRoute GivenDefaultRoute(int port) => new() + { + DownstreamPathTemplate = "/", + DownstreamHostAndPorts = new() { Localhost(port) }, + DownstreamScheme = Uri.UriSchemeHttp, + UpstreamPathTemplate = "/", + UpstreamHttpMethod = new() { HttpMethods.Get }, + }; + + public async Task ThenConfigShouldBe(FileConfiguration fileConfig) + { + var internalConfigCreator = _ocelotServer.Host.Services.GetService(); + var internalConfigRepo = _ocelotServer.Host.Services.GetService(); + + var internalConfig = internalConfigRepo.Get(); + var config = await internalConfigCreator.Create(fileConfig); + + internalConfig.Data.RequestId.ShouldBe(config.Data.RequestId); + } + + //public async Task ThenConfigShouldBeWithTimeout(FileConfiguration fileConfig, int timeoutMs) + //{ + // var result = await Wait.WaitFor(timeoutMs).UntilAsync(async () => + // { + // var internalConfigCreator = _ocelotServer.Host.Services.GetService(); + // var internalConfigRepo = _ocelotServer.Host.Services.GetService(); + // var internalConfig = internalConfigRepo.Get(); + // var config = await internalConfigCreator.Create(fileConfig); + // return internalConfig.Data.RequestId == config.Data.RequestId; + // }); + // result.ShouldBe(true); + //} + + /// + /// TODO Move to . See references. + /// + /// Task. + public async Task StartFakeOcelotWithWebSockets() + { + _ocelotBuilder = TestHostBuilder.Create(); + _ocelotBuilder.ConfigureServices(s => + { + s.AddSingleton(_ocelotBuilder); + s.AddOcelot(); + }); + _ocelotBuilder.UseKestrel() + .UseUrls("http://localhost:5000") + .UseContentRoot(Directory.GetCurrentDirectory()) + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, false, false); + config.AddEnvironmentVariables(); + }) + .ConfigureLogging((hostingContext, logging) => + { + logging.AddConfiguration(hostingContext.Configuration.GetSection("Logging")); + logging.AddConsole(); + }) + .Configure(async app => + { + app.UseWebSockets(); + await app.UseOcelot(); + }) + .UseIISIntegration(); + _ocelotHost = _ocelotBuilder.Build(); + await _ocelotHost.StartAsync(); + } + + public void GivenThereIsAConfiguration(FileConfiguration fileConfiguration) + => GivenThereIsAConfiguration(fileConfiguration, _ocelotConfigFileName); + public void GivenThereIsAConfiguration(FileConfiguration from, string toFile) + { + var json = SerializeJson(from, ref toFile); + File.WriteAllText(toFile, json); + } + public Task GivenThereIsAConfigurationAsync(FileConfiguration from, string toFile) + { + var json = SerializeJson(from, ref toFile); + return File.WriteAllTextAsync(toFile, json); + } + protected string SerializeJson(FileConfiguration from, ref string toFile) + { + toFile ??= _ocelotConfigFileName; + Files.Add(toFile); // register for disposing + return JsonConvert.SerializeObject(from, Formatting.Indented); + } + + protected virtual void DeleteFiles() + { + foreach (var file in Files) + { + if (!File.Exists(file)) + { + continue; + } + + try + { + File.Delete(file); + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + } + + protected virtual void DeleteFolders() + { + foreach (var folder in Folders) + { + try + { + var f = new DirectoryInfo(folder); + if (f.Exists && f.FullName != AppContext.BaseDirectory) + { + f.Delete(true); + } + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + } + + public void ThenTheResponseBodyHeaderIs(string key, string value) + { + var header = _response.Content.Headers.GetValues(key); + header.First().ShouldBe(value); + } + + public void GivenOcelotIsRunningReloadingConfig(bool shouldReload) + { + StartOcelot((_, config) => config.AddJsonFile(_ocelotConfigFileName, false, shouldReload)); + } + + public void GivenIHaveAChangeToken() + { + _changeToken = _ocelotServer.Host.Services.GetRequiredService(); + } + + /// + /// This is annoying cos it should be in the constructor but we need to set up the file before calling startup so its a step. + /// + public void GivenOcelotIsRunning() + { + StartOcelot((_, config) => config.AddJsonFile(_ocelotConfigFileName, false, false)); + } + + protected void StartOcelot(Action configureAddOcelot, string environmentName = null) + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + var env = hostingContext.HostingEnvironment; + config.SetBasePath(env.ContentRootPath); + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + configureAddOcelot.Invoke(hostingContext, config); // config.AddOcelot(...); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(WithAddOcelot) + .Configure(WithUseOcelot) + .UseEnvironment(environmentName ?? nameof(AcceptanceTests)); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + public void ThenTheTraceHeaderIsSet(string key) + { + var header = _response.Headers.GetValues(key); + header.First().ShouldNotBeNullOrEmpty(); + } + + //internal void GivenOcelotIsRunningUsingButterfly(string butterflyUrl) + //{ + // _webHostBuilder = TestHostBuilder.Create() + // .ConfigureAppConfiguration((hostingContext, config) => + // { + // config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + // var env = hostingContext.HostingEnvironment; + // config.AddJsonFile("appsettings.json", true, false) + // .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + // config.AddJsonFile(_ocelotConfigFileName, true, false); + // config.AddEnvironmentVariables(); + // }) + // .ConfigureServices(s => + // { + // s.AddOcelot() + // .AddButterfly(option => + // { + // //this is the url that the butterfly collector server is running on... + // option.CollectorUrl = butterflyUrl; + // option.Service = "Ocelot"; + // }); + // }) + // .Configure(async app => + // { + // app.Use(async (_, next) => { await next.Invoke(); }); + // await app.UseOcelot(); + // }); + + // _ocelotServer = new TestServer(_webHostBuilder); + // _ocelotClient = _ocelotServer.CreateClient(); + //} + + //public async Task WhenIGetUrlOnTheApiGatewayWaitingForTheResponseToBeOk(string url) + //{ + // var result = await Wait.WaitFor(2000).UntilAsync(async () => + // { + // try + // { + // _response = await _ocelotClient.GetAsync(url); + // _response.EnsureSuccessStatusCode(); + // return true; + // } + // catch (Exception) + // { + // return false; + // } + // }); + + // result.ShouldBeTrue(); + //} + + //public void GivenOcelotIsRunningUsingJsonSerializedCache() + //{ + // _webHostBuilder = TestHostBuilder.Create() + // .ConfigureAppConfiguration((hostingContext, config) => + // { + // config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + // var env = hostingContext.HostingEnvironment; + // config.AddJsonFile("appsettings.json", true, false) + // .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + // config.AddJsonFile(_ocelotConfigFileName, false, false); + // config.AddEnvironmentVariables(); + // }) + // .ConfigureServices(s => + // { + // s.AddOcelot() + // .AddCacheManager((x) => + // { + // x.WithMicrosoftLogging(_ => + // { + // //log.AddConsole(LogLevel.Debug); + // }) + // .WithJsonSerializer() + // .WithHandle(typeof(InMemoryJsonHandle<>)); + // }); + // }) + // .Configure(async app => await app.UseOcelot()); + + // _ocelotServer = new TestServer(_webHostBuilder); + // _ocelotClient = _ocelotServer.CreateClient(); + //} + + public static void GivenIWait(int wait) => Thread.Sleep(wait); + + public void GivenOcelotIsRunningWithMiddlewareBeforePipeline(Func callback) + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, true, false); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(s => { s.AddOcelot(); }) + .Configure(async app => + { + app.UseMiddleware(callback); + await app.UseOcelot(); + }); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + public void GivenOcelotIsRunningWithSpecificHandlersRegisteredInDi() + where TOne : DelegatingHandler + where TWo : DelegatingHandler + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, true, false); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(s => + { + s.AddSingleton(_webHostBuilder); + s.AddOcelot() + .AddDelegatingHandler() + .AddDelegatingHandler(); + }) + .Configure(async a => await a.UseOcelot()); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + public void GivenOcelotIsRunningWithGlobalHandlersRegisteredInDi() + where TOne : DelegatingHandler + where TWo : DelegatingHandler + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, true, false); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(s => + { + s.AddSingleton(_webHostBuilder); + s.AddOcelot() + .AddDelegatingHandler(true) + .AddDelegatingHandler(true); + }) + .Configure(async a => await a.UseOcelot()); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + public void GivenOcelotIsRunningWithHandlerRegisteredInDi(bool global = false) + where TOne : DelegatingHandler + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, true, false); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(s => + { + s.AddSingleton(_webHostBuilder); + s.AddOcelot() + .AddDelegatingHandler(global); + }) + .Configure(async a => await a.UseOcelot()); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + public class FakeDependency + { + public bool Called; + } + public void GivenOcelotIsRunningWithGlobalHandlersRegisteredInDi(FakeDependency dependency) + where TOne : DelegatingHandler + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, true, false); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(s => + { + s.AddSingleton(_webHostBuilder); + s.AddSingleton(dependency); + s.AddOcelot() + .AddDelegatingHandler(true); + }) + .Configure(async a => await a.UseOcelot()); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + // # + // # Cookies helpers + // # + public void GivenIAddCookieToMyRequest(string cookie) + => _ocelotClient.DefaultRequestHeaders.Add("Set-Cookie", cookie); + public async Task WhenIGetUrlOnTheApiGatewayWithCookie(string url, string cookie, string value) + => _response = await WhenIGetUrlOnTheApiGateway(url, cookie, value); + public async Task WhenIGetUrlOnTheApiGatewayWithCookie(string url, CookieHeaderValue cookie) + => _response = await WhenIGetUrlOnTheApiGateway(url, cookie); + + public Task WhenIGetUrlOnTheApiGateway(string url, string cookie, string value) + { + var header = new CookieHeaderValue(cookie, value); + return WhenIGetUrlOnTheApiGateway(url, header); + } + + public Task WhenIGetUrlOnTheApiGateway(string url, CookieHeaderValue cookie) + { + var requestMessage = new HttpRequestMessage(HttpMethod.Get, url); + requestMessage.Headers.Add("Cookie", cookie.ToString()); + return _ocelotClient.SendAsync(requestMessage); + } + + // END of Cookies helpers + + ///// + ///// This is annoying cos it should be in the constructor but we need to set up the file before calling startup so its a step. + ///// + //public void GivenOcelotIsRunning(Action options, + // string authenticationProviderKey) + //{ + // _webHostBuilder = TestHostBuilder.Create() + // .ConfigureAppConfiguration((hostingContext, config) => + // { + // config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + // var env = hostingContext.HostingEnvironment; + // config.AddJsonFile("appsettings.json", true, false) + // .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + // config.AddJsonFile(_ocelotConfigFileName, true, false); + // config.AddEnvironmentVariables(); + // }) + // .ConfigureServices(s => + // { + // s.AddOcelot(); + // s.AddAuthentication() + // .AddIdentityServerAuthentication(authenticationProviderKey, options); + // }) + // .Configure(async app => await app.UseOcelot()); + + // _ocelotServer = new TestServer(_webHostBuilder); + // _ocelotClient = _ocelotServer.CreateClient(); + //} + + public void ThenTheResponseHeaderIs(string key, string value) + { + var header = _response.Headers.GetValues(key); + header.First().ShouldBe(value); + } + + public void ThenTheReasonPhraseIs(string expected) + { + _response.ReasonPhrase.ShouldBe(expected); + } + + public void GivenOcelotIsRunningWithServices(Action configureServices) + => GivenOcelotIsRunningWithServices(configureServices, null); + + public void GivenOcelotIsRunningWithServices(Action configureServices, Action configureApp/*, bool validateScopes*/) + { + _webHostBuilder = TestHostBuilder.Create() // ValidateScopes = true + .ConfigureAppConfiguration(WithBasicConfiguration) + .ConfigureServices(configureServices ?? WithAddOcelot) + .Configure(configureApp ?? WithUseOcelot); + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + public void WithBasicConfiguration(WebHostBuilderContext hosting, IConfigurationBuilder config) + { + var env = hosting.HostingEnvironment; + config.SetBasePath(env.ContentRootPath); + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, true, false); + config.AddEnvironmentVariables(); + } + + public static void WithAddOcelot(IServiceCollection services) => services.AddOcelot(); + public static void WithUseOcelot(IApplicationBuilder app) => app.UseOcelot().Wait(); + + /// + /// This is annoying cos it should be in the constructor but we need to set up the file before calling startup so its a step. + /// + public void GivenOcelotIsRunning(OcelotPipelineConfiguration ocelotPipelineConfig) + { + var builder = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", true, false) + .AddJsonFile(_ocelotConfigFileName, false, false) + .AddEnvironmentVariables(); + + var configuration = builder.Build(); + _webHostBuilder = TestHostBuilder.Create() + .ConfigureServices(s => { s.AddSingleton(_webHostBuilder); }); + + _ocelotServer = new TestServer(_webHostBuilder + .UseConfiguration(configuration) + .ConfigureServices(s => { s.AddOcelot(configuration); }) + .ConfigureLogging(l => + { + l.AddConsole(); + l.AddDebug(); + }) + .Configure(async a => await a.UseOcelot(ocelotPipelineConfig))); + + _ocelotClient = _ocelotServer.CreateClient(); + } + + public void GivenIHaveAddedATokenToMyRequest() + { + _ocelotClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _token.AccessToken); + } + + public static List> GivenDefaultAuthTokenForm() => new() + { + new ("client_id", "client"), + new ("client_secret", "secret"), + new ("scope", "api"), + new ("username", "test"), + new ("password", "test"), + new ("grant_type", "password"), + }; + + internal Task GivenIHaveAToken(string url) + { + var form = GivenDefaultAuthTokenForm(); + return GivenIHaveATokenWithForm(url, form); + } + + internal async Task GivenIHaveATokenWithForm(string url, IEnumerable> form) + { + var tokenUrl = $"{url}/connect/token"; + var formData = form ?? Enumerable.Empty>(); + var content = new FormUrlEncodedContent(formData); + + using var httpClient = new HttpClient(); + var response = await httpClient.PostAsync(tokenUrl, content); + var responseContent = await response.Content.ReadAsStringAsync(); + response.EnsureSuccessStatusCode(); + _token = JsonConvert.DeserializeObject(responseContent); + return _token; + } + + public static async Task VerifyIdentityServerStarted(string url) + { + using var httpClient = new HttpClient(); + var response = await httpClient.GetAsync($"{url}/.well-known/openid-configuration"); + await response.Content.ReadAsStringAsync(); + response.EnsureSuccessStatusCode(); + } + + //public void GivenOcelotIsRunningWithMinimumLogLevel(Logger logger, string appsettingsFileName) + //{ + // _webHostBuilder = TestHostBuilder.Create() + // .UseKestrel() + // .ConfigureAppConfiguration((_, config) => + // { + // config.AddJsonFile(appsettingsFileName, false, false); + // config.AddJsonFile(_ocelotConfigFileName, false, false); + // config.AddEnvironmentVariables(); + // }) + // .ConfigureServices(s => { s.AddOcelot(); }) + // .ConfigureLogging(logging => + // { + // logging.ClearProviders(); + // logging.AddSerilog(logger); + // }) + // .Configure(async app => + // { + // app.Use(async (context, next) => + // { + // var loggerFactory = context.RequestServices.GetService(); + // var ocelotLogger = loggerFactory.CreateLogger(); + // ocelotLogger.LogDebug(() => $"DEBUG: {nameof(ocelotLogger)}, {nameof(loggerFactory)}"); + // ocelotLogger.LogTrace(() => $"TRACE: {nameof(ocelotLogger)}, {nameof(loggerFactory)}"); + // ocelotLogger.LogInformation(() => + // $"INFORMATION: {nameof(ocelotLogger)}, {nameof(loggerFactory)}"); + // ocelotLogger.LogWarning(() => $"WARNING: {nameof(ocelotLogger)}, {nameof(loggerFactory)}"); + // ocelotLogger.LogError(() => $"ERROR: {nameof(ocelotLogger)}, {nameof(loggerFactory)}", + // new Exception("test")); + // ocelotLogger.LogCritical(() => $"CRITICAL: {nameof(ocelotLogger)}, {nameof(loggerFactory)}", + // new Exception("test")); + + // await next.Invoke(); + // }); + // await app.UseOcelot(); + // }); + + // _ocelotServer = new TestServer(_webHostBuilder); + // _ocelotClient = _ocelotServer.CreateClient(); + //} + + //public void GivenOcelotIsRunningWithEureka() + // => GivenOcelotIsRunningWithServices(s => s.AddOcelot().AddEureka()); + + //public void GivenOcelotIsRunningWithPolly() => GivenOcelotIsRunningWithServices(WithPolly); + //public static void WithPolly(IServiceCollection services) => services.AddOcelot().AddPolly(); + + public async Task WhenIGetUrlOnTheApiGateway(string url) + => _response = await _ocelotClient.GetAsync(url); + + public Task WhenIGetUrl(string url) + => _ocelotClient.GetAsync(url); + + public async Task WhenIGetUrlWithBodyOnTheApiGateway(string url, string body) + { + var request = new HttpRequestMessage(HttpMethod.Get, url) + { + Content = new StringContent(body), + }; + _response = await _ocelotClient.SendAsync(request); + } + + public async Task WhenIGetUrlWithFormOnTheApiGateway(string url, string name, IEnumerable> values) + { + var content = new MultipartFormDataContent(); + var dataContent = new FormUrlEncodedContent(values); + content.Add(dataContent, name); + content.Headers.ContentDisposition = new ContentDispositionHeaderValue("form-data"); + + var request = new HttpRequestMessage(HttpMethod.Get, url) + { + Content = content, + }; + _response = await _ocelotClient.SendAsync(request); + } + + public async Task WhenIGetUrlOnTheApiGateway(string url, HttpContent content) + { + var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, url) { Content = content }; + _response = await _ocelotClient.SendAsync(httpRequestMessage); + } + + public async Task WhenIPostUrlOnTheApiGateway(string url, HttpContent content) + { + var httpRequestMessage = new HttpRequestMessage(HttpMethod.Post, url) { Content = content }; + _response = await _ocelotClient.SendAsync(httpRequestMessage); + } + + public void GivenIAddAHeader(string key, string value) + { + _ocelotClient.DefaultRequestHeaders.TryAddWithoutValidation(key, value); + } + + public static void WhenIDoActionMultipleTimes(int times, Action action) + { + for (int i = 0; i < times; i++) + action?.Invoke(i); + } + + public static async Task WhenIDoActionMultipleTimes(int times, Func action) + { + for (int i = 0; i < times; i++) + await action.Invoke(i); + } + public static async Task WhenIDoActionForTime(TimeSpan time, Func action) + { + var watcher = Stopwatch.StartNew(); + for (int i = 0; watcher.Elapsed < time; i++) + { + await action.Invoke(i); + } + watcher.Stop(); + } + + public async Task WhenIGetUrlOnTheApiGateway(string url, string requestId) + { + _ocelotClient.DefaultRequestHeaders.TryAddWithoutValidation(RequestIdKey, requestId); + _response = await _ocelotClient.GetAsync(url); + } + + public async Task WhenIPostUrlOnTheApiGateway(string url) + { + _response = await _ocelotClient.PostAsync(url, _postContent); + } + + public void GivenThePostHasContent(string postContent) + { + _postContent = new StringContent(postContent); + } + + public void GivenThePostHasContentType(string postContent) + { + _postContent.Headers.ContentType = new MediaTypeHeaderValue(postContent); + } + + public void GivenThePostHasGzipContent(object input) + { + var json = JsonConvert.SerializeObject(input); + var jsonBytes = Encoding.UTF8.GetBytes(json); + var ms = new MemoryStream(); + using (var gzip = new GZipStream(ms, CompressionMode.Compress, true)) + { + gzip.Write(jsonBytes, 0, jsonBytes.Length); + } + + ms.Position = 0; + var content = new StreamContent(ms); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + content.Headers.ContentEncoding.Add("gzip"); + _postContent = content; + } + + public void ThenTheResponseBodyShouldBe(string expectedBody) + => _response.Content.ReadAsStringAsync().GetAwaiter().GetResult().ShouldBe(expectedBody); + public void ThenTheResponseBodyShouldBe(string expectedBody, string customMessage) + => _response.Content.ReadAsStringAsync().GetAwaiter().GetResult().ShouldBe(expectedBody, customMessage); + + public void ThenTheContentLengthIs(int expected) + => _response.Content.Headers.ContentLength.ShouldBe(expected); + + public void ThenTheStatusCodeShouldBe(HttpStatusCode expected) + => _response.StatusCode.ShouldBe(expected); + public void ThenTheStatusCodeShouldBe(int expected) + => ((int)_response.StatusCode).ShouldBe(expected); + + public void ThenTheRequestIdIsReturned() + => _response.Headers.GetValues(RequestIdKey).First().ShouldNotBeNullOrEmpty(); + + public void ThenTheRequestIdIsReturned(string expected) + => _response.Headers.GetValues(RequestIdKey).First().ShouldBe(expected); + + public void WhenIMakeLotsOfDifferentRequestsToTheApiGateway() + { + var numberOfRequests = 100; + var aggregateUrl = "/"; + var aggregateExpected = "{\"Laura\":{Hello from Laura},\"Tom\":{Hello from Tom}}"; + var tomUrl = "/tom"; + var tomExpected = "{Hello from Tom}"; + var lauraUrl = "/laura"; + var lauraExpected = "{Hello from Laura}"; + var random = new Random(); + + var aggregateTasks = new Task[numberOfRequests]; + + for (var i = 0; i < numberOfRequests; i++) + { + aggregateTasks[i] = Fire(aggregateUrl, aggregateExpected, random); + } + + var tomTasks = new Task[numberOfRequests]; + + for (var i = 0; i < numberOfRequests; i++) + { + tomTasks[i] = Fire(tomUrl, tomExpected, random); + } + + var lauraTasks = new Task[numberOfRequests]; + + for (var i = 0; i < numberOfRequests; i++) + { + lauraTasks[i] = Fire(lauraUrl, lauraExpected, random); + } + + Task.WaitAll(lauraTasks); + Task.WaitAll(tomTasks); + Task.WaitAll(aggregateTasks); + } + + private async Task Fire(string url, string expectedBody, Random random) + { + var request = new HttpRequestMessage(new HttpMethod("GET"), url); + await Task.Delay(random.Next(0, 2)); + var response = await _ocelotClient.SendAsync(request); + var content = await response.Content.ReadAsStringAsync(); + content.ShouldBe(expectedBody); + } + + public void GivenOcelotIsRunningWithBlowingUpDiskRepo(IFileConfigurationRepository fake) + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, false, false); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(s => + { + s.AddSingleton(fake); + s.AddOcelot(); + }) + .Configure(async app => await app.UseOcelot()); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + public void TheChangeTokenShouldBeActive(bool itShouldBeActive) + { + _changeToken.ChangeToken.HasChanged.ShouldBe(itShouldBeActive); + } + + public void GivenOcelotIsRunningWithLogger() + { + _webHostBuilder = TestHostBuilder.Create() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + var env = hostingContext.HostingEnvironment; + config.AddJsonFile("appsettings.json", true, false) + .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + config.AddJsonFile(_ocelotConfigFileName, false, false); + config.AddEnvironmentVariables(); + }) + .ConfigureServices(s => + { + s.AddOcelot(); + s.AddSingleton(); + }) + .Configure(async app => await app.UseOcelot()); + + _ocelotServer = new TestServer(_webHostBuilder); + _ocelotClient = _ocelotServer.CreateClient(); + } + + //internal void GivenOcelotIsRunningUsingOpenTracing(OpenTracing.ITracer fakeTracer) + //{ + // _webHostBuilder = TestHostBuilder.Create() + // .ConfigureAppConfiguration((hostingContext, config) => + // { + // config.SetBasePath(hostingContext.HostingEnvironment.ContentRootPath); + // var env = hostingContext.HostingEnvironment; + // config.AddJsonFile("appsettings.json", true, false) + // .AddJsonFile($"appsettings.{env.EnvironmentName}.json", true, false); + // config.AddJsonFile(_ocelotConfigFileName, true, false); + // config.AddEnvironmentVariables(); + // }) + // .ConfigureServices(s => + // { + // s.AddOcelot() + // .AddOpenTracing(); + + // s.AddSingleton(fakeTracer); + // }) + // .Configure(async app => + // { + // app.Use(async (_, next) => { await next.Invoke(); }); + // await app.UseOcelot(); + // }); + + // _ocelotServer = new TestServer(_webHostBuilder); + // _ocelotClient = _ocelotServer.CreateClient(); + //} + + public void ThenWarningShouldBeLogged(int howMany) + { + var loggerFactory = (MockLoggerFactory)_ocelotServer.Host.Services.GetService(); + loggerFactory.Verify(Times.Exactly(howMany)); + } + + internal class MockLoggerFactory : IOcelotLoggerFactory + { + private Mock _logger; + + public IOcelotLogger CreateLogger() + { + if (_logger != null) + { + return _logger.Object; + } + + _logger = new Mock(); + _logger.Setup(x => x.LogWarning(It.IsAny())).Verifiable(); + _logger.Setup(x => x.LogWarning(It.IsAny>())).Verifiable(); + + return _logger.Object; + } + + public void Verify(Times howMany) + { + _logger.Verify(x => x.LogWarning(It.IsAny>()), howMany); + } + } + + /// + /// Public implementation of Dispose pattern callable by consumers. + /// + public virtual void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private bool _disposedValue; + + /// + /// Protected implementation of Dispose pattern. + /// + /// Flag to trigger actual disposing operation. + protected virtual void Dispose(bool disposing) + { + if (_disposedValue) + { + return; + } + + if (disposing) + { + _ocelotClient?.Dispose(); + _ocelotServer?.Dispose(); + _ocelotHost?.Dispose(); + _response?.Dispose(); + DeleteFiles(); + DeleteFolders(); + } + + _disposedValue = true; + } +} + +internal class BearerToken +{ + [JsonProperty("access_token")] + public string AccessToken { get; set; } + + [JsonProperty("expires_in")] + public int ExpiresIn { get; set; } + + [JsonProperty("token_type")] + public string TokenType { get; set; } +} diff --git a/acceptance/TestHostBuilder.cs b/acceptance/TestHostBuilder.cs new file mode 100644 index 0000000..6b8175c --- /dev/null +++ b/acceptance/TestHostBuilder.cs @@ -0,0 +1,16 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; + +namespace Ocelot.Discovery.Nacos.AcceptanceTests; + +public sealed class TestHostBuilder : WebHostBuilder +{ + public static IWebHostBuilder Create() + => new WebHostBuilder().UseDefaultServiceProvider(WithEnabledValidateScopes); + + public static IWebHostBuilder Create(Action configure) + => new WebHostBuilder().UseDefaultServiceProvider(configure + WithEnabledValidateScopes); + + public static void WithEnabledValidateScopes(ServiceProviderOptions options) + => options.ValidateScopes = true; +} diff --git a/src/Nacos.cs b/src/Nacos.cs new file mode 100644 index 0000000..3edda18 --- /dev/null +++ b/src/Nacos.cs @@ -0,0 +1,73 @@ +using Ocelot.Logging; +using Service = Ocelot.Values.Service; + +namespace Ocelot.Discovery.Nacos; + +public class Nacos : IServiceDiscoveryProvider +{ + private readonly INacosNamingService _client; + private readonly string _serviceName; + private readonly IOcelotLogger _logger; + + public Nacos(string serviceName, INacosNamingService client, IOcelotLoggerFactory factory) + { + _client = client; + _serviceName = serviceName; + _logger = factory.CreateLogger(); + ; + } + + public async Task> GetAsync() + { + try + { + var instances = await _client.GetAllInstances(_serviceName) + .ConfigureAwait(false); + + return instances? + .Where(i => i.Healthy && i.Enabled && i.Weight > 0) // Filter out unhealthy instances + .Select(TransformInstance) + .ToList() ?? new(); + } + catch (NacosException ex) + { + _logger.LogError( + () => $"{nameof(Nacos)} discovery: An exception occurred while fetching instances for service:{_serviceName} from Nacos.", + ex); + return new(); + } + } + + private Service TransformInstance(Instance instance) + { + var metadata = instance.Metadata ?? new(); + + return new Service( + id: instance.InstanceId, + hostAndPort: new(instance.Ip, instance.Port), + name: instance.ServiceName, + version: metadata.GetValueOrDefault("version", "default"), + tags: ProcessMetadataTags(metadata) + ); + } + + private static List ProcessMetadataTags(IDictionary metadata) => metadata + .Where(kv => !ReservedKeys.Contains(kv.Key)) + .Select(FormatTag) + .ToList(); + + private static string FormatTag(KeyValuePair kv) + => $"{WebUtility.UrlEncode(kv.Key)}={WebUtility.UrlEncode(kv.Value)}"; + + /// + /// Reserved keys that should not be included in the metadata tags. + /// These keys are used internally by the Nacos service discovery provider + /// and should not be exposed as part of the service metadata tags. + /// Version key used to specify the version of the service. + /// Group key used to specify the group of the service. + /// Cluster key used to specify the cluster of the service. + /// Namespace key used to specify the namespace of the service. + /// Weight key used to specify the weight of the service instance. + /// + public static readonly string[] ReservedKeys = { "version", "group", "cluster", "namespace", "weight" }; +} \ No newline at end of file diff --git a/src/NacosMiddlewareConfigurationProvider.cs b/src/NacosMiddlewareConfigurationProvider.cs new file mode 100644 index 0000000..3a8511f --- /dev/null +++ b/src/NacosMiddlewareConfigurationProvider.cs @@ -0,0 +1,26 @@ +using Microsoft.AspNetCore.Builder; +using Ocelot.Configuration.Repository; +using Ocelot.Middleware; + +namespace Ocelot.Discovery.Nacos; + +public class NacosMiddlewareConfigurationProvider +{ + public static OcelotMiddlewareConfigurationDelegate Get { get; } = GetInternal; + + private static Task GetInternal(IApplicationBuilder builder) + { + var internalConfigRepo = builder.ApplicationServices.GetService(); + var config = internalConfigRepo?.Get(); + if (config != null && UsingNacosServiceDiscoveryProvider(config.Data)) + { + var log = builder.ApplicationServices.GetService>(); + log?.LogInformation("Using Nacos service discovery provider."); + } + + return Task.CompletedTask; + } + + private static bool UsingNacosServiceDiscoveryProvider(IInternalConfiguration configuration) + => nameof(Nacos).Equals(configuration?.ServiceProviderConfiguration?.Type, StringComparison.OrdinalIgnoreCase); +} diff --git a/src/NacosProviderFactory.cs b/src/NacosProviderFactory.cs new file mode 100644 index 0000000..61634f7 --- /dev/null +++ b/src/NacosProviderFactory.cs @@ -0,0 +1,32 @@ +using Ocelot.Logging; +using Ocelot.ServiceDiscovery; + +namespace Ocelot.Discovery.Nacos; + +public static class NacosProviderFactory +{ + /// + /// String constant used for provider type definition. + /// + public const string Nacos = nameof(Discovery.Nacos.Nacos); + + public static ServiceDiscoveryFinderDelegate Get { get; } = CreateProvider; + + private static IServiceDiscoveryProvider? CreateProvider(IServiceProvider provider, ServiceProviderConfiguration config, DownstreamRoute route) + { + var client = provider.GetService(); + var loggerFactory = provider.GetService(); + if (client == null) + { + throw new NullReferenceException($"{Nacos} discovery: Cannot get an {nameof(INacosNamingService)} service during {nameof(CreateProvider)} operation to instantiate the {nameof(Nacos)} provider!"); + } + if (loggerFactory != null) + { + return Nacos.Equals(config.Type, StringComparison.OrdinalIgnoreCase) + ? new Nacos(route.ServiceName, client, loggerFactory) + : null; + } + + return null; + } +} diff --git a/src/Ocelot.Discovery.Nacos.csproj b/src/Ocelot.Discovery.Nacos.csproj new file mode 100644 index 0000000..3e0106c --- /dev/null +++ b/src/Ocelot.Discovery.Nacos.csproj @@ -0,0 +1,13 @@ + + + + net6.0;net8.0 + enable + enable + + + + + + + diff --git a/src/OcelotBuilderExtensions.cs b/src/OcelotBuilderExtensions.cs new file mode 100644 index 0000000..d9fcab7 --- /dev/null +++ b/src/OcelotBuilderExtensions.cs @@ -0,0 +1,16 @@ +using Ocelot.DependencyInjection; +using Nacos.AspNetCore.V2; + +namespace Ocelot.Discovery.Nacos; + +public static class OcelotBuilderExtensions +{ + public static IOcelotBuilder AddNacos(this IOcelotBuilder builder, string section = "nacos") + { + builder.Services + .AddNacosAspNet(builder.Configuration, section) + .AddSingleton(NacosProviderFactory.Get) + .AddSingleton(NacosMiddlewareConfigurationProvider.Get); + return builder; + } +} diff --git a/src/Usings.cs b/src/Usings.cs new file mode 100644 index 0000000..f404911 --- /dev/null +++ b/src/Usings.cs @@ -0,0 +1,11 @@ +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using Ocelot.Configuration; +global using Ocelot.ServiceDiscovery.Providers; +global using Nacos.V2; +global using Nacos.V2.Exceptions; +global using Nacos.V2.Naming.Dtos; +global using System.Net; +global using System.Collections.Generic; +global using System.Linq; +global using System.Threading.Tasks; diff --git a/unit/MSTestSettings.cs b/unit/MSTestSettings.cs new file mode 100644 index 0000000..aaf278c --- /dev/null +++ b/unit/MSTestSettings.cs @@ -0,0 +1 @@ +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] diff --git a/unit/NacosMiddlewareConfigurationProviderTests.cs b/unit/NacosMiddlewareConfigurationProviderTests.cs new file mode 100644 index 0000000..4472526 --- /dev/null +++ b/unit/NacosMiddlewareConfigurationProviderTests.cs @@ -0,0 +1,42 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using Ocelot.Configuration; +using Ocelot.Configuration.Builder; +using Ocelot.Configuration.Repository; +using Ocelot.Responses; + +namespace Ocelot.Discovery.Nacos.UnitTests; + +[TestClass] +public class NacosMiddlewareConfigurationProviderTests +{ + [TestMethod] + public void ShouldNotBuild() + { + var configRepo = new Mock(); + configRepo.Setup(x => x.Get()) + .Returns(new OkResponse( + new InternalConfiguration(null, null, null, null, null, null, null, null,null,null))); + var services = new ServiceCollection(); + services.AddSingleton(configRepo.Object); + var sp = services.BuildServiceProvider(); + var provider = NacosMiddlewareConfigurationProvider.Get(new ApplicationBuilder(sp)); + Assert.IsInstanceOfType(provider, typeof(Task)); + } + + [TestMethod] + public void ShouldBuild() + { + var serviceProviderConfig = new ServiceProviderConfigurationBuilder().WithType("nacos").Build(); + var configRepo = new Mock(); + configRepo.Setup(x => x.Get()) + .Returns(new OkResponse( + new InternalConfiguration(null, null, serviceProviderConfig, null, null, null, null, null, null, null))); + var services = new ServiceCollection(); + services.AddSingleton(configRepo.Object); + var sp = services.BuildServiceProvider(); + var provider = NacosMiddlewareConfigurationProvider.Get(new ApplicationBuilder(sp)); + Assert.IsInstanceOfType(provider, typeof(Task)); + } +} diff --git a/unit/NacosProviderFactoryTests.cs b/unit/NacosProviderFactoryTests.cs new file mode 100644 index 0000000..0e6a18b --- /dev/null +++ b/unit/NacosProviderFactoryTests.cs @@ -0,0 +1,138 @@ +using Microsoft.Extensions.Logging; +using Moq; +using Nacos.V2; +using Ocelot.Configuration; +using Ocelot.Configuration.Builder; +using Ocelot.Logging; + +namespace Ocelot.Discovery.Nacos.UnitTests; + +[TestClass] +public class NacosProviderFactoryTests +{ + [TestMethod] + public void CreateProvider_ShouldReturnNacosProvider_WhenConfigTypeIsNacos() + { + // Arrange + var serviceProviderMock = new Mock(); + var loggerMock = new Mock(); + serviceProviderMock.Setup(sp => sp.GetService(typeof(IOcelotLoggerFactory))) + .Returns(loggerMock.Object); + var nacosNamingServiceMock = new Mock(); + serviceProviderMock.Setup(sp => sp.GetService(typeof(INacosNamingService))) + .Returns(nacosNamingServiceMock.Object); + + var config = new ServiceProviderConfiguration( + type: "nacos", + scheme: "http", + host: "localhost", + port: 8848, + token: null, + configurationKey: "nacos", + pollingInterval: 5000 + ); + var route = new DownstreamRouteBuilder() + .WithKey("testKey") + .WithServiceName("testService") + .Build(); + + // Act + var provider = NacosProviderFactory.Get(serviceProviderMock.Object, config, route); + + // Assert + Assert.IsNotNull(provider); + Assert.IsInstanceOfType(provider, typeof(Nacos)); + } + + [TestMethod] + public void CreateProvider_ShouldThrowException_WhenNacosNamingServiceIsNull() + { + // Arrange + var serviceProviderMock = new Mock(); + serviceProviderMock.Setup(sp => sp.GetService(typeof(INacosNamingService))) + .Returns(null!); + + var config = new ServiceProviderConfiguration( + type: "nacos", + scheme: "http", + host: "localhost", + port: 8848, + token: null, + configurationKey: "nacos", + pollingInterval: 5000 + ); + var route = new DownstreamRouteBuilder() + .WithKey("testKey") + .WithServiceName("testService") + .Build(); + + // Act, Assert + Assert.ThrowsException(() => + NacosProviderFactory.Get(serviceProviderMock.Object, config, route)); + } + + [TestMethod] + public void CreateProvider_ShouldReturnNull_WhenLoggerIsNull() + { + // Arrange + var serviceProviderMock = new Mock(); + serviceProviderMock.Setup(sp => sp.GetService(typeof(ILogger))) + .Returns(null!); + var nacosNamingServiceMock = new Mock(); + serviceProviderMock.Setup(sp => sp.GetService(typeof(INacosNamingService))) + .Returns(nacosNamingServiceMock.Object); + + var config = new ServiceProviderConfiguration( + type: "nacos", + scheme: "http", + host: "localhost", + port: 8848, + token: null, + configurationKey: "nacos", + pollingInterval: 5000 + ); + var route = new DownstreamRouteBuilder() + .WithKey("testKey") + .WithServiceName("testService") + .Build(); + + // Act + var provider = NacosProviderFactory.Get(serviceProviderMock.Object, config, route); + + // Assert + Assert.IsNull(provider); + } + + [TestMethod] + public void CreateProvider_ShouldReturnNull_WhenConfigTypeIsNotNacos() + { + // Arrange + var serviceProviderMock = new Mock(); + var loggerMock = new Mock>(); + serviceProviderMock.Setup(sp => sp.GetService(typeof(ILogger))) + .Returns(loggerMock.Object); + var nacosNamingServiceMock = new Mock(); + serviceProviderMock.Setup(sp => sp.GetService(typeof(INacosNamingService))) + .Returns(nacosNamingServiceMock.Object); + + var config = new ServiceProviderConfiguration( + type: "other", + scheme: "http", + host: "localhost", + port: 8848, + token: null, + configurationKey: "nacos", + pollingInterval: 5000 + ); + var route = new DownstreamRouteBuilder() + .WithKey("testKey") + .WithServiceName("testService") + .Build(); + + // Act + var provider = NacosProviderFactory.Get(serviceProviderMock.Object, config, route); + + // Assert + Assert.IsNull(provider); + } +} diff --git a/unit/NacosTests.cs b/unit/NacosTests.cs new file mode 100644 index 0000000..618819a --- /dev/null +++ b/unit/NacosTests.cs @@ -0,0 +1,124 @@ +using Microsoft.Extensions.Logging; +using Moq; +using Nacos.V2; +using Nacos.V2.Exceptions; +using Nacos.V2.Naming.Dtos; +using Ocelot.Logging; + +namespace Ocelot.Discovery.Nacos.UnitTests; + +[TestClass] +public class NacosTests +{ + private readonly Mock _mockNacosNamingService; + private readonly Mock _mockLogger; + private Nacos _nacos; + + public NacosTests() + { + _mockNacosNamingService = new Mock(); + _mockLogger = new Mock(); + _nacos = new Nacos("testService", _mockNacosNamingService.Object, _mockLogger.Object); + } + + [TestInitialize] + public void Setup() + { + } + + [TestMethod] + public async Task GetAsync_ShouldReturnServices_WhenInstancesAreHealthy() + { + // Arrange + var instances = new List + { + new() { InstanceId = "1", Ip = "127.0.0.1", Port = 80, ServiceName = "testService", Healthy = true, Enabled = true, Weight = 1 }, + new() { InstanceId = "2", Ip = "127.0.0.2", Port = 81, ServiceName = "testService", Healthy = true, Enabled = true, Weight = 1 } + }; + _mockNacosNamingService?.Setup(x => x.GetAllInstances("testService")).ReturnsAsync(instances); + + // Act + var result = await _nacos?.GetAsync()!; + + // Assert + Assert.AreEqual(2, result.Count); + Assert.AreEqual("1", result[0].Id); + Assert.AreEqual("2", result[1].Id); + } + + [TestMethod] + public async Task GetAsync_ShouldReturnEmptyList_WhenNoInstancesAreHealthy() + { + // Arrange + var instances = new List + { + new() { InstanceId = "1", Ip = "127.0.0.1", Port = 80, ServiceName = "testService", Healthy = false, Enabled = true, Weight = 1 }, + new() { InstanceId = "2", Ip = "127.0.0.2", Port = 81, ServiceName = "testService", Healthy = false, Enabled = true, Weight = 1 } + }; + _mockNacosNamingService?.Setup(x => x.GetAllInstances("testService")).ReturnsAsync(instances); + + // Act + var result = await _nacos?.GetAsync()!; + + // Assert + Assert.AreEqual(0, result.Count); + } + + [TestMethod] + public async Task GetAsync_ShouldReturnEmptyList_WhenExceptionOccurs() + { + // Arrange + var logger = new Mock(); + logger.Setup(l => l.LogError(It.IsAny>(), It.IsAny())) + .Callback, Exception>(AssertMessage) + .Verifiable($"{nameof(IOcelotLogger)}.{nameof(IOcelotLogger.LogError)}() was not called."); + var factory = new Mock(); + factory.Setup(f => f.CreateLogger()).Returns(logger.Object); + + _nacos = new Nacos("testService", _mockNacosNamingService.Object, factory.Object); + var ex = new NacosException("Test exception"); + _mockNacosNamingService?.Setup(x => x.GetAllInstances("testService")) + .ThrowsAsync(ex); + + // Act + var result = await _nacos.GetAsync(); + + // Assert + Assert.AreEqual(0, result.Count); + logger.Verify(l => l.LogError(It.IsAny>(), It.IsAny()), + Times.Once); + static void AssertMessage(Func messageFactory, Exception exception) + { + Assert.IsNotNull(messageFactory); + Assert.IsNotNull(exception); + string message = messageFactory.Invoke(); + Assert.AreEqual("Nacos discovery: An exception occurred while fetching instances for service:testService from Nacos.", message); + Assert.IsInstanceOfType(exception); + } + } + + [TestMethod] + public async Task GetAsync_ShouldReturnServices_WithCorrectMetadataTags() + { + // Arrange + var instances = new List + { + new() + { + InstanceId = "1", Ip = "127.0.0.1", Port = 80, ServiceName = "testService", Healthy = true, Enabled = true, Weight = 1, + Metadata = new Dictionary { { "version", "1.0" }, { "customTag", "customValue" } } + } + }; + _mockNacosNamingService?.Setup(x => x.GetAllInstances("testService")).ReturnsAsync(instances); + + // Act + var result = await _nacos?.GetAsync()!; + + // Assert + Assert.AreEqual(1, result.Count); + Assert.AreEqual("1", result[0].Id); + Assert.AreEqual("1.0", result[0].Version); + Assert.IsTrue(result[0].Tags.Contains("customTag=customValue")); + Assert.IsFalse(result[0].Tags.Contains("version=1.0")); + } +} diff --git a/unit/Ocelot.Discovery.Nacos.UnitTests.csproj b/unit/Ocelot.Discovery.Nacos.UnitTests.csproj new file mode 100644 index 0000000..cbf045c --- /dev/null +++ b/unit/Ocelot.Discovery.Nacos.UnitTests.csproj @@ -0,0 +1,40 @@ + + + + net6.0;net8.0 + latest + enable + enable + false + true + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + + + + diff --git a/unit/OcelotBuilderExtensionsTests.cs b/unit/OcelotBuilderExtensionsTests.cs new file mode 100644 index 0000000..d910d42 --- /dev/null +++ b/unit/OcelotBuilderExtensionsTests.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using Ocelot.DependencyInjection; + +namespace Ocelot.Discovery.Nacos.UnitTests; + +[TestClass] +public class OcelotBuilderExtensionsTests +{ + [TestMethod] + public void AddNacos_ShouldRegisterNacosServices() + { + // Arrange + var services = new ServiceCollection(); + var configuration = new Mock(); + var builder = new OcelotBuilder(services, configuration.Object); + + // Act + builder.Services.AddOcelot().AddNacos(); + + // Assert + var serviceProvider = services.BuildServiceProvider(); + Assert.IsNotNull(serviceProvider.GetService(NacosProviderFactory.Get.GetType())); + Assert.IsNotNull(serviceProvider.GetService(NacosMiddlewareConfigurationProvider.Get.GetType())); + } +}