Skip to content

Commit 7c910d6

Browse files
neilreesrobertcoltheart
neilrees
and
robertcoltheart
authored
Performance Improvement - Perform test filtering earlier in process (#433)
* Add test to run one out of 10000 specs * Prevent multiple enumeration of Contexts * Move filtering to before Context is created * Simplify FindContexts to use same pattern as FindContextsIn * Use IEnumerable extension method * Display compile diagnostic when compilation fails * Make parameter optional * Extend test to track instantiations * Update src/Machine.Specifications/Utility/KeyValuePairExtensions.cs Co-authored-by: Robert Coltheart <[email protected]> * Update src/Machine.Specifications/Explorers/AssemblyExplorer.cs Co-authored-by: Robert Coltheart <[email protected]> * Update src/Machine.Specifications/Explorers/AssemblyExplorer.cs Co-authored-by: Robert Coltheart <[email protected]> Co-authored-by: Robert Coltheart <[email protected]>
1 parent c4eb55a commit 7c910d6

File tree

8 files changed

+235
-81
lines changed

8 files changed

+235
-81
lines changed

‎src/Machine.Specifications.Runner.Console.Specs/ProgramSpecs.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ public class when_a_specification_fails_and_silent_is_set : FailingSpecs
162162
console.Lines.ShouldContain(l => l.Contains("hi scott, love you, miss you."));
163163

164164
It should_separate_failures_from_the_rest_of_the_test_run = () =>
165-
console.Output.ShouldMatchRegex(String.Format("\\S{0}{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
165+
console.Output.ShouldMatchRegex(String.Format("{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
166166
}
167167

168168
[Subject("Console runner")]
@@ -189,7 +189,7 @@ public class when_a_specification_fails_and_progress_is_set : FailingSpecs
189189
console.Lines.ShouldContain(l => l.Contains("hi scott, love you, miss you."));
190190

191191
It should_separate_failures_from_the_rest_of_the_test_run = () =>
192-
console.Output.ShouldMatchRegex(String.Format("\\S{0}{0}{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
192+
console.Output.ShouldMatchRegex(String.Format("{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
193193
}
194194

195195
[Subject("Console runner")]

‎src/Machine.Specifications.Specs/CompileContext.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public string Compile(string code)
3434
.Emit(filename);
3535

3636
if (!result.Success)
37-
throw new InvalidOperationException();
37+
throw new InvalidOperationException(result.Diagnostics[0].GetMessage());
3838
#else
3939
var parameters = new CompilerParameters
4040
{

‎src/Machine.Specifications.Specs/Fixtures/LargeFixture.cs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
using System.Text;
2+
3+
namespace Machine.Specifications.Specs.Fixtures
4+
{
5+
public class LargeFixture
6+
{
7+
public static string CreateCode(int specCount)
8+
{
9+
var sb = new StringBuilder();
10+
11+
sb.AppendLine(@"
12+
using System;
13+
using System.Diagnostics;
14+
using System.Reflection;
15+
using System.Threading.Tasks;
16+
using Machine.Specifications;
17+
18+
namespace Example.Large
19+
{
20+
public class when_there_are_many_contexts
21+
{
22+
public static bool Created = false;
23+
24+
public when_there_are_many_contexts()
25+
{
26+
Created = true;
27+
}
28+
29+
It spec = () => {};
30+
}
31+
32+
public static class OtherTests
33+
{
34+
public static bool Created = false;
35+
}
36+
37+
");
38+
39+
for (var i = 1; i <= specCount; i++)
40+
{
41+
sb.AppendLine($@"
42+
public class when_there_are_many_contexts_{i}
43+
{{
44+
public when_there_are_many_contexts_{i}()
45+
{{
46+
OtherTests.Created = true;
47+
}}
48+
49+
It spec = () => {{}};
50+
}}");
51+
}
52+
53+
sb.AppendLine(@"
54+
}");
55+
56+
return sb.ToString();
57+
}
58+
}
59+
}

‎src/Machine.Specifications.Specs/Runner/SpecificationRunnerSpecs.cs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using System;
2+
using System.Diagnostics;
3+
using System.Linq;
24
using System.Reflection;
35
using Machine.Specifications.Runner;
46
using Machine.Specifications.Runner.Impl;
@@ -691,6 +693,62 @@ public class when_running_a_context_inside_a_static_class_that_is_nested_in_a_no
691693
testListener.LastResult.Passed.ShouldBeTrue();
692694
}
693695

696+
[Subject("Specification Runner")]
697+
public class when_running_a_single_spec_out_of_a_large_number_of_specifications : RunnerSpecs
698+
{
699+
static Type when_a_context_has_many_specifications;
700+
static Type filtered_out_spec;
701+
static TimeSpan elapsed { get; set; }
702+
703+
Establish context = () =>
704+
{
705+
using (var compiler = new CompileContext())
706+
{
707+
var assemblyPath = compiler.Compile(LargeFixture.CreateCode(10000));
708+
var assembly = Assembly.LoadFile(assemblyPath);
709+
710+
when_a_context_has_many_specifications = assembly.GetType("Example.Large.when_there_are_many_contexts");
711+
filtered_out_spec = assembly.GetType("Example.Large.OtherTests");
712+
}
713+
};
714+
715+
Because of = () =>
716+
{
717+
var runner = new DefaultRunner(testListener, new RunOptions(
718+
Enumerable.Empty<string>(),
719+
Enumerable.Empty<string>(),
720+
new[] {when_a_context_has_many_specifications.FullName})
721+
);
722+
723+
var sw = Stopwatch.StartNew();
724+
runner.RunAssembly(when_a_context_has_many_specifications.Assembly);
725+
sw.Stop();
726+
elapsed = sw.Elapsed;
727+
};
728+
729+
It should_run_the_single_specification = () =>
730+
{
731+
testListener.SpecCount.ShouldEqual(1);
732+
};
733+
734+
It should_run_in_a_reasonable_period_of_time = () =>
735+
{
736+
elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1));
737+
};
738+
739+
It should_have_created_the_test_instance = () =>
740+
{
741+
var fieldInfo = when_a_context_has_many_specifications.GetField("Created");
742+
((bool) fieldInfo.GetValue(null)).ShouldBeTrue();
743+
};
744+
745+
It should_have_not_have_created_any_of_the_filtered_out_tests = () =>
746+
{
747+
var fieldInfo = filtered_out_spec.GetField("Created");
748+
((bool) fieldInfo.GetValue(null)).ShouldBeFalse();
749+
};
750+
}
751+
694752
public class RandomRunnerSpecs : RunnerSpecs
695753
{
696754
static CompileContext compiler;

‎src/Machine.Specifications/Explorers/AssemblyExplorer.cs‎

Lines changed: 82 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Reflection;
55
using Machine.Specifications.Factories;
66
using Machine.Specifications.Model;
7+
using Machine.Specifications.Runner;
78
using Machine.Specifications.Sdk;
89
using Machine.Specifications.Utility;
910

@@ -18,37 +19,73 @@ public AssemblyExplorer()
1819
_contextFactory = new ContextFactory();
1920
}
2021

22+
public Context FindContexts(Type type, RunOptions options = null)
23+
{
24+
var types = new[] {type};
25+
26+
return types
27+
.Where(IsContext)
28+
.FilterBy(options)
29+
.Select(CreateContextFrom)
30+
.FirstOrDefault();
31+
}
32+
33+
public Context FindContexts(FieldInfo info, RunOptions options = null)
34+
{
35+
var types = new[] {info.DeclaringType};
36+
37+
return types
38+
.Where(IsContext)
39+
.FilterBy(options)
40+
.Select(t => CreateContextFrom(t, info))
41+
.FirstOrDefault();
42+
}
43+
2144
public IEnumerable<Context> FindContextsIn(Assembly assembly)
2245
{
23-
return EnumerateContextsIn(assembly).Select(CreateContextFrom);
46+
return FindContextsIn(assembly, options: null);
47+
}
48+
49+
public IEnumerable<Context> FindContextsIn(Assembly assembly, RunOptions options)
50+
{
51+
return EnumerateContextsIn(assembly)
52+
.FilterBy(options)
53+
.OrderBy(t => t.Namespace)
54+
.Select(CreateContextFrom);
2455
}
2556

2657
public IEnumerable<Context> FindContextsIn(Assembly assembly, string targetNamespace)
58+
{
59+
return FindContextsIn(assembly, targetNamespace, options: null);
60+
}
61+
62+
public IEnumerable<Context> FindContextsIn(Assembly assembly, string targetNamespace, RunOptions options)
2763
{
2864
return EnumerateContextsIn(assembly)
29-
.Where(x => x.Namespace == targetNamespace)
30-
.Select(CreateContextFrom);
65+
.Where(x => x.Namespace == targetNamespace)
66+
.FilterBy(options)
67+
.Select(CreateContextFrom);
3168
}
3269

3370
public IEnumerable<ICleanupAfterEveryContextInAssembly> FindAssemblyWideContextCleanupsIn(Assembly assembly)
3471
{
3572
return assembly.GetExportedTypes()
36-
.Where(x => x.GetInterfaces().Contains(typeof(ICleanupAfterEveryContextInAssembly)))
37-
.Select(x => (ICleanupAfterEveryContextInAssembly)Activator.CreateInstance(x));
73+
.Where(x => x.GetInterfaces().Contains(typeof(ICleanupAfterEveryContextInAssembly)))
74+
.Select(x => (ICleanupAfterEveryContextInAssembly) Activator.CreateInstance(x));
3875
}
3976

4077
public IEnumerable<ISupplementSpecificationResults> FindSpecificationSupplementsIn(Assembly assembly)
4178
{
4279
return assembly.GetExportedTypes()
43-
.Where(x => x.GetInterfaces().Contains(typeof(ISupplementSpecificationResults)))
44-
.Select(x => (ISupplementSpecificationResults)Activator.CreateInstance(x));
80+
.Where(x => x.GetInterfaces().Contains(typeof(ISupplementSpecificationResults)))
81+
.Select(x => (ISupplementSpecificationResults) Activator.CreateInstance(x));
4582
}
4683

4784
public IEnumerable<IAssemblyContext> FindAssemblyContextsIn(Assembly assembly)
4885
{
4986
return assembly.GetExportedTypes()
50-
.Where(x => x.GetTypeInfo().IsClass && !x.GetTypeInfo().IsAbstract && x.GetInterfaces().Contains(typeof(IAssemblyContext)))
51-
.Select(x => (IAssemblyContext)Activator.CreateInstance(x));
87+
.Where(x => x.GetTypeInfo().IsClass && !x.GetTypeInfo().IsAbstract && x.GetInterfaces().Contains(typeof(IAssemblyContext)))
88+
.Select(x => (IAssemblyContext) Activator.CreateInstance(x));
5289
}
5390

5491
Context CreateContextFrom(Type type)
@@ -76,30 +113,52 @@ static bool HasSpecificationMembers(Type type)
76113
static IEnumerable<Type> EnumerateContextsIn(Assembly assembly)
77114
{
78115
return assembly
79-
.GetTypes()
80-
.Where(IsContext)
81-
.OrderBy(t => t.Namespace);
116+
.GetTypes()
117+
.Where(IsContext);
82118
}
119+
}
83120

84-
public Context FindContexts(Type type)
121+
public static class FilteringExtensions
122+
{
123+
public static IEnumerable<Type> FilterBy(this IEnumerable<Type> types, RunOptions options)
85124
{
86-
if (IsContext(type))
125+
if (options == null)
87126
{
88-
return CreateContextFrom(type);
127+
return types;
89128
}
90129

91-
return null;
92-
}
130+
var filteredTypes = types;
93131

94-
public Context FindContexts(FieldInfo info)
95-
{
96-
Type type = info.DeclaringType;
97-
if (IsContext(type))
132+
var restrictToTypes = new HashSet<string>(options.Filters, StringComparer.OrdinalIgnoreCase);
133+
134+
if (restrictToTypes.Any())
135+
{
136+
filteredTypes = filteredTypes.Where(x => restrictToTypes.Contains(x.FullName));
137+
}
138+
139+
var includeTags = new HashSet<Tag>(options.IncludeTags.Select(tag => new Tag(tag)));
140+
var excludeTags = new HashSet<Tag>(options.ExcludeTags.Select(tag => new Tag(tag)));
141+
142+
if (includeTags.Any() || excludeTags.Any())
98143
{
99-
return CreateContextFrom(type, info);
144+
var extractor = new AttributeTagExtractor();
145+
146+
var filteredTypesWithTags = filteredTypes.Select(type => (Type: type, Tags: extractor.ExtractTags(type)));
147+
148+
if (includeTags.Any())
149+
{
150+
filteredTypesWithTags = filteredTypesWithTags.Where(x => x.Tags.Intersect(includeTags).Any());
151+
}
152+
153+
if (excludeTags.Any())
154+
{
155+
filteredTypesWithTags = filteredTypesWithTags.Where(x => !x.Tags.Intersect(excludeTags).Any());
156+
}
157+
158+
filteredTypes = filteredTypesWithTags.Select(x => x.Type);
100159
}
101160

102-
return null;
161+
return filteredTypes;
103162
}
104163
}
105164
}

‎src/Machine.Specifications/Runner/Impl/AssemblyRunner.cs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,18 +44,17 @@ public void Run(Assembly assembly, IEnumerable<Context> contexts)
4444

4545
try
4646
{
47-
hasExecutableSpecifications = contexts.Any(x => x.HasExecutableSpecifications);
48-
4947
var globalCleanups = _explorer.FindAssemblyWideContextCleanupsIn(assembly).ToList();
5048
var specificationSupplements = _explorer.FindSpecificationSupplementsIn(assembly).ToList();
5149

52-
if (hasExecutableSpecifications)
53-
{
54-
_assemblyStart(assembly);
55-
}
56-
5750
foreach (var context in contexts)
5851
{
52+
if (!hasExecutableSpecifications)
53+
{
54+
_assemblyStart(assembly);
55+
hasExecutableSpecifications = true;
56+
}
57+
5958
RunContext(context, globalCleanups, specificationSupplements);
6059
}
6160
}

0 commit comments

Comments
 (0)