Skip to content

Commit d749335

Browse files
tamasvajktamasvajk
committed
C#: Limit extracted compilation and extraction messages
1 parent e8e1dc0 commit d749335

File tree

10 files changed

+100
-55
lines changed

10 files changed

+100
-55
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,7 @@ public override void Populate(TextWriter trapFile)
8080
// Diagnostics
8181
Context.Compilation
8282
.GetDiagnostics()
83-
.Select(d => new Diagnostic(Context, d))
84-
.ForEach((diag, index) => trapFile.diagnostic_for(diag, this, 0, index));
83+
.ForEach((diag, index) => new CompilerDiagnostic(Context, diag, this, index));
8584
}
8685

8786
public void PopulatePerformance(PerformanceMetrics p)

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/CompilerDiagnostic.cs

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
using System.Collections.Concurrent;
2+
using System.IO;
3+
using Semmle.Util;
4+
5+
namespace Semmle.Extraction.CSharp.Entities
6+
{
7+
internal class CompilerDiagnostic : FreshEntity
8+
{
9+
private static readonly int limit = EnvironmentVariables.TryGetExtractorOption<int>("COMPILER_DIAGNOSTIC_LIMIT") ?? 1000;
10+
private static readonly ConcurrentDictionary<string, int> messageCounts = new();
11+
12+
private readonly Microsoft.CodeAnalysis.Diagnostic diagnostic;
13+
private readonly Compilation compilation;
14+
private readonly int index;
15+
16+
public CompilerDiagnostic(Context cx, Microsoft.CodeAnalysis.Diagnostic diag, Compilation compilation, int index) : base(cx)
17+
{
18+
diagnostic = diag;
19+
this.compilation = compilation;
20+
this.index = index;
21+
TryPopulate();
22+
}
23+
24+
protected override void Populate(TextWriter trapFile)
25+
{
26+
// The below doesn't limit the extractor messages to the exact limit, but it's good enough.
27+
var key = diagnostic.Id;
28+
var messageCount = messageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
29+
if (messageCount > limit)
30+
{
31+
if (messageCount == limit + 1)
32+
{
33+
Context.Extractor.Logger.LogWarning($"Stopped logging {key} compiler diagnostics after reaching {limit}");
34+
}
35+
36+
return;
37+
}
38+
39+
trapFile.diagnostics(this, (int)diagnostic.Severity, key, diagnostic.Descriptor.Title.ToString(),
40+
diagnostic.GetMessage(), Context.CreateLocation(diagnostic.Location));
41+
42+
trapFile.diagnostic_for(this, compilation, 0, index);
43+
}
44+
}
45+
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Diagnostic.cs

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,10 @@ internal static void destructor_location(this TextWriter trapFile, Destructor de
122122
internal static void destructors(this TextWriter trapFile, Destructor destructor, string name, Type containingType, Destructor original) =>
123123
trapFile.WriteTuple("destructors", destructor, name, containingType, original);
124124

125-
internal static void diagnostic_for(this TextWriter trapFile, Diagnostic diag, Compilation comp, int fileNo, int index) =>
125+
internal static void diagnostic_for(this TextWriter trapFile, CompilerDiagnostic diag, Compilation comp, int fileNo, int index) =>
126126
trapFile.WriteTuple("diagnostic_for", diag, comp, fileNo, index);
127127

128-
internal static void diagnostics(this TextWriter trapFile, Diagnostic diag, int severity, string errorTag, string errorMessage, string fullErrorMessage, Location location) =>
128+
internal static void diagnostics(this TextWriter trapFile, CompilerDiagnostic diag, int severity, string errorTag, string errorMessage, string fullErrorMessage, Location location) =>
129129
trapFile.WriteTuple("diagnostics", diag, severity, errorTag, errorMessage, fullErrorMessage, location);
130130

131131
internal static void dynamic_member_name(this TextWriter trapFile, Expression e, string name) =>

‎csharp/extractor/Semmle.Extraction/Entities/ExtractionError.cs

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
using System.IO;
2+
using System.Threading;
3+
using Semmle.Util;
4+
5+
namespace Semmle.Extraction.Entities
6+
{
7+
internal class ExtractionMessage : FreshEntity
8+
{
9+
private static readonly int limit = EnvironmentVariables.TryGetExtractorOption<int>("MESSAGE_LIMIT") ?? 10000;
10+
private static int messageCount = 0;
11+
12+
private readonly Message msg;
13+
14+
public ExtractionMessage(Context cx, Message msg) : base(cx)
15+
{
16+
this.msg = msg;
17+
TryPopulate();
18+
}
19+
20+
protected override void Populate(TextWriter trapFile)
21+
{
22+
// The below doesn't limit the extractor messages to the exact limit, but it's good enough.
23+
Interlocked.Increment(ref messageCount);
24+
if (messageCount > limit)
25+
{
26+
if (messageCount == limit + 1)
27+
{
28+
Context.Extractor.Logger.LogWarning($"Stopped logging extractor messages after reaching {limit}");
29+
}
30+
return;
31+
}
32+
33+
trapFile.extractor_messages(this, msg.Severity, "C# extractor", msg.Text, msg.EntityText ?? string.Empty,
34+
msg.Location ?? Context.CreateLocation(), msg.StackTrace ?? string.Empty);
35+
}
36+
}
37+
}

‎csharp/extractor/Semmle.Util/EnvironmentVariables.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using System;
2+
using System.Globalization;
3+
using System.Numerics;
24

35
namespace Semmle.Util
46
{
@@ -7,6 +9,16 @@ public class EnvironmentVariables
79
public static string? GetExtractorOption(string name) =>
810
Environment.GetEnvironmentVariable($"CODEQL_EXTRACTOR_CSHARP_OPTION_{name.ToUpper()}");
911

12+
public static T? TryGetExtractorOption<T>(string name) where T : struct, INumberBase<T>
13+
{
14+
var value = GetExtractorOption(name);
15+
if (T.TryParse(value, NumberStyles.Number, CultureInfo.InvariantCulture, out var result))
16+
{
17+
return result;
18+
}
19+
return null;
20+
}
21+
1022
public static int GetDefaultNumberOfThreads()
1123
{
1224
if (!int.TryParse(Environment.GetEnvironmentVariable("CODEQL_THREADS"), out var threads) || threads == -1)

‎csharp/ql/integration-tests/all-platforms/standalone/CompilerMessage.expected

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,4 @@
11
| Program.cs:2:9:2:9 | CS0103: The name 'M' does not exist in the current context | Error CS0103 The name 'M' does not exist in the current context |
22
| Program.cs:2:15:2:15 | CS0103: The name 'M' does not exist in the current context | Error CS0103 The name 'M' does not exist in the current context |
3-
| Program.cs:2:21:2:21 | CS0103: The name 'M' does not exist in the current context | Error CS0103 The name 'M' does not exist in the current context |
4-
| test-db/working/implicitUsings/GlobalUsings.g.cs:3:1:3:28 | CS8019: Unnecessary using directive. | Hidden CS8019 Unnecessary using directive. |
5-
| test-db/working/implicitUsings/GlobalUsings.g.cs:4:1:4:48 | CS8019: Unnecessary using directive. | Hidden CS8019 Unnecessary using directive. |
6-
| test-db/working/implicitUsings/GlobalUsings.g.cs:5:1:5:31 | CS8019: Unnecessary using directive. | Hidden CS8019 Unnecessary using directive. |
7-
| test-db/working/implicitUsings/GlobalUsings.g.cs:6:1:6:33 | CS8019: Unnecessary using directive. | Hidden CS8019 Unnecessary using directive. |
83
| test-db/working/implicitUsings/GlobalUsings.g.cs:7:1:7:37 | CS8019: Unnecessary using directive. | Hidden CS8019 Unnecessary using directive. |
94
| test-db/working/implicitUsings/GlobalUsings.g.cs:8:1:8:38 | CS8019: Unnecessary using directive. | Hidden CS8019 Unnecessary using directive. |
10-
| test-db/working/implicitUsings/GlobalUsings.g.cs:9:1:9:44 | CS8019: Unnecessary using directive. | Hidden CS8019 Unnecessary using directive. |

‎csharp/ql/integration-tests/all-platforms/standalone/ExtractortMessages.expected

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
| Program.cs:2:9:2:11 | Failed to determine type | 5 | M() |
2-
| Program.cs:2:9:2:11 | Unable to resolve target for call. (Compilation error?) | 5 | M() |
32
| Program.cs:2:9:2:17 | Failed to determine type | 5 | M() + M() |
43
| Program.cs:2:9:2:23 | Failed to determine type | 5 | M() + M() + M() |
5-
| Program.cs:2:15:2:17 | Failed to determine type | 5 | M() |
6-
| Program.cs:2:15:2:17 | Unable to resolve target for call. (Compilation error?) | 5 | M() |
74
| Program.cs:2:21:2:23 | Failed to determine type | 5 | M() |
85
| Program.cs:2:21:2:23 | Unable to resolve target for call. (Compilation error?) | 5 | M() |

‎csharp/ql/integration-tests/all-platforms/standalone/test.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import os
12
from create_database_utils import *
23

4+
os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_COMPILER_DIAGNOSTIC_LIMIT'] = '2'
5+
os.environ['CODEQL_EXTRACTOR_CSHARP_OPTION_MESSAGE_LIMIT'] = '5'
36
run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])

0 commit comments

Comments
 (0)