Skip to content

Commit c777f1d

Browse files
yoffyoff
committed
Merge branch 'main' of github.com:github/codeql into python-api-enhancements
2 parents a9af135 + a23d8de commit c777f1d

File tree

827 files changed

+21601
-13204
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

827 files changed

+21601
-13204
lines changed

‎.github/workflows/check-change-note.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
name: Check change note
2+
13
on:
24
pull_request_target:
35
types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]

‎.github/workflows/docs-review.yml

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎config/identical-files.json

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
"cpp/ql/src/semmle/code/cpp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
3737
"cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
3838
"cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
39+
"cpp/ql/src/semmle/code/cpp/ir/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
3940
"csharp/ql/src/semmle/code/csharp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
4041
"csharp/ql/src/semmle/code/csharp/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
4142
"csharp/ql/src/semmle/code/csharp/dataflow/internal/tainttracking3/TaintTrackingImpl.qll",
@@ -376,7 +377,6 @@
376377
],
377378
"DuplicationProblems.inc.qhelp": [
378379
"cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
379-
"csharp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
380380
"javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",
381381
"python/ql/src/Metrics/DuplicationProblems.inc.qhelp"
382382
],
@@ -429,10 +429,11 @@
429429
"SSA C#": [
430430
"csharp/ql/src/semmle/code/csharp/dataflow/internal/SsaImplCommon.qll",
431431
"csharp/ql/src/semmle/code/csharp/controlflow/internal/pressa/SsaImplCommon.qll",
432-
"csharp/ql/src/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll"
432+
"csharp/ql/src/semmle/code/csharp/dataflow/internal/basessa/SsaImplCommon.qll",
433+
"csharp/ql/src/semmle/code/cil/internal/SsaImplCommon.qll"
433434
],
434435
"CryptoAlgorithms Python/JS": [
435436
"javascript/ql/src/semmle/javascript/security/CryptoAlgorithms.qll",
436437
"python/ql/src/semmle/crypto/Crypto.qll"
437438
]
438-
}
439+
}

‎cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Linq;
66
using Microsoft.Build.Construction;
77
using System.Xml;
8+
using System.IO;
89

910
namespace Semmle.Autobuild.Cpp.Tests
1011
{
@@ -43,6 +44,8 @@ bool IBuildActions.FileExists(string file)
4344
public IDictionary<string, int> RunProcess = new Dictionary<string, int>();
4445
public IDictionary<string, string> RunProcessOut = new Dictionary<string, string>();
4546
public IDictionary<string, string> RunProcessWorkingDirectory = new Dictionary<string, string>();
47+
public HashSet<string> CreateDirectories { get; } = new HashSet<string>();
48+
public HashSet<(string, string)> DownloadFiles { get; } = new HashSet<(string, string)>();
4649

4750
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
4851
{
@@ -135,6 +138,14 @@ string IBuildActions.PathCombine(params string[] parts)
135138

136139
string IBuildActions.GetFullPath(string path) => path;
137140

141+
string? IBuildActions.GetFileName(string? path) => Path.GetFileName(path?.Replace('\\', '/'));
142+
143+
public string? GetDirectoryName(string? path)
144+
{
145+
var dir = Path.GetDirectoryName(path?.Replace('\\', '/'));
146+
return dir is null ? path : path?.Substring(0, dir.Length);
147+
}
148+
138149
void IBuildActions.WriteAllText(string filename, string contents)
139150
{
140151
}
@@ -153,6 +164,18 @@ public string EnvironmentExpandEnvironmentVariables(string s)
153164
s = s.Replace($"%{kvp.Key}%", kvp.Value);
154165
return s;
155166
}
167+
168+
public void CreateDirectory(string path)
169+
{
170+
if (!CreateDirectories.Contains(path))
171+
throw new ArgumentException($"Missing CreateDirectory, {path}");
172+
}
173+
174+
public void DownloadFile(string address, string fileName)
175+
{
176+
if (!DownloadFiles.Contains((address, fileName)))
177+
throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
178+
}
156179
}
157180

158181
/// <summary>
@@ -213,6 +236,7 @@ CppAutobuilder CreateAutoBuilder(bool isWindows,
213236
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
214237
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
215238
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
239+
Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
216240
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
217241
Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
218242
Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
@@ -273,7 +297,8 @@ public void TestDefaultCppAutobuilder()
273297
[Fact]
274298
public void TestCppAutobuilderSuccess()
275299
{
276-
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1;
300+
Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
301+
Actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
277302
Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
278303
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
279304
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
@@ -286,11 +311,13 @@ public void TestCppAutobuilderSuccess()
286311
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
287312
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
288313
Actions.EnumerateDirectories[@"C:\Project"] = "";
314+
Actions.CreateDirectories.Add(@"C:\Project\.nuget");
315+
Actions.DownloadFiles.Add(("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", @"C:\Project\.nuget\nuget.exe"));
289316

290317
var autobuilder = CreateAutoBuilder(true);
291318
var solution = new TestSolution(@"C:\Project\test.sln");
292319
autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
293-
TestAutobuilderScript(autobuilder, 0, 2);
320+
TestAutobuilderScript(autobuilder, 0, 3);
294321
}
295322
}
296323
}

‎cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
</ItemGroup>
1818

1919
<ItemGroup>
20-
<PackageReference Include="Microsoft.Build" Version="16.0.461" />
20+
<PackageReference Include="Microsoft.Build" Version="16.9.0" />
2121
</ItemGroup>
2222

2323
<ItemGroup>

‎cpp/change-notes/2021-03-17-av-rule-79.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
lgtm,codescanning
2+
* The 'Resource not released in destructor' (cpp/resource-not-released-in-destructor) query has been improved to recognize more releases of resources.

‎cpp/ql/src/Diagnostics/ExtractionErrors.ql

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/**
2+
* @name Extraction errors
3+
* @description List all extraction errors for files in the source code directory.
4+
* @kind diagnostic
5+
* @id cpp/diagnostics/extraction-errors
6+
*/
7+
8+
import cpp
9+
import ExtractionErrors
10+
11+
from ExtractionError error
12+
where
13+
error instanceof ExtractionUnknownError or
14+
exists(error.getFile().getRelativePath())
15+
select error, "Extraction failed in " + error.getFile() + " with error " + error.getErrorMessage(),
16+
error.getSeverity()

‎cpp/ql/src/Diagnostics/ExtractionErrors.qll

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
/**
2+
* Provides a common hierarchy of all types of errors that can occur during extraction.
3+
*/
4+
5+
import cpp
6+
7+
/*
8+
* A note about how the C/C++ extractor emits diagnostics:
9+
* When the extractor frontend encounters an error, it emits a diagnostic message,
10+
* that includes a message, location and severity.
11+
* However, that process is best-effort and may fail (e.g. due to lack of memory).
12+
* Thus, if the extractor emitted at least one diagnostic of severity discretionary
13+
* error (or higher), it *also* emits a simple "There was an error during this compilation"
14+
* error diagnostic, without location information.
15+
* In the common case, this means that a compilation during which one or more errors happened also gets
16+
* the catch-all diagnostic.
17+
* This diagnostic has the empty string as file path.
18+
* We filter out these useless diagnostics if there is at least one error-level diagnostic
19+
* for the affected compilation in the database.
20+
* Otherwise, we show it to indicate that something went wrong and that we
21+
* don't know what exactly happened.
22+
*/
23+
24+
/**
25+
* An error that, if present, leads to a file being marked as non-successfully extracted.
26+
*/
27+
class ReportableError extends Diagnostic {
28+
ReportableError() {
29+
(
30+
this instanceof CompilerDiscretionaryError or
31+
this instanceof CompilerError or
32+
this instanceof CompilerCatastrophe
33+
) and
34+
// Filter for the catch-all diagnostic, see note above.
35+
not this.getFile().getAbsolutePath() = ""
36+
}
37+
}
38+
39+
private newtype TExtractionError =
40+
TReportableError(ReportableError err) or
41+
TCompilationFailed(Compilation c, File f) {
42+
f = c.getAFileCompiled() and not c.normalTermination()
43+
} or
44+
// Show the catch-all diagnostic (see note above) only if we haven't seen any other error-level diagnostic
45+
// for that compilation
46+
TUnknownError(CompilerError err) {
47+
not exists(ReportableError e | e.getCompilation() = err.getCompilation())
48+
}
49+
50+
/**
51+
* Superclass for the extraction error hierarchy.
52+
*/
53+
class ExtractionError extends TExtractionError {
54+
/** Gets the string representation of the error. */
55+
string toString() { none() }
56+
57+
/** Gets the error message for this error. */
58+
string getErrorMessage() { none() }
59+
60+
/** Gets the file this error occured in. */
61+
File getFile() { none() }
62+
63+
/** Gets the location this error occured in. */
64+
Location getLocation() { none() }
65+
66+
/** Gets the SARIF severity of this error. */
67+
int getSeverity() {
68+
// Unfortunately, we can't distinguish between errors and fatal errors in SARIF,
69+
// so all errors have severity 2.
70+
result = 2
71+
}
72+
}
73+
74+
/**
75+
* An unrecoverable extraction error, where extraction was unable to finish.
76+
* This can be caused by a multitude of reasons, for example:
77+
* - hitting a frontend assertion
78+
* - crashing due to dereferencing an invalid pointer
79+
* - stack overflow
80+
* - out of memory
81+
*/
82+
class ExtractionUnrecoverableError extends ExtractionError, TCompilationFailed {
83+
Compilation c;
84+
File f;
85+
86+
ExtractionUnrecoverableError() { this = TCompilationFailed(c, f) }
87+
88+
override string toString() {
89+
result = "Unrecoverable extraction error while compiling " + f.toString()
90+
}
91+
92+
override string getErrorMessage() { result = "unrecoverable compilation failure." }
93+
94+
override File getFile() { result = f }
95+
96+
override Location getLocation() { result = f.getLocation() }
97+
}
98+
99+
/**
100+
* A recoverable extraction error.
101+
* These are compiler errors from the frontend.
102+
* Upon encountering one of these, we still continue extraction, but the
103+
* database will be incomplete for that file.
104+
*/
105+
class ExtractionRecoverableError extends ExtractionError, TReportableError {
106+
ReportableError err;
107+
108+
ExtractionRecoverableError() { this = TReportableError(err) }
109+
110+
override string toString() { result = "Recoverable extraction error: " + err }
111+
112+
override string getErrorMessage() { result = err.getFullMessage() }
113+
114+
override File getFile() { result = err.getFile() }
115+
116+
override Location getLocation() { result = err.getLocation() }
117+
}
118+
119+
/**
120+
* An unknown error happened during extraction.
121+
* These are only displayed if we know that we encountered an error during extraction,
122+
* but, for some reason, failed to emit a proper diagnostic with location information
123+
* and error message.
124+
*/
125+
class ExtractionUnknownError extends ExtractionError, TUnknownError {
126+
CompilerError err;
127+
128+
ExtractionUnknownError() { this = TUnknownError(err) }
129+
130+
override string toString() { result = "Unknown extraction error: " + err }
131+
132+
override string getErrorMessage() { result = err.getFullMessage() }
133+
134+
override File getFile() { result = err.getFile() }
135+
136+
override Location getLocation() { result = err.getLocation() }
137+
}

‎cpp/ql/src/Diagnostics/FailedExtractions.ql renamed to ‎cpp/ql/src/Diagnostics/FailedExtractorInvocations.ql

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
/**
2-
* @name Failed extractions
3-
* @description Gives the command-line of compilations for which extraction did not run to completion.
2+
* @name Failed extractor invocations
3+
* @description Gives the command line of compilations for which extraction did not run to completion.
44
* @kind diagnostic
5-
* @id cpp/diagnostics/failed-extractions
5+
* @id cpp/diagnostics/failed-extractor-invocations
66
*/
77

88
import cpp
@@ -19,4 +19,4 @@ string describe(Compilation c) {
1919

2020
from Compilation c
2121
where not c.normalTermination()
22-
select c, "Extraction failed for " + describe(c), 2
22+
select c, "Extraction aborted for " + describe(c), 2

‎cpp/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/**
2+
* @name Successfully extracted files
3+
* @description Lists all files in the source code directory that were extracted without encountering an error in the file.
4+
* @kind diagnostic
5+
* @id cpp/diagnostics/successfully-extracted-files
6+
*/
7+
8+
import cpp
9+
import ExtractionErrors
10+
11+
from File f
12+
where
13+
not exists(ExtractionError e | e.getFile() = f) and
14+
exists(f.getRelativePath())
15+
select f, ""

0 commit comments

Comments
 (0)