Skip to content

Commit 901db73

Browse files
authored
Merge pull request github#12745 from github/redsun82/swift-logging
Swift: introduce usage of binlog
2 parents 5272810 + cbe247e commit 901db73

File tree

17 files changed

+564
-16
lines changed

17 files changed

+564
-16
lines changed

misc/codegen/templates/cpp_classes_h.mustache

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
#include <iostream>
66
#include <optional>
77
#include <vector>
8+
#include <binlog/binlog.hpp>
9+
#include <binlog/adapt_stdoptional.hpp>
810

911
#include "{{trap_library}}/TrapLabel.h"
1012
#include "{{trap_library}}/TrapTagTraits.h"
@@ -80,3 +82,9 @@ struct detail::ToTrapClassFunctor<{{name}}Tag> {
8082
};
8183
{{/classes}}
8284
}
85+
86+
{{#classes}}
87+
{{#final}}
88+
BINLOG_ADAPT_STRUCT(codeql::{{name}}, id{{> cpp_list_fields}});
89+
{{/final}}
90+
{{/classes}}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{{#bases}}{{#ref}}{{> cpp_list_fields}}{{/ref}}{{/bases}}{{#fields}}, {{field_name}}{{/fields}}

misc/codegen/templates/trap_traps_h.mustache

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
#include <iostream>
66
#include <string>
7+
#include <binlog/binlog.hpp>
78

89
#include "{{trap_library_dir}}/TrapLabel.h"
910
#include "{{trap_library_dir}}/TrapTagTraits.h"
@@ -43,3 +44,7 @@ struct ToBindingTrapFunctor<{{type}}> {
4344
{{/id}}
4445
{{/traps}}
4546
}
47+
48+
{{#traps}}
49+
BINLOG_ADAPT_STRUCT(codeql::{{name}}Trap{{#fields}}, {{field_name}}{{/fields}});
50+
{{/traps}}

swift/README.md

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
The Swift CodeQL package is an experimental and unsupported work in progress.
66

7+
##
8+
79
## Building the Swift extractor
810

911
First ensure you have Bazel installed, for example with
@@ -28,7 +30,9 @@ set up the search path
2830
in [the per-user CodeQL configuration file](https://docs.github.com/en/code-security/codeql-cli/using-the-codeql-cli/specifying-command-options-in-a-codeql-configuration-file#using-a-codeql-configuration-file)
2931
.
3032

31-
## Code generation
33+
## Development
34+
35+
### Code generation
3236

3337
Run
3438

@@ -41,7 +45,27 @@ to update generated files. This can be shortened to
4145

4246
You can also run `../misc/codegen/codegen.py`, as long as you are beneath the `swift` directory.
4347

44-
## IDE setup
48+
### Logging configuration
49+
50+
A log file is produced for each run under `CODEQL_EXTRACTOR_SWIFT_LOG_DIR` (the usual DB log directory).
51+
52+
You can use the environment variable `CODEQL_EXTRACTOR_SWIFT_LOG_LEVELS` to configure levels for
53+
loggers and outputs. This must have the form of a comma separated `spec:min_level` list, where
54+
`spec` is either a glob pattern (made up of alphanumeric, `/`, `*` and `.` characters) for
55+
matching logger names or one of `out:bin`, `out:text` or `out:console`, and `min_level` is one
56+
of `trace`, `debug`, `info`, `warning`, `error`, `critical` or `no_logs` to turn logs completely off.
57+
58+
Current output default levels are no binary logs, `info` logs or higher in the text file and `warning` logs or higher on
59+
standard error. By default, all loggers are configured with the lowest logging level of all outputs (`info` by default).
60+
Logger names are visible in the textual logs between `[...]`. Examples are `extractor/dispatcher`
61+
or `extractor/<source filename>.trap`. An example of `CODEQL_EXTRACTOR_SWIFT_LOG_LEVELS` usage is the following:
62+
63+
```bash
64+
export CODEQL_EXTRACTOR_SWIFT_LOG_LEVELS=out:console:trace,out:text:no_logs,*:warning,*.trap:trace
65+
```
66+
67+
This will turn off generation of a text log file, redirecting all logs to standard error, but will make all loggers only
68+
write warnings or above, except for trap emission logs which will output all logs.
4569

4670
### CLion and the native bazel plugin
4771

@@ -84,3 +108,7 @@ In particular for breakpoints to work you might need to setup the following remo
84108
|-------------|--------------------------------------|
85109
| `swift` | `/absolute/path/to/codeql/swift` |
86110
| `bazel-out` | `/absolute/path/to/codeql/bazel-out` |
111+
112+
### Thread safety
113+
114+
The extractor is single-threaded, and there was no effort to make anything in it thread-safe.

swift/extractor/infra/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ swift_cc_library(
88
deps = [
99
"//swift/extractor/config",
1010
"//swift/extractor/infra/file",
11+
"//swift/extractor/infra/log",
1112
"//swift/extractor/trap",
1213
"//swift/third_party/swift-llvm-support",
1314
],

swift/extractor/infra/SwiftDispatcher.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "swift/extractor/infra/SwiftLocationExtractor.h"
1414
#include "swift/extractor/infra/SwiftBodyEmissionStrategy.h"
1515
#include "swift/extractor/config/SwiftExtractorState.h"
16+
#include "swift/extractor/infra/log/SwiftLogging.h"
1617

1718
namespace codeql {
1819

@@ -151,7 +152,9 @@ class SwiftDispatcher {
151152
return *l;
152153
}
153154
waitingForNewLabel = e;
155+
// TODO: add tracing logs for visited stuff, maybe within the translators?
154156
visit(e, std::forward<Args>(args)...);
157+
Log::flush();
155158
// TODO when everything is moved to structured C++ classes, this should be moved to createEntry
156159
if (auto l = store.get(e)) {
157160
if constexpr (IsLocatable<E>) {
@@ -329,6 +332,7 @@ class SwiftDispatcher {
329332
SwiftBodyEmissionStrategy& bodyEmissionStrategy;
330333
Store::Handle waitingForNewLabel{std::monostate{}};
331334
std::unordered_set<swift::ModuleDecl*> encounteredModules;
335+
Logger logger{"dispatcher"};
332336
};
333337

334338
} // namespace codeql

swift/extractor/infra/file/TargetFile.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ class TargetFile {
3636
return *this;
3737
}
3838

39+
const std::filesystem::path& target() const { return targetPath; }
40+
3941
private:
4042
TargetFile(const std::filesystem::path& target,
4143
const std::filesystem::path& targetDir,

swift/extractor/infra/log/BUILD.bazel

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
cc_library(
2+
name = "log",
3+
srcs = glob(["*.cpp"]),
4+
hdrs = glob(["*.h"]),
5+
visibility = ["//visibility:public"],
6+
deps = ["@binlog"],
7+
)
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
#include "swift/extractor/infra/log/SwiftLogging.h"
2+
3+
#include <filesystem>
4+
#include <stdlib.h>
5+
#include <optional>
6+
7+
#define LEVEL_REGEX_PATTERN "trace|debug|info|warning|error|critical|no_logs"
8+
9+
BINLOG_ADAPT_ENUM(codeql::Log::Level, trace, debug, info, warning, error, critical, no_logs)
10+
11+
namespace codeql {
12+
13+
namespace {
14+
using LevelRule = std::pair<std::regex, Log::Level>;
15+
using LevelRules = std::vector<LevelRule>;
16+
17+
Log::Level getLevelFor(std::string_view name, const LevelRules& rules, Log::Level dflt) {
18+
for (auto it = rules.rbegin(); it != rules.rend(); ++it) {
19+
if (std::regex_match(std::begin(name), std::end(name), it->first)) {
20+
return it->second;
21+
}
22+
}
23+
return dflt;
24+
}
25+
26+
const char* getEnvOr(const char* var, const char* dflt) {
27+
if (const char* ret = getenv(var)) {
28+
return ret;
29+
}
30+
return dflt;
31+
}
32+
33+
std::string_view matchToView(std::csub_match m) {
34+
return {m.first, static_cast<size_t>(m.length())};
35+
}
36+
37+
Log::Level stringToLevel(std::string_view v) {
38+
if (v == "trace") return Log::Level::trace;
39+
if (v == "debug") return Log::Level::debug;
40+
if (v == "info") return Log::Level::info;
41+
if (v == "warning") return Log::Level::warning;
42+
if (v == "error") return Log::Level::error;
43+
if (v == "critical") return Log::Level::critical;
44+
return Log::Level::no_logs;
45+
}
46+
47+
Log::Level matchToLevel(std::csub_match m) {
48+
return stringToLevel(matchToView(m));
49+
}
50+
51+
} // namespace
52+
53+
std::vector<std::string> Log::collectSeverityRulesAndReturnProblems(const char* envVar) {
54+
std::vector<std::string> problems;
55+
if (auto levels = getEnvOr(envVar, nullptr)) {
56+
// expect comma-separated <glob pattern>:<log severity>
57+
std::regex comma{","};
58+
std::regex levelAssignment{R"((?:([*./\w]+)|(?:out:(bin|text|console))):()" LEVEL_REGEX_PATTERN
59+
")"};
60+
std::cregex_token_iterator begin{levels, levels + strlen(levels), comma, -1};
61+
std::cregex_token_iterator end{};
62+
for (auto it = begin; it != end; ++it) {
63+
std::cmatch match;
64+
if (std::regex_match(it->first, it->second, match, levelAssignment)) {
65+
auto level = matchToLevel(match[3]);
66+
if (match[1].matched) {
67+
auto pattern = match[1].str();
68+
// replace all "*" with ".*" and all "." with "\.", turning the glob pattern into a regex
69+
std::string::size_type pos = 0;
70+
while ((pos = pattern.find_first_of("*.", pos)) != std::string::npos) {
71+
pattern.insert(pos, (pattern[pos] == '*') ? "." : "\\");
72+
pos += 2;
73+
}
74+
sourceRules.emplace_back(pattern, level);
75+
} else {
76+
auto out = matchToView(match[2]);
77+
if (out == "bin") {
78+
binary.level = level;
79+
} else if (out == "text") {
80+
text.level = level;
81+
} else if (out == "console") {
82+
console.level = level;
83+
}
84+
}
85+
} else {
86+
problems.emplace_back("Malformed log level rule: " + it->str());
87+
}
88+
}
89+
}
90+
return problems;
91+
}
92+
93+
void Log::configure() {
94+
// as we are configuring logging right now, we collect problems and log them at the end
95+
auto problems = collectSeverityRulesAndReturnProblems("CODEQL_EXTRACTOR_SWIFT_LOG_LEVELS");
96+
if (text || binary) {
97+
std::filesystem::path logFile = getEnvOr("CODEQL_EXTRACTOR_SWIFT_LOG_DIR", ".");
98+
logFile /= logRootName;
99+
logFile /= std::to_string(std::chrono::system_clock::now().time_since_epoch().count());
100+
std::error_code ec;
101+
std::filesystem::create_directories(logFile.parent_path(), ec);
102+
if (!ec) {
103+
if (text) {
104+
logFile.replace_extension(".log");
105+
textFile.open(logFile);
106+
if (!textFile) {
107+
problems.emplace_back("Unable to open text log file " + logFile.string());
108+
text.level = Level::no_logs;
109+
}
110+
}
111+
if (binary) {
112+
logFile.replace_extension(".blog");
113+
binary.output.open(logFile, std::fstream::out | std::fstream::binary);
114+
if (!binary.output) {
115+
problems.emplace_back("Unable to open binary log file " + logFile.string());
116+
binary.level = Level::no_logs;
117+
}
118+
}
119+
} else {
120+
problems.emplace_back("Unable to create log directory " + logFile.parent_path().string() +
121+
": " + ec.message());
122+
binary.level = Level::no_logs;
123+
text.level = Level::no_logs;
124+
}
125+
}
126+
for (const auto& problem : problems) {
127+
LOG_ERROR("{}", problem);
128+
}
129+
LOG_INFO("Logging configured (binary: {}, text: {}, console: {})", binary.level, text.level,
130+
console.level);
131+
flushImpl();
132+
}
133+
134+
void Log::flushImpl() {
135+
session.consume(*this);
136+
}
137+
138+
Log::LoggerConfiguration Log::getLoggerConfigurationImpl(std::string_view name) {
139+
LoggerConfiguration ret{session, std::string{logRootName}};
140+
ret.fullyQualifiedName += '/';
141+
ret.fullyQualifiedName += name;
142+
ret.level = std::min({binary.level, text.level, console.level});
143+
ret.level = getLevelFor(ret.fullyQualifiedName, sourceRules, ret.level);
144+
// avoid Logger constructor loop
145+
if (name != "logging") {
146+
LOG_DEBUG("Configuring logger {} with level {}", ret.fullyQualifiedName, ret.level);
147+
}
148+
return ret;
149+
}
150+
151+
Log& Log::write(const char* buffer, std::streamsize size) {
152+
if (text) text.write(buffer, size);
153+
if (binary) binary.write(buffer, size);
154+
if (console) console.write(buffer, size);
155+
return *this;
156+
}
157+
158+
Logger& Log::logger() {
159+
static Logger ret{getLoggerConfigurationImpl("logging")};
160+
return ret;
161+
}
162+
163+
} // namespace codeql

0 commit comments

Comments
 (0)