Skip to content

Commit 4e71155

Browse files
author
Paolo Tranquilli
committed
Rust: move steps breakdown from JSON diagnostics to the DB
1 parent 5251dc2 commit 4e71155

18 files changed

+236
-320
lines changed

‎rust/extractor/src/diagnostics.rs

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ pub struct Diagnostics<T> {
7676

7777
#[derive(Default, Debug, Clone, Copy, Serialize, PartialEq, Eq, Hash)]
7878
#[serde(rename_all = "camelCase")]
79-
enum ExtractionStepKind {
79+
pub enum ExtractionStepKind {
8080
#[default]
8181
LoadManifest,
8282
LoadSource,
@@ -87,9 +87,9 @@ enum ExtractionStepKind {
8787
#[derive(Debug, Clone, Serialize)]
8888
#[serde(rename_all = "camelCase")]
8989
pub struct ExtractionStep {
90-
action: ExtractionStepKind,
91-
file: PathBuf,
92-
ms: u128,
90+
pub action: ExtractionStepKind,
91+
pub file: PathBuf,
92+
pub ms: u128,
9393
}
9494

9595
impl ExtractionStep {
@@ -186,14 +186,7 @@ struct ExtractionSummary {
186186
durations: DurationsSummary,
187187
}
188188

189-
#[derive(Debug, Default, Clone, Serialize)]
190-
#[serde(rename_all = "camelCase")]
191-
struct ExtractionAttributes {
192-
steps: Vec<ExtractionStep>,
193-
summary: ExtractionSummary,
194-
}
195-
196-
type ExtractionDiagnostics = Diagnostics<ExtractionAttributes>;
189+
type ExtractionDiagnostics = Diagnostics<ExtractionSummary>;
197190

198191
fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary {
199192
let mut number_of_manifests = 0;
@@ -229,9 +222,9 @@ fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary {
229222
pub fn emit_extraction_diagnostics(
230223
start: Instant,
231224
config: &Config,
232-
steps: Vec<ExtractionStep>,
225+
steps: &[ExtractionStep],
233226
) -> anyhow::Result<()> {
234-
let summary = summary(start, &steps);
227+
let summary = summary(start, steps);
235228
let diagnostics = ExtractionDiagnostics {
236229
source: Source {
237230
id: "rust/extractor/telemetry".to_owned(),
@@ -243,7 +236,7 @@ pub fn emit_extraction_diagnostics(
243236
..Default::default()
244237
},
245238
timestamp: Utc::now(),
246-
attributes: ExtractionAttributes { steps, summary },
239+
attributes: summary,
247240
..Default::default()
248241
};
249242

‎rust/extractor/src/main.rs

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use crate::diagnostics::{emit_extraction_diagnostics, ExtractionStep};
22
use crate::rust_analyzer::path_to_file_id;
33
use anyhow::Context;
44
use archive::Archiver;
5-
use log::info;
5+
use log::{info, warn};
66
use ra_ap_hir::Semantics;
77
use ra_ap_ide_db::line_index::{LineCol, LineIndex};
88
use ra_ap_ide_db::RootDatabase;
@@ -140,7 +140,27 @@ impl<'a> Extractor<'a> {
140140
start: Instant,
141141
cfg: &config::Config,
142142
) -> anyhow::Result<()> {
143-
emit_extraction_diagnostics(start, cfg, self.steps)
143+
emit_extraction_diagnostics(start, cfg, &self.steps)?;
144+
let mut trap = self.traps.create("diagnostics", "extraction");
145+
for step in self.steps {
146+
let file_label = trap.emit_file(&step.file);
147+
let step_label = trap.writer.fresh_id();
148+
let ms = usize::try_from(step.ms).unwrap_or_else(|_e| {
149+
warn!("extraction step duration overflowed ({step:?})");
150+
i32::MAX as usize
151+
});
152+
trap.writer.add_tuple(
153+
"extractor_steps",
154+
vec![
155+
step_label.into(),
156+
format!("{:?}", step.action).into(),
157+
file_label.into(),
158+
ms.into(),
159+
],
160+
);
161+
}
162+
trap.commit()?;
163+
Ok(())
144164
}
145165
}
146166

‎rust/extractor/src/trap.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,8 +243,8 @@ impl TrapFileProvider {
243243
})
244244
}
245245

246-
pub fn create(&self, category: &str, key: &Path) -> TrapFile {
247-
let path = file_paths::path_for(&self.trap_dir.join(category), key, "trap");
246+
pub fn create(&self, category: &str, key: impl AsRef<Path>) -> TrapFile {
247+
let path = file_paths::path_for(&self.trap_dir.join(category), key.as_ref(), "trap");
248248
debug!("creating trap file {}", path.display());
249249
let mut writer = trap::Writer::new();
250250
extractor::populate_empty_location(&mut writer);

‎rust/prefix.dbscheme

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,10 @@ locatable_locations(
33
int id: @locatable ref,
44
int location: @location_default ref
55
);
6+
7+
extractor_steps(
8+
unique int id: @extractor_step,
9+
string action: string ref,
10+
int file: @file ref,
11+
int duration_ms: int ref
12+
)

‎rust/ql/integration-tests/conftest.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,8 @@ def manifests(cwd):
1616

1717
@pytest.fixture
1818
def rust_check_diagnostics(check_diagnostics):
19-
check_diagnostics.replacements += [
20-
("Cargo.toml|rust-project.json", "<manifest-file>"),
21-
]
2219
check_diagnostics.redact += [
23-
"attributes.summary.durations.*.ms",
24-
"attributes.summary.durations.*.pretty",
25-
"attributes.steps.ms",
20+
"attributes.durations.*.ms",
21+
"attributes.durations.*.pretty",
2622
]
27-
check_diagnostics.sort = True # the order of the steps is not stable
2823
return check_diagnostics

‎rust/ql/integration-tests/hello-project/diagnostics.expected

Lines changed: 23 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -1,108 +1,29 @@
11
{
22
"attributes": {
3-
"steps": [
4-
{
5-
"action": "extract",
6-
"file": "<test-root-directory>/src/directory_module/mod.rs",
7-
"ms": "__REDACTED__"
8-
},
9-
{
10-
"action": "extract",
11-
"file": "<test-root-directory>/src/directory_module/nested_module.rs",
12-
"ms": "__REDACTED__"
13-
},
14-
{
15-
"action": "extract",
16-
"file": "<test-root-directory>/src/directory_module/not_loaded.rs",
17-
"ms": "__REDACTED__"
18-
},
19-
{
20-
"action": "extract",
21-
"file": "<test-root-directory>/src/file_module.rs",
22-
"ms": "__REDACTED__"
23-
},
24-
{
25-
"action": "extract",
26-
"file": "<test-root-directory>/src/main.rs",
27-
"ms": "__REDACTED__"
28-
},
29-
{
30-
"action": "loadManifest",
31-
"file": "<test-root-directory>/<manifest-file>",
32-
"ms": "__REDACTED__"
33-
},
34-
{
35-
"action": "loadSource",
36-
"file": "<test-root-directory>/src/directory_module/mod.rs",
37-
"ms": "__REDACTED__"
38-
},
39-
{
40-
"action": "loadSource",
41-
"file": "<test-root-directory>/src/directory_module/nested_module.rs",
42-
"ms": "__REDACTED__"
43-
},
44-
{
45-
"action": "loadSource",
46-
"file": "<test-root-directory>/src/file_module.rs",
47-
"ms": "__REDACTED__"
48-
},
49-
{
50-
"action": "loadSource",
51-
"file": "<test-root-directory>/src/main.rs",
52-
"ms": "__REDACTED__"
53-
},
54-
{
55-
"action": "parse",
56-
"file": "<test-root-directory>/src/directory_module/mod.rs",
57-
"ms": "__REDACTED__"
58-
},
59-
{
60-
"action": "parse",
61-
"file": "<test-root-directory>/src/directory_module/nested_module.rs",
62-
"ms": "__REDACTED__"
63-
},
64-
{
65-
"action": "parse",
66-
"file": "<test-root-directory>/src/directory_module/not_loaded.rs",
67-
"ms": "__REDACTED__"
68-
},
69-
{
70-
"action": "parse",
71-
"file": "<test-root-directory>/src/file_module.rs",
72-
"ms": "__REDACTED__"
73-
},
74-
{
75-
"action": "parse",
76-
"file": "<test-root-directory>/src/main.rs",
77-
"ms": "__REDACTED__"
3+
"durations": {
4+
"extract": {
5+
"ms": "__REDACTED__",
6+
"pretty": "__REDACTED__"
7+
},
8+
"loadManifest": {
9+
"ms": "__REDACTED__",
10+
"pretty": "__REDACTED__"
11+
},
12+
"loadSource": {
13+
"ms": "__REDACTED__",
14+
"pretty": "__REDACTED__"
15+
},
16+
"parse": {
17+
"ms": "__REDACTED__",
18+
"pretty": "__REDACTED__"
19+
},
20+
"total": {
21+
"ms": "__REDACTED__",
22+
"pretty": "__REDACTED__"
7823
}
79-
],
80-
"summary": {
81-
"durations": {
82-
"extract": {
83-
"ms": "__REDACTED__",
84-
"pretty": "__REDACTED__"
85-
},
86-
"loadManifest": {
87-
"ms": "__REDACTED__",
88-
"pretty": "__REDACTED__"
89-
},
90-
"loadSource": {
91-
"ms": "__REDACTED__",
92-
"pretty": "__REDACTED__"
93-
},
94-
"parse": {
95-
"ms": "__REDACTED__",
96-
"pretty": "__REDACTED__"
97-
},
98-
"total": {
99-
"ms": "__REDACTED__",
100-
"pretty": "__REDACTED__"
101-
}
102-
},
103-
"numberOfFiles": 5,
104-
"numberOfManifests": 1
105-
}
24+
},
25+
"numberOfFiles": 5,
26+
"numberOfManifests": 1
10627
},
10728
"severity": "note",
10829
"source": {

‎rust/ql/integration-tests/hello-project/steps.cargo.expected

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
| Cargo.toml:0:0:0:0 | LoadManifest(Cargo.toml) |
2+
| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) |
3+
| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) |
4+
| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) |
5+
| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) |
6+
| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) |
7+
| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) |
8+
| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) |
9+
| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) |
10+
| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) |
11+
| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) |
12+
| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) |
13+
| src/main.rs:0:0:0:0 | Extract(src/main.rs) |
14+
| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) |
15+
| src/main.rs:0:0:0:0 | Parse(src/main.rs) |

‎rust/ql/integration-tests/hello-project/steps.ql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
import codeql.rust.internal.ExtractorStep
2+
3+
from ExtractorStep step
4+
select step

‎rust/ql/integration-tests/hello-project/steps.rust-project.expected

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
| rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) |
2+
| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) |
3+
| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) |
4+
| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) |
5+
| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) |
6+
| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) |
7+
| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) |
8+
| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) |
9+
| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) |
10+
| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) |
11+
| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) |
12+
| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) |
13+
| src/main.rs:0:0:0:0 | Extract(src/main.rs) |
14+
| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) |
15+
| src/main.rs:0:0:0:0 | Parse(src/main.rs) |

‎rust/ql/integration-tests/hello-project/test_project.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1+
import pytest
2+
3+
4+
@pytest.mark.ql_test("steps.ql", expected=".cargo.expected")
15
def test_cargo(codeql, rust, manifests, check_source_archive, rust_check_diagnostics):
26
manifests.select("Cargo.toml")
37
codeql.database.create()
48

9+
@pytest.mark.ql_test("steps.ql", expected=".rust-project.expected")
510
def test_rust_project(codeql, rust, manifests, check_source_archive, rust_check_diagnostics):
611
manifests.select("rust-project.json")
712
codeql.database.create()

0 commit comments

Comments
 (0)