Skip to content

Commit 00f68d5

Browse files
clippy_dev: parsing revamp 1/N (#15866)
Supercedes #15270 This first PR just moves the existing code and simplifies a few things. The switch to use the new range API isn't really used in this PR , but it's needed by future changes to avoid having to add `.clone()` all over the place. changelog: none
2 parents 0c592df + 1b31b09 commit 00f68d5

File tree

10 files changed

+564
-504
lines changed

10 files changed

+564
-504
lines changed

clippy_dev/src/deprecate_lint.rs

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
use crate::update_lints::{DeprecatedLint, Lint, find_lint_decls, generate_lint_files, read_deprecated_lints};
1+
use crate::parse::{DeprecatedLint, Lint, find_lint_decls, read_deprecated_lints};
2+
use crate::update_lints::generate_lint_files;
23
use crate::utils::{UpdateMode, Version};
34
use std::ffi::OsStr;
45
use std::path::{Path, PathBuf};
@@ -14,10 +15,6 @@ use std::{fs, io};
1415
///
1516
/// If a file path could not read from or written to
1617
pub fn deprecate(clippy_version: Version, name: &str, reason: &str) {
17-
if let Some((prefix, _)) = name.split_once("::") {
18-
panic!("`{name}` should not contain the `{prefix}` prefix");
19-
}
20-
2118
let mut lints = find_lint_decls();
2219
let (mut deprecated_lints, renamed_lints) = read_deprecated_lints();
2320

@@ -135,14 +132,14 @@ fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec<Lint>) -> io
135132
);
136133

137134
assert!(
138-
content[lint.declaration_range.clone()].contains(&name.to_uppercase()),
135+
content[lint.declaration_range].contains(&name.to_uppercase()),
139136
"error: `{}` does not contain lint `{}`'s declaration",
140137
path.display(),
141138
lint.name
142139
);
143140

144141
// Remove lint declaration (declare_clippy_lint!)
145-
content.replace_range(lint.declaration_range.clone(), "");
142+
content.replace_range(lint.declaration_range, "");
146143

147144
// Remove the module declaration (mod xyz;)
148145
let mod_decl = format!("\nmod {name};");

clippy_dev/src/lib.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#![feature(
2-
rustc_private,
32
exit_status_error,
43
if_let_guard,
4+
new_range,
5+
new_range_api,
56
os_str_slice,
67
os_string_truncate,
8+
rustc_private,
79
slice_split_once
810
)]
911
#![warn(
@@ -34,3 +36,5 @@ pub mod update_lints;
3436

3537
mod utils;
3638
pub use utils::{ClippyInfo, UpdateMode};
39+
40+
mod parse;

clippy_dev/src/main.rs

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ use clippy_dev::{
77
ClippyInfo, UpdateMode, deprecate_lint, dogfood, fmt, lint, new_lint, release, rename_lint, serve, setup, sync,
88
update_lints,
99
};
10-
use std::convert::Infallible;
1110
use std::env;
1211

1312
fn main() {
@@ -95,6 +94,20 @@ fn main() {
9594
}
9695
}
9796

97+
fn lint_name(name: &str) -> Result<String, String> {
98+
let name = name.replace('-', "_");
99+
if let Some((pre, _)) = name.split_once("::") {
100+
Err(format!("lint name should not contain the `{pre}` prefix"))
101+
} else if name
102+
.bytes()
103+
.any(|x| !matches!(x, b'_' | b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z'))
104+
{
105+
Err("lint name contains invalid characters".to_owned())
106+
} else {
107+
Ok(name)
108+
}
109+
}
110+
98111
#[derive(Parser)]
99112
#[command(name = "dev", about)]
100113
struct Dev {
@@ -150,7 +163,7 @@ enum DevCommand {
150163
#[arg(
151164
short,
152165
long,
153-
value_parser = |name: &str| Ok::<_, Infallible>(name.replace('-', "_")),
166+
value_parser = lint_name,
154167
)]
155168
/// Name of the new lint in snake case, ex: `fn_too_long`
156169
name: String,
@@ -223,8 +236,12 @@ enum DevCommand {
223236
/// Rename a lint
224237
RenameLint {
225238
/// The name of the lint to rename
239+
#[arg(value_parser = lint_name)]
226240
old_name: String,
227-
#[arg(required_unless_present = "uplift")]
241+
#[arg(
242+
required_unless_present = "uplift",
243+
value_parser = lint_name,
244+
)]
228245
/// The new name of the lint
229246
new_name: Option<String>,
230247
#[arg(long)]
@@ -234,6 +251,7 @@ enum DevCommand {
234251
/// Deprecate the given lint
235252
Deprecate {
236253
/// The name of the lint to deprecate
254+
#[arg(value_parser = lint_name)]
237255
name: String,
238256
#[arg(long, short)]
239257
/// The reason for deprecation

clippy_dev/src/new_lint.rs

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
use crate::utils::{RustSearcher, Token, Version};
1+
use crate::parse::cursor::{self, Capture, Cursor};
2+
use crate::utils::Version;
23
use clap::ValueEnum;
34
use indoc::{formatdoc, writedoc};
45
use std::fmt::{self, Write as _};
@@ -516,22 +517,22 @@ fn setup_mod_file(path: &Path, lint: &LintData<'_>) -> io::Result<&'static str>
516517
// Find both the last lint declaration (declare_clippy_lint!) and the lint pass impl
517518
fn parse_mod_file(path: &Path, contents: &str) -> (&'static str, usize) {
518519
#[allow(clippy::enum_glob_use)]
519-
use Token::*;
520+
use cursor::Pat::*;
520521

521522
let mut context = None;
522523
let mut decl_end = None;
523-
let mut searcher = RustSearcher::new(contents);
524-
while let Some(name) = searcher.find_capture_token(CaptureIdent) {
525-
match name {
524+
let mut cursor = Cursor::new(contents);
525+
let mut captures = [Capture::EMPTY];
526+
while let Some(name) = cursor.find_any_ident() {
527+
match cursor.get_text(name) {
526528
"declare_clippy_lint" => {
527-
if searcher.match_tokens(&[Bang, OpenBrace], &mut []) && searcher.find_token(CloseBrace) {
528-
decl_end = Some(searcher.pos());
529+
if cursor.match_all(&[Bang, OpenBrace], &mut []) && cursor.find_pat(CloseBrace) {
530+
decl_end = Some(cursor.pos());
529531
}
530532
},
531533
"impl" => {
532-
let mut capture = "";
533-
if searcher.match_tokens(&[Lt, Lifetime, Gt, CaptureIdent], &mut [&mut capture]) {
534-
match capture {
534+
if cursor.match_all(&[Lt, Lifetime, Gt, CaptureIdent], &mut captures) {
535+
match cursor.get_text(captures[0]) {
535536
"LateLintPass" => context = Some("LateContext"),
536537
"EarlyLintPass" => context = Some("EarlyContext"),
537538
_ => {},

clippy_dev/src/parse.rs

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
pub mod cursor;
2+
3+
use self::cursor::{Capture, Cursor};
4+
use crate::utils::{ErrAction, File, expect_action};
5+
use core::range::Range;
6+
use std::fs;
7+
use std::path::{Path, PathBuf};
8+
use walkdir::{DirEntry, WalkDir};
9+
10+
pub struct Lint {
11+
pub name: String,
12+
pub group: String,
13+
pub module: String,
14+
pub path: PathBuf,
15+
pub declaration_range: Range<usize>,
16+
}
17+
18+
pub struct DeprecatedLint {
19+
pub name: String,
20+
pub reason: String,
21+
pub version: String,
22+
}
23+
24+
pub struct RenamedLint {
25+
pub old_name: String,
26+
pub new_name: String,
27+
pub version: String,
28+
}
29+
30+
/// Finds all lint declarations (`declare_clippy_lint!`)
31+
#[must_use]
32+
pub fn find_lint_decls() -> Vec<Lint> {
33+
let mut lints = Vec::with_capacity(1000);
34+
let mut contents = String::new();
35+
for e in expect_action(fs::read_dir("."), ErrAction::Read, ".") {
36+
let e = expect_action(e, ErrAction::Read, ".");
37+
if !expect_action(e.file_type(), ErrAction::Read, ".").is_dir() {
38+
continue;
39+
}
40+
let Ok(mut name) = e.file_name().into_string() else {
41+
continue;
42+
};
43+
if name.starts_with("clippy_lints") && name != "clippy_lints_internal" {
44+
name.push_str("/src");
45+
for (file, module) in read_src_with_module(name.as_ref()) {
46+
parse_clippy_lint_decls(
47+
file.path(),
48+
File::open_read_to_cleared_string(file.path(), &mut contents),
49+
&module,
50+
&mut lints,
51+
);
52+
}
53+
}
54+
}
55+
lints.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name));
56+
lints
57+
}
58+
59+
/// Reads the source files from the given root directory
60+
fn read_src_with_module(src_root: &Path) -> impl use<'_> + Iterator<Item = (DirEntry, String)> {
61+
WalkDir::new(src_root).into_iter().filter_map(move |e| {
62+
let e = expect_action(e, ErrAction::Read, src_root);
63+
let path = e.path().as_os_str().as_encoded_bytes();
64+
if let Some(path) = path.strip_suffix(b".rs")
65+
&& let Some(path) = path.get(src_root.as_os_str().len() + 1..)
66+
{
67+
if path == b"lib" {
68+
Some((e, String::new()))
69+
} else {
70+
let path = if let Some(path) = path.strip_suffix(b"mod")
71+
&& let Some(path) = path.strip_suffix(b"/").or_else(|| path.strip_suffix(b"\\"))
72+
{
73+
path
74+
} else {
75+
path
76+
};
77+
if let Ok(path) = str::from_utf8(path) {
78+
let path = path.replace(['/', '\\'], "::");
79+
Some((e, path))
80+
} else {
81+
None
82+
}
83+
}
84+
} else {
85+
None
86+
}
87+
})
88+
}
89+
90+
/// Parse a source file looking for `declare_clippy_lint` macro invocations.
91+
fn parse_clippy_lint_decls(path: &Path, contents: &str, module: &str, lints: &mut Vec<Lint>) {
92+
#[allow(clippy::enum_glob_use)]
93+
use cursor::Pat::*;
94+
#[rustfmt::skip]
95+
static DECL_TOKENS: &[cursor::Pat<'_>] = &[
96+
// !{ /// docs
97+
Bang, OpenBrace, AnyComment,
98+
// #[clippy::version = "version"]
99+
Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, LitStr, CloseBracket,
100+
// pub NAME, GROUP,
101+
Ident("pub"), CaptureIdent, Comma, AnyComment, CaptureIdent, Comma,
102+
];
103+
104+
let mut cursor = Cursor::new(contents);
105+
let mut captures = [Capture::EMPTY; 2];
106+
while let Some(start) = cursor.find_ident("declare_clippy_lint") {
107+
if cursor.match_all(DECL_TOKENS, &mut captures) && cursor.find_pat(CloseBrace) {
108+
lints.push(Lint {
109+
name: cursor.get_text(captures[0]).to_lowercase(),
110+
group: cursor.get_text(captures[1]).into(),
111+
module: module.into(),
112+
path: path.into(),
113+
declaration_range: start as usize..cursor.pos() as usize,
114+
});
115+
}
116+
}
117+
}
118+
119+
#[must_use]
120+
pub fn read_deprecated_lints() -> (Vec<DeprecatedLint>, Vec<RenamedLint>) {
121+
#[allow(clippy::enum_glob_use)]
122+
use cursor::Pat::*;
123+
#[rustfmt::skip]
124+
static DECL_TOKENS: &[cursor::Pat<'_>] = &[
125+
// #[clippy::version = "version"]
126+
Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, CaptureLitStr, CloseBracket,
127+
// ("first", "second"),
128+
OpenParen, CaptureLitStr, Comma, CaptureLitStr, CloseParen, Comma,
129+
];
130+
#[rustfmt::skip]
131+
static DEPRECATED_TOKENS: &[cursor::Pat<'_>] = &[
132+
// !{ DEPRECATED(DEPRECATED_VERSION) = [
133+
Bang, OpenBrace, Ident("DEPRECATED"), OpenParen, Ident("DEPRECATED_VERSION"), CloseParen, Eq, OpenBracket,
134+
];
135+
#[rustfmt::skip]
136+
static RENAMED_TOKENS: &[cursor::Pat<'_>] = &[
137+
// !{ RENAMED(RENAMED_VERSION) = [
138+
Bang, OpenBrace, Ident("RENAMED"), OpenParen, Ident("RENAMED_VERSION"), CloseParen, Eq, OpenBracket,
139+
];
140+
141+
let path = "clippy_lints/src/deprecated_lints.rs";
142+
let mut deprecated = Vec::with_capacity(30);
143+
let mut renamed = Vec::with_capacity(80);
144+
let mut contents = String::new();
145+
File::open_read_to_cleared_string(path, &mut contents);
146+
147+
let mut cursor = Cursor::new(&contents);
148+
let mut captures = [Capture::EMPTY; 3];
149+
150+
// First instance is the macro definition.
151+
assert!(
152+
cursor.find_ident("declare_with_version").is_some(),
153+
"error reading deprecated lints"
154+
);
155+
156+
if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(DEPRECATED_TOKENS, &mut []) {
157+
while cursor.match_all(DECL_TOKENS, &mut captures) {
158+
deprecated.push(DeprecatedLint {
159+
name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])),
160+
reason: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])),
161+
version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])),
162+
});
163+
}
164+
} else {
165+
panic!("error reading deprecated lints");
166+
}
167+
168+
if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(RENAMED_TOKENS, &mut []) {
169+
while cursor.match_all(DECL_TOKENS, &mut captures) {
170+
renamed.push(RenamedLint {
171+
old_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])),
172+
new_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])),
173+
version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])),
174+
});
175+
}
176+
} else {
177+
panic!("error reading renamed lints");
178+
}
179+
180+
deprecated.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name));
181+
renamed.sort_by(|lhs, rhs| lhs.old_name.cmp(&rhs.old_name));
182+
(deprecated, renamed)
183+
}
184+
185+
/// Removes the line splices and surrounding quotes from a string literal
186+
fn parse_str_lit(s: &str) -> String {
187+
let (s, is_raw) = if let Some(s) = s.strip_prefix("r") {
188+
(s.trim_matches('#'), true)
189+
} else {
190+
(s, false)
191+
};
192+
let s = s
193+
.strip_prefix('"')
194+
.and_then(|s| s.strip_suffix('"'))
195+
.unwrap_or_else(|| panic!("expected quoted string, found `{s}`"));
196+
197+
if is_raw {
198+
s.into()
199+
} else {
200+
let mut res = String::with_capacity(s.len());
201+
rustc_literal_escaper::unescape_str(s, &mut |_, ch| {
202+
if let Ok(ch) = ch {
203+
res.push(ch);
204+
}
205+
});
206+
res
207+
}
208+
}
209+
210+
fn parse_str_single_line(path: &Path, s: &str) -> String {
211+
let value = parse_str_lit(s);
212+
assert!(
213+
!value.contains('\n'),
214+
"error parsing `{}`: `{s}` should be a single line string",
215+
path.display(),
216+
);
217+
value
218+
}

0 commit comments

Comments
 (0)