From 6308a8af11999ef74afc956f6bfd089684371e0f Mon Sep 17 00:00:00 2001 From: Jeremy Smart Date: Thu, 21 Aug 2025 20:20:43 -0400 Subject: [PATCH] add Iterator::dedup and friends --- compiler/rustc_codegen_ssa/src/back/link.rs | 13 +- library/core/src/iter/adapters/dedup.rs | 70 +++++++++ library/core/src/iter/adapters/mod.rs | 5 + library/core/src/iter/mod.rs | 2 + library/core/src/iter/traits/iterator.rs | 137 ++++++++++++++++++ .../src/macro_metavars_in_unsafe.rs | 64 ++++---- 6 files changed, 252 insertions(+), 39 deletions(-) create mode 100644 library/core/src/iter/adapters/dedup.rs diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index c3777f64e9e93..9f81dc4984c8a 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -1457,10 +1457,8 @@ fn print_native_static_libs( all_native_libs: &[NativeLib], all_rust_dylibs: &[&Path], ) { - let mut lib_args: Vec<_> = all_native_libs - .iter() - .filter(|l| relevant_lib(sess, l)) - .filter_map(|lib| { + let mut lib_args: Vec<_> = Itertools::dedup( + all_native_libs.iter().filter(|l| relevant_lib(sess, l)).filter_map(|lib| { let name = lib.name; match lib.kind { NativeLibKind::Static { bundle: Some(false), .. } @@ -1486,10 +1484,9 @@ fn print_native_static_libs( | NativeLibKind::WasmImportModule | NativeLibKind::RawDylib => None, } - }) - // deduplication of consecutive repeated libraries, see rust-lang/rust#113209 - .dedup() - .collect(); + }), // deduplication of consecutive repeated libraries, see rust-lang/rust#113209 + ) + .collect(); for path in all_rust_dylibs { // FIXME deduplicate with add_dynamic_crate diff --git a/library/core/src/iter/adapters/dedup.rs b/library/core/src/iter/adapters/dedup.rs new file mode 100644 index 0000000000000..3bd483a40177e --- /dev/null +++ b/library/core/src/iter/adapters/dedup.rs @@ -0,0 +1,70 @@ +trait DedupPredicate { + fn eq(&mut self, a: &T, b: &T) -> bool; +} + +impl bool> DedupPredicate for F { + fn eq(&mut self, a: &T, b: &T) -> bool { + self(a, b) + } +} + +#[unstable(feature = "iter_dedup", issue = "83747")] +#[doc(hidden)] +#[derive(Debug)] +pub struct DedupEq; + +impl DedupPredicate for DedupEq { + fn eq(&mut self, a: &T, b: &T) -> bool { + a == b + } +} + +#[unstable(feature = "iter_dedup", issue = "83747")] +#[doc(hidden)] +#[derive(Debug)] +pub struct DedupKey(pub F); + +impl K> DedupPredicate for DedupKey { + fn eq(&mut self, a: &T, b: &T) -> bool { + (self.0)(a) == (self.0)(b) + } +} + +/// An iterator to deduplicate adjacent items in another iterator. +/// +/// This `struct` is created by the [`dedup`], [`dedup_by`], and +/// [`dedup_by_key`] methods on [`Iterator`]. See their documentation for more. +/// +/// [`dedup`]: Iterator::dedup +/// [`dedup_by`]: Iterator::dedup_by +/// [`dedup_by_key`]: Iterator::dedup_by_key +#[unstable(feature = "iter_dedup", issue = "83747")] +#[derive(Debug)] +pub struct Dedup { + inner: I, + f: F, + last: Option, +} + +impl Dedup { + pub(in crate::iter) fn new(mut it: I, f: F) -> Self { + let first = it.next(); + Self { inner: it, f, last: first } + } +} + +#[unstable(feature = "iter_dedup", issue = "83747")] +impl Iterator for Dedup +where + I: Iterator, + I::Item: Clone, + F: DedupPredicate, +{ + type Item = I::Item; + + fn next(&mut self) -> Option { + let last = self.last.as_ref()?; + self.last = self.inner.find(|e| self.f.eq(e, last)); + return self.last.clone(); + } +} diff --git a/library/core/src/iter/adapters/mod.rs b/library/core/src/iter/adapters/mod.rs index 6c6de0a4e5c98..4084491dfdb7c 100644 --- a/library/core/src/iter/adapters/mod.rs +++ b/library/core/src/iter/adapters/mod.rs @@ -8,6 +8,7 @@ mod chain; mod cloned; mod copied; mod cycle; +mod dedup; mod enumerate; mod filter; mod filter_map; @@ -38,6 +39,10 @@ pub use self::chain::chain; pub use self::cloned::Cloned; #[stable(feature = "iter_copied", since = "1.36.0")] pub use self::copied::Copied; +#[unstable(feature = "iter_dedup", issue = "83747")] +pub use self::dedup::Dedup; +#[unstable(feature = "iter_dedup", issue = "83747")] +pub use self::dedup::{DedupEq, DedupKey}; #[stable(feature = "iterator_flatten", since = "1.29.0")] pub use self::flatten::Flatten; #[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] diff --git a/library/core/src/iter/mod.rs b/library/core/src/iter/mod.rs index bc07324f5204c..4e10e72f42bba 100644 --- a/library/core/src/iter/mod.rs +++ b/library/core/src/iter/mod.rs @@ -414,6 +414,8 @@ pub use self::adapters::{ Chain, Cycle, Enumerate, Filter, FilterMap, FlatMap, Fuse, Inspect, Map, Peekable, Rev, Scan, Skip, SkipWhile, Take, TakeWhile, Zip, }; +#[unstable(feature = "iter_dedup", issue = "83747")] +pub use self::adapters::{Dedup, DedupEq, DedupKey}; #[unstable(feature = "iter_intersperse", reason = "recently added", issue = "79524")] pub use self::adapters::{Intersperse, IntersperseWith}; #[unstable( diff --git a/library/core/src/iter/traits/iterator.rs b/library/core/src/iter/traits/iterator.rs index 7fb162a653fb9..11c859192bba4 100644 --- a/library/core/src/iter/traits/iterator.rs +++ b/library/core/src/iter/traits/iterator.rs @@ -6,6 +6,7 @@ use super::super::{ }; use crate::array; use crate::cmp::{self, Ordering}; +use crate::iter::adapters::{Dedup, DedupEq, DedupKey}; use crate::num::NonZero; use crate::ops::{ChangeOutputType, ControlFlow, FromResidual, Residual, Try}; @@ -1863,6 +1864,142 @@ pub trait Iterator { Inspect::new(self, f) } + /// Removes all but the first of consecutive repeated elements in the iterator + /// according to the [`PartialEq`] trait implementation. + /// + /// For an iterator yielding infinitely many consecutive duplicates, + /// calling [`next`][Iterator::next] on this iterator may never halt. + /// + /// If the iterator is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_dedup)] + /// + /// let vec = vec![1, 2, 2, 3, 2]; + /// + /// let mut iter = vec.into_iter().dedup(); + /// + /// assert_eq!(iter.next(), Some(1)); + /// assert_eq!(iter.next(), Some(2)); + /// assert_eq!(iter.next(), Some(3)); + /// assert_eq!(iter.next(), Some(2)); + /// assert_eq!(iter.next(), None); + /// ``` + /// + /// Example of an infinite loop: + /// + /// ```no_run + /// #![feature(iter_dedup)] + /// + /// // this will never terminate + /// let _ = std::iter::repeat(2).dedup().next(); + /// ``` + #[unstable(feature = "iter_dedup", issue = "83747")] + #[inline] + fn dedup(self) -> Dedup + where + Self: Sized, + Self::Item: PartialEq, + { + Dedup::new(self, DedupEq) + } + + /// Removes all but the first of consecutive elements in the iterator + /// satisfying a given equality relation. + /// + /// The `same_bucket` function is passed a references to two elements from + /// the iterator and must determine if the elements compare equal. + /// + /// For an iterator yielding infinitely many consecutive duplicates, + /// calling [`next`][Iterator::next] on this iterator may never halt. + /// + /// If the iterator is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_dedup)] + /// + /// let vec = vec!["foo", "bar", "Bar", "baz", "bar"]; + /// + /// let mut iter = vec.into_iter().dedup_by(|a, b| a.eq_ignore_ascii_case(b)); + /// + /// assert_eq!(iter.next(), Some("foo")); + /// assert_eq!(iter.next(), Some("bar")); + /// assert_eq!(iter.next(), Some("baz")); + /// assert_eq!(iter.next(), Some("bar")); + /// assert_eq!(iter.next(), None); + /// ``` + /// + /// Example of an infinite loop: + /// + /// ```no_run + /// #![feature(iter_dedup)] + /// + /// // this will never terminate + /// let _ = std::iter::repeat(2).dedup_by(|a, b| a == b).next(); + /// ``` + #[unstable(feature = "iter_dedup", issue = "83747")] + #[inline] + fn dedup_by(self, f: F) -> Dedup + where + Self: Sized, + F: FnMut(&Self::Item, &Self::Item) -> bool, + { + Dedup::new(self, f) + } + + /// Removes all but the first of consecutive elements in the iterator + /// that resolve to the same key. + /// + /// For an iterator yielding infinitely many consecutive duplicates, + /// calling [`next`][Iterator::next] on this iterator may never halt. + /// + /// If the iterator is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(iter_dedup)] + /// + /// let vec = vec![10, 20, 21, 30, 20]; + /// + /// let mut iter = vec.into_iter().dedup_by_key(|&i| i / 10); + /// + /// assert_eq!(iter.next(), Some(10)); + /// assert_eq!(iter.next(), Some(20)); + /// assert_eq!(iter.next(), Some(30)); + /// assert_eq!(iter.next(), Some(20)); + /// assert_eq!(iter.next(), None); + /// ``` + /// + /// Example of an infinite loop: + /// + /// ```no_run + /// #![feature(iter_dedup)] + /// + /// // this will never terminate + /// let _ = std::iter::repeat(2).dedup_by_key(|&n| n).next(); + /// ``` + #[unstable(feature = "iter_dedup", issue = "83747")] + #[inline] + fn dedup_by_key(self, f: F) -> Dedup> + where + Self: Sized, + F: FnMut(&Self::Item) -> K, + K: PartialEq, + { + Dedup::new(self, DedupKey(f)) + } + /// Creates a "by reference" adapter for this instance of `Iterator`. /// /// Consuming method calls (direct or indirect calls to `next`) diff --git a/src/tools/clippy/clippy_lints/src/macro_metavars_in_unsafe.rs b/src/tools/clippy/clippy_lints/src/macro_metavars_in_unsafe.rs index 9071c9c95f9d7..565e27d329d5b 100644 --- a/src/tools/clippy/clippy_lints/src/macro_metavars_in_unsafe.rs +++ b/src/tools/clippy/clippy_lints/src/macro_metavars_in_unsafe.rs @@ -241,38 +241,40 @@ impl<'tcx> LateLintPass<'tcx> for ExprMetavarsInUnsafe { // $y: [unsafe#1] // ``` // We want to lint unsafe blocks #0 and #1 - let bad_unsafe_blocks = self - .metavar_expns - .iter() - .filter_map(|(_, state)| match state { - MetavarState::ReferencedInUnsafe { unsafe_blocks } => Some(unsafe_blocks.as_slice()), - MetavarState::ReferencedInSafe => None, - }) - .flatten() - .copied() - .inspect(|&unsafe_block| { - if let LevelAndSource { - level: Level::Expect, - lint_id: Some(id), - .. - } = cx.tcx.lint_level_at_node(MACRO_METAVARS_IN_UNSAFE, unsafe_block) - { - // Since we're going to deduplicate expanded unsafe blocks by its enclosing macro definition soon, - // which would lead to unfulfilled `#[expect()]`s in all other unsafe blocks that are filtered out - // except for the one we emit the warning at, we must manually fulfill the lint - // for all unsafe blocks here. - cx.fulfill_expectation(id); - } - }) - .map(|id| { - // Remove the syntax context to hide "in this macro invocation" in the diagnostic. - // The invocation doesn't matter. Also we want to dedupe by the unsafe block and not by anything - // related to the callsite. - let span = cx.tcx.hir_span(id); + let bad_unsafe_blocks = Itertools::dedup_by( + self.metavar_expns + .iter() + .filter_map(|(_, state)| match state { + MetavarState::ReferencedInUnsafe { unsafe_blocks } => Some(unsafe_blocks.as_slice()), + MetavarState::ReferencedInSafe => None, + }) + .flatten() + .copied() + .inspect(|&unsafe_block| { + if let LevelAndSource { + level: Level::Expect, + lint_id: Some(id), + .. + } = cx.tcx.lint_level_at_node(MACRO_METAVARS_IN_UNSAFE, unsafe_block) + { + // Since we're going to deduplicate expanded unsafe blocks by its enclosing macro definition + // soon, which would lead to unfulfilled `#[expect()]`s in all other + // unsafe blocks that are filtered out except for the one we emit the + // warning at, we must manually fulfill the lint for all unsafe blocks + // here. + cx.fulfill_expectation(id); + } + }) + .map(|id| { + // Remove the syntax context to hide "in this macro invocation" in the diagnostic. + // The invocation doesn't matter. Also we want to dedupe by the unsafe block and not by anything + // related to the callsite. + let span = cx.tcx.hir_span(id); - (id, Span::new(span.lo(), span.hi(), SyntaxContext::root(), None)) - }) - .dedup_by(|&(_, a), &(_, b)| a == b); + (id, Span::new(span.lo(), span.hi(), SyntaxContext::root(), None)) + }), + |&(_, a), &(_, b)| a == b, + ); for (id, span) in bad_unsafe_blocks { span_lint_hir_and_then(