From 45bda3b45d760183711d24cc2f0b6d98475219b0 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Fri, 15 Aug 2025 22:43:31 +0200 Subject: [PATCH 01/10] Remove `Container::push`. Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 8 +------- timely/src/dataflow/operators/core/capture/extract.rs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index 839f80167..f7bd00b67 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -22,12 +22,6 @@ pub trait Container: Default { /// The type of elements when draining the container. type Item<'a> where Self: 'a; - /// Push `item` into self - #[inline] - fn push(&mut self, item: T) where Self: PushInto { - self.push_into(item) - } - /// The number of elements in this container /// /// This number is used in progress tracking to confirm the receipt of some number @@ -165,7 +159,7 @@ impl> PushInto for CapacityContainerBuil self.current.ensure_capacity(&mut self.empty); // Push item - self.current.push(item); + self.current.push_into(item); // Maybe flush if self.current.at_capacity() { diff --git a/timely/src/dataflow/operators/core/capture/extract.rs b/timely/src/dataflow/operators/core/capture/extract.rs index fcf024f31..e937ca3a5 100644 --- a/timely/src/dataflow/operators/core/capture/extract.rs +++ b/timely/src/dataflow/operators/core/capture/extract.rs @@ -71,7 +71,7 @@ where to_sort.sort(); let mut sorted = C::default(); for datum in to_sort.into_iter() { - sorted.push(datum); + sorted.push_into(datum); } if !sorted.is_empty() { result.push((time, sorted)); From 922902c490ca0e8aa1eeada69933d22a63ecf243 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Fri, 15 Aug 2025 23:17:14 +0200 Subject: [PATCH 02/10] Remove `Container::clear` I validated all call sites to handle non-empty containers gracefully. Container builders already need to handle it, and inputs should do so, too. Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 33 +++++---------------- timely/examples/columnar.rs | 9 ------ timely/src/dataflow/channels/mod.rs | 1 - timely/src/dataflow/operators/core/input.rs | 1 - 4 files changed, 7 insertions(+), 37 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index f7bd00b67..07d2bd725 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -7,7 +7,7 @@ use std::collections::VecDeque; /// A container transferring data through dataflow edges /// /// A container stores a number of elements and thus is able to describe it length (`len()`) and -/// whether it is empty (`is_empty()`). It supports removing all elements (`clear`). +/// whether it is empty (`is_empty()`). /// /// A container must implement default. The default implementation is not required to allocate /// memory for variable-length components. @@ -15,6 +15,10 @@ use std::collections::VecDeque; /// We require the container to be cloneable to enable efficient copies when providing references /// of containers to operators. Care must be taken that the type's `clone_from` implementation /// is efficient (which is not necessarily the case when deriving `Clone`.) +// The container is `Default` because `CapacityContainerBuilder` only implements `ContainerBuilder` +// for containers that implement `Default`, and we use the associated `::Container` all over Timely. +// We can only access the type if all requirements for the `ContainerBuilder` implementation are +// satisfied. pub trait Container: Default { /// The type of elements when reading non-destructively from the container. type ItemRef<'a> where Self: 'a; @@ -35,10 +39,6 @@ pub trait Container: Default { self.len() == 0 } - /// Remove all contents from `self` while retaining allocated memory. - /// After calling `clear`, `is_empty` must return `true` and `len` 0. - fn clear(&mut self); - /// Iterator type when reading from the container. type Iter<'a>: Iterator> where Self: 'a; @@ -96,6 +96,8 @@ pub trait PushInto { /// decide to represent a push order for `extract` and `finish`, or not. pub trait ContainerBuilder: Default + 'static { /// The container type we're building. + // The container is `Clone` because `Tee` requires it, otherwise we need to repeat it + // all over Timely. `'static` because we don't want lifetimes everywhere. type Container: Container + Clone + 'static; /// Extract assembled containers, potentially leaving unfinished data behind. Can /// be called repeatedly, for example while the caller can send data. @@ -117,7 +119,6 @@ pub trait ContainerBuilder: Default + 'static { let index = index(&datum); builders[index].push_into(datum); } - container.clear(); } /// Indicates a good moment to release resources. @@ -205,8 +206,6 @@ impl Container for Vec { Vec::is_empty(self) } - fn clear(&mut self) { Vec::clear(self) } - type Iter<'a> = std::slice::Iter<'a, T> where Self: 'a; fn iter(&self) -> Self::Iter<'_> { @@ -276,15 +275,6 @@ mod rc { std::ops::Deref::deref(self).is_empty() } - fn clear(&mut self) { - // Try to reuse the allocation if possible - if let Some(inner) = Rc::get_mut(self) { - inner.clear(); - } else { - *self = Self::default(); - } - } - type Iter<'a> = T::Iter<'a> where Self: 'a; fn iter(&self) -> Self::Iter<'_> { @@ -317,15 +307,6 @@ mod arc { std::ops::Deref::deref(self).is_empty() } - fn clear(&mut self) { - // Try to reuse the allocation if possible - if let Some(inner) = Arc::get_mut(self) { - inner.clear(); - } else { - *self = Self::default(); - } - } - type Iter<'a> = T::Iter<'a> where Self: 'a; fn iter(&self) -> Self::Iter<'_> { diff --git a/timely/examples/columnar.rs b/timely/examples/columnar.rs index b190cd64e..3f3ecfa32 100644 --- a/timely/examples/columnar.rs +++ b/timely/examples/columnar.rs @@ -179,15 +179,6 @@ mod container { impl timely::Container for Column { fn len(&self) -> usize { self.borrow().len() } - // This sets `self` to be an empty `Typed` variant, appropriate for pushing into. - fn clear(&mut self) { - match self { - Column::Typed(t) => t.clear(), - Column::Bytes(_) => *self = Column::Typed(Default::default()), - Column::Align(_) => *self = Column::Typed(Default::default()), - } - } - type ItemRef<'a> = C::Ref<'a>; type Iter<'a> = IterOwn>; fn iter<'a>(&'a self) -> Self::Iter<'a> { self.borrow().into_index_iter() } diff --git a/timely/src/dataflow/channels/mod.rs b/timely/src/dataflow/channels/mod.rs index 6a351753b..405a31b0e 100644 --- a/timely/src/dataflow/channels/mod.rs +++ b/timely/src/dataflow/channels/mod.rs @@ -51,7 +51,6 @@ impl Message { if let Some(message) = bundle { *buffer = message.data; - buffer.clear(); } } } diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index 5088f6eba..633e425bd 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -392,7 +392,6 @@ impl Handle { Message::push_at(container, now_at.clone(), &mut pushers[index]); } } - container.clear(); } /// Closes the current epoch, flushing if needed, shutting if needed, and updating the frontier. From 5095ff4f98f71b2dce1d2fb39f22360f65d1e74d Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Fri, 15 Aug 2025 23:35:42 +0200 Subject: [PATCH 03/10] Split Container into Container, IterContainer, DrainContainer Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 89 ++++++++----------- timely/examples/columnar.rs | 20 +++-- timely/src/dataflow/channels/pact.rs | 13 +-- .../src/dataflow/channels/pushers/exchange.rs | 14 +-- .../operators/core/capture/extract.rs | 8 +- .../src/dataflow/operators/core/exchange.rs | 12 +-- timely/src/dataflow/operators/core/filter.rs | 6 +- timely/src/dataflow/operators/core/inspect.rs | 9 +- timely/src/dataflow/operators/core/map.rs | 6 +- timely/src/dataflow/operators/core/ok_err.rs | 8 +- .../src/dataflow/operators/core/partition.rs | 9 +- 11 files changed, 96 insertions(+), 98 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index 07d2bd725..c3c51cefa 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -20,12 +20,6 @@ use std::collections::VecDeque; // We can only access the type if all requirements for the `ContainerBuilder` implementation are // satisfied. pub trait Container: Default { - /// The type of elements when reading non-destructively from the container. - type ItemRef<'a> where Self: 'a; - - /// The type of elements when draining the container. - type Item<'a> where Self: 'a; - /// The number of elements in this container /// /// This number is used in progress tracking to confirm the receipt of some number @@ -38,16 +32,24 @@ pub trait Container: Default { fn is_empty(&self) -> bool { self.len() == 0 } +} +/// TODO +pub trait IterContainer { + /// The type of elements when reading non-destructively from the container. + type ItemRef<'a> where Self: 'a; /// Iterator type when reading from the container. type Iter<'a>: Iterator> where Self: 'a; - /// Returns an iterator that reads the contents of this container. fn iter(&self) -> Self::Iter<'_>; +} +/// TODO +pub trait DrainContainer { + /// The type of elements when draining the container. + type Item<'a> where Self: 'a; /// Iterator type when draining the container. type DrainIter<'a>: Iterator> where Self: 'a; - /// Returns an iterator that drains the contents of this container. /// Drain leaves the container in an undefined state. fn drain(&mut self) -> Self::DrainIter<'_>; @@ -112,8 +114,9 @@ pub trait ContainerBuilder: Default + 'static { /// Partitions `container` among `builders`, using the function `index` to direct items. fn partition(container: &mut Self::Container, builders: &mut [Self], mut index: I) where - Self: for<'a> PushInto<::Item<'a>>, - I: for<'a> FnMut(&::Item<'a>) -> usize, + Self::Container: DrainContainer, + Self: for<'a> PushInto<::Item<'a>>, + I: for<'a> FnMut(&::Item<'a>) -> usize, { for datum in container.drain() { let index = index(&datum); @@ -195,25 +198,25 @@ impl ContainerBuilder for CapacityContainerBuild impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } impl Container for Vec { - type ItemRef<'a> = &'a T where T: 'a; - type Item<'a> = T where T: 'a; - fn len(&self) -> usize { Vec::len(self) } - fn is_empty(&self) -> bool { Vec::is_empty(self) } +} +impl IterContainer for Vec { + type ItemRef<'a> = &'a T where T: 'a; type Iter<'a> = std::slice::Iter<'a, T> where Self: 'a; - fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter() } +} +impl DrainContainer for Vec { + type Item<'a> = T where T: 'a; type DrainIter<'a> = std::vec::Drain<'a, T> where Self: 'a; - fn drain(&mut self) -> Self::DrainIter<'_> { self.drain(..) } @@ -261,31 +264,25 @@ mod rc { use std::ops::Deref; use std::rc::Rc; - use crate::Container; + use crate::{Container, IterContainer, DrainContainer}; impl Container for Rc { - type ItemRef<'a> = T::ItemRef<'a> where Self: 'a; - type Item<'a> = T::ItemRef<'a> where Self: 'a; - fn len(&self) -> usize { std::ops::Deref::deref(self).len() } - fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } - + } + impl IterContainer for Rc { + type ItemRef<'a> = T::ItemRef<'a> where Self: 'a; type Iter<'a> = T::Iter<'a> where Self: 'a; - - fn iter(&self) -> Self::Iter<'_> { - self.deref().iter() - } - + fn iter(&self) -> Self::Iter<'_> { self.deref().iter() } + } + impl DrainContainer for Rc { + type Item<'a> = T::ItemRef<'a> where Self: 'a; type DrainIter<'a> = T::Iter<'a> where Self: 'a; - - fn drain(&mut self) -> Self::DrainIter<'_> { - self.iter() - } + fn drain(&mut self) -> Self::DrainIter<'_> { self.iter() } } } @@ -293,31 +290,21 @@ mod arc { use std::ops::Deref; use std::sync::Arc; - use crate::Container; + use crate::{Container, IterContainer, DrainContainer}; impl Container for Arc { + fn len(&self) -> usize { std::ops::Deref::deref(self).len() } + fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } + } + impl IterContainer for Arc { type ItemRef<'a> = T::ItemRef<'a> where Self: 'a; - type Item<'a> = T::ItemRef<'a> where Self: 'a; - - fn len(&self) -> usize { - std::ops::Deref::deref(self).len() - } - - fn is_empty(&self) -> bool { - std::ops::Deref::deref(self).is_empty() - } - type Iter<'a> = T::Iter<'a> where Self: 'a; - - fn iter(&self) -> Self::Iter<'_> { - self.deref().iter() - } - + fn iter(&self) -> Self::Iter<'_> { self.deref().iter() } + } + impl DrainContainer for Arc { + type Item<'a> = T::ItemRef<'a> where Self: 'a; type DrainIter<'a> = T::Iter<'a> where Self: 'a; - - fn drain(&mut self) -> Self::DrainIter<'_> { - self.iter() - } + fn drain(&mut self) -> Self::DrainIter<'_> { self.iter() } } } diff --git a/timely/examples/columnar.rs b/timely/examples/columnar.rs index 3f3ecfa32..f9a17bc65 100644 --- a/timely/examples/columnar.rs +++ b/timely/examples/columnar.rs @@ -1,13 +1,12 @@ //! Wordcount based on the `columnar` crate. -use { - std::collections::HashMap, - timely::{Container, container::CapacityContainerBuilder}, - timely::dataflow::channels::pact::{ExchangeCore, Pipeline}, - timely::dataflow::InputHandleCore, - timely::dataflow::operators::{Inspect, Operator, Probe}, - timely::dataflow::ProbeHandle, -}; +use std::collections::HashMap; + +use timely::container::{IterContainer, CapacityContainerBuilder}; +use timely::dataflow::channels::pact::{ExchangeCore, Pipeline}; +use timely::dataflow::InputHandleCore; +use timely::dataflow::operators::{Inspect, Operator, Probe}; +use timely::dataflow::ProbeHandle; // Creates `WordCountContainer` and `WordCountReference` structs, // as well as various implementations relating them to `WordCount`. @@ -179,10 +178,13 @@ mod container { impl timely::Container for Column { fn len(&self) -> usize { self.borrow().len() } + } + impl timely::container::IterContainer for Column { type ItemRef<'a> = C::Ref<'a>; type Iter<'a> = IterOwn>; fn iter<'a>(&'a self) -> Self::Iter<'a> { self.borrow().into_index_iter() } - + } + impl timely::container::DrainContainer for Column { type Item<'a> = C::Ref<'a>; type DrainIter<'a> = IterOwn>; fn drain<'a>(&'a mut self) -> Self::DrainIter<'a> { self.borrow().into_index_iter() } diff --git a/timely/src/dataflow/channels/pact.rs b/timely/src/dataflow/channels/pact.rs index fb2f95e7d..8c550b561 100644 --- a/timely/src/dataflow/channels/pact.rs +++ b/timely/src/dataflow/channels/pact.rs @@ -10,7 +10,8 @@ use std::{fmt::{self, Debug}, marker::PhantomData}; use std::rc::Rc; -use crate::{Container, container::{ContainerBuilder, LengthPreservingContainerBuilder, SizableContainer, CapacityContainerBuilder, PushInto}}; +use crate::Container; +use crate::container::{ContainerBuilder, DrainContainer, LengthPreservingContainerBuilder, SizableContainer, CapacityContainerBuilder, PushInto}; use crate::communication::allocator::thread::{ThreadPusher, ThreadPuller}; use crate::communication::{Push, Pull}; use crate::dataflow::channels::pushers::Exchange as ExchangePusher; @@ -53,7 +54,8 @@ pub type Exchange = ExchangeCore>, F>; impl ExchangeCore where CB: LengthPreservingContainerBuilder, - for<'a> F: FnMut(&::Item<'a>)->u64 + CB::Container: DrainContainer, + for<'a> F: FnMut(&::Item<'a>)->u64 { /// Allocates a new `Exchange` pact from a distribution function. pub fn new_core(func: F) -> ExchangeCore { @@ -66,7 +68,7 @@ where impl ExchangeCore, F> where - C: SizableContainer, + C: SizableContainer + DrainContainer, for<'a> F: FnMut(&C::Item<'a>)->u64 { /// Allocates a new `Exchange` pact from a distribution function. @@ -82,9 +84,10 @@ where impl ParallelizationContract for ExchangeCore where CB: ContainerBuilder, - CB: for<'a> PushInto<::Item<'a>>, + CB::Container: DrainContainer, + CB: for<'a> PushInto<::Item<'a>>, CB::Container: Data + Send + crate::dataflow::channels::ContainerBytes, - for<'a> H: FnMut(&::Item<'a>) -> u64 + for<'a> H: FnMut(&::Item<'a>) -> u64 { type Pusher = ExchangePusher>>>, H>; type Puller = LogPuller>>>; diff --git a/timely/src/dataflow/channels/pushers/exchange.rs b/timely/src/dataflow/channels/pushers/exchange.rs index 9b7baeece..9dc0eb60b 100644 --- a/timely/src/dataflow/channels/pushers/exchange.rs +++ b/timely/src/dataflow/channels/pushers/exchange.rs @@ -3,15 +3,17 @@ use crate::communication::Push; use crate::container::{ContainerBuilder, PushInto}; use crate::dataflow::channels::Message; -use crate::{Container, Data}; +use crate::Data; +use crate::container::DrainContainer; // TODO : Software write combining /// Distributes records among target pushees according to a distribution function. pub struct Exchange where CB: ContainerBuilder, + CB::Container: DrainContainer, P: Push>, - for<'a> H: FnMut(&::Item<'a>) -> u64 + for<'a> H: FnMut(&::Item<'a>) -> u64 { pushers: Vec

, builders: Vec, @@ -22,8 +24,9 @@ where impl Exchange where CB: ContainerBuilder, + CB::Container: DrainContainer, P: Push>, - for<'a> H: FnMut(&::Item<'a>) -> u64 + for<'a> H: FnMut(&::Item<'a>) -> u64 { /// Allocates a new `Exchange` from a supplied set of pushers and a distribution function. pub fn new(pushers: Vec

, key: H) -> Exchange { @@ -48,9 +51,10 @@ where impl Push> for Exchange where CB: ContainerBuilder, - CB: for<'a> PushInto<::Item<'a>>, + CB::Container: DrainContainer, + CB: for<'a> PushInto<::Item<'a>>, P: Push>, - for<'a> H: FnMut(&::Item<'a>) -> u64 + for<'a> H: FnMut(&::Item<'a>) -> u64 { #[inline(never)] fn push(&mut self, message: &mut Option>) { diff --git a/timely/src/dataflow/operators/core/capture/extract.rs b/timely/src/dataflow/operators/core/capture/extract.rs index e937ca3a5..808098a7e 100644 --- a/timely/src/dataflow/operators/core/capture/extract.rs +++ b/timely/src/dataflow/operators/core/capture/extract.rs @@ -1,7 +1,7 @@ //! Traits and types for extracting captured timely dataflow streams. use super::Event; -use crate::{container::{SizableContainer, PushInto}}; +use crate::container::{SizableContainer, DrainContainer, PushInto}; /// Supports extracting a sequence of timestamp and data. pub trait Extract { @@ -48,10 +48,10 @@ pub trait Extract { fn extract(self) -> Vec<(T, C)>; } -impl Extract for ::std::sync::mpsc::Receiver> +impl Extract for ::std::sync::mpsc::Receiver> where - for<'a> C: PushInto>, - for<'a> C::Item<'a>: Ord, + for<'a> C: SizableContainer + DrainContainer: Ord> + PushInto>, + T: Ord, { fn extract(self) -> Vec<(T, C)> { let mut staged = std::collections::BTreeMap::new(); diff --git a/timely/src/dataflow/operators/core/exchange.rs b/timely/src/dataflow/operators/core/exchange.rs index bb021a5d5..6e28fc1c1 100644 --- a/timely/src/dataflow/operators/core/exchange.rs +++ b/timely/src/dataflow/operators/core/exchange.rs @@ -1,13 +1,13 @@ //! Exchange records between workers. use crate::ExchangeData; -use crate::container::{Container, SizableContainer, PushInto}; +use crate::container::{DrainContainer, SizableContainer, PushInto}; use crate::dataflow::channels::pact::ExchangeCore; use crate::dataflow::operators::generic::operator::Operator; use crate::dataflow::{Scope, StreamCore}; /// Exchange records between workers. -pub trait Exchange { +pub trait Exchange { /// Exchange records between workers. /// /// The closure supplied should map a reference to a record to a `u64`, @@ -30,9 +30,11 @@ pub trait Exchange { impl Exchange for StreamCore where - C: SizableContainer + ExchangeData + crate::dataflow::channels::ContainerBytes, - C: for<'a> PushInto>, - + C: SizableContainer + + DrainContainer + + ExchangeData + + crate::dataflow::channels::ContainerBytes + + for<'a> PushInto>, { fn exchange(&self, route: F) -> StreamCore where diff --git a/timely/src/dataflow/operators/core/filter.rs b/timely/src/dataflow/operators/core/filter.rs index 80ef564b0..017262024 100644 --- a/timely/src/dataflow/operators/core/filter.rs +++ b/timely/src/dataflow/operators/core/filter.rs @@ -1,12 +1,12 @@ //! Filters a stream by a predicate. -use crate::container::{Container, SizableContainer, PushInto}; +use crate::container::{DrainContainer, SizableContainer, PushInto}; use crate::Data; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for filtering. -pub trait Filter { +pub trait Filter { /// Returns a new instance of `self` containing only records satisfying `predicate`. /// /// # Examples @@ -23,7 +23,7 @@ pub trait Filter { fn filter)->bool+'static>(&self, predicate: P) -> Self; } -impl Filter for StreamCore +impl Filter for StreamCore where for<'a> C: PushInto> { diff --git a/timely/src/dataflow/operators/core/inspect.rs b/timely/src/dataflow/operators/core/inspect.rs index b8c41f97b..781a0e8a8 100644 --- a/timely/src/dataflow/operators/core/inspect.rs +++ b/timely/src/dataflow/operators/core/inspect.rs @@ -1,12 +1,13 @@ //! Extension trait and implementation for observing and action on streamed data. use crate::{Container, Data}; +use crate::container::IterContainer; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::operators::generic::Operator; /// Methods to inspect records and batches of records on a stream. -pub trait Inspect: InspectCore + Sized { +pub trait Inspect: InspectCore + Sized { /// Runs a supplied closure on each observed data element. /// /// # Examples @@ -90,14 +91,14 @@ pub trait Inspect: InspectCore + Sized { fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl Inspect for StreamCore { +impl Inspect for StreamCore { fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { self.inspect_container(func) } } /// Inspect containers -pub trait InspectCore { +pub trait InspectCore { /// Runs a supplied closure on each observed container, and each frontier advancement. /// /// Rust's `Result` type is used to distinguish the events, with `Ok` for time and data, @@ -120,7 +121,7 @@ pub trait InspectCore { fn inspect_container(&self, func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl InspectCore for StreamCore { +impl InspectCore for StreamCore { fn inspect_container(&self, mut func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static diff --git a/timely/src/dataflow/operators/core/map.rs b/timely/src/dataflow/operators/core/map.rs index 8af70e4a4..94aabbd7f 100644 --- a/timely/src/dataflow/operators/core/map.rs +++ b/timely/src/dataflow/operators/core/map.rs @@ -1,13 +1,13 @@ //! Extension methods for `StreamCore` based on record-by-record transformation. -use crate::container::{Container, SizableContainer, PushInto}; +use crate::container::{Container, DrainContainer, SizableContainer, PushInto}; use crate::Data; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; /// Extension trait for `Stream`. -pub trait Map { +pub trait Map { /// Consumes each element of the stream and yields a new element. /// /// # Examples @@ -51,7 +51,7 @@ pub trait Map { ; } -impl Map for StreamCore { +impl Map for StreamCore { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. diff --git a/timely/src/dataflow/operators/core/ok_err.rs b/timely/src/dataflow/operators/core/ok_err.rs index 77fd1f527..ea1e70341 100644 --- a/timely/src/dataflow/operators/core/ok_err.rs +++ b/timely/src/dataflow/operators/core/ok_err.rs @@ -1,13 +1,13 @@ //! Operators that separate one stream into two streams based on some condition -use crate::container::{Container, SizableContainer, PushInto}; -use crate::Data; +use crate::{Container, Data}; +use crate::container::{DrainContainer, SizableContainer, PushInto}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::{Scope, StreamCore}; /// Extension trait for `Stream`. -pub trait OkErr { +pub trait OkErr { /// Takes one input stream and splits it into two output streams. /// For each record, the supplied closure is called with the data. /// If it returns `Ok(x)`, then `x` will be sent @@ -39,7 +39,7 @@ pub trait OkErr { ; } -impl OkErr for StreamCore { +impl OkErr for StreamCore { fn ok_err( &self, mut logic: L, diff --git a/timely/src/dataflow/operators/core/partition.rs b/timely/src/dataflow/operators/core/partition.rs index 58205364b..811e0360c 100644 --- a/timely/src/dataflow/operators/core/partition.rs +++ b/timely/src/dataflow/operators/core/partition.rs @@ -1,15 +1,14 @@ //! Partition a stream of records into multiple streams. -use timely_container::{Container, ContainerBuilder, PushInto}; - +use crate::container::{DrainContainer, ContainerBuilder, PushInto}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::operators::InputCapability; use crate::dataflow::{Scope, StreamCore}; -use crate::Data; +use crate::{Container, Data}; /// Partition a stream of records into multiple streams. -pub trait Partition { +pub trait Partition { /// Produces `parts` output streams, containing records produced and assigned by `route`. /// /// # Examples @@ -35,7 +34,7 @@ pub trait Partition { F: FnMut(C::Item<'_>) -> (u64, D2) + 'static; } -impl Partition for StreamCore { +impl Partition for StreamCore { fn partition(&self, parts: u64, mut route: F) -> Vec> where CB: ContainerBuilder + PushInto, From 13a2b43fd8c671bacd6d571dd90e43a1977d318f Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Sat, 16 Aug 2025 07:39:47 +0200 Subject: [PATCH 04/10] Remove unneeded is_empty check Signed-off-by: Moritz Hoffmann --- timely/src/dataflow/operators/core/filter.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/timely/src/dataflow/operators/core/filter.rs b/timely/src/dataflow/operators/core/filter.rs index 017262024..8167feca8 100644 --- a/timely/src/dataflow/operators/core/filter.rs +++ b/timely/src/dataflow/operators/core/filter.rs @@ -30,9 +30,7 @@ where fn filter)->bool+'static>(&self, mut predicate: P) -> StreamCore { self.unary(Pipeline, "Filter", move |_,_| move |input, output| { input.for_each(|time, data| { - if !data.is_empty() { - output.session(&time).give_iterator(data.drain().filter(&mut predicate)); - } + output.session(&time).give_iterator(data.drain().filter(&mut predicate)); }); }) } From 59680ec23e7cd79207d2af59a710d96fa823ef92 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Sat, 16 Aug 2025 07:52:39 +0200 Subject: [PATCH 05/10] Rework Container::len to Container::update_count Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 53 +++++++------------ timely/examples/columnar.rs | 3 +- timely/src/dataflow/channels/pact.rs | 4 +- .../src/dataflow/channels/pullers/counter.rs | 6 +-- .../src/dataflow/channels/pushers/counter.rs | 2 +- .../src/dataflow/operators/core/enterleave.rs | 2 +- timely/src/logging.rs | 2 +- 7 files changed, 30 insertions(+), 42 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index c3c51cefa..5c5587254 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -6,32 +6,27 @@ use std::collections::VecDeque; /// A container transferring data through dataflow edges /// -/// A container stores a number of elements and thus is able to describe it length (`len()`) and -/// whether it is empty (`is_empty()`). +/// A container stores a number of updates and thus is able to describe it count +/// (`update_count()`) and whether it is empty (`is_empty()`). It is empty if the +/// update count is zero. /// /// A container must implement default. The default implementation is not required to allocate /// memory for variable-length components. -/// -/// We require the container to be cloneable to enable efficient copies when providing references -/// of containers to operators. Care must be taken that the type's `clone_from` implementation -/// is efficient (which is not necessarily the case when deriving `Clone`.) // The container is `Default` because `CapacityContainerBuilder` only implements `ContainerBuilder` // for containers that implement `Default`, and we use the associated `::Container` all over Timely. // We can only access the type if all requirements for the `ContainerBuilder` implementation are // satisfied. pub trait Container: Default { - /// The number of elements in this container + /// The number of updates in this container /// /// This number is used in progress tracking to confirm the receipt of some number - /// of outstanding records, and it is highly load bearing. The main restriction is + /// of outstanding updates, and it is highly load bearing. The main restriction is /// imposed on the `LengthPreservingContainerBuilder` trait, whose implementors /// must preserve the number of items. - fn len(&self) -> usize; + fn update_count(&self) -> i64; - /// Determine if the container contains any elements, corresponding to `len() == 0`. - fn is_empty(&self) -> bool { - self.len() == 0 - } + /// Determine if the container contains any updates, corresponding to `update_count() == 0`. + #[inline] fn is_empty(&self) -> bool { self.update_count() == 0 } } /// TODO @@ -198,18 +193,14 @@ impl ContainerBuilder for CapacityContainerBuild impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } impl Container for Vec { - fn len(&self) -> usize { - Vec::len(self) - } - fn is_empty(&self) -> bool { - Vec::is_empty(self) - } + #[inline] fn update_count(&self) -> i64 { i64::try_from(Vec::len(self)).unwrap() } + #[inline] fn is_empty(&self) -> bool { Vec::is_empty(self) } } impl IterContainer for Vec { type ItemRef<'a> = &'a T where T: 'a; type Iter<'a> = std::slice::Iter<'a, T> where Self: 'a; - fn iter(&self) -> Self::Iter<'_> { + #[inline] fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter() } } @@ -217,7 +208,7 @@ impl IterContainer for Vec { impl DrainContainer for Vec { type Item<'a> = T where T: 'a; type DrainIter<'a> = std::vec::Drain<'a, T> where Self: 'a; - fn drain(&mut self) -> Self::DrainIter<'_> { + #[inline] fn drain(&mut self) -> Self::DrainIter<'_> { self.drain(..) } } @@ -267,22 +258,18 @@ mod rc { use crate::{Container, IterContainer, DrainContainer}; impl Container for Rc { - fn len(&self) -> usize { - std::ops::Deref::deref(self).len() - } - fn is_empty(&self) -> bool { - std::ops::Deref::deref(self).is_empty() - } + #[inline] fn update_count(&self) -> i64 { std::ops::Deref::deref(self).update_count() } + #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } impl IterContainer for Rc { type ItemRef<'a> = T::ItemRef<'a> where Self: 'a; type Iter<'a> = T::Iter<'a> where Self: 'a; - fn iter(&self) -> Self::Iter<'_> { self.deref().iter() } + #[inline] fn iter(&self) -> Self::Iter<'_> { self.deref().iter() } } impl DrainContainer for Rc { type Item<'a> = T::ItemRef<'a> where Self: 'a; type DrainIter<'a> = T::Iter<'a> where Self: 'a; - fn drain(&mut self) -> Self::DrainIter<'_> { self.iter() } + #[inline] fn drain(&mut self) -> Self::DrainIter<'_> { self.iter() } } } @@ -293,18 +280,18 @@ mod arc { use crate::{Container, IterContainer, DrainContainer}; impl Container for Arc { - fn len(&self) -> usize { std::ops::Deref::deref(self).len() } - fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } + #[inline] fn update_count(&self) -> i64 { std::ops::Deref::deref(self).update_count() } + #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } impl IterContainer for Arc { type ItemRef<'a> = T::ItemRef<'a> where Self: 'a; type Iter<'a> = T::Iter<'a> where Self: 'a; - fn iter(&self) -> Self::Iter<'_> { self.deref().iter() } + #[inline] fn iter(&self) -> Self::Iter<'_> { self.deref().iter() } } impl DrainContainer for Arc { type Item<'a> = T::ItemRef<'a> where Self: 'a; type DrainIter<'a> = T::Iter<'a> where Self: 'a; - fn drain(&mut self) -> Self::DrainIter<'_> { self.iter() } + #[inline] fn drain(&mut self) -> Self::DrainIter<'_> { self.iter() } } } diff --git a/timely/examples/columnar.rs b/timely/examples/columnar.rs index f9a17bc65..23cfc939b 100644 --- a/timely/examples/columnar.rs +++ b/timely/examples/columnar.rs @@ -177,7 +177,8 @@ mod container { } impl timely::Container for Column { - fn len(&self) -> usize { self.borrow().len() } + #[inline] fn update_count(&self) -> i64 { i64::try_from(self.borrow().len()).unwrap() } + #[inline] fn is_empty(&self) -> bool { self.borrow().is_empty() } } impl timely::container::IterContainer for Column { type ItemRef<'a> = C::Ref<'a>; diff --git a/timely/src/dataflow/channels/pact.rs b/timely/src/dataflow/channels/pact.rs index 8c550b561..ad226e6f3 100644 --- a/timely/src/dataflow/channels/pact.rs +++ b/timely/src/dataflow/channels/pact.rs @@ -148,7 +148,7 @@ impl>> Push> for LogPusher< source: self.source, target: self.target, seq_no: self.counter - 1, - length: bundle.data.len(), + update_count: bundle.data.update_count(), }) } } @@ -193,7 +193,7 @@ impl>> Pull> for LogPuller< source: bundle.from, target, seq_no: bundle.seq, - length: bundle.data.len(), + update_count: bundle.data.update_count(), }); } } diff --git a/timely/src/dataflow/channels/pullers/counter.rs b/timely/src/dataflow/channels/pullers/counter.rs index 6f71848a4..9c26499ea 100644 --- a/timely/src/dataflow/channels/pullers/counter.rs +++ b/timely/src/dataflow/channels/pullers/counter.rs @@ -19,7 +19,7 @@ pub struct Counter>> { pub struct ConsumedGuard { consumed: Rc>>, time: Option, - len: usize, + update_count: i64, } impl ConsumedGuard { @@ -32,7 +32,7 @@ impl Drop for ConsumedGuard { fn drop(&mut self) { // SAFETY: we're in a Drop impl, so this runs at most once let time = self.time.take().unwrap(); - self.consumed.borrow_mut().update(time, self.len as i64); + self.consumed.borrow_mut().update(time, self.update_count); } } @@ -49,7 +49,7 @@ impl>> Counter let guard = ConsumedGuard { consumed: Rc::clone(&self.consumed), time: Some(message.time.clone()), - len: message.data.len(), + update_count: message.data.update_count(), }; Some((guard, message)) } diff --git a/timely/src/dataflow/channels/pushers/counter.rs b/timely/src/dataflow/channels/pushers/counter.rs index a6a16ce5d..5e94969f4 100644 --- a/timely/src/dataflow/channels/pushers/counter.rs +++ b/timely/src/dataflow/channels/pushers/counter.rs @@ -21,7 +21,7 @@ impl Push> for Counter whe #[inline] fn push(&mut self, message: &mut Option>) { if let Some(message) = message { - self.produced.borrow_mut().update(message.time.clone(), message.data.len() as i64); + self.produced.borrow_mut().update(message.time.clone(), message.data.update_count()); } // only propagate `None` if dirty (indicates flush) diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index bc221c4a6..ecaa150cc 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -218,7 +218,7 @@ where source: self.index, target: self.index, seq_no: self.counter, - length: bundle.data.len(), + update_count: bundle.data.update_count(), }; let recv_event = MessagesEvent { is_send: false, diff --git a/timely/src/logging.rs b/timely/src/logging.rs index 2c2439319..b812dd4d2 100644 --- a/timely/src/logging.rs +++ b/timely/src/logging.rs @@ -135,7 +135,7 @@ pub struct MessagesEvent { /// Message sequence number. pub seq_no: usize, /// Number of typed records in the message. - pub length: usize, + pub update_count: i64, } /// Records the starting and stopping of an operator. From d941d0ff0eeafb0859d25e341f7a76c0e08453f6 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Sat, 16 Aug 2025 07:57:54 +0200 Subject: [PATCH 06/10] Rename Container to WithProgress Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 33 ++++++++++--------- timely/examples/columnar.rs | 2 +- timely/src/dataflow/channels/mod.rs | 4 +-- timely/src/dataflow/channels/pact.rs | 8 ++--- .../src/dataflow/channels/pullers/counter.rs | 4 +-- .../src/dataflow/channels/pushers/buffer.rs | 4 +-- .../src/dataflow/channels/pushers/counter.rs | 4 +-- timely/src/dataflow/channels/pushers/tee.rs | 8 ++--- timely/src/dataflow/operators/branch.rs | 4 +-- .../operators/core/capture/capture.rs | 6 ++-- .../dataflow/operators/core/capture/replay.rs | 4 +-- timely/src/dataflow/operators/core/concat.rs | 12 +++---- .../src/dataflow/operators/core/enterleave.rs | 18 +++++----- .../src/dataflow/operators/core/feedback.rs | 16 ++++----- timely/src/dataflow/operators/core/input.rs | 8 ++--- timely/src/dataflow/operators/core/inspect.rs | 6 ++-- timely/src/dataflow/operators/core/map.rs | 4 +-- timely/src/dataflow/operators/core/ok_err.rs | 4 +-- .../src/dataflow/operators/core/partition.rs | 4 +-- timely/src/dataflow/operators/core/probe.rs | 6 ++-- timely/src/dataflow/operators/core/rc.rs | 6 ++-- timely/src/dataflow/operators/core/reclock.rs | 8 ++--- .../src/dataflow/operators/core/to_stream.rs | 4 +-- .../operators/core/unordered_input.rs | 4 +-- .../dataflow/operators/generic/builder_raw.rs | 10 +++--- .../dataflow/operators/generic/builder_rc.rs | 6 ++-- .../src/dataflow/operators/generic/handles.rs | 16 ++++----- .../dataflow/operators/generic/operator.rs | 20 +++++------ timely/src/dataflow/stream.rs | 6 ++-- timely/src/lib.rs | 2 +- timely/src/logging.rs | 4 +-- 31 files changed, 123 insertions(+), 122 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index 5c5587254..7774c40a3 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -4,20 +4,21 @@ use std::collections::VecDeque; -/// A container transferring data through dataflow edges +/// An object with effects on progress /// -/// A container stores a number of updates and thus is able to describe it count +/// The object stores a number of updates and thus is able to describe it count /// (`update_count()`) and whether it is empty (`is_empty()`). It is empty if the /// update count is zero. /// -/// A container must implement default. The default implementation is not required to allocate -/// memory for variable-length components. +/// It must implement default for historic reason. The default implementation is not required +/// to allocate memory for variable-length components. +// TODO: Remove `Default` requirement in the future. // The container is `Default` because `CapacityContainerBuilder` only implements `ContainerBuilder` // for containers that implement `Default`, and we use the associated `::Container` all over Timely. // We can only access the type if all requirements for the `ContainerBuilder` implementation are // satisfied. -pub trait Container: Default { - /// The number of updates in this container +pub trait WithProgress: Default { + /// The number of updates /// /// This number is used in progress tracking to confirm the receipt of some number /// of outstanding updates, and it is highly load bearing. The main restriction is @@ -25,7 +26,7 @@ pub trait Container: Default { /// must preserve the number of items. fn update_count(&self) -> i64; - /// Determine if the container contains any updates, corresponding to `update_count() == 0`. + /// Determine if this contains any updates, corresponding to `update_count() == 0`. #[inline] fn is_empty(&self) -> bool { self.update_count() == 0 } } @@ -51,7 +52,7 @@ pub trait DrainContainer { } /// A container that can be sized and reveals its capacity. -pub trait SizableContainer: Container { +pub trait SizableContainer: WithProgress { /// Indicates that the container is "full" and should be shipped. fn at_capacity(&self) -> bool; /// Restores `self` to its desired capacity, if it has one. @@ -95,7 +96,7 @@ pub trait ContainerBuilder: Default + 'static { /// The container type we're building. // The container is `Clone` because `Tee` requires it, otherwise we need to repeat it // all over Timely. `'static` because we don't want lifetimes everywhere. - type Container: Container + Clone + 'static; + type Container: WithProgress + Clone + 'static; /// Extract assembled containers, potentially leaving unfinished data behind. Can /// be called repeatedly, for example while the caller can send data. /// @@ -167,7 +168,7 @@ impl> PushInto for CapacityContainerBuil } } -impl ContainerBuilder for CapacityContainerBuilder { +impl ContainerBuilder for CapacityContainerBuilder { type Container = C; #[inline] @@ -190,9 +191,9 @@ impl ContainerBuilder for CapacityContainerBuild } } -impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } +impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } -impl Container for Vec { +impl WithProgress for Vec { #[inline] fn update_count(&self) -> i64 { i64::try_from(Vec::len(self)).unwrap() } #[inline] fn is_empty(&self) -> bool { Vec::is_empty(self) } } @@ -255,9 +256,9 @@ mod rc { use std::ops::Deref; use std::rc::Rc; - use crate::{Container, IterContainer, DrainContainer}; + use crate::{WithProgress, IterContainer, DrainContainer}; - impl Container for Rc { + impl WithProgress for Rc { #[inline] fn update_count(&self) -> i64 { std::ops::Deref::deref(self).update_count() } #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } @@ -277,9 +278,9 @@ mod arc { use std::ops::Deref; use std::sync::Arc; - use crate::{Container, IterContainer, DrainContainer}; + use crate::{WithProgress, IterContainer, DrainContainer}; - impl Container for Arc { + impl WithProgress for Arc { #[inline] fn update_count(&self) -> i64 { std::ops::Deref::deref(self).update_count() } #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } diff --git a/timely/examples/columnar.rs b/timely/examples/columnar.rs index 23cfc939b..2b2b68e55 100644 --- a/timely/examples/columnar.rs +++ b/timely/examples/columnar.rs @@ -176,7 +176,7 @@ mod container { } } - impl timely::Container for Column { + impl timely::WithProgress for Column { #[inline] fn update_count(&self) -> i64 { i64::try_from(self.borrow().len()).unwrap() } #[inline] fn is_empty(&self) -> bool { self.borrow().is_empty() } } diff --git a/timely/src/dataflow/channels/mod.rs b/timely/src/dataflow/channels/mod.rs index 405a31b0e..2b0e72851 100644 --- a/timely/src/dataflow/channels/mod.rs +++ b/timely/src/dataflow/channels/mod.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use crate::communication::Push; -use crate::Container; +use crate::WithProgress; /// A collection of types that may be pushed at. pub mod pushers; @@ -32,7 +32,7 @@ impl Message { } } -impl Message { +impl Message { /// Creates a new message instance from arguments. pub fn new(time: T, data: C, from: usize, seq: usize) -> Self { Message { time, data, from, seq } diff --git a/timely/src/dataflow/channels/pact.rs b/timely/src/dataflow/channels/pact.rs index ad226e6f3..0b6203b42 100644 --- a/timely/src/dataflow/channels/pact.rs +++ b/timely/src/dataflow/channels/pact.rs @@ -10,7 +10,7 @@ use std::{fmt::{self, Debug}, marker::PhantomData}; use std::rc::Rc; -use crate::Container; +use crate::WithProgress; use crate::container::{ContainerBuilder, DrainContainer, LengthPreservingContainerBuilder, SizableContainer, CapacityContainerBuilder, PushInto}; use crate::communication::allocator::thread::{ThreadPusher, ThreadPuller}; use crate::communication::{Push, Pull}; @@ -35,7 +35,7 @@ pub trait ParallelizationContract { #[derive(Debug)] pub struct Pipeline; -impl ParallelizationContract for Pipeline { +impl ParallelizationContract for Pipeline { type Pusher = LogPusher>>; type Puller = LogPuller>>; fn connect(self, allocator: &mut A, identifier: usize, address: Rc<[usize]>, logging: Option) -> (Self::Pusher, Self::Puller) { @@ -130,7 +130,7 @@ impl

LogPusher

{ } } -impl>> Push> for LogPusher

{ +impl>> Push> for LogPusher

{ #[inline] fn push(&mut self, pair: &mut Option>) { if let Some(bundle) = pair { @@ -178,7 +178,7 @@ impl

LogPuller

{ } } -impl>> Pull> for LogPuller

{ +impl>> Pull> for LogPuller

{ #[inline] fn pull(&mut self) -> &mut Option> { let result = self.puller.pull(); diff --git a/timely/src/dataflow/channels/pullers/counter.rs b/timely/src/dataflow/channels/pullers/counter.rs index 9c26499ea..0504fc06c 100644 --- a/timely/src/dataflow/channels/pullers/counter.rs +++ b/timely/src/dataflow/channels/pullers/counter.rs @@ -6,7 +6,7 @@ use std::cell::RefCell; use crate::dataflow::channels::Message; use crate::progress::ChangeBatch; use crate::communication::Pull; -use crate::Container; +use crate::WithProgress; /// A wrapper which accounts records pulled past in a shared count map. pub struct Counter>> { @@ -36,7 +36,7 @@ impl Drop for ConsumedGuard { } } -impl>> Counter { +impl>> Counter { /// Retrieves the next timestamp and batch of data. #[inline] pub fn next(&mut self) -> Option<&mut Message> { diff --git a/timely/src/dataflow/channels/pushers/buffer.rs b/timely/src/dataflow/channels/pushers/buffer.rs index be579936c..d08a36ae3 100644 --- a/timely/src/dataflow/channels/pushers/buffer.rs +++ b/timely/src/dataflow/channels/pushers/buffer.rs @@ -6,7 +6,7 @@ use crate::container::{ContainerBuilder, CapacityContainerBuilder, PushInto}; use crate::dataflow::channels::Message; use crate::dataflow::operators::Capability; use crate::progress::Timestamp; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; /// Buffers data sent at the same time, for efficient communication. /// @@ -44,7 +44,7 @@ impl Buffer { } } -impl>> Buffer, P> where T: Eq+Clone { +impl>> Buffer, P> where T: Eq+Clone { /// Returns a `Session`, which accepts data to send at the associated time #[inline] pub fn session(&mut self, time: &T) -> Session<'_, T, CapacityContainerBuilder, P> { diff --git a/timely/src/dataflow/channels/pushers/counter.rs b/timely/src/dataflow/channels/pushers/counter.rs index 5e94969f4..ee5596d87 100644 --- a/timely/src/dataflow/channels/pushers/counter.rs +++ b/timely/src/dataflow/channels/pushers/counter.rs @@ -7,7 +7,7 @@ use std::cell::RefCell; use crate::progress::{ChangeBatch, Timestamp}; use crate::dataflow::channels::Message; use crate::communication::Push; -use crate::Container; +use crate::WithProgress; /// A wrapper which updates shared `produced` based on the number of records pushed. #[derive(Debug)] @@ -17,7 +17,7 @@ pub struct Counter>> { phantom: PhantomData, } -impl Push> for Counter where P: Push> { +impl Push> for Counter where P: Push> { #[inline] fn push(&mut self, message: &mut Option>) { if let Some(message) = message { diff --git a/timely/src/dataflow/channels/pushers/tee.rs b/timely/src/dataflow/channels/pushers/tee.rs index 49d9e17d0..3f9e2c877 100644 --- a/timely/src/dataflow/channels/pushers/tee.rs +++ b/timely/src/dataflow/channels/pushers/tee.rs @@ -7,7 +7,7 @@ use std::rc::Rc; use crate::dataflow::channels::Message; use crate::communication::Push; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; type PushList = Rc>>>>>; @@ -17,7 +17,7 @@ pub struct Tee { shared: PushList, } -impl Push> for Tee { +impl Push> for Tee { #[inline] fn push(&mut self, message: &mut Option>) { let mut pushers = self.shared.borrow_mut(); @@ -39,7 +39,7 @@ impl Push> for Tee { } } -impl Tee { +impl Tee { /// Allocates a new pair of `Tee` and `TeeHelper`. pub fn new() -> (Tee, TeeHelper) { let shared = Rc::new(RefCell::new(Vec::new())); @@ -52,7 +52,7 @@ impl Tee { } } -impl Clone for Tee { +impl Clone for Tee { fn clone(&self) -> Self { Self { buffer: Default::default(), diff --git a/timely/src/dataflow/operators/branch.rs b/timely/src/dataflow/operators/branch.rs index 924bd196a..0a008f6fa 100644 --- a/timely/src/dataflow/operators/branch.rs +++ b/timely/src/dataflow/operators/branch.rs @@ -3,7 +3,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::{Scope, Stream, StreamCore}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; /// Extension trait for `Stream`. pub trait Branch { @@ -93,7 +93,7 @@ pub trait BranchWhen: Sized { fn branch_when(&self, condition: impl Fn(&T) -> bool + 'static) -> (Self, Self); } -impl BranchWhen for StreamCore { +impl BranchWhen for StreamCore { fn branch_when(&self, condition: impl Fn(&S::Timestamp) -> bool + 'static) -> (Self, Self) { let mut builder = OperatorBuilder::new("Branch".to_owned(), self.scope()); builder.set_notify(false); diff --git a/timely/src/dataflow/operators/core/capture/capture.rs b/timely/src/dataflow/operators/core/capture/capture.rs index 410fe486e..7f90f02cd 100644 --- a/timely/src/dataflow/operators/core/capture/capture.rs +++ b/timely/src/dataflow/operators/core/capture/capture.rs @@ -10,14 +10,14 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pullers::Counter as PullCounter; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::progress::ChangeBatch; use crate::progress::Timestamp; use super::{Event, EventPusher}; /// Capture a stream of timestamped data for later replay. -pub trait Capture { +pub trait Capture { /// Captures a stream of timestamped data for later replay. /// /// # Examples @@ -113,7 +113,7 @@ pub trait Capture { } } -impl Capture for StreamCore { +impl Capture for StreamCore { fn capture_into+'static>(&self, mut event_pusher: P) { let mut builder = OperatorBuilder::new("Capture".to_owned(), self.scope()); diff --git a/timely/src/dataflow/operators/core/capture/replay.rs b/timely/src/dataflow/operators/core/capture/replay.rs index 49878b887..b66fa6a7f 100644 --- a/timely/src/dataflow/operators/core/capture/replay.rs +++ b/timely/src/dataflow/operators/core/capture/replay.rs @@ -46,7 +46,7 @@ use crate::progress::Timestamp; use super::Event; use super::event::EventIterator; -use crate::Container; +use crate::WithProgress; /// Replay a capture stream into a scope with the same timestamp. pub trait Replay : Sized { @@ -62,7 +62,7 @@ pub trait Replay : Sized { fn replay_core>(self, scope: &mut S, period: Option) -> StreamCore; } -impl Replay for I +impl Replay for I where I : IntoIterator, ::Item: EventIterator+'static, diff --git a/timely/src/dataflow/operators/core/concat.rs b/timely/src/dataflow/operators/core/concat.rs index 24fcef532..a09df603f 100644 --- a/timely/src/dataflow/operators/core/concat.rs +++ b/timely/src/dataflow/operators/core/concat.rs @@ -1,12 +1,12 @@ //! Merges the contents of multiple streams. -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{StreamCore, Scope}; /// Merge the contents of two streams. -pub trait Concat { +pub trait Concat { /// Merge the contents of two streams. /// /// # Examples @@ -23,14 +23,14 @@ pub trait Concat { fn concat(&self, _: &StreamCore) -> StreamCore; } -impl Concat for StreamCore { +impl Concat for StreamCore { fn concat(&self, other: &StreamCore) -> StreamCore { self.scope().concatenate([self.clone(), other.clone()]) } } /// Merge the contents of multiple streams. -pub trait Concatenate { +pub trait Concatenate { /// Merge the contents of multiple streams. /// /// # Examples @@ -52,7 +52,7 @@ pub trait Concatenate { I: IntoIterator>; } -impl Concatenate for StreamCore { +impl Concatenate for StreamCore { fn concatenate(&self, sources: I) -> StreamCore where I: IntoIterator> @@ -62,7 +62,7 @@ impl Concatenate for StreamCore { } } -impl Concatenate for G { +impl Concatenate for G { fn concatenate(&self, sources: I) -> StreamCore where I: IntoIterator> diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index ecaa150cc..e2e428483 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -26,7 +26,7 @@ use crate::logging::{TimelyLogger, MessagesEvent}; use crate::progress::Timestamp; use crate::progress::timestamp::Refines; use crate::progress::{Source, Target}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::communication::Push; use crate::dataflow::channels::pushers::{Counter, Tee}; use crate::dataflow::channels::Message; @@ -35,7 +35,7 @@ use crate::dataflow::{StreamCore, Scope}; use crate::dataflow::scopes::Child; /// Extension trait to move a `Stream` into a child of its current `Scope`. -pub trait Enter, C: Container> { +pub trait Enter, C: WithProgress> { /// Moves the `Stream` argument into a child of its current `Scope`. /// /// # Examples @@ -53,7 +53,7 @@ pub trait Enter, C: Container> { fn enter<'a>(&self, _: &Child<'a, G, T>) -> StreamCore, C>; } -impl, C: Data+Container> Enter for StreamCore { +impl, C: Data+ WithProgress> Enter for StreamCore { fn enter<'a>(&self, scope: &Child<'a, G, T>) -> StreamCore, C> { use crate::scheduling::Scheduler; @@ -85,7 +85,7 @@ impl, C: Data+Container> Enter { +pub trait Leave { /// Moves a `Stream` to the parent of its current `Scope`. /// /// # Examples @@ -103,7 +103,7 @@ pub trait Leave { fn leave(&self) -> StreamCore; } -impl> Leave for StreamCore, C> { +impl> Leave for StreamCore, C> { fn leave(&self) -> StreamCore { let scope = self.scope(); @@ -130,14 +130,14 @@ impl> Leave, TContainer: Container + Data> { +struct IngressNub, TContainer: WithProgress + Data> { targets: Counter>, phantom: ::std::marker::PhantomData, activator: crate::scheduling::Activator, active: bool, } -impl, TContainer: Container + Data> Push> for IngressNub { +impl, TContainer: WithProgress + Data> Push> for IngressNub { fn push(&mut self, element: &mut Option>) { if let Some(outer_message) = element { let data = ::std::mem::take(&mut outer_message.data); @@ -164,7 +164,7 @@ struct EgressNub, TContaine phantom: PhantomData, } -impl Push> for EgressNub +impl Push> for EgressNub where TOuter: Timestamp, TInner: Timestamp+Refines, TContainer: Data { fn push(&mut self, message: &mut Option>) { if let Some(inner_message) = message { @@ -207,7 +207,7 @@ impl

LogPusher

{ impl Push> for LogPusher

where - C: Container, + C: WithProgress, P: Push>, { fn push(&mut self, element: &mut Option>) { diff --git a/timely/src/dataflow/operators/core/feedback.rs b/timely/src/dataflow/operators/core/feedback.rs index 61bd5c196..3ca8bb076 100644 --- a/timely/src/dataflow/operators/core/feedback.rs +++ b/timely/src/dataflow/operators/core/feedback.rs @@ -1,6 +1,6 @@ //! Create cycles in a timely dataflow graph. -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::container::CapacityContainerBuilder; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pushers::Tee; @@ -36,7 +36,7 @@ pub trait Feedback { /// .connect_loop(handle); /// }); /// ``` - fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore); + fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore); } /// Creates a `StreamCore` and a `Handle` to later bind the source of that `StreamCore`. @@ -64,12 +64,12 @@ pub trait LoopVariable<'a, G: Scope, T: Timestamp> { /// }); /// }); /// ``` - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>); + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>); } impl Feedback for G { - fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore) { + fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore) { let mut builder = OperatorBuilder::new("Feedback".to_owned(), self.clone()); builder.set_notify(false); @@ -80,13 +80,13 @@ impl Feedback for G { } impl<'a, G: Scope, T: Timestamp> LoopVariable<'a, G, T> for Iterative<'a, G, T> { - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>) { + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>) { self.feedback(Product::new(Default::default(), summary)) } } /// Connect a `Stream` to the input of a loop variable. -pub trait ConnectLoop { +pub trait ConnectLoop { /// Connect a `Stream` to be the input of a loop variable. /// /// # Examples @@ -107,7 +107,7 @@ pub trait ConnectLoop { fn connect_loop(&self, handle: Handle); } -impl ConnectLoop for StreamCore { +impl ConnectLoop for StreamCore { fn connect_loop(&self, handle: Handle) { let mut builder = handle.builder; @@ -132,7 +132,7 @@ impl ConnectLoop for StreamCore { /// A handle used to bind the source of a loop variable. #[derive(Debug)] -pub struct Handle { +pub struct Handle { builder: OperatorBuilder, summary: ::Summary, output: OutputWrapper, Tee>, diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index 633e425bd..7c564405c 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -10,7 +10,7 @@ use crate::scheduling::{Schedule, Activator}; use crate::progress::{Operate, operate::SharedProgress, Timestamp, ChangeBatch}; use crate::progress::Source; use crate::progress::operate::Connectivity; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::communication::Push; use crate::dataflow::{Scope, ScopeParent, StreamCore}; use crate::dataflow::channels::pushers::{Tee, Counter}; @@ -59,7 +59,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore); + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore); /// Create a new [StreamCore] and [Handle] through which to supply input. /// @@ -134,7 +134,7 @@ pub trait Input : Scope { use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore) { + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore) { let mut handle = Handle::new(); let stream = self.input_from(&mut handle); (handle, stream) @@ -224,7 +224,7 @@ pub struct Handle { now_at: T, } -impl Handle> { +impl Handle> { /// Allocates a new input handle, from which one can create timely streams. /// /// # Examples diff --git a/timely/src/dataflow/operators/core/inspect.rs b/timely/src/dataflow/operators/core/inspect.rs index 781a0e8a8..6c2dbc19e 100644 --- a/timely/src/dataflow/operators/core/inspect.rs +++ b/timely/src/dataflow/operators/core/inspect.rs @@ -1,6 +1,6 @@ //! Extension trait and implementation for observing and action on streamed data. -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::container::IterContainer; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{Scope, StreamCore}; @@ -91,7 +91,7 @@ pub trait Inspect: InspectCore + Sized { fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl Inspect for StreamCore { +impl Inspect for StreamCore { fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { self.inspect_container(func) } @@ -121,7 +121,7 @@ pub trait InspectCore { fn inspect_container(&self, func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl InspectCore for StreamCore { +impl InspectCore for StreamCore { fn inspect_container(&self, mut func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static diff --git a/timely/src/dataflow/operators/core/map.rs b/timely/src/dataflow/operators/core/map.rs index 94aabbd7f..ee1abb073 100644 --- a/timely/src/dataflow/operators/core/map.rs +++ b/timely/src/dataflow/operators/core/map.rs @@ -1,6 +1,6 @@ //! Extension methods for `StreamCore` based on record-by-record transformation. -use crate::container::{Container, DrainContainer, SizableContainer, PushInto}; +use crate::container::{WithProgress, DrainContainer, SizableContainer, PushInto}; use crate::Data; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; @@ -51,7 +51,7 @@ pub trait Map { ; } -impl Map for StreamCore { +impl Map for StreamCore { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. diff --git a/timely/src/dataflow/operators/core/ok_err.rs b/timely/src/dataflow/operators/core/ok_err.rs index ea1e70341..f4d1d756d 100644 --- a/timely/src/dataflow/operators/core/ok_err.rs +++ b/timely/src/dataflow/operators/core/ok_err.rs @@ -1,6 +1,6 @@ //! Operators that separate one stream into two streams based on some condition -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::container::{DrainContainer, SizableContainer, PushInto}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; @@ -39,7 +39,7 @@ pub trait OkErr { ; } -impl OkErr for StreamCore { +impl OkErr for StreamCore { fn ok_err( &self, mut logic: L, diff --git a/timely/src/dataflow/operators/core/partition.rs b/timely/src/dataflow/operators/core/partition.rs index 811e0360c..84b998e3c 100644 --- a/timely/src/dataflow/operators/core/partition.rs +++ b/timely/src/dataflow/operators/core/partition.rs @@ -5,7 +5,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::operators::InputCapability; use crate::dataflow::{Scope, StreamCore}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; /// Partition a stream of records into multiple streams. pub trait Partition { @@ -34,7 +34,7 @@ pub trait Partition { F: FnMut(C::Item<'_>) -> (u64, D2) + 'static; } -impl Partition for StreamCore { +impl Partition for StreamCore { fn partition(&self, parts: u64, mut route: F) -> Vec> where CB: ContainerBuilder + PushInto, diff --git a/timely/src/dataflow/operators/core/probe.rs b/timely/src/dataflow/operators/core/probe.rs index cb3c96b34..33bde7497 100644 --- a/timely/src/dataflow/operators/core/probe.rs +++ b/timely/src/dataflow/operators/core/probe.rs @@ -13,10 +13,10 @@ use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; use crate::dataflow::{StreamCore, Scope}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; /// Monitors progress at a `Stream`. -pub trait Probe { +pub trait Probe { /// Constructs a progress probe which indicates which timestamps have elapsed at the operator. /// /// # Examples @@ -79,7 +79,7 @@ pub trait Probe { fn probe_with(&self, handle: &Handle) -> StreamCore; } -impl Probe for StreamCore { +impl Probe for StreamCore { fn probe(&self) -> Handle { // the frontier is shared state; scope updates, handle reads. diff --git a/timely/src/dataflow/operators/core/rc.rs b/timely/src/dataflow/operators/core/rc.rs index fdc68b9d4..0ad58d998 100644 --- a/timely/src/dataflow/operators/core/rc.rs +++ b/timely/src/dataflow/operators/core/rc.rs @@ -3,11 +3,11 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::Operator; use crate::dataflow::{Scope, StreamCore}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use std::rc::Rc; /// Convert a stream into a stream of shared containers -pub trait SharedStream { +pub trait SharedStream { /// Convert a stream into a stream of shared data /// /// # Examples @@ -24,7 +24,7 @@ pub trait SharedStream { fn shared(&self) -> StreamCore>; } -impl SharedStream for StreamCore { +impl SharedStream for StreamCore { fn shared(&self) -> StreamCore> { self.unary(Pipeline, "Shared", move |_, _| { move |input, output| { diff --git a/timely/src/dataflow/operators/core/reclock.rs b/timely/src/dataflow/operators/core/reclock.rs index e74bedbc4..5af558f3d 100644 --- a/timely/src/dataflow/operators/core/reclock.rs +++ b/timely/src/dataflow/operators/core/reclock.rs @@ -1,6 +1,6 @@ //! Extension methods for `Stream` based on record-by-record transformation. -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::order::PartialOrder; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; @@ -45,11 +45,11 @@ pub trait Reclock { /// assert_eq!(extracted[1], (5, vec![4,5])); /// assert_eq!(extracted[2], (8, vec![6,7,8])); /// ``` - fn reclock(&self, clock: &StreamCore) -> Self; + fn reclock(&self, clock: &StreamCore) -> Self; } -impl Reclock for StreamCore { - fn reclock(&self, clock: &StreamCore) -> StreamCore { +impl Reclock for StreamCore { + fn reclock(&self, clock: &StreamCore) -> StreamCore { let mut stash = vec![]; diff --git a/timely/src/dataflow/operators/core/to_stream.rs b/timely/src/dataflow/operators/core/to_stream.rs index cfc6f429b..983671c0d 100644 --- a/timely/src/dataflow/operators/core/to_stream.rs +++ b/timely/src/dataflow/operators/core/to_stream.rs @@ -1,7 +1,7 @@ //! Conversion to the `StreamCore` type from iterators. use crate::container::{CapacityContainerBuilder, ContainerBuilder, SizableContainer, PushInto}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::dataflow::operators::generic::operator::source; use crate::dataflow::{StreamCore, Scope}; @@ -61,7 +61,7 @@ impl ToStreamBuilder for I wh /// Converts to a timely [StreamCore]. Equivalent to [`ToStreamBuilder`] but /// uses a [`CapacityContainerBuilder`]. -pub trait ToStream { +pub trait ToStream { /// Converts to a timely [StreamCore]. /// /// # Examples diff --git a/timely/src/dataflow/operators/core/unordered_input.rs b/timely/src/dataflow/operators/core/unordered_input.rs index 77c6ba81b..a0d1dacdd 100644 --- a/timely/src/dataflow/operators/core/unordered_input.rs +++ b/timely/src/dataflow/operators/core/unordered_input.rs @@ -2,7 +2,7 @@ use std::rc::Rc; use std::cell::RefCell; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; use crate::scheduling::{Schedule, ActivateOnDrop}; @@ -164,7 +164,7 @@ impl UnorderedHandle { } } -impl UnorderedHandle> { +impl UnorderedHandle> { /// Allocates a new automatically flushing session based on the supplied capability. #[inline] pub fn session(&mut self, cap: ActivateCapability) -> ActivateOnDrop, Counter>>> { diff --git a/timely/src/dataflow/operators/generic/builder_raw.rs b/timely/src/dataflow/operators/generic/builder_raw.rs index 4659383a9..47bd0ba95 100644 --- a/timely/src/dataflow/operators/generic/builder_raw.rs +++ b/timely/src/dataflow/operators/generic/builder_raw.rs @@ -13,7 +13,7 @@ use crate::scheduling::{Schedule, Activations}; use crate::progress::{Source, Target}; use crate::progress::{Timestamp, Operate, operate::SharedProgress, Antichain}; use crate::progress::operate::{Connectivity, PortConnectivity}; -use crate::Container; +use crate::WithProgress; use crate::dataflow::{StreamCore, Scope}; use crate::dataflow::channels::pushers::Tee; use crate::dataflow::channels::pact::ParallelizationContract; @@ -104,7 +104,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> P::Puller + pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> P::Puller where P: ParallelizationContract { @@ -113,7 +113,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> P::Puller + pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> P::Puller where P: ParallelizationContract, I: IntoIterator::Summary>)>, @@ -133,14 +133,14 @@ impl OperatorBuilder { } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output(&mut self) -> (Tee, StreamCore) { + pub fn new_output(&mut self) -> (Tee, StreamCore) { let connection = (0 .. self.shape.inputs).map(|i| (i, Antichain::from_elem(Default::default()))); self.new_output_connection(connection) } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output_connection(&mut self, connection: I) -> (Tee, StreamCore) + pub fn new_output_connection(&mut self, connection: I) -> (Tee, StreamCore) where I: IntoIterator::Summary>)>, { diff --git a/timely/src/dataflow/operators/generic/builder_rc.rs b/timely/src/dataflow/operators/generic/builder_rc.rs index fee122014..9218f5725 100644 --- a/timely/src/dataflow/operators/generic/builder_rc.rs +++ b/timely/src/dataflow/operators/generic/builder_rc.rs @@ -8,7 +8,7 @@ use crate::progress::{ChangeBatch, Timestamp}; use crate::progress::operate::SharedProgress; use crate::progress::frontier::{Antichain, MutableAntichain}; -use crate::Container; +use crate::WithProgress; use crate::container::ContainerBuilder; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pushers::Tee; @@ -56,7 +56,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> InputHandleCore + pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> InputHandleCore where P: ParallelizationContract { @@ -72,7 +72,7 @@ impl OperatorBuilder { /// /// Commonly the connections are either the unit summary, indicating the same timestamp might be produced as output, or an empty /// antichain indicating that there is no connection from the input to the output. - pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> InputHandleCore + pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> InputHandleCore where P: ParallelizationContract, I: IntoIterator::Summary>)> + Clone, diff --git a/timely/src/dataflow/operators/generic/handles.rs b/timely/src/dataflow/operators/generic/handles.rs index dc9db01ce..81607f981 100644 --- a/timely/src/dataflow/operators/generic/handles.rs +++ b/timely/src/dataflow/operators/generic/handles.rs @@ -15,14 +15,14 @@ use crate::dataflow::channels::pushers::Counter as PushCounter; use crate::dataflow::channels::pushers::buffer::{Buffer, Session}; use crate::dataflow::channels::Message; use crate::communication::{Push, Pull}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; use crate::dataflow::operators::InputCapability; use crate::dataflow::operators::capability::CapabilityTrait; /// Handle to an operator's input stream. -pub struct InputHandleCore>> { +pub struct InputHandleCore>> { pull_counter: PullCounter, internal: Rc>>>>>, /// Timestamp summaries from this input to each output. @@ -36,7 +36,7 @@ pub struct InputHandleCore>> { pub type InputHandle = InputHandleCore, P>; /// Handle to an operator's input stream and frontier. -pub struct FrontieredInputHandleCore<'a, T: Timestamp, C: Container+'a, P: Pull>+'a> { +pub struct FrontieredInputHandleCore<'a, T: Timestamp, C: WithProgress +'a, P: Pull>+'a> { /// The underlying input handle. pub handle: &'a mut InputHandleCore, /// The frontier as reported by timely progress tracking. @@ -46,7 +46,7 @@ pub struct FrontieredInputHandleCore<'a, T: Timestamp, C: Container+'a, P: Pull< /// Handle to an operator's input stream and frontier, specialized to vectors. pub type FrontieredInputHandle<'a, T, D, P> = FrontieredInputHandleCore<'a, T, Vec, P>; -impl>> InputHandleCore { +impl>> InputHandleCore { /// Reads the next input buffer (at some timestamp `t`) and a corresponding capability for `t`. /// The timestamp `t` of the input buffer can be retrieved by invoking `.time()` on the capability. @@ -87,7 +87,7 @@ impl>> InputHandleCore>+'a> FrontieredInputHandleCore<'a, T, C, P> { +impl<'a, T: Timestamp, C: WithProgress, P: Pull>+'a> FrontieredInputHandleCore<'a, T, C, P> { /// Allocate a new frontiered input handle. pub fn new(handle: &'a mut InputHandleCore, frontier: &'a MutableAntichain) -> Self { FrontieredInputHandleCore { @@ -134,13 +134,13 @@ impl<'a, T: Timestamp, C: Container, P: Pull>+'a> FrontieredInputH } } -pub fn _access_pull_counter>>(input: &mut InputHandleCore) -> &mut PullCounter { +pub fn _access_pull_counter>>(input: &mut InputHandleCore) -> &mut PullCounter { &mut input.pull_counter } /// Constructs an input handle. /// Declared separately so that it can be kept private when `InputHandle` is re-exported. -pub fn new_input_handle>>( +pub fn new_input_handle>>( pull_counter: PullCounter, internal: Rc>>>>>, summaries: Rc>>, @@ -231,7 +231,7 @@ impl<'a, T: Timestamp, CB: ContainerBuilder, P: Push>> } } -impl<'a, T: Timestamp, C: Container + Data, P: Push>> OutputHandleCore<'a, T, CapacityContainerBuilder, P> { +impl<'a, T: Timestamp, C: WithProgress + Data, P: Push>> OutputHandleCore<'a, T, CapacityContainerBuilder, P> { /// Obtains a session that can send data at the timestamp associated with capability `cap`. /// /// In order to send data at a future timestamp, obtain a capability for the new timestamp diff --git a/timely/src/dataflow/operators/generic/operator.rs b/timely/src/dataflow/operators/generic/operator.rs index 5f76bc59c..356c6000e 100644 --- a/timely/src/dataflow/operators/generic/operator.rs +++ b/timely/src/dataflow/operators/generic/operator.rs @@ -12,11 +12,11 @@ use crate::dataflow::{Scope, StreamCore}; use super::builder_rc::OperatorBuilder; use crate::dataflow::operators::generic::OperatorInfo; use crate::dataflow::operators::generic::notificator::{Notificator, FrontierNotificator}; -use crate::{Container, Data}; +use crate::{WithProgress, Data}; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; /// Methods to construct generic streaming and blocking operators. -pub trait Operator { +pub trait Operator { /// Creates a new dataflow operator that partitions its input stream by a parallelization /// strategy `pact`, and repeatedly invokes `logic`, the function returned by the function passed as `constructor`. /// `logic` can read from the input stream, write to the output stream, and inspect the frontier at the input. @@ -177,7 +177,7 @@ pub trait Operator { /// ``` fn binary_frontier(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: Container + Data, + C2: WithProgress + Data, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, @@ -227,7 +227,7 @@ pub trait Operator { /// } /// }).unwrap(); /// ``` - fn binary_notify, &mut InputHandleCore, @@ -269,7 +269,7 @@ pub trait Operator { /// ``` fn binary(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: Container + Data, + C2: WithProgress + Data, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, @@ -307,7 +307,7 @@ pub trait Operator { P: ParallelizationContract; } -impl Operator for StreamCore { +impl Operator for StreamCore { fn unary_frontier(&self, pact: P, name: &str, constructor: B) -> StreamCore where @@ -388,7 +388,7 @@ impl Operator for StreamCore { fn binary_frontier(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: Container + Data, + C2: WithProgress + Data, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, @@ -419,7 +419,7 @@ impl Operator for StreamCore { stream } - fn binary_notify, &mut InputHandleCore, @@ -447,7 +447,7 @@ impl Operator for StreamCore { fn binary(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: Container + Data, + C2: WithProgress + Data, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, @@ -580,7 +580,7 @@ where /// /// }); /// ``` -pub fn empty(scope: &G) -> StreamCore { +pub fn empty(scope: &G) -> StreamCore { source::<_, CapacityContainerBuilder, _, _>(scope, "Empty", |_capability, _info| |_output| { // drop capability, do nothing }) diff --git a/timely/src/dataflow/stream.rs b/timely/src/dataflow/stream.rs index 7959c842a..2d1dfb4a7 100644 --- a/timely/src/dataflow/stream.rs +++ b/timely/src/dataflow/stream.rs @@ -11,7 +11,7 @@ use crate::dataflow::Scope; use crate::dataflow::channels::pushers::tee::TeeHelper; use crate::dataflow::channels::Message; use std::fmt::{self, Debug}; -use crate::Container; +use crate::WithProgress; // use dataflow::scopes::root::loggers::CHANNELS_Q; @@ -47,7 +47,7 @@ impl Clone for StreamCore { /// A stream batching data in vectors. pub type Stream = StreamCore>; -impl StreamCore { +impl StreamCore { /// Connects the stream to a destination. /// /// The destination is described both by a `Target`, for progress tracking information, and a `P: Push` where the @@ -76,7 +76,7 @@ impl StreamCore { pub fn scope(&self) -> S { self.scope.clone() } /// Allows the assertion of a container type, for the benefit of type inference. - pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } + pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } } /// A type that can be translated to a [StreamCore]. diff --git a/timely/src/lib.rs b/timely/src/lib.rs index 557b7c7e1..83447d09f 100644 --- a/timely/src/lib.rs +++ b/timely/src/lib.rs @@ -65,7 +65,7 @@ pub use timely_communication::Config as CommunicationConfig; pub use worker::Config as WorkerConfig; pub use execute::Config as Config; -pub use timely_container::Container; +pub use timely_container::WithProgress; /// Re-export of the `timely_container` crate. pub mod container { pub use timely_container::*; diff --git a/timely/src/logging.rs b/timely/src/logging.rs index b812dd4d2..e9f918312 100644 --- a/timely/src/logging.rs +++ b/timely/src/logging.rs @@ -19,7 +19,7 @@ use std::time::Duration; use columnar::Columnar; use serde::{Deserialize, Serialize}; -use crate::Container; +use crate::WithProgress; use crate::container::CapacityContainerBuilder; use crate::dataflow::operators::capture::{Event, EventPusher}; use crate::progress::operate::Connectivity; @@ -31,7 +31,7 @@ pub struct BatchLogger where P: EventPusher { _phantom: ::std::marker::PhantomData, } -impl BatchLogger where P: EventPusher, C: Container { +impl BatchLogger where P: EventPusher, C: WithProgress { /// Creates a new batch logger. pub fn new(event_pusher: P) -> Self { BatchLogger { From 711258b7e6f6d9e7c27f20f1e9d78cef81d8c61f Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Sun, 17 Aug 2025 21:59:41 +0200 Subject: [PATCH 07/10] Bring back Container trait It's a composite trait requiring WithProgress, Default, Clone, 'static Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 12 +++++------ timely/src/dataflow/channels/mod.rs | 4 ++-- timely/src/dataflow/channels/pact.rs | 3 +-- .../src/dataflow/channels/pullers/counter.rs | 2 +- .../src/dataflow/channels/pushers/buffer.rs | 4 ++-- timely/src/dataflow/channels/pushers/tee.rs | 10 +++++----- timely/src/dataflow/operators/branch.rs | 4 ++-- .../operators/core/capture/capture.rs | 6 +++--- .../operators/core/capture/extract.rs | 3 ++- .../dataflow/operators/core/capture/replay.rs | 4 ++-- timely/src/dataflow/operators/core/concat.rs | 13 ++++++------ .../src/dataflow/operators/core/enterleave.rs | 20 +++++++++---------- .../src/dataflow/operators/core/exchange.rs | 7 ++++--- .../src/dataflow/operators/core/feedback.rs | 16 +++++++-------- timely/src/dataflow/operators/core/filter.rs | 4 ++-- timely/src/dataflow/operators/core/input.rs | 8 ++++---- timely/src/dataflow/operators/core/inspect.rs | 6 +++--- timely/src/dataflow/operators/core/map.rs | 12 +++++------ timely/src/dataflow/operators/core/ok_err.rs | 12 +++++------ .../src/dataflow/operators/core/partition.rs | 6 ++---- timely/src/dataflow/operators/core/probe.rs | 6 +++--- timely/src/dataflow/operators/core/rc.rs | 6 +++--- timely/src/dataflow/operators/core/reclock.rs | 8 ++++---- .../src/dataflow/operators/core/to_stream.rs | 6 +++--- .../operators/core/unordered_input.rs | 5 +++-- .../dataflow/operators/generic/builder_raw.rs | 10 +++++----- .../dataflow/operators/generic/builder_rc.rs | 6 +++--- .../src/dataflow/operators/generic/handles.rs | 8 ++++---- .../dataflow/operators/generic/operator.rs | 20 +++++++++---------- timely/src/dataflow/stream.rs | 5 ++--- timely/src/lib.rs | 6 ++++++ timely/src/logging.rs | 4 ++-- 32 files changed, 125 insertions(+), 121 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index 7774c40a3..7d4134183 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -17,7 +17,7 @@ use std::collections::VecDeque; // for containers that implement `Default`, and we use the associated `::Container` all over Timely. // We can only access the type if all requirements for the `ContainerBuilder` implementation are // satisfied. -pub trait WithProgress: Default { +pub trait WithProgress { /// The number of updates /// /// This number is used in progress tracking to confirm the receipt of some number @@ -52,7 +52,7 @@ pub trait DrainContainer { } /// A container that can be sized and reveals its capacity. -pub trait SizableContainer: WithProgress { +pub trait SizableContainer: Sized { /// Indicates that the container is "full" and should be shipped. fn at_capacity(&self) -> bool; /// Restores `self` to its desired capacity, if it has one. @@ -96,7 +96,7 @@ pub trait ContainerBuilder: Default + 'static { /// The container type we're building. // The container is `Clone` because `Tee` requires it, otherwise we need to repeat it // all over Timely. `'static` because we don't want lifetimes everywhere. - type Container: WithProgress + Clone + 'static; + type Container: WithProgress + Default + Clone + 'static; /// Extract assembled containers, potentially leaving unfinished data behind. Can /// be called repeatedly, for example while the caller can send data. /// @@ -152,7 +152,7 @@ pub struct CapacityContainerBuilder{ pending: VecDeque, } -impl> PushInto for CapacityContainerBuilder { +impl> PushInto for CapacityContainerBuilder { #[inline] fn push_into(&mut self, item: T) { // Ensure capacity @@ -168,7 +168,7 @@ impl> PushInto for CapacityContainerBuil } } -impl ContainerBuilder for CapacityContainerBuilder { +impl ContainerBuilder for CapacityContainerBuilder { type Container = C; #[inline] @@ -191,7 +191,7 @@ impl ContainerBuilder for CapacityContainerBu } } -impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } +impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } impl WithProgress for Vec { #[inline] fn update_count(&self) -> i64 { i64::try_from(Vec::len(self)).unwrap() } diff --git a/timely/src/dataflow/channels/mod.rs b/timely/src/dataflow/channels/mod.rs index 2b0e72851..405a31b0e 100644 --- a/timely/src/dataflow/channels/mod.rs +++ b/timely/src/dataflow/channels/mod.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use crate::communication::Push; -use crate::WithProgress; +use crate::Container; /// A collection of types that may be pushed at. pub mod pushers; @@ -32,7 +32,7 @@ impl Message { } } -impl Message { +impl Message { /// Creates a new message instance from arguments. pub fn new(time: T, data: C, from: usize, seq: usize) -> Self { Message { time, data, from, seq } diff --git a/timely/src/dataflow/channels/pact.rs b/timely/src/dataflow/channels/pact.rs index 0b6203b42..f03fdb639 100644 --- a/timely/src/dataflow/channels/pact.rs +++ b/timely/src/dataflow/channels/pact.rs @@ -19,7 +19,6 @@ use crate::dataflow::channels::Message; use crate::logging::{TimelyLogger as Logger, MessagesEvent}; use crate::progress::Timestamp; use crate::worker::AsWorker; -use crate::Data; /// A `ParallelizationContract` allocates paired `Push` and `Pull` implementors. pub trait ParallelizationContract { @@ -86,7 +85,7 @@ where CB: ContainerBuilder, CB::Container: DrainContainer, CB: for<'a> PushInto<::Item<'a>>, - CB::Container: Data + Send + crate::dataflow::channels::ContainerBytes, + CB::Container: Send + crate::dataflow::channels::ContainerBytes, for<'a> H: FnMut(&::Item<'a>) -> u64 { type Pusher = ExchangePusher>>>, H>; diff --git a/timely/src/dataflow/channels/pullers/counter.rs b/timely/src/dataflow/channels/pullers/counter.rs index 0504fc06c..20b6d0da7 100644 --- a/timely/src/dataflow/channels/pullers/counter.rs +++ b/timely/src/dataflow/channels/pullers/counter.rs @@ -9,7 +9,7 @@ use crate::communication::Pull; use crate::WithProgress; /// A wrapper which accounts records pulled past in a shared count map. -pub struct Counter>> { +pub struct Counter { pullable: P, consumed: Rc>>, phantom: ::std::marker::PhantomData, diff --git a/timely/src/dataflow/channels/pushers/buffer.rs b/timely/src/dataflow/channels/pushers/buffer.rs index d08a36ae3..23efd934e 100644 --- a/timely/src/dataflow/channels/pushers/buffer.rs +++ b/timely/src/dataflow/channels/pushers/buffer.rs @@ -6,7 +6,7 @@ use crate::container::{ContainerBuilder, CapacityContainerBuilder, PushInto}; use crate::dataflow::channels::Message; use crate::dataflow::operators::Capability; use crate::progress::Timestamp; -use crate::{WithProgress, Data}; +use crate::{Container, WithProgress}; /// Buffers data sent at the same time, for efficient communication. /// @@ -44,7 +44,7 @@ impl Buffer { } } -impl>> Buffer, P> where T: Eq+Clone { +impl>> Buffer, P> where T: Eq+Clone { /// Returns a `Session`, which accepts data to send at the associated time #[inline] pub fn session(&mut self, time: &T) -> Session<'_, T, CapacityContainerBuilder, P> { diff --git a/timely/src/dataflow/channels/pushers/tee.rs b/timely/src/dataflow/channels/pushers/tee.rs index 3f9e2c877..df2e2090d 100644 --- a/timely/src/dataflow/channels/pushers/tee.rs +++ b/timely/src/dataflow/channels/pushers/tee.rs @@ -7,7 +7,7 @@ use std::rc::Rc; use crate::dataflow::channels::Message; use crate::communication::Push; -use crate::{WithProgress, Data}; +use crate::{Container, Data}; type PushList = Rc>>>>>; @@ -17,7 +17,7 @@ pub struct Tee { shared: PushList, } -impl Push> for Tee { +impl Push> for Tee { #[inline] fn push(&mut self, message: &mut Option>) { let mut pushers = self.shared.borrow_mut(); @@ -32,14 +32,14 @@ impl Push> for Tee { pushers[index-1].push(&mut None); } } - if pushers.len() > 0 { + if !pushers.is_empty() { let last = pushers.len() - 1; pushers[last].push(message); } } } -impl Tee { +impl Tee { /// Allocates a new pair of `Tee` and `TeeHelper`. pub fn new() -> (Tee, TeeHelper) { let shared = Rc::new(RefCell::new(Vec::new())); @@ -52,7 +52,7 @@ impl Tee { } } -impl Clone for Tee { +impl Clone for Tee { fn clone(&self) -> Self { Self { buffer: Default::default(), diff --git a/timely/src/dataflow/operators/branch.rs b/timely/src/dataflow/operators/branch.rs index 0a008f6fa..dad88be2b 100644 --- a/timely/src/dataflow/operators/branch.rs +++ b/timely/src/dataflow/operators/branch.rs @@ -3,7 +3,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::{Scope, Stream, StreamCore}; -use crate::{WithProgress, Data}; +use crate::{Container, Data}; /// Extension trait for `Stream`. pub trait Branch { @@ -93,7 +93,7 @@ pub trait BranchWhen: Sized { fn branch_when(&self, condition: impl Fn(&T) -> bool + 'static) -> (Self, Self); } -impl BranchWhen for StreamCore { +impl BranchWhen for StreamCore { fn branch_when(&self, condition: impl Fn(&S::Timestamp) -> bool + 'static) -> (Self, Self) { let mut builder = OperatorBuilder::new("Branch".to_owned(), self.scope()); builder.set_notify(false); diff --git a/timely/src/dataflow/operators/core/capture/capture.rs b/timely/src/dataflow/operators/core/capture/capture.rs index 7f90f02cd..4d8e3053e 100644 --- a/timely/src/dataflow/operators/core/capture/capture.rs +++ b/timely/src/dataflow/operators/core/capture/capture.rs @@ -10,14 +10,14 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pullers::Counter as PullCounter; use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; -use crate::{WithProgress, Data}; +use crate::Container; use crate::progress::ChangeBatch; use crate::progress::Timestamp; use super::{Event, EventPusher}; /// Capture a stream of timestamped data for later replay. -pub trait Capture { +pub trait Capture { /// Captures a stream of timestamped data for later replay. /// /// # Examples @@ -113,7 +113,7 @@ pub trait Capture { } } -impl Capture for StreamCore { +impl Capture for StreamCore { fn capture_into+'static>(&self, mut event_pusher: P) { let mut builder = OperatorBuilder::new("Capture".to_owned(), self.scope()); diff --git a/timely/src/dataflow/operators/core/capture/extract.rs b/timely/src/dataflow/operators/core/capture/extract.rs index 808098a7e..318f3957b 100644 --- a/timely/src/dataflow/operators/core/capture/extract.rs +++ b/timely/src/dataflow/operators/core/capture/extract.rs @@ -1,6 +1,7 @@ //! Traits and types for extracting captured timely dataflow streams. use super::Event; +use crate::Container; use crate::container::{SizableContainer, DrainContainer, PushInto}; /// Supports extracting a sequence of timestamp and data. @@ -50,7 +51,7 @@ pub trait Extract { impl Extract for ::std::sync::mpsc::Receiver> where - for<'a> C: SizableContainer + DrainContainer: Ord> + PushInto>, + for<'a> C: Container + SizableContainer + DrainContainer: Ord> + PushInto>, T: Ord, { fn extract(self) -> Vec<(T, C)> { diff --git a/timely/src/dataflow/operators/core/capture/replay.rs b/timely/src/dataflow/operators/core/capture/replay.rs index b66fa6a7f..41ff50d12 100644 --- a/timely/src/dataflow/operators/core/capture/replay.rs +++ b/timely/src/dataflow/operators/core/capture/replay.rs @@ -46,7 +46,7 @@ use crate::progress::Timestamp; use super::Event; use super::event::EventIterator; -use crate::WithProgress; +use crate::Container; /// Replay a capture stream into a scope with the same timestamp. pub trait Replay : Sized { @@ -62,7 +62,7 @@ pub trait Replay : Sized { fn replay_core>(self, scope: &mut S, period: Option) -> StreamCore; } -impl Replay for I +impl Replay for I where I : IntoIterator, ::Item: EventIterator+'static, diff --git a/timely/src/dataflow/operators/core/concat.rs b/timely/src/dataflow/operators/core/concat.rs index a09df603f..f377d3626 100644 --- a/timely/src/dataflow/operators/core/concat.rs +++ b/timely/src/dataflow/operators/core/concat.rs @@ -1,12 +1,11 @@ //! Merges the contents of multiple streams. - -use crate::{WithProgress, Data}; +use crate::Container; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{StreamCore, Scope}; /// Merge the contents of two streams. -pub trait Concat { +pub trait Concat { /// Merge the contents of two streams. /// /// # Examples @@ -23,14 +22,14 @@ pub trait Concat { fn concat(&self, _: &StreamCore) -> StreamCore; } -impl Concat for StreamCore { +impl Concat for StreamCore { fn concat(&self, other: &StreamCore) -> StreamCore { self.scope().concatenate([self.clone(), other.clone()]) } } /// Merge the contents of multiple streams. -pub trait Concatenate { +pub trait Concatenate { /// Merge the contents of multiple streams. /// /// # Examples @@ -52,7 +51,7 @@ pub trait Concatenate { I: IntoIterator>; } -impl Concatenate for StreamCore { +impl Concatenate for StreamCore { fn concatenate(&self, sources: I) -> StreamCore where I: IntoIterator> @@ -62,7 +61,7 @@ impl Concatenate for StreamCore { } } -impl Concatenate for G { +impl Concatenate for G { fn concatenate(&self, sources: I) -> StreamCore where I: IntoIterator> diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index e2e428483..acff0fa6a 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -26,7 +26,7 @@ use crate::logging::{TimelyLogger, MessagesEvent}; use crate::progress::Timestamp; use crate::progress::timestamp::Refines; use crate::progress::{Source, Target}; -use crate::{WithProgress, Data}; +use crate::{WithProgress, Container}; use crate::communication::Push; use crate::dataflow::channels::pushers::{Counter, Tee}; use crate::dataflow::channels::Message; @@ -35,7 +35,7 @@ use crate::dataflow::{StreamCore, Scope}; use crate::dataflow::scopes::Child; /// Extension trait to move a `Stream` into a child of its current `Scope`. -pub trait Enter, C: WithProgress> { +pub trait Enter, C> { /// Moves the `Stream` argument into a child of its current `Scope`. /// /// # Examples @@ -53,7 +53,7 @@ pub trait Enter, C: WithProgress> { fn enter<'a>(&self, _: &Child<'a, G, T>) -> StreamCore, C>; } -impl, C: Data+ WithProgress> Enter for StreamCore { +impl, C: Container> Enter for StreamCore { fn enter<'a>(&self, scope: &Child<'a, G, T>) -> StreamCore, C> { use crate::scheduling::Scheduler; @@ -85,7 +85,7 @@ impl, C: Data+ WithProgress> Enter< } /// Extension trait to move a `Stream` to the parent of its current `Scope`. -pub trait Leave { +pub trait Leave { /// Moves a `Stream` to the parent of its current `Scope`. /// /// # Examples @@ -103,7 +103,7 @@ pub trait Leave { fn leave(&self) -> StreamCore; } -impl> Leave for StreamCore, C> { +impl> Leave for StreamCore, C> { fn leave(&self) -> StreamCore { let scope = self.scope(); @@ -130,14 +130,14 @@ impl> Leave } -struct IngressNub, TContainer: WithProgress + Data> { +struct IngressNub, TContainer: Container> { targets: Counter>, phantom: ::std::marker::PhantomData, activator: crate::scheduling::Activator, active: bool, } -impl, TContainer: WithProgress + Data> Push> for IngressNub { +impl, TContainer: Container> Push> for IngressNub { fn push(&mut self, element: &mut Option>) { if let Some(outer_message) = element { let data = ::std::mem::take(&mut outer_message.data); @@ -159,13 +159,13 @@ impl, TContainer: WithProgr } -struct EgressNub, TContainer: Data> { +struct EgressNub, TContainer> { targets: Tee, phantom: PhantomData, } -impl Push> for EgressNub -where TOuter: Timestamp, TInner: Timestamp+Refines, TContainer: Data { +impl Push> for EgressNub +where TOuter: Timestamp, TInner: Timestamp+Refines, { fn push(&mut self, message: &mut Option>) { if let Some(inner_message) = message { let data = ::std::mem::take(&mut inner_message.data); diff --git a/timely/src/dataflow/operators/core/exchange.rs b/timely/src/dataflow/operators/core/exchange.rs index 6e28fc1c1..92c9d78f5 100644 --- a/timely/src/dataflow/operators/core/exchange.rs +++ b/timely/src/dataflow/operators/core/exchange.rs @@ -1,6 +1,6 @@ //! Exchange records between workers. -use crate::ExchangeData; +use crate::Container; use crate::container::{DrainContainer, SizableContainer, PushInto}; use crate::dataflow::channels::pact::ExchangeCore; use crate::dataflow::operators::generic::operator::Operator; @@ -30,9 +30,10 @@ pub trait Exchange { impl Exchange for StreamCore where - C: SizableContainer + C: Container + + SizableContainer + DrainContainer - + ExchangeData + + Send + crate::dataflow::channels::ContainerBytes + for<'a> PushInto>, { diff --git a/timely/src/dataflow/operators/core/feedback.rs b/timely/src/dataflow/operators/core/feedback.rs index 3ca8bb076..067ea3db8 100644 --- a/timely/src/dataflow/operators/core/feedback.rs +++ b/timely/src/dataflow/operators/core/feedback.rs @@ -1,6 +1,6 @@ //! Create cycles in a timely dataflow graph. -use crate::{WithProgress, Data}; +use crate::Container; use crate::container::CapacityContainerBuilder; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::channels::pushers::Tee; @@ -36,7 +36,7 @@ pub trait Feedback { /// .connect_loop(handle); /// }); /// ``` - fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore); + fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore); } /// Creates a `StreamCore` and a `Handle` to later bind the source of that `StreamCore`. @@ -64,12 +64,12 @@ pub trait LoopVariable<'a, G: Scope, T: Timestamp> { /// }); /// }); /// ``` - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>); + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>); } impl Feedback for G { - fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore) { + fn feedback(&mut self, summary: ::Summary) -> (Handle, StreamCore) { let mut builder = OperatorBuilder::new("Feedback".to_owned(), self.clone()); builder.set_notify(false); @@ -80,13 +80,13 @@ impl Feedback for G { } impl<'a, G: Scope, T: Timestamp> LoopVariable<'a, G, T> for Iterative<'a, G, T> { - fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>) { + fn loop_variable(&mut self, summary: T::Summary) -> (Handle, C>, StreamCore, C>) { self.feedback(Product::new(Default::default(), summary)) } } /// Connect a `Stream` to the input of a loop variable. -pub trait ConnectLoop { +pub trait ConnectLoop { /// Connect a `Stream` to be the input of a loop variable. /// /// # Examples @@ -107,7 +107,7 @@ pub trait ConnectLoop { fn connect_loop(&self, handle: Handle); } -impl ConnectLoop for StreamCore { +impl ConnectLoop for StreamCore { fn connect_loop(&self, handle: Handle) { let mut builder = handle.builder; @@ -132,7 +132,7 @@ impl ConnectLoop for StreamCore { /// A handle used to bind the source of a loop variable. #[derive(Debug)] -pub struct Handle { +pub struct Handle { builder: OperatorBuilder, summary: ::Summary, output: OutputWrapper, Tee>, diff --git a/timely/src/dataflow/operators/core/filter.rs b/timely/src/dataflow/operators/core/filter.rs index 8167feca8..f82a76339 100644 --- a/timely/src/dataflow/operators/core/filter.rs +++ b/timely/src/dataflow/operators/core/filter.rs @@ -1,6 +1,6 @@ //! Filters a stream by a predicate. use crate::container::{DrainContainer, SizableContainer, PushInto}; -use crate::Data; +use crate::Container; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::operators::generic::operator::Operator; @@ -23,7 +23,7 @@ pub trait Filter { fn filter)->bool+'static>(&self, predicate: P) -> Self; } -impl Filter for StreamCore +impl Filter for StreamCore where for<'a> C: PushInto> { diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index 7c564405c..9c26f1ae4 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -10,7 +10,7 @@ use crate::scheduling::{Schedule, Activator}; use crate::progress::{Operate, operate::SharedProgress, Timestamp, ChangeBatch}; use crate::progress::Source; use crate::progress::operate::Connectivity; -use crate::{WithProgress, Data}; +use crate::{WithProgress, Container}; use crate::communication::Push; use crate::dataflow::{Scope, ScopeParent, StreamCore}; use crate::dataflow::channels::pushers::{Tee, Counter}; @@ -59,7 +59,7 @@ pub trait Input : Scope { /// } /// }); /// ``` - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore); + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore); /// Create a new [StreamCore] and [Handle] through which to supply input. /// @@ -134,7 +134,7 @@ pub trait Input : Scope { use crate::order::TotalOrder; impl Input for G where ::Timestamp: TotalOrder { - fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore) { + fn new_input(&mut self) -> (Handle<::Timestamp, CapacityContainerBuilder>, StreamCore) { let mut handle = Handle::new(); let stream = self.input_from(&mut handle); (handle, stream) @@ -224,7 +224,7 @@ pub struct Handle { now_at: T, } -impl Handle> { +impl Handle> { /// Allocates a new input handle, from which one can create timely streams. /// /// # Examples diff --git a/timely/src/dataflow/operators/core/inspect.rs b/timely/src/dataflow/operators/core/inspect.rs index 6c2dbc19e..1eea57488 100644 --- a/timely/src/dataflow/operators/core/inspect.rs +++ b/timely/src/dataflow/operators/core/inspect.rs @@ -1,6 +1,6 @@ //! Extension trait and implementation for observing and action on streamed data. -use crate::{WithProgress, Data}; +use crate::Container; use crate::container::IterContainer; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::{Scope, StreamCore}; @@ -91,7 +91,7 @@ pub trait Inspect: InspectCore + Sized { fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl Inspect for StreamCore { +impl Inspect for StreamCore { fn inspect_core(&self, func: F) -> Self where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>) + 'static { self.inspect_container(func) } @@ -121,7 +121,7 @@ pub trait InspectCore { fn inspect_container(&self, func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl InspectCore for StreamCore { +impl InspectCore for StreamCore { fn inspect_container(&self, mut func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static diff --git a/timely/src/dataflow/operators/core/map.rs b/timely/src/dataflow/operators/core/map.rs index ee1abb073..140eeba62 100644 --- a/timely/src/dataflow/operators/core/map.rs +++ b/timely/src/dataflow/operators/core/map.rs @@ -1,7 +1,7 @@ //! Extension methods for `StreamCore` based on record-by-record transformation. -use crate::container::{WithProgress, DrainContainer, SizableContainer, PushInto}; -use crate::Data; +use crate::container::{DrainContainer, SizableContainer, PushInto}; +use crate::Container; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::operator::Operator; @@ -24,7 +24,7 @@ pub trait Map { /// ``` fn map(&self, mut logic: L) -> StreamCore where - C2: SizableContainer + PushInto + Data, + C2: Container + SizableContainer + PushInto, L: FnMut(C::Item<'_>)->D2 + 'static, { self.flat_map(move |x| std::iter::once(logic(x))) @@ -46,19 +46,19 @@ pub trait Map { fn flat_map(&self, logic: L) -> StreamCore where I: IntoIterator, - C2: SizableContainer + PushInto + Data, + C2: Container + SizableContainer + PushInto, L: FnMut(C::Item<'_>)->I + 'static, ; } -impl Map for StreamCore { +impl Map for StreamCore { // TODO : This would be more robust if it captured an iterator and then pulled an appropriate // TODO : number of elements from the iterator. This would allow iterators that produce many // TODO : records without taking arbitrarily long and arbitrarily much memory. fn flat_map(&self, mut logic: L) -> StreamCore where I: IntoIterator, - C2: SizableContainer + PushInto + Data, + C2: Container + SizableContainer + PushInto, L: FnMut(C::Item<'_>)->I + 'static, { self.unary(Pipeline, "FlatMap", move |_,_| move |input, output| { diff --git a/timely/src/dataflow/operators/core/ok_err.rs b/timely/src/dataflow/operators/core/ok_err.rs index f4d1d756d..339b51576 100644 --- a/timely/src/dataflow/operators/core/ok_err.rs +++ b/timely/src/dataflow/operators/core/ok_err.rs @@ -1,6 +1,6 @@ //! Operators that separate one stream into two streams based on some condition -use crate::{WithProgress, Data}; +use crate::Container; use crate::container::{DrainContainer, SizableContainer, PushInto}; use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; @@ -33,20 +33,20 @@ pub trait OkErr { logic: L, ) -> (StreamCore, StreamCore) where - C1: SizableContainer + PushInto + Data, - C2: SizableContainer + PushInto + Data, + C1: Container + SizableContainer + PushInto, + C2: Container + SizableContainer + PushInto, L: FnMut(C::Item<'_>) -> Result+'static ; } -impl OkErr for StreamCore { +impl OkErr for StreamCore { fn ok_err( &self, mut logic: L, ) -> (StreamCore, StreamCore) where - C1: SizableContainer + PushInto + Data, - C2: SizableContainer + PushInto + Data, + C1: Container + SizableContainer + PushInto, + C2: Container + SizableContainer + PushInto, L: FnMut(C::Item<'_>) -> Result+'static { let mut builder = OperatorBuilder::new("OkErr".to_owned(), self.scope()); diff --git a/timely/src/dataflow/operators/core/partition.rs b/timely/src/dataflow/operators/core/partition.rs index 84b998e3c..7602b4892 100644 --- a/timely/src/dataflow/operators/core/partition.rs +++ b/timely/src/dataflow/operators/core/partition.rs @@ -5,7 +5,7 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::dataflow::operators::InputCapability; use crate::dataflow::{Scope, StreamCore}; -use crate::{WithProgress, Data}; +use crate::Container; /// Partition a stream of records into multiple streams. pub trait Partition { @@ -30,15 +30,13 @@ pub trait Partition { fn partition(&self, parts: u64, route: F) -> Vec> where CB: ContainerBuilder + PushInto, - CB::Container: Data, F: FnMut(C::Item<'_>) -> (u64, D2) + 'static; } -impl Partition for StreamCore { +impl Partition for StreamCore { fn partition(&self, parts: u64, mut route: F) -> Vec> where CB: ContainerBuilder + PushInto, - CB::Container: Data, F: FnMut(C::Item<'_>) -> (u64, D2) + 'static, { let mut builder = OperatorBuilder::new("Partition".to_owned(), self.scope()); diff --git a/timely/src/dataflow/operators/core/probe.rs b/timely/src/dataflow/operators/core/probe.rs index 33bde7497..324061991 100644 --- a/timely/src/dataflow/operators/core/probe.rs +++ b/timely/src/dataflow/operators/core/probe.rs @@ -13,10 +13,10 @@ use crate::dataflow::operators::generic::builder_raw::OperatorBuilder; use crate::dataflow::{StreamCore, Scope}; -use crate::{WithProgress, Data}; +use crate::Container; /// Monitors progress at a `Stream`. -pub trait Probe { +pub trait Probe { /// Constructs a progress probe which indicates which timestamps have elapsed at the operator. /// /// # Examples @@ -79,7 +79,7 @@ pub trait Probe { fn probe_with(&self, handle: &Handle) -> StreamCore; } -impl Probe for StreamCore { +impl Probe for StreamCore { fn probe(&self) -> Handle { // the frontier is shared state; scope updates, handle reads. diff --git a/timely/src/dataflow/operators/core/rc.rs b/timely/src/dataflow/operators/core/rc.rs index 0ad58d998..e9c622bc7 100644 --- a/timely/src/dataflow/operators/core/rc.rs +++ b/timely/src/dataflow/operators/core/rc.rs @@ -3,11 +3,11 @@ use crate::dataflow::channels::pact::Pipeline; use crate::dataflow::operators::Operator; use crate::dataflow::{Scope, StreamCore}; -use crate::{WithProgress, Data}; +use crate::Container; use std::rc::Rc; /// Convert a stream into a stream of shared containers -pub trait SharedStream { +pub trait SharedStream { /// Convert a stream into a stream of shared data /// /// # Examples @@ -24,7 +24,7 @@ pub trait SharedStream { fn shared(&self) -> StreamCore>; } -impl SharedStream for StreamCore { +impl SharedStream for StreamCore { fn shared(&self) -> StreamCore> { self.unary(Pipeline, "Shared", move |_, _| { move |input, output| { diff --git a/timely/src/dataflow/operators/core/reclock.rs b/timely/src/dataflow/operators/core/reclock.rs index 5af558f3d..86ec5ae69 100644 --- a/timely/src/dataflow/operators/core/reclock.rs +++ b/timely/src/dataflow/operators/core/reclock.rs @@ -1,6 +1,6 @@ //! Extension methods for `Stream` based on record-by-record transformation. -use crate::{WithProgress, Data}; +use crate::Container; use crate::order::PartialOrder; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pact::Pipeline; @@ -45,11 +45,11 @@ pub trait Reclock { /// assert_eq!(extracted[1], (5, vec![4,5])); /// assert_eq!(extracted[2], (8, vec![6,7,8])); /// ``` - fn reclock(&self, clock: &StreamCore) -> Self; + fn reclock(&self, clock: &StreamCore) -> Self; } -impl Reclock for StreamCore { - fn reclock(&self, clock: &StreamCore) -> StreamCore { +impl Reclock for StreamCore { + fn reclock(&self, clock: &StreamCore) -> StreamCore { let mut stash = vec![]; diff --git a/timely/src/dataflow/operators/core/to_stream.rs b/timely/src/dataflow/operators/core/to_stream.rs index 983671c0d..4d2dea525 100644 --- a/timely/src/dataflow/operators/core/to_stream.rs +++ b/timely/src/dataflow/operators/core/to_stream.rs @@ -1,7 +1,7 @@ //! Conversion to the `StreamCore` type from iterators. use crate::container::{CapacityContainerBuilder, ContainerBuilder, SizableContainer, PushInto}; -use crate::{WithProgress, Data}; +use crate::Container; use crate::dataflow::operators::generic::operator::source; use crate::dataflow::{StreamCore, Scope}; @@ -61,7 +61,7 @@ impl ToStreamBuilder for I wh /// Converts to a timely [StreamCore]. Equivalent to [`ToStreamBuilder`] but /// uses a [`CapacityContainerBuilder`]. -pub trait ToStream { +pub trait ToStream { /// Converts to a timely [StreamCore]. /// /// # Examples @@ -81,7 +81,7 @@ pub trait ToStream { fn to_stream(self, scope: &mut S) -> StreamCore; } -impl ToStream for I where C: PushInto { +impl ToStream for I where C: PushInto { fn to_stream(self, scope: &mut S) -> StreamCore { ToStreamBuilder::>::to_stream_with_builder(self, scope) } diff --git a/timely/src/dataflow/operators/core/unordered_input.rs b/timely/src/dataflow/operators/core/unordered_input.rs index a0d1dacdd..703a461d3 100644 --- a/timely/src/dataflow/operators/core/unordered_input.rs +++ b/timely/src/dataflow/operators/core/unordered_input.rs @@ -2,7 +2,8 @@ use std::rc::Rc; use std::cell::RefCell; -use crate::{WithProgress, Data}; + +use crate::Container; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; use crate::scheduling::{Schedule, ActivateOnDrop}; @@ -164,7 +165,7 @@ impl UnorderedHandle { } } -impl UnorderedHandle> { +impl UnorderedHandle> { /// Allocates a new automatically flushing session based on the supplied capability. #[inline] pub fn session(&mut self, cap: ActivateCapability) -> ActivateOnDrop, Counter>>> { diff --git a/timely/src/dataflow/operators/generic/builder_raw.rs b/timely/src/dataflow/operators/generic/builder_raw.rs index 47bd0ba95..4659383a9 100644 --- a/timely/src/dataflow/operators/generic/builder_raw.rs +++ b/timely/src/dataflow/operators/generic/builder_raw.rs @@ -13,7 +13,7 @@ use crate::scheduling::{Schedule, Activations}; use crate::progress::{Source, Target}; use crate::progress::{Timestamp, Operate, operate::SharedProgress, Antichain}; use crate::progress::operate::{Connectivity, PortConnectivity}; -use crate::WithProgress; +use crate::Container; use crate::dataflow::{StreamCore, Scope}; use crate::dataflow::channels::pushers::Tee; use crate::dataflow::channels::pact::ParallelizationContract; @@ -104,7 +104,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> P::Puller + pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> P::Puller where P: ParallelizationContract { @@ -113,7 +113,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> P::Puller + pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> P::Puller where P: ParallelizationContract, I: IntoIterator::Summary>)>, @@ -133,14 +133,14 @@ impl OperatorBuilder { } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output(&mut self) -> (Tee, StreamCore) { + pub fn new_output(&mut self) -> (Tee, StreamCore) { let connection = (0 .. self.shape.inputs).map(|i| (i, Antichain::from_elem(Default::default()))); self.new_output_connection(connection) } /// Adds a new output to a generic operator builder, returning the `Push` implementor to use. - pub fn new_output_connection(&mut self, connection: I) -> (Tee, StreamCore) + pub fn new_output_connection(&mut self, connection: I) -> (Tee, StreamCore) where I: IntoIterator::Summary>)>, { diff --git a/timely/src/dataflow/operators/generic/builder_rc.rs b/timely/src/dataflow/operators/generic/builder_rc.rs index 9218f5725..fee122014 100644 --- a/timely/src/dataflow/operators/generic/builder_rc.rs +++ b/timely/src/dataflow/operators/generic/builder_rc.rs @@ -8,7 +8,7 @@ use crate::progress::{ChangeBatch, Timestamp}; use crate::progress::operate::SharedProgress; use crate::progress::frontier::{Antichain, MutableAntichain}; -use crate::WithProgress; +use crate::Container; use crate::container::ContainerBuilder; use crate::dataflow::{Scope, StreamCore}; use crate::dataflow::channels::pushers::Tee; @@ -56,7 +56,7 @@ impl OperatorBuilder { } /// Adds a new input to a generic operator builder, returning the `Pull` implementor to use. - pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> InputHandleCore + pub fn new_input(&mut self, stream: &StreamCore, pact: P) -> InputHandleCore where P: ParallelizationContract { @@ -72,7 +72,7 @@ impl OperatorBuilder { /// /// Commonly the connections are either the unit summary, indicating the same timestamp might be produced as output, or an empty /// antichain indicating that there is no connection from the input to the output. - pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> InputHandleCore + pub fn new_input_connection(&mut self, stream: &StreamCore, pact: P, connection: I) -> InputHandleCore where P: ParallelizationContract, I: IntoIterator::Summary>)> + Clone, diff --git a/timely/src/dataflow/operators/generic/handles.rs b/timely/src/dataflow/operators/generic/handles.rs index 81607f981..064b3d47a 100644 --- a/timely/src/dataflow/operators/generic/handles.rs +++ b/timely/src/dataflow/operators/generic/handles.rs @@ -15,14 +15,14 @@ use crate::dataflow::channels::pushers::Counter as PushCounter; use crate::dataflow::channels::pushers::buffer::{Buffer, Session}; use crate::dataflow::channels::Message; use crate::communication::{Push, Pull}; -use crate::{WithProgress, Data}; +use crate::{Container, WithProgress}; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; use crate::dataflow::operators::InputCapability; use crate::dataflow::operators::capability::CapabilityTrait; /// Handle to an operator's input stream. -pub struct InputHandleCore>> { +pub struct InputHandleCore>> { pull_counter: PullCounter, internal: Rc>>>>>, /// Timestamp summaries from this input to each output. @@ -36,7 +36,7 @@ pub struct InputHandleCore> pub type InputHandle = InputHandleCore, P>; /// Handle to an operator's input stream and frontier. -pub struct FrontieredInputHandleCore<'a, T: Timestamp, C: WithProgress +'a, P: Pull>+'a> { +pub struct FrontieredInputHandleCore<'a, T: Timestamp, C: 'a, P: Pull>+'a> { /// The underlying input handle. pub handle: &'a mut InputHandleCore, /// The frontier as reported by timely progress tracking. @@ -231,7 +231,7 @@ impl<'a, T: Timestamp, CB: ContainerBuilder, P: Push>> } } -impl<'a, T: Timestamp, C: WithProgress + Data, P: Push>> OutputHandleCore<'a, T, CapacityContainerBuilder, P> { +impl<'a, T: Timestamp, C: Container, P: Push>> OutputHandleCore<'a, T, CapacityContainerBuilder, P> { /// Obtains a session that can send data at the timestamp associated with capability `cap`. /// /// In order to send data at a future timestamp, obtain a capability for the new timestamp diff --git a/timely/src/dataflow/operators/generic/operator.rs b/timely/src/dataflow/operators/generic/operator.rs index 356c6000e..79f6076e4 100644 --- a/timely/src/dataflow/operators/generic/operator.rs +++ b/timely/src/dataflow/operators/generic/operator.rs @@ -12,11 +12,11 @@ use crate::dataflow::{Scope, StreamCore}; use super::builder_rc::OperatorBuilder; use crate::dataflow::operators::generic::OperatorInfo; use crate::dataflow::operators::generic::notificator::{Notificator, FrontierNotificator}; -use crate::{WithProgress, Data}; +use crate::Container; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; /// Methods to construct generic streaming and blocking operators. -pub trait Operator { +pub trait Operator { /// Creates a new dataflow operator that partitions its input stream by a parallelization /// strategy `pact`, and repeatedly invokes `logic`, the function returned by the function passed as `constructor`. /// `logic` can read from the input stream, write to the output stream, and inspect the frontier at the input. @@ -177,7 +177,7 @@ pub trait Operator { /// ``` fn binary_frontier(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: WithProgress + Data, + C2: Container, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, @@ -227,7 +227,7 @@ pub trait Operator { /// } /// }).unwrap(); /// ``` - fn binary_notify, &mut InputHandleCore, @@ -269,7 +269,7 @@ pub trait Operator { /// ``` fn binary(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: WithProgress + Data, + C2: Container, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, @@ -307,7 +307,7 @@ pub trait Operator { P: ParallelizationContract; } -impl Operator for StreamCore { +impl Operator for StreamCore { fn unary_frontier(&self, pact: P, name: &str, constructor: B) -> StreamCore where @@ -388,7 +388,7 @@ impl Operator for StreamCore { fn binary_frontier(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: WithProgress + Data, + C2: Container, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut FrontieredInputHandleCore, @@ -419,7 +419,7 @@ impl Operator for StreamCore { stream } - fn binary_notify, &mut InputHandleCore, @@ -447,7 +447,7 @@ impl Operator for StreamCore { fn binary(&self, other: &StreamCore, pact1: P1, pact2: P2, name: &str, constructor: B) -> StreamCore where - C2: WithProgress + Data, + C2: Container, CB: ContainerBuilder, B: FnOnce(Capability, OperatorInfo) -> L, L: FnMut(&mut InputHandleCore, @@ -580,7 +580,7 @@ where /// /// }); /// ``` -pub fn empty(scope: &G) -> StreamCore { +pub fn empty(scope: &G) -> StreamCore { source::<_, CapacityContainerBuilder, _, _>(scope, "Empty", |_capability, _info| |_output| { // drop capability, do nothing }) diff --git a/timely/src/dataflow/stream.rs b/timely/src/dataflow/stream.rs index 2d1dfb4a7..b5bb93a01 100644 --- a/timely/src/dataflow/stream.rs +++ b/timely/src/dataflow/stream.rs @@ -11,7 +11,6 @@ use crate::dataflow::Scope; use crate::dataflow::channels::pushers::tee::TeeHelper; use crate::dataflow::channels::Message; use std::fmt::{self, Debug}; -use crate::WithProgress; // use dataflow::scopes::root::loggers::CHANNELS_Q; @@ -47,7 +46,7 @@ impl Clone for StreamCore { /// A stream batching data in vectors. pub type Stream = StreamCore>; -impl StreamCore { +impl StreamCore { /// Connects the stream to a destination. /// /// The destination is described both by a `Target`, for progress tracking information, and a `P: Push` where the @@ -76,7 +75,7 @@ impl StreamCore { pub fn scope(&self) -> S { self.scope.clone() } /// Allows the assertion of a container type, for the benefit of type inference. - pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } + pub fn container(self) -> StreamCore where Self: AsStream { self.as_stream() } } /// A type that can be translated to a [StreamCore]. diff --git a/timely/src/lib.rs b/timely/src/lib.rs index 83447d09f..cd92e8a35 100644 --- a/timely/src/lib.rs +++ b/timely/src/lib.rs @@ -104,6 +104,12 @@ pub mod scheduling; pub trait Data: Clone+'static { } impl Data for T { } +/// A composite trait for types usable as containers in timely dataflow. +/// +/// The `Container` trait is necessary for all containers in timely dataflow channels. +pub trait Container: WithProgress + Default + Clone + 'static { } +impl Container for C { } + /// A composite trait for types usable on exchange channels in timely dataflow. /// /// The `ExchangeData` trait extends `Data` with any requirements imposed by the `timely_communication` diff --git a/timely/src/logging.rs b/timely/src/logging.rs index e9f918312..b812dd4d2 100644 --- a/timely/src/logging.rs +++ b/timely/src/logging.rs @@ -19,7 +19,7 @@ use std::time::Duration; use columnar::Columnar; use serde::{Deserialize, Serialize}; -use crate::WithProgress; +use crate::Container; use crate::container::CapacityContainerBuilder; use crate::dataflow::operators::capture::{Event, EventPusher}; use crate::progress::operate::Connectivity; @@ -31,7 +31,7 @@ pub struct BatchLogger where P: EventPusher { _phantom: ::std::marker::PhantomData, } -impl BatchLogger where P: EventPusher, C: WithProgress { +impl BatchLogger where P: EventPusher, C: Container { /// Creates a new batch logger. pub fn new(event_pusher: P) -> Self { BatchLogger { From a8b41d988d9a1958cb2a2d1be38464d5bf9890bd Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Sun, 17 Aug 2025 22:16:36 +0200 Subject: [PATCH 08/10] Relax InspectCore trait bounds Signed-off-by: Moritz Hoffmann --- timely/src/dataflow/operators/core/inspect.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timely/src/dataflow/operators/core/inspect.rs b/timely/src/dataflow/operators/core/inspect.rs index 1eea57488..6491b9d54 100644 --- a/timely/src/dataflow/operators/core/inspect.rs +++ b/timely/src/dataflow/operators/core/inspect.rs @@ -98,7 +98,7 @@ impl Inspect for StreamCore } /// Inspect containers -pub trait InspectCore { +pub trait InspectCore { /// Runs a supplied closure on each observed container, and each frontier advancement. /// /// Rust's `Result` type is used to distinguish the events, with `Ok` for time and data, @@ -121,7 +121,7 @@ pub trait InspectCore { fn inspect_container(&self, func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static; } -impl InspectCore for StreamCore { +impl InspectCore for StreamCore { fn inspect_container(&self, mut func: F) -> StreamCore where F: FnMut(Result<(&G::Timestamp, &C), &[G::Timestamp]>)+'static From e89e020432884658dbbe450d51a97f0fe3a42482 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Thu, 28 Aug 2025 17:46:55 +0200 Subject: [PATCH 09/10] Address my own feedback: * Properly document where containers are left in an undefined state. * Change update count to record count. * Add some missing documentation and remove outdated comments Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 41 +++++++++++-------- timely/examples/columnar.rs | 2 +- timely/src/dataflow/channels/mod.rs | 2 +- timely/src/dataflow/channels/pact.rs | 4 +- .../src/dataflow/channels/pullers/counter.rs | 6 +-- .../src/dataflow/channels/pushers/counter.rs | 2 +- .../operators/core/capture/extract.rs | 5 +-- .../src/dataflow/operators/core/enterleave.rs | 2 +- timely/src/dataflow/operators/core/input.rs | 2 +- timely/src/logging.rs | 2 +- 10 files changed, 37 insertions(+), 31 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index 7d4134183..b14d86f1f 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -12,25 +12,23 @@ use std::collections::VecDeque; /// /// It must implement default for historic reason. The default implementation is not required /// to allocate memory for variable-length components. -// TODO: Remove `Default` requirement in the future. -// The container is `Default` because `CapacityContainerBuilder` only implements `ContainerBuilder` -// for containers that implement `Default`, and we use the associated `::Container` all over Timely. -// We can only access the type if all requirements for the `ContainerBuilder` implementation are -// satisfied. pub trait WithProgress { - /// The number of updates + /// The number of records /// /// This number is used in progress tracking to confirm the receipt of some number - /// of outstanding updates, and it is highly load bearing. The main restriction is + /// of outstanding records, and it is highly load bearing. The main restriction is /// imposed on the `LengthPreservingContainerBuilder` trait, whose implementors - /// must preserve the number of items. - fn update_count(&self) -> i64; + /// must preserve the number of records. + fn record_count(&self) -> i64; /// Determine if this contains any updates, corresponding to `update_count() == 0`. - #[inline] fn is_empty(&self) -> bool { self.update_count() == 0 } + #[inline] fn is_empty(&self) -> bool { self.record_count() == 0 } } -/// TODO +/// A container that allows iteration. +/// +/// Iterating the container presents items in an implmentation-specific order. +/// The container's contents are not changed. pub trait IterContainer { /// The type of elements when reading non-destructively from the container. type ItemRef<'a> where Self: 'a; @@ -40,7 +38,11 @@ pub trait IterContainer { fn iter(&self) -> Self::Iter<'_>; } -/// TODO +/// A container that can drain itself. +/// +/// Draining the container presents items in an implementation-specific order. +/// The container is in an undefined state after calling [`drain`]. Dropping +/// the iterator also leaves the container in an undefined state. pub trait DrainContainer { /// The type of elements when draining the container. type Item<'a> where Self: 'a; @@ -60,6 +62,9 @@ pub trait SizableContainer: Sized { /// The `stash` argument is available, and may have the intended capacity. /// However, it may be non-empty, and may be of the wrong capacity. The /// method should guard against these cases. + /// + /// Assume that the `stash` is in an undefined state, and properly clear it + /// before re-using it. fn ensure_capacity(&mut self, stash: &mut Option); } @@ -85,7 +90,8 @@ pub trait PushInto { /// /// The caller should consume the containers returned by [`Self::extract`] and /// [`Self::finish`]. Implementations can recycle buffers, but should ensure that they clear -/// any remaining elements. +/// any remaining elements. It is up to the implementation of this trait to ensure that +/// containers are properly cleared before recycling them. /// /// For example, a consolidating builder can aggregate differences in-place, but it has /// to ensure that it preserves the intended information. @@ -108,6 +114,7 @@ pub trait ContainerBuilder: Default + 'static { #[must_use] fn finish(&mut self) -> Option<&mut Self::Container>; /// Partitions `container` among `builders`, using the function `index` to direct items. + /// Drains the container. The container is left in an undefined state. fn partition(container: &mut Self::Container, builders: &mut [Self], mut index: I) where Self::Container: DrainContainer, @@ -131,7 +138,7 @@ pub trait ContainerBuilder: Default + 'static { /// A wrapper trait indicating that the container building will preserve the number of records. /// -/// Specifically, the sum of lengths of all extracted and finished containers must equal the +/// Specifically, the sum of record counts of all extracted and finished containers must equal the /// number of times that `push_into` is called on the container builder. /// If you have any questions about this trait you are best off not implementing it. pub trait LengthPreservingContainerBuilder : ContainerBuilder { } @@ -194,7 +201,7 @@ impl ContainerBuilder for CapacityC impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } impl WithProgress for Vec { - #[inline] fn update_count(&self) -> i64 { i64::try_from(Vec::len(self)).unwrap() } + #[inline] fn record_count(&self) -> i64 { i64::try_from(Vec::len(self)).unwrap() } #[inline] fn is_empty(&self) -> bool { Vec::is_empty(self) } } @@ -259,7 +266,7 @@ mod rc { use crate::{WithProgress, IterContainer, DrainContainer}; impl WithProgress for Rc { - #[inline] fn update_count(&self) -> i64 { std::ops::Deref::deref(self).update_count() } + #[inline] fn record_count(&self) -> i64 { std::ops::Deref::deref(self).record_count() } #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } impl IterContainer for Rc { @@ -281,7 +288,7 @@ mod arc { use crate::{WithProgress, IterContainer, DrainContainer}; impl WithProgress for Arc { - #[inline] fn update_count(&self) -> i64 { std::ops::Deref::deref(self).update_count() } + #[inline] fn record_count(&self) -> i64 { std::ops::Deref::deref(self).record_count() } #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } impl IterContainer for Arc { diff --git a/timely/examples/columnar.rs b/timely/examples/columnar.rs index 2b2b68e55..bf3daca00 100644 --- a/timely/examples/columnar.rs +++ b/timely/examples/columnar.rs @@ -177,7 +177,7 @@ mod container { } impl timely::WithProgress for Column { - #[inline] fn update_count(&self) -> i64 { i64::try_from(self.borrow().len()).unwrap() } + #[inline] fn record_count(&self) -> i64 { i64::try_from(self.borrow().len()).unwrap() } #[inline] fn is_empty(&self) -> bool { self.borrow().is_empty() } } impl timely::container::IterContainer for Column { diff --git a/timely/src/dataflow/channels/mod.rs b/timely/src/dataflow/channels/mod.rs index 405a31b0e..36fb44bb1 100644 --- a/timely/src/dataflow/channels/mod.rs +++ b/timely/src/dataflow/channels/mod.rs @@ -39,7 +39,7 @@ impl Message { } /// Forms a message, and pushes contents at `pusher`. Replaces `buffer` with what the pusher - /// leaves in place, or the container's default element. The buffer is cleared. + /// leaves in place, or the container's default element. The buffer is left in an undefined state. #[inline] pub fn push_at>>(buffer: &mut C, time: T, pusher: &mut P) { diff --git a/timely/src/dataflow/channels/pact.rs b/timely/src/dataflow/channels/pact.rs index f03fdb639..85e8b9781 100644 --- a/timely/src/dataflow/channels/pact.rs +++ b/timely/src/dataflow/channels/pact.rs @@ -147,7 +147,7 @@ impl>> Push> for LogPush source: self.source, target: self.target, seq_no: self.counter - 1, - update_count: bundle.data.update_count(), + record_count: bundle.data.record_count(), }) } } @@ -192,7 +192,7 @@ impl>> Pull> for LogPull source: bundle.from, target, seq_no: bundle.seq, - update_count: bundle.data.update_count(), + record_count: bundle.data.record_count(), }); } } diff --git a/timely/src/dataflow/channels/pullers/counter.rs b/timely/src/dataflow/channels/pullers/counter.rs index 20b6d0da7..044bc1fd5 100644 --- a/timely/src/dataflow/channels/pullers/counter.rs +++ b/timely/src/dataflow/channels/pullers/counter.rs @@ -19,7 +19,7 @@ pub struct Counter { pub struct ConsumedGuard { consumed: Rc>>, time: Option, - update_count: i64, + record_count: i64, } impl ConsumedGuard { @@ -32,7 +32,7 @@ impl Drop for ConsumedGuard { fn drop(&mut self) { // SAFETY: we're in a Drop impl, so this runs at most once let time = self.time.take().unwrap(); - self.consumed.borrow_mut().update(time, self.update_count); + self.consumed.borrow_mut().update(time, self.record_count); } } @@ -49,7 +49,7 @@ impl>> Counter Push> for Counter #[inline] fn push(&mut self, message: &mut Option>) { if let Some(message) = message { - self.produced.borrow_mut().update(message.time.clone(), message.data.update_count()); + self.produced.borrow_mut().update(message.time.clone(), message.data.record_count()); } // only propagate `None` if dirty (indicates flush) diff --git a/timely/src/dataflow/operators/core/capture/extract.rs b/timely/src/dataflow/operators/core/capture/extract.rs index 318f3957b..c0af57d37 100644 --- a/timely/src/dataflow/operators/core/capture/extract.rs +++ b/timely/src/dataflow/operators/core/capture/extract.rs @@ -49,10 +49,9 @@ pub trait Extract { fn extract(self) -> Vec<(T, C)>; } -impl Extract for ::std::sync::mpsc::Receiver> +impl Extract for ::std::sync::mpsc::Receiver> where - for<'a> C: Container + SizableContainer + DrainContainer: Ord> + PushInto>, - T: Ord, + for<'a> C: Container + DrainContainer: Ord> + PushInto>, { fn extract(self) -> Vec<(T, C)> { let mut staged = std::collections::BTreeMap::new(); diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index acff0fa6a..4778dd3a2 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -218,7 +218,7 @@ where source: self.index, target: self.index, seq_no: self.counter, - update_count: bundle.data.update_count(), + record_count: bundle.data.record_count(), }; let recv_event = MessagesEvent { is_send: false, diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index 9c26f1ae4..c5e7e2274 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -374,7 +374,7 @@ impl Handle { /// Sends a container at each of the destinations. There can be more than one; clone if needed. /// Does not take `self` because `flush` and `extract` borrow `self` mutably. - /// Clears the container. + /// Leaves the container in an undefined state. // TODO: Find a better name for this function. #[inline] fn send_container( diff --git a/timely/src/logging.rs b/timely/src/logging.rs index b812dd4d2..9122f6da4 100644 --- a/timely/src/logging.rs +++ b/timely/src/logging.rs @@ -135,7 +135,7 @@ pub struct MessagesEvent { /// Message sequence number. pub seq_no: usize, /// Number of typed records in the message. - pub update_count: i64, + pub record_count: i64, } /// Records the starting and stopping of an operator. From 153f3d8ad21765ae751005ce524554c51cd593a2 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Thu, 28 Aug 2025 20:47:17 +0200 Subject: [PATCH 10/10] Address feedback Signed-off-by: Moritz Hoffmann --- container/src/lib.rs | 40 +++++++++---------- timely/examples/columnar.rs | 2 +- timely/src/dataflow/channels/pact.rs | 8 ++-- .../src/dataflow/channels/pullers/counter.rs | 4 +- .../src/dataflow/channels/pushers/buffer.rs | 2 +- .../src/dataflow/channels/pushers/counter.rs | 4 +- .../src/dataflow/operators/core/enterleave.rs | 4 +- timely/src/dataflow/operators/core/input.rs | 2 +- .../src/dataflow/operators/generic/handles.rs | 10 ++--- timely/src/lib.rs | 6 +-- 10 files changed, 41 insertions(+), 41 deletions(-) diff --git a/container/src/lib.rs b/container/src/lib.rs index b14d86f1f..94d1cb421 100644 --- a/container/src/lib.rs +++ b/container/src/lib.rs @@ -4,15 +4,12 @@ use std::collections::VecDeque; -/// An object with effects on progress +/// An type containing a number of records accounted for by progress tracking. /// /// The object stores a number of updates and thus is able to describe it count /// (`update_count()`) and whether it is empty (`is_empty()`). It is empty if the /// update count is zero. -/// -/// It must implement default for historic reason. The default implementation is not required -/// to allocate memory for variable-length components. -pub trait WithProgress { +pub trait Accountable { /// The number of records /// /// This number is used in progress tracking to confirm the receipt of some number @@ -22,12 +19,13 @@ pub trait WithProgress { fn record_count(&self) -> i64; /// Determine if this contains any updates, corresponding to `update_count() == 0`. + /// It is a correctness error for this to by anything other than `self.record_count() == 0`. #[inline] fn is_empty(&self) -> bool { self.record_count() == 0 } } -/// A container that allows iteration. +/// A container that allows iteration morally equivalent to [`IntoIterator`]. /// -/// Iterating the container presents items in an implmentation-specific order. +/// Iterating the container presents items in an implementation-specific order. /// The container's contents are not changed. pub trait IterContainer { /// The type of elements when reading non-destructively from the container. @@ -54,7 +52,7 @@ pub trait DrainContainer { } /// A container that can be sized and reveals its capacity. -pub trait SizableContainer: Sized { +pub trait SizableContainer { /// Indicates that the container is "full" and should be shipped. fn at_capacity(&self) -> bool; /// Restores `self` to its desired capacity, if it has one. @@ -65,7 +63,7 @@ pub trait SizableContainer: Sized { /// /// Assume that the `stash` is in an undefined state, and properly clear it /// before re-using it. - fn ensure_capacity(&mut self, stash: &mut Option); + fn ensure_capacity(&mut self, stash: &mut Option) where Self: Sized; } /// A container that can absorb items of a specific type. @@ -90,8 +88,10 @@ pub trait PushInto { /// /// The caller should consume the containers returned by [`Self::extract`] and /// [`Self::finish`]. Implementations can recycle buffers, but should ensure that they clear -/// any remaining elements. It is up to the implementation of this trait to ensure that -/// containers are properly cleared before recycling them. +/// any remaining elements. +/// +/// Implementations are allowed to re-use the contents of the mutable references left by the caller, +/// but they should ensure that they clear the contents before doing so. /// /// For example, a consolidating builder can aggregate differences in-place, but it has /// to ensure that it preserves the intended information. @@ -102,7 +102,7 @@ pub trait ContainerBuilder: Default + 'static { /// The container type we're building. // The container is `Clone` because `Tee` requires it, otherwise we need to repeat it // all over Timely. `'static` because we don't want lifetimes everywhere. - type Container: WithProgress + Default + Clone + 'static; + type Container: Accountable + Default + Clone + 'static; /// Extract assembled containers, potentially leaving unfinished data behind. Can /// be called repeatedly, for example while the caller can send data. /// @@ -139,7 +139,7 @@ pub trait ContainerBuilder: Default + 'static { /// A wrapper trait indicating that the container building will preserve the number of records. /// /// Specifically, the sum of record counts of all extracted and finished containers must equal the -/// number of times that `push_into` is called on the container builder. +/// number of accounted records that are pushed into the container builder. /// If you have any questions about this trait you are best off not implementing it. pub trait LengthPreservingContainerBuilder : ContainerBuilder { } @@ -175,7 +175,7 @@ impl> PushInto for CapacityCon } } -impl ContainerBuilder for CapacityContainerBuilder { +impl ContainerBuilder for CapacityContainerBuilder { type Container = C; #[inline] @@ -198,9 +198,9 @@ impl ContainerBuilder for CapacityC } } -impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } +impl LengthPreservingContainerBuilder for CapacityContainerBuilder { } -impl WithProgress for Vec { +impl Accountable for Vec { #[inline] fn record_count(&self) -> i64 { i64::try_from(Vec::len(self)).unwrap() } #[inline] fn is_empty(&self) -> bool { Vec::is_empty(self) } } @@ -263,9 +263,9 @@ mod rc { use std::ops::Deref; use std::rc::Rc; - use crate::{WithProgress, IterContainer, DrainContainer}; + use crate::{IterContainer, DrainContainer}; - impl WithProgress for Rc { + impl crate::Accountable for Rc { #[inline] fn record_count(&self) -> i64 { std::ops::Deref::deref(self).record_count() } #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } @@ -285,9 +285,9 @@ mod arc { use std::ops::Deref; use std::sync::Arc; - use crate::{WithProgress, IterContainer, DrainContainer}; + use crate::{IterContainer, DrainContainer}; - impl WithProgress for Arc { + impl crate::Accountable for Arc { #[inline] fn record_count(&self) -> i64 { std::ops::Deref::deref(self).record_count() } #[inline] fn is_empty(&self) -> bool { std::ops::Deref::deref(self).is_empty() } } diff --git a/timely/examples/columnar.rs b/timely/examples/columnar.rs index bf3daca00..d858aac79 100644 --- a/timely/examples/columnar.rs +++ b/timely/examples/columnar.rs @@ -176,7 +176,7 @@ mod container { } } - impl timely::WithProgress for Column { + impl timely::Accountable for Column { #[inline] fn record_count(&self) -> i64 { i64::try_from(self.borrow().len()).unwrap() } #[inline] fn is_empty(&self) -> bool { self.borrow().is_empty() } } diff --git a/timely/src/dataflow/channels/pact.rs b/timely/src/dataflow/channels/pact.rs index 85e8b9781..743df855c 100644 --- a/timely/src/dataflow/channels/pact.rs +++ b/timely/src/dataflow/channels/pact.rs @@ -10,7 +10,7 @@ use std::{fmt::{self, Debug}, marker::PhantomData}; use std::rc::Rc; -use crate::WithProgress; +use crate::Accountable; use crate::container::{ContainerBuilder, DrainContainer, LengthPreservingContainerBuilder, SizableContainer, CapacityContainerBuilder, PushInto}; use crate::communication::allocator::thread::{ThreadPusher, ThreadPuller}; use crate::communication::{Push, Pull}; @@ -34,7 +34,7 @@ pub trait ParallelizationContract { #[derive(Debug)] pub struct Pipeline; -impl ParallelizationContract for Pipeline { +impl ParallelizationContract for Pipeline { type Pusher = LogPusher>>; type Puller = LogPuller>>; fn connect(self, allocator: &mut A, identifier: usize, address: Rc<[usize]>, logging: Option) -> (Self::Pusher, Self::Puller) { @@ -129,7 +129,7 @@ impl

LogPusher

{ } } -impl>> Push> for LogPusher

{ +impl>> Push> for LogPusher

{ #[inline] fn push(&mut self, pair: &mut Option>) { if let Some(bundle) = pair { @@ -177,7 +177,7 @@ impl

LogPuller

{ } } -impl>> Pull> for LogPuller

{ +impl>> Pull> for LogPuller

{ #[inline] fn pull(&mut self) -> &mut Option> { let result = self.puller.pull(); diff --git a/timely/src/dataflow/channels/pullers/counter.rs b/timely/src/dataflow/channels/pullers/counter.rs index 044bc1fd5..5903b2e4c 100644 --- a/timely/src/dataflow/channels/pullers/counter.rs +++ b/timely/src/dataflow/channels/pullers/counter.rs @@ -6,7 +6,7 @@ use std::cell::RefCell; use crate::dataflow::channels::Message; use crate::progress::ChangeBatch; use crate::communication::Pull; -use crate::WithProgress; +use crate::Accountable; /// A wrapper which accounts records pulled past in a shared count map. pub struct Counter { @@ -36,7 +36,7 @@ impl Drop for ConsumedGuard { } } -impl>> Counter { +impl>> Counter { /// Retrieves the next timestamp and batch of data. #[inline] pub fn next(&mut self) -> Option<&mut Message> { diff --git a/timely/src/dataflow/channels/pushers/buffer.rs b/timely/src/dataflow/channels/pushers/buffer.rs index 23efd934e..c99d76060 100644 --- a/timely/src/dataflow/channels/pushers/buffer.rs +++ b/timely/src/dataflow/channels/pushers/buffer.rs @@ -6,7 +6,7 @@ use crate::container::{ContainerBuilder, CapacityContainerBuilder, PushInto}; use crate::dataflow::channels::Message; use crate::dataflow::operators::Capability; use crate::progress::Timestamp; -use crate::{Container, WithProgress}; +use crate::{Container, Accountable}; /// Buffers data sent at the same time, for efficient communication. /// diff --git a/timely/src/dataflow/channels/pushers/counter.rs b/timely/src/dataflow/channels/pushers/counter.rs index ce2bdc7ac..67d39318d 100644 --- a/timely/src/dataflow/channels/pushers/counter.rs +++ b/timely/src/dataflow/channels/pushers/counter.rs @@ -7,7 +7,7 @@ use std::cell::RefCell; use crate::progress::{ChangeBatch, Timestamp}; use crate::dataflow::channels::Message; use crate::communication::Push; -use crate::WithProgress; +use crate::Accountable; /// A wrapper which updates shared `produced` based on the number of records pushed. #[derive(Debug)] @@ -17,7 +17,7 @@ pub struct Counter>> { phantom: PhantomData, } -impl Push> for Counter where P: Push> { +impl Push> for Counter where P: Push> { #[inline] fn push(&mut self, message: &mut Option>) { if let Some(message) = message { diff --git a/timely/src/dataflow/operators/core/enterleave.rs b/timely/src/dataflow/operators/core/enterleave.rs index 4778dd3a2..c0889498e 100644 --- a/timely/src/dataflow/operators/core/enterleave.rs +++ b/timely/src/dataflow/operators/core/enterleave.rs @@ -26,7 +26,7 @@ use crate::logging::{TimelyLogger, MessagesEvent}; use crate::progress::Timestamp; use crate::progress::timestamp::Refines; use crate::progress::{Source, Target}; -use crate::{WithProgress, Container}; +use crate::{Accountable, Container}; use crate::communication::Push; use crate::dataflow::channels::pushers::{Counter, Tee}; use crate::dataflow::channels::Message; @@ -207,7 +207,7 @@ impl

LogPusher

{ impl Push> for LogPusher

where - C: WithProgress, + C: Accountable, P: Push>, { fn push(&mut self, element: &mut Option>) { diff --git a/timely/src/dataflow/operators/core/input.rs b/timely/src/dataflow/operators/core/input.rs index c5e7e2274..8db72755e 100644 --- a/timely/src/dataflow/operators/core/input.rs +++ b/timely/src/dataflow/operators/core/input.rs @@ -10,7 +10,7 @@ use crate::scheduling::{Schedule, Activator}; use crate::progress::{Operate, operate::SharedProgress, Timestamp, ChangeBatch}; use crate::progress::Source; use crate::progress::operate::Connectivity; -use crate::{WithProgress, Container}; +use crate::{Accountable, Container}; use crate::communication::Push; use crate::dataflow::{Scope, ScopeParent, StreamCore}; use crate::dataflow::channels::pushers::{Tee, Counter}; diff --git a/timely/src/dataflow/operators/generic/handles.rs b/timely/src/dataflow/operators/generic/handles.rs index 064b3d47a..a10ff2f40 100644 --- a/timely/src/dataflow/operators/generic/handles.rs +++ b/timely/src/dataflow/operators/generic/handles.rs @@ -15,7 +15,7 @@ use crate::dataflow::channels::pushers::Counter as PushCounter; use crate::dataflow::channels::pushers::buffer::{Buffer, Session}; use crate::dataflow::channels::Message; use crate::communication::{Push, Pull}; -use crate::{Container, WithProgress}; +use crate::{Container, Accountable}; use crate::container::{ContainerBuilder, CapacityContainerBuilder}; use crate::dataflow::operators::InputCapability; @@ -46,7 +46,7 @@ pub struct FrontieredInputHandleCore<'a, T: Timestamp, C: 'a, P: Pull = FrontieredInputHandleCore<'a, T, Vec, P>; -impl>> InputHandleCore { +impl>> InputHandleCore { /// Reads the next input buffer (at some timestamp `t`) and a corresponding capability for `t`. /// The timestamp `t` of the input buffer can be retrieved by invoking `.time()` on the capability. @@ -87,7 +87,7 @@ impl>> InputHandleCore>+'a> FrontieredInputHandleCore<'a, T, C, P> { +impl<'a, T: Timestamp, C: Accountable, P: Pull>+'a> FrontieredInputHandleCore<'a, T, C, P> { /// Allocate a new frontiered input handle. pub fn new(handle: &'a mut InputHandleCore, frontier: &'a MutableAntichain) -> Self { FrontieredInputHandleCore { @@ -134,13 +134,13 @@ impl<'a, T: Timestamp, C: WithProgress, P: Pull>+'a> FrontieredInp } } -pub fn _access_pull_counter>>(input: &mut InputHandleCore) -> &mut PullCounter { +pub fn _access_pull_counter>>(input: &mut InputHandleCore) -> &mut PullCounter { &mut input.pull_counter } /// Constructs an input handle. /// Declared separately so that it can be kept private when `InputHandle` is re-exported. -pub fn new_input_handle>>( +pub fn new_input_handle>>( pull_counter: PullCounter, internal: Rc>>>>>, summaries: Rc>>, diff --git a/timely/src/lib.rs b/timely/src/lib.rs index cd92e8a35..20f3f995e 100644 --- a/timely/src/lib.rs +++ b/timely/src/lib.rs @@ -65,7 +65,7 @@ pub use timely_communication::Config as CommunicationConfig; pub use worker::Config as WorkerConfig; pub use execute::Config as Config; -pub use timely_container::WithProgress; +pub use timely_container::Accountable; /// Re-export of the `timely_container` crate. pub mod container { pub use timely_container::*; @@ -107,8 +107,8 @@ impl Data for T { } /// A composite trait for types usable as containers in timely dataflow. /// /// The `Container` trait is necessary for all containers in timely dataflow channels. -pub trait Container: WithProgress + Default + Clone + 'static { } -impl Container for C { } +pub trait Container: Accountable + Default + Clone + 'static { } +impl Container for C { } /// A composite trait for types usable on exchange channels in timely dataflow. ///