Skip to content

Commit 304ed9d

Browse files
antigurufrankmcsherry
antiguru
and
frankmcsherry
authored
Let Layout specify OffsetContainer (#449)
* Let Layout specify OffsetContainer Signed-off-by: Moritz Hoffmann <[email protected]> * Generalize ReadItem type --------- Signed-off-by: Moritz Hoffmann <[email protected]> Co-authored-by: Frank McSherry <[email protected]>
1 parent 30ac63d commit 304ed9d

File tree

3 files changed

+130
-34
lines changed

3 files changed

+130
-34
lines changed

‎src/trace/implementations/mod.rs

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ pub use self::ord_neu::OrdValSpine as ValSpine;
5454
pub use self::ord_neu::OrdKeySpine as KeySpine;
5555

5656
use std::borrow::{ToOwned};
57+
use std::cmp::Ordering;
5758

5859
use timely::container::columnation::{Columnation, TimelyStack};
5960
use crate::lattice::Lattice;
@@ -97,6 +98,8 @@ pub trait Layout {
9798
/// Container for update vals.
9899
type UpdContainer:
99100
for<'a> BatchContainer<PushItem=(<Self::Target as Update>::Time, <Self::Target as Update>::Diff), ReadItem<'a> = &'a (<Self::Target as Update>::Time, <Self::Target as Update>::Diff)>;
101+
/// Container for offsets.
102+
type OffsetContainer: BatchContainer<PushItem=usize>;
100103
}
101104

102105
/// A layout that uses vectors
@@ -113,6 +116,7 @@ where
113116
type KeyContainer = Vec<U::Key>;
114117
type ValContainer = Vec<U::Val>;
115118
type UpdContainer = Vec<(U::Time, U::Diff)>;
119+
type OffsetContainer = OffsetList;
116120
}
117121

118122
/// A layout based on timely stacks
@@ -131,6 +135,7 @@ where
131135
type KeyContainer = TimelyStack<U::Key>;
132136
type ValContainer = TimelyStack<U::Val>;
133137
type UpdContainer = TimelyStack<(U::Time, U::Diff)>;
138+
type OffsetContainer = OffsetList;
134139
}
135140

136141
/// A type with a preferred container.
@@ -183,10 +188,13 @@ where
183188
type KeyContainer = K::Container;
184189
type ValContainer = V::Container;
185190
type UpdContainer = Vec<(T, D)>;
191+
type OffsetContainer = OffsetList;
186192
}
187193

188194
use std::convert::TryInto;
195+
use std::ops::Deref;
189196
use abomonation_derive::Abomonation;
197+
use crate::trace::cursor::MyTrait;
190198

191199
/// A list of unsigned integers that uses `u32` elements as long as they are small enough, and switches to `u64` once they are not.
192200
#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Debug, Abomonation)]
@@ -234,6 +242,87 @@ impl OffsetList {
234242
}
235243
}
236244

245+
/// Helper struct to provide `MyTrait` for `Copy` types.
246+
#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Copy)]
247+
pub struct Wrapper<T: Copy>(T);
248+
249+
impl<T: Copy> Deref for Wrapper<T> {
250+
type Target = T;
251+
252+
fn deref(&self) -> &Self::Target {
253+
&self.0
254+
}
255+
}
256+
257+
impl<'a, T: Copy + Ord> MyTrait<'a> for Wrapper<T> {
258+
type Owned = T;
259+
260+
fn into_owned(self) -> Self::Owned {
261+
self.0
262+
}
263+
264+
fn clone_onto(&self, other: &mut Self::Owned) {
265+
*other = self.0;
266+
}
267+
268+
fn compare(&self, other: &Self::Owned) -> Ordering {
269+
self.0.cmp(other)
270+
}
271+
272+
fn borrow_as(other: &'a Self::Owned) -> Self {
273+
Self(*other)
274+
}
275+
}
276+
277+
impl BatchContainer for OffsetList {
278+
type PushItem = usize;
279+
type ReadItem<'a> = Wrapper<usize>;
280+
281+
fn push(&mut self, item: Self::PushItem) {
282+
self.push(item);
283+
}
284+
285+
fn copy_push(&mut self, item: &Self::PushItem) {
286+
self.push(*item);
287+
}
288+
289+
fn copy(&mut self, item: Self::ReadItem<'_>) {
290+
self.push(item.0);
291+
}
292+
293+
fn copy_slice(&mut self, slice: &[Self::PushItem]) {
294+
for index in slice {
295+
self.push(*index);
296+
}
297+
}
298+
299+
fn copy_range(&mut self, other: &Self, start: usize, end: usize) {
300+
for offset in start..end {
301+
self.push(other.index(offset));
302+
}
303+
}
304+
305+
fn with_capacity(size: usize) -> Self {
306+
Self::with_capacity(size)
307+
}
308+
309+
fn reserve(&mut self, _additional: usize) {
310+
// Nop
311+
}
312+
313+
fn merge_capacity(cont1: &Self, cont2: &Self) -> Self {
314+
Self::with_capacity(cont1.len() + cont2.len())
315+
}
316+
317+
fn index(&self, index: usize) -> Self::ReadItem<'_> {
318+
Wrapper(self.index(index))
319+
}
320+
321+
fn len(&self) -> usize {
322+
self.len()
323+
}
324+
}
325+
237326
pub use self::containers::{BatchContainer, SliceContainer, SliceContainer2};
238327

239328
/// Containers for data that resemble `Vec<T>`, with leaner implementations.
@@ -245,7 +334,7 @@ pub mod containers {
245334
use crate::trace::MyTrait;
246335

247336
/// A general-purpose container resembling `Vec<T>`.
248-
pub trait BatchContainer: Default + 'static {
337+
pub trait BatchContainer: 'static {
249338
/// The type of contained item.
250339
///
251340
/// The container only supplies references to the item, so it needn't be sized.

‎src/trace/implementations/ord_neu.rs

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ mod val_batch {
7070
use timely::progress::{Antichain, frontier::AntichainRef};
7171

7272
use crate::trace::{Batch, BatchReader, Builder, Cursor, Description, Merger};
73-
use crate::trace::implementations::{BatchContainer, OffsetList};
73+
use crate::trace::implementations::BatchContainer;
7474
use crate::trace::cursor::MyTrait;
7575

7676
use super::{Layout, Update};
@@ -83,7 +83,7 @@ mod val_batch {
8383
/// Offsets used to provide indexes from keys to values.
8484
///
8585
/// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
86-
pub keys_offs: OffsetList,
86+
pub keys_offs: L::OffsetContainer,
8787
/// Concatenated ordered lists of values, bracketed by offsets in `keys_offs`.
8888
pub vals: L::ValContainer,
8989
/// Offsets used to provide indexes from values to updates.
@@ -94,20 +94,20 @@ mod val_batch {
9494
/// single common update values (e.g. in a snapshot, the minimal time and a diff of one).
9595
///
9696
/// The length of this list is one longer than `vals`, so that we can avoid bounds logic.
97-
pub vals_offs: OffsetList,
97+
pub vals_offs: L::OffsetContainer,
9898
/// Concatenated ordered lists of updates, bracketed by offsets in `vals_offs`.
9999
pub updates: L::UpdContainer,
100100
}
101101

102102
impl<L: Layout> OrdValStorage<L> {
103103
/// Lower and upper bounds in `self.vals` corresponding to the key at `index`.
104104
fn values_for_key(&self, index: usize) -> (usize, usize) {
105-
(self.keys_offs.index(index), self.keys_offs.index(index+1))
105+
(self.keys_offs.index(index).into_owned(), self.keys_offs.index(index+1).into_owned())
106106
}
107107
/// Lower and upper bounds in `self.updates` corresponding to the value at `index`.
108108
fn updates_for_value(&self, index: usize) -> (usize, usize) {
109-
let mut lower = self.vals_offs.index(index);
110-
let upper = self.vals_offs.index(index+1);
109+
let mut lower = self.vals_offs.index(index).into_owned();
110+
let upper = self.vals_offs.index(index+1).into_owned();
111111
// We use equal lower and upper to encode "singleton update; just before here".
112112
// It should only apply when there is a prior element, so `lower` should be greater than zero.
113113
if lower == upper {
@@ -206,14 +206,17 @@ mod val_batch {
206206

207207
let mut storage = OrdValStorage {
208208
keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
209-
keys_offs: OffsetList::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
209+
keys_offs: L::OffsetContainer::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
210210
vals: L::ValContainer::merge_capacity(&batch1.vals, &batch2.vals),
211-
vals_offs: OffsetList::with_capacity(batch1.vals_offs.len() + batch2.vals_offs.len()),
211+
vals_offs: L::OffsetContainer::with_capacity(batch1.vals_offs.len() + batch2.vals_offs.len()),
212212
updates: L::UpdContainer::merge_capacity(&batch1.updates, &batch2.updates),
213213
};
214214

215-
storage.keys_offs.push(0);
216-
storage.vals_offs.push(0);
215+
// Mark explicit types because type inference fails to resolve it.
216+
let keys_offs: &mut L::OffsetContainer = &mut storage.keys_offs;
217+
keys_offs.push(0);
218+
let vals_offs: &mut L::OffsetContainer = &mut storage.vals_offs;
219+
vals_offs.push(0);
217220

218221
OrdValMerger {
219222
key_cursor1: 0,
@@ -546,9 +549,9 @@ mod val_batch {
546549
Self {
547550
result: OrdValStorage {
548551
keys: L::KeyContainer::with_capacity(keys),
549-
keys_offs: OffsetList::with_capacity(keys + 1),
552+
keys_offs: L::OffsetContainer::with_capacity(keys + 1),
550553
vals: L::ValContainer::with_capacity(vals),
551-
vals_offs: OffsetList::with_capacity(vals + 1),
554+
vals_offs: L::OffsetContainer::with_capacity(vals + 1),
552555
updates: L::UpdContainer::with_capacity(upds),
553556
},
554557
singleton: None,
@@ -636,7 +639,7 @@ mod key_batch {
636639
use timely::progress::{Antichain, frontier::AntichainRef};
637640

638641
use crate::trace::{Batch, BatchReader, Builder, Cursor, Description, Merger};
639-
use crate::trace::implementations::{BatchContainer, OffsetList};
642+
use crate::trace::implementations::BatchContainer;
640643
use crate::trace::cursor::MyTrait;
641644

642645
use super::{Layout, Update};
@@ -654,16 +657,16 @@ mod key_batch {
654657
/// single common update values (e.g. in a snapshot, the minimal time and a diff of one).
655658
///
656659
/// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
657-
pub keys_offs: OffsetList,
660+
pub keys_offs: L::OffsetContainer,
658661
/// Concatenated ordered lists of updates, bracketed by offsets in `vals_offs`.
659662
pub updates: L::UpdContainer,
660663
}
661664

662665
impl<L: Layout> OrdKeyStorage<L> {
663666
/// Lower and upper bounds in `self.vals` corresponding to the key at `index`.
664667
fn updates_for_key(&self, index: usize) -> (usize, usize) {
665-
let mut lower = self.keys_offs.index(index);
666-
let upper = self.keys_offs.index(index+1);
668+
let mut lower = self.keys_offs.index(index).into_owned();
669+
let upper = self.keys_offs.index(index+1).into_owned();
667670
// We use equal lower and upper to encode "singleton update; just before here".
668671
// It should only apply when there is a prior element, so `lower` should be greater than zero.
669672
if lower == upper {
@@ -763,11 +766,12 @@ mod key_batch {
763766

764767
let mut storage = OrdKeyStorage {
765768
keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
766-
keys_offs: OffsetList::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
769+
keys_offs: L::OffsetContainer::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
767770
updates: L::UpdContainer::merge_capacity(&batch1.updates, &batch2.updates),
768771
};
769772

770-
storage.keys_offs.push(0);
773+
let keys_offs: &mut L::OffsetContainer = &mut storage.keys_offs;
774+
keys_offs.push(0);
771775

772776
OrdKeyMerger {
773777
key_cursor1: 0,
@@ -1011,7 +1015,7 @@ mod key_batch {
10111015
Self {
10121016
result: OrdKeyStorage {
10131017
keys: L::KeyContainer::with_capacity(keys),
1014-
keys_offs: OffsetList::with_capacity(keys + 1),
1018+
keys_offs: L::OffsetContainer::with_capacity(keys + 1),
10151019
updates: L::UpdContainer::with_capacity(upds),
10161020
},
10171021
singleton: None,

‎src/trace/implementations/rhh.rs

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ mod val_batch {
8484
use crate::hashable::Hashable;
8585

8686
use crate::trace::{Batch, BatchReader, Builder, Cursor, Description, Merger};
87-
use crate::trace::implementations::{BatchContainer, OffsetList};
87+
use crate::trace::implementations::BatchContainer;
8888
use crate::trace::cursor::MyTrait;
8989

9090
use super::{Layout, Update, HashOrdered};
@@ -122,7 +122,7 @@ mod val_batch {
122122
/// Offsets used to provide indexes from keys to values.
123123
///
124124
/// The length of this list is one longer than `keys`, so that we can avoid bounds logic.
125-
pub keys_offs: OffsetList,
125+
pub keys_offs: L::OffsetContainer,
126126
/// Concatenated ordered lists of values, bracketed by offsets in `keys_offs`.
127127
pub vals: L::ValContainer,
128128
/// Offsets used to provide indexes from values to updates.
@@ -133,7 +133,7 @@ mod val_batch {
133133
/// single common update values (e.g. in a snapshot, the minimal time and a diff of one).
134134
///
135135
/// The length of this list is one longer than `vals`, so that we can avoid bounds logic.
136-
pub vals_offs: OffsetList,
136+
pub vals_offs: L::OffsetContainer,
137137
/// Concatenated ordered lists of updates, bracketed by offsets in `vals_offs`.
138138
pub updates: L::UpdContainer,
139139
}
@@ -144,16 +144,16 @@ mod val_batch {
144144
{
145145
/// Lower and upper bounds in `self.vals` corresponding to the key at `index`.
146146
fn values_for_key(&self, index: usize) -> (usize, usize) {
147-
let lower = self.keys_offs.index(index);
148-
let upper = self.keys_offs.index(index+1);
147+
let lower = self.keys_offs.index(index).into_owned();
148+
let upper = self.keys_offs.index(index+1).into_owned();
149149
// Looking up values for an invalid key indicates something is wrong.
150150
assert!(lower < upper, "{:?} v {:?} at {:?}", lower, upper, index);
151151
(lower, upper)
152152
}
153153
/// Lower and upper bounds in `self.updates` corresponding to the value at `index`.
154154
fn updates_for_value(&self, index: usize) -> (usize, usize) {
155-
let mut lower = self.vals_offs.index(index);
156-
let upper = self.vals_offs.index(index+1);
155+
let mut lower = self.vals_offs.index(index).into_owned();
156+
let upper = self.vals_offs.index(index+1).into_owned();
157157
// We use equal lower and upper to encode "singleton update; just before here".
158158
// It should only apply when there is a prior element, so `lower` should be greater than zero.
159159
if lower == upper {
@@ -178,7 +178,7 @@ mod val_batch {
178178
// push additional blank entries in.
179179
while self.keys.len() < desired {
180180
// We insert a default (dummy) key and repeat the offset to indicate this.
181-
let current_offset = self.keys_offs.index(self.keys.len());
181+
let current_offset = self.keys_offs.index(self.keys.len()).into_owned();
182182
self.keys.push(Default::default());
183183
self.keys_offs.push(current_offset);
184184
}
@@ -339,17 +339,20 @@ mod val_batch {
339339

340340
let mut storage = RhhValStorage {
341341
keys: L::KeyContainer::merge_capacity(&batch1.keys, &batch2.keys),
342-
keys_offs: OffsetList::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
342+
keys_offs: L::OffsetContainer::with_capacity(batch1.keys_offs.len() + batch2.keys_offs.len()),
343343
vals: L::ValContainer::merge_capacity(&batch1.vals, &batch2.vals),
344-
vals_offs: OffsetList::with_capacity(batch1.vals_offs.len() + batch2.vals_offs.len()),
344+
vals_offs: L::OffsetContainer::with_capacity(batch1.vals_offs.len() + batch2.vals_offs.len()),
345345
updates: L::UpdContainer::merge_capacity(&batch1.updates, &batch2.updates),
346346
key_count: 0,
347347
key_capacity: rhh_cap,
348348
divisor: RhhValStorage::<L>::divisor_for_capacity(rhh_cap),
349349
};
350350

351-
storage.keys_offs.push(0);
352-
storage.vals_offs.push(0);
351+
// Mark explicit types because type inference fails to resolve it.
352+
let keys_offs: &mut L::OffsetContainer = &mut storage.keys_offs;
353+
keys_offs.push(0);
354+
let vals_offs: &mut L::OffsetContainer = &mut storage.vals_offs;
355+
vals_offs.push(0);
353356

354357
RhhValMerger {
355358
key_cursor1: 0,
@@ -746,9 +749,9 @@ mod val_batch {
746749
Self {
747750
result: RhhValStorage {
748751
keys: L::KeyContainer::with_capacity(keys),
749-
keys_offs: OffsetList::with_capacity(keys + 1),
752+
keys_offs: L::OffsetContainer::with_capacity(keys + 1),
750753
vals: L::ValContainer::with_capacity(vals),
751-
vals_offs: OffsetList::with_capacity(vals + 1),
754+
vals_offs: L::OffsetContainer::with_capacity(vals + 1),
752755
updates: L::UpdContainer::with_capacity(upds),
753756
key_count: 0,
754757
key_capacity: rhh_capacity,

0 commit comments

Comments
 (0)