Cache lookups into the borrow stack

This adds a very simple LRU-like cache which stores the locations of
often-used tags. While the implementation is very simple, the cache hit
rate is incredible at ~99.9% on most programs, and often the element at
position 0 in the cache has a hit rate of 90%. So the sub-optimality of
this cache basicaly vanishes into the noise in a profile.

Additionally, we keep a range which denotes where there might be an item
granting Unique permission in the stack, so that when we invalidate
Uniques we do not need to scan much of the stack, and often scan nothing
at all.

Author: saethlin

src/stacked_borrows.rs

@@ -3,6 +3,7 @@
 
 use log::trace;
 use std::cell::RefCell;
+use std::collections::VecDeque;
 use std::fmt;
 use std::num::NonZeroU64;
 use std::rc::Rc;
@@ -82,16 +83,34 @@ impl fmt::Debug for Item {
     }
 }
 
+const CACHE_MAX_LEN: usize = 16;
+
 /// Extra per-location state.
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug)]
 pub struct Stack {
     /// Used *mostly* as a stack; never empty.
     /// Invariants:
     /// * Above a `SharedReadOnly` there can only be more `SharedReadOnly`.
     /// * Except for `Untagged`, no tag occurs in the stack more than once.
     borrows: Vec<Item>,
+    cache: VecDeque<(SbTag, usize)>,
+    /// Index of the most recently inserted Untagged
+    top_untagged: Option<usize>,
+    /// On a read, we need to disable all `Unique` above the granting item. We can avoid most of
+    /// this scan by keeping track of the region of the borrow stack that may contain `Unique`s.
+    first_unique: usize,
+    last_unique: usize,
+}
+
+impl PartialEq for Stack {
+    fn eq(&self, other: &Self) -> bool {
+        // All the semantics of Stack are in self.borrows, everything else is caching
+        self.borrows == other.borrows
+    }
 }
 
+impl Eq for Stack {}
+
 /// Extra per-allocation state.
 #[derive(Clone, Debug)]
 pub struct Stacks {
@@ -276,18 +295,60 @@ impl Permission {
 impl<'tcx> Stack {
     /// Find the item granting the given kind of access to the given tag, and return where
     /// it is on the stack.
-    fn find_granting(&self, access: AccessKind, tag: SbTag) -> Option<usize> {
-        self.borrows
-            .iter()
-            .enumerate() // we also need to know *where* in the stack
-            .rev() // search top-to-bottom
-            // Return permission of first item that grants access.
-            // We require a permission with the right tag, ensuring U3 and F3.
-            .find_map(
-                |(idx, item)| {
-                    if tag == item.tag && item.perm.grants(access) { Some(idx) } else { None }
-                },
-            )
+    fn find_granting(&mut self, access: AccessKind, tag: SbTag) -> Option<usize> {
+        match tag {
+            SbTag::Untagged => {
+                let end = if let Some(idx) = self.top_untagged {
+                    if self.borrows[idx].perm.grants(access) {
+                        return Some(idx);
+                    } else {
+                        idx
+                    }
+                } else {
+                    self.borrows.len()
+                };
+                // Search top-to-bottom
+                for (idx, item) in self.borrows[..end]
+                    .iter()
+                    .enumerate()
+                    .rev()
+                    .filter(|(_, item)| item.tag == SbTag::Untagged)
+                {
+                    if self.top_untagged.is_none() {
+                        self.top_untagged = Some(idx);
+                    }
+                    // Return permission of the first item that grants access.
+                    // We require a permission with the right tag, ensuring U3 and F3.
+                    if item.perm.grants(access) {
+                        return Some(idx);
+                    }
+                }
+                return None;
+            }
+            SbTag::Tagged(_) => {
+                for cache_idx in 0..self.cache.len() {
+                    let stack_idx = self.cache[cache_idx].1;
+                    if self.cache[cache_idx].0 == tag && self.borrows[stack_idx].perm.grants(access)
+                    {
+                        if cache_idx != 0 {
+                            let element = self.cache.remove(cache_idx).unwrap();
+                            self.cache.push_front(element);
+                        }
+                        return Some(stack_idx);
+                    }
+                }
+                for (stack_idx, item) in self.borrows.iter().enumerate().rev() {
+                    if tag == item.tag && item.perm.grants(access) {
+                        if self.cache.len() == CACHE_MAX_LEN {
+                            self.cache.pop_back();
+                        }
+                        self.cache.push_front((tag, stack_idx));
+                        return Some(stack_idx);
+                    }
+                }
+                None
+            }
+        }
     }
 
     /// Find the first write-incompatible item above the given one --
@@ -402,6 +463,32 @@ impl<'tcx> Stack {
                 )?;
                 alloc_history.log_invalidation(item.tag, alloc_range, threads);
             }
+
+            // The drain removes elements from `borrows`, but we need to remove those elements
+            // from the lookup cache too.
+            let mut i = 0;
+            while i < self.cache.len() {
+                if self.cache[i].1 >= first_incompatible_idx {
+                    self.cache.remove(i);
+                } else {
+                    i += 1;
+                }
+            }
+
+            if let Some(idx) = self.top_untagged {
+                if idx >= first_incompatible_idx {
+                    self.top_untagged = None;
+                }
+            }
+
+            if first_incompatible_idx <= self.first_unique {
+                // We removed all the Unique items
+                self.first_unique = 0;
+                self.last_unique = 0;
+            } else {
+                // Ensure the range doesn't extend past the new top of the stack
+                self.last_unique = self.last_unique.min(first_incompatible_idx.saturating_sub(1));
+            }
         } else {
             // On a read, *disable* all `Unique` above the granting item.  This ensures U2 for read accesses.
             // The reason this is not following the stack discipline (by removing the first Unique and
@@ -411,20 +498,32 @@ impl<'tcx> Stack {
             // This pattern occurs a lot in the standard library: create a raw pointer, then also create a shared
             // reference and use that.
             // We *disable* instead of removing `Unique` to avoid "connecting" two neighbouring blocks of SRWs.
-            for idx in ((granting_idx + 1)..self.borrows.len()).rev() {
-                let item = &mut self.borrows[idx];
-                if item.perm == Permission::Unique {
-                    trace!("access: disabling item {:?}", item);
-                    Stack::check_protector(
-                        item,
-                        Some((tag, alloc_range, offset, access)),
-                        global,
-                        alloc_history,
-                    )?;
-                    item.perm = Permission::Disabled;
-                    alloc_history.log_invalidation(item.tag, alloc_range, threads);
+            if granting_idx < self.last_unique {
+                // add 1 so we don't disable the granting item
+                let lower = self.first_unique.max(granting_idx + 1);
+                //let upper = (self.last_unique + 1).min(self.borrows.len());
+                for item in &mut self.borrows[lower..=self.last_unique] {
+                    if item.perm == Permission::Unique {
+                        trace!("access: disabling item {:?}", item);
+                        Stack::check_protector(
+                            item,
+                            Some((tag, alloc_range, offset, access)),
+                            global,
+                            alloc_history,
+                        )?;
+                        item.perm = Permission::Disabled;
+                        alloc_history.log_invalidation(item.tag, alloc_range, threads);
+                    }
                 }
             }
+            if granting_idx < self.first_unique {
+                // We disabled all Unique items
+                self.first_unique = 0;
+                self.last_unique = 0;
+            } else {
+                // Truncate the range to granting_idx
+                self.last_unique = self.last_unique.min(granting_idx);
+            }
         }
 
         // Done.
@@ -456,6 +555,10 @@ impl<'tcx> Stack {
             Stack::check_protector(&item, None, global, alloc_history)?;
         }
 
+        self.cache.clear();
+        self.first_unique = 0;
+        self.last_unique = 0;
+
         Ok(())
     }
 
@@ -523,6 +626,31 @@ impl<'tcx> Stack {
         } else {
             trace!("reborrow: adding item {:?}", new);
             self.borrows.insert(new_idx, new);
+            // The above insert changes the meaning of every index in the cache >= new_idx, so now
+            // we need to find every one of those indexes and increment it.
+
+            if new.tag == SbTag::Untagged && new_idx >= self.top_untagged.unwrap_or(0) {
+                self.top_untagged = Some(new_idx);
+            }
+
+            // Adjust the possibly-unique range if an insert occurs before or within it
+            if self.first_unique >= new_idx {
+                self.first_unique += 1;
+            }
+            if self.last_unique >= new_idx {
+                self.last_unique += 1;
+            }
+            if new.perm == Permission::Unique {
+                // Make sure the possibly-unique range contains the new borrow
+                self.first_unique = self.first_unique.min(new_idx);
+                self.last_unique = self.last_unique.max(new_idx);
+            }
+
+            for (_tag, idx) in &mut self.cache {
+                if *idx >= new_idx {
+                    *idx += 1;
+                }
+            }
         }
 
         Ok(())
@@ -535,7 +663,15 @@ impl<'tcx> Stacks {
     /// Creates new stack with initial tag.
     fn new(size: Size, perm: Permission, tag: SbTag, local_crates: Rc<[CrateNum]>) -> Self {
         let item = Item { perm, tag, protector: None };
-        let stack = Stack { borrows: vec![item] };
+        let mut cache = VecDeque::default();
+        cache.push_front((tag, 0));
+        let stack = Stack {
+            borrows: vec![item],
+            cache,
+            first_unique: 0,
+            last_unique: 0,
+            top_untagged: None,
+        };
 
         Stacks {
             stacks: RefCell::new(RangeMap::new(size, stack)),