Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 20 additions & 76 deletions lib/bench/kv_router/active_sequences_open_loop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
use std::cell::RefCell;
use std::collections::{BTreeMap, HashMap};
use std::hint::black_box;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, mpsc};
use std::time::{Duration, Instant};

use dynamo_bench::kv_router_common::fixed_lane::{EMPTY_OPERATION_ID, FixedLane, LaneOverflow};
use dynamo_bench::kv_router_common::issuer::pin_current_thread;
use dynamo_bench::kv_router_common::replay::NoopSequencePublisher;
use dynamo_bench::kv_router_common::trace_gen::WorkerTimelines;
Expand All @@ -18,12 +19,11 @@ use dynamo_kv_router::{
use dynamo_tokens::SequenceHash;
use rustc_hash::FxHashMap;
use serde::Serialize;
use tokio::sync::{Notify, oneshot};
use tokio::sync::oneshot;

use super::active_sequences_shared::{SequenceTrace, SequenceTraceEntry};

const RESULT_SCHEMA_VERSION: u32 = 2;
const EMPTY_OPERATION_ID: u32 = u32::MAX;
const START_LEAD_NS: u64 = 20_000_000;

#[derive(Clone, Debug)]
Expand Down Expand Up @@ -562,53 +562,6 @@ pub(crate) fn prepare_active_sequences_trial(
})
}

struct OperationLane {
slots: Box<[AtomicU32]>,
published: AtomicUsize,
consumed: AtomicUsize,
closed: AtomicBool,
notify: Notify,
}

impl OperationLane {
fn new(capacity: usize) -> Self {
Self {
slots: (0..capacity)
.map(|_| AtomicU32::new(EMPTY_OPERATION_ID))
.collect::<Vec<_>>()
.into_boxed_slice(),
published: AtomicUsize::new(0),
consumed: AtomicUsize::new(0),
closed: AtomicBool::new(false),
notify: Notify::new(),
}
}

fn write(&self, cursor: usize, id: u32) -> Result<(), IssuerFailure> {
let Some(slot) = self.slots.get(cursor) else {
return Err(IssuerFailure::LaneOverflow);
};
slot.store(id, Ordering::Relaxed);
Ok(())
}

fn publish(&self, count: usize) {
self.published.store(count, Ordering::Release);
self.notify.notify_one();
}

fn close(&self) {
self.closed.store(true, Ordering::Release);
self.notify.notify_one();
}

fn depth(&self) -> usize {
self.published
.load(Ordering::Acquire)
.saturating_sub(self.consumed.load(Ordering::Acquire))
}
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
enum CompletionFailure {
#[default]
Expand Down Expand Up @@ -809,7 +762,7 @@ fn execute_payload(

async fn operation_lane_worker(
sequences: Arc<ActiveSequencesMultiWorker<NoopSequencePublisher>>,
lane: Arc<OperationLane>,
lane: Arc<FixedLane>,
mut payloads: Box<[LanePayloadSlot]>,
request_ids: Box<[String]>,
clock: Arc<BenchmarkClock>,
Expand All @@ -821,9 +774,9 @@ async fn operation_lane_worker(
let _ = ready.send(());

loop {
let published = lane.published.load(Ordering::Acquire);
let published = lane.published();
while consumed < published {
let id = lane.slots[consumed].load(Ordering::Relaxed);
let id = lane.id(consumed);
if id == EMPTY_OPERATION_ID {
failure = LaneFailure::MissingPublishedId;
break;
Expand All @@ -848,23 +801,15 @@ async fn operation_lane_worker(
payload_slot.payload.take(),
);
consumed += 1;
lane.consumed.store(consumed, Ordering::Release);
lane.record_consumed(consumed);
}

if failure != LaneFailure::None {
break;
}
if lane.closed.load(Ordering::Acquire) && consumed == lane.published.load(Ordering::Acquire)
{
if !lane.wait_for_more(consumed).await {
break;
}

let notified = lane.notify.notified();
if consumed == lane.published.load(Ordering::Acquire)
&& !lane.closed.load(Ordering::Acquire)
{
notified.await;
}
}

LaneResult {
Expand Down Expand Up @@ -892,6 +837,12 @@ enum IssuerFailure {
DispatchMismatch,
}

impl From<LaneOverflow> for IssuerFailure {
fn from(_: LaneOverflow) -> Self {
IssuerFailure::LaneOverflow
}
}

struct IssuerOutput {
records: Box<[IssueRecord]>,
producer_stop_ns: u64,
Expand All @@ -901,7 +852,7 @@ struct IssuerOutput {

fn issue_operations(
dispatch: Vec<ScheduledOperation>,
lanes: Vec<Arc<OperationLane>>,
lanes: Vec<Arc<FixedLane>>,
operation_count: usize,
clock: Arc<BenchmarkClock>,
start_signal: Arc<AtomicU64>,
Expand Down Expand Up @@ -954,7 +905,7 @@ fn issue_operations(
break;
};
if let Err(error) = lane.write(lane_cursors[operation.lane], operation.id) {
failure = Some(error);
failure = Some(error.into());
break;
}
lane_cursors[operation.lane] += 1;
Expand Down Expand Up @@ -1104,7 +1055,7 @@ pub(crate) async fn run_active_sequences_trial(
let lanes = trial
.lane_capacities
.iter()
.map(|&capacity| Arc::new(OperationLane::new(capacity)))
.map(|&capacity| Arc::new(FixedLane::new(capacity)))
.collect::<Vec<_>>();

let mut ready_receivers = Vec::with_capacity(lanes.len());
Expand Down Expand Up @@ -1739,8 +1690,8 @@ mod tests {

use super::{
ActiveOperationKind, ActiveSequencesRunConfig, BenchmarkClock, CompletionFailure,
IssuerFailure, LanePayload, LanePayloadSlot, OperationLane,
prepare_active_sequences_corpus, run_active_sequences_benchmark,
FixedLane, LanePayload, LanePayloadSlot, prepare_active_sequences_corpus,
run_active_sequences_benchmark,
};
use crate::active_sequences_shared::{SequenceTrace, SequenceTraceEntry};
use dynamo_bench::kv_router_common::replay::NoopSequencePublisher;
Expand Down Expand Up @@ -1892,16 +1843,9 @@ mod tests {
assert!(error.contains("replay window must be positive"));
}

#[test]
fn fixed_lane_reports_overflow() {
let lane = OperationLane::new(1);
assert!(lane.write(0, 1).is_ok());
assert_eq!(lane.write(1, 2), Err(IssuerFailure::LaneOverflow));
}

#[tokio::test(flavor = "current_thread")]
async fn fixed_lane_does_not_lose_publish_before_wait() {
let lane = Arc::new(OperationLane::new(1));
let lane = Arc::new(FixedLane::new(1));
lane.write(0, 7).unwrap();
lane.publish(1);
lane.close();
Expand Down
78 changes: 18 additions & 60 deletions lib/bench/kv_router/approximate_lru_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@

use std::collections::{HashMap, HashSet};
use std::hint::black_box;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, mpsc};
use std::time::{Duration, Instant};

use anyhow::Context;
use clap::{Parser, ValueEnum};
use dynamo_bench::kv_router_common::args::CommonArgs;
use dynamo_bench::kv_router_common::fixed_lane::{EMPTY_OPERATION_ID, FixedLane};
use dynamo_bench::kv_router_common::issuer::{
parse_cpu_list, pin_current_thread, pin_current_thread_to_cpus,
};
Expand All @@ -23,10 +24,9 @@ use dynamo_kv_router::scheduling::AttemptId;
use dynamo_kv_router::{ConcurrentRadixTreeCompressed, ThreadPoolIndexer, approx::PruneConfig};
use dynamo_tokens::SequenceHash;
use serde::Serialize;
use tokio::sync::{Notify, oneshot};
use tokio::sync::oneshot;

const RESULT_SCHEMA_VERSION: u32 = 1;
const EMPTY_OPERATION_ID: u32 = u32::MAX;
const START_LEAD_NS: u64 = 20_000_000;
const RANK_INCARNATION: u64 = 1;

Expand Down Expand Up @@ -459,40 +459,6 @@ fn prepare_trial(
})
}

struct OperationLane {
slots: Box<[AtomicU32]>,
published: AtomicUsize,
consumed: AtomicUsize,
closed: AtomicBool,
notify: Notify,
}

impl OperationLane {
fn new(capacity: usize) -> Self {
Self {
slots: (0..capacity)
.map(|_| AtomicU32::new(EMPTY_OPERATION_ID))
.collect::<Vec<_>>()
.into_boxed_slice(),
published: AtomicUsize::new(0),
consumed: AtomicUsize::new(0),
closed: AtomicBool::new(false),
notify: Notify::new(),
}
}

fn close(&self) {
self.closed.store(true, Ordering::Release);
self.notify.notify_one();
}

fn depth(&self) -> usize {
self.published
.load(Ordering::Acquire)
.saturating_sub(self.consumed.load(Ordering::Acquire))
}
}

#[derive(Debug)]
struct CompletionRecord {
id: u32,
Expand Down Expand Up @@ -590,7 +556,7 @@ async fn execute_payload(
async fn lane_worker(
indexer: Arc<ApproximateIndexer>,
policy: Policy,
lane: Arc<OperationLane>,
lane: Arc<FixedLane>,
mut payloads: Box<[LanePayloadSlot]>,
request_count: usize,
ready: oneshot::Sender<()>,
Expand All @@ -601,9 +567,9 @@ async fn lane_worker(
let _ = ready.send(());

loop {
let published = lane.published.load(Ordering::Acquire);
let published = lane.published();
while consumed < published {
let id = lane.slots[consumed].load(Ordering::Relaxed);
let id = lane.id(consumed);
let (mode, failure) = match payloads.get_mut(consumed) {
Some(slot) if slot.id == id => match slot.payload.take() {
Some(payload) => {
Expand All @@ -615,27 +581,20 @@ async fn lane_worker(
};
completions.push(CompletionRecord { id, mode, failure });
consumed += 1;
lane.consumed.store(consumed, Ordering::Release);
lane.record_consumed(consumed);
}

if lane.closed.load(Ordering::Acquire) && consumed == lane.published.load(Ordering::Acquire)
{
if !lane.wait_for_more(consumed).await {
break;
}
let notified = lane.notify.notified();
if consumed == lane.published.load(Ordering::Acquire)
&& !lane.closed.load(Ordering::Acquire)
{
notified.await;
}
}
drop(leases);
LaneResult { completions }
}

fn issue_operations(
dispatch: Vec<ScheduledOperation>,
lanes: Vec<Arc<OperationLane>>,
lanes: Vec<Arc<FixedLane>>,
clock: Arc<BenchmarkClock>,
start_signal: Arc<AtomicU64>,
ready: mpsc::SyncSender<()>,
Expand Down Expand Up @@ -664,16 +623,15 @@ fn issue_operations(
failure = Some("issuer lane out of range".to_string());
break;
};
let cursor = lane_cursors[operation.lane];
let Some(slot) = lane.slots.get(cursor) else {
if lane
.write(lane_cursors[operation.lane], operation.id)
.is_err()
{
failure = Some("issuer fixed lane overflow".to_string());
break;
};
slot.store(operation.id, Ordering::Relaxed);
}
lane_cursors[operation.lane] += 1;
lane.published
.store(lane_cursors[operation.lane], Ordering::Release);
lane.notify.notify_one();
lane.publish(lane_cursors[operation.lane]);
issued_operations += 1;
}

Expand Down Expand Up @@ -822,7 +780,7 @@ async fn run_trial(
let lanes = trial
.lane_payloads
.iter()
.map(|payloads| Arc::new(OperationLane::new(payloads.len())))
.map(|payloads| Arc::new(FixedLane::new(payloads.len())))
.collect::<Vec<_>>();
let mut ready_receivers = Vec::with_capacity(lanes.len());
let mut tasks = Vec::with_capacity(lanes.len());
Expand Down Expand Up @@ -930,8 +888,8 @@ async fn run_trial(
push_failure(&mut failure_reasons, "duplicate_completion");
}
}
let expected = lanes[lane_id].slots.len();
let actual = lanes[lane_id].consumed.load(Ordering::Acquire);
let expected = lanes[lane_id].capacity();
let actual = lanes[lane_id].consumed();
if expected != actual {
push_failure(&mut failure_reasons, "lane_incomplete");
}
Expand Down
Loading
Loading