Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 9 additions & 19 deletions lib/llm/src/local_model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,6 @@ impl LocalModelBuilder {
migration_limit: self.migration_limit,
migration_max_seq_len: self.migration_max_seq_len,
self_host_metadata: self.self_host_metadata,
attached_self_host_suffix: None,
});
}

Expand Down Expand Up @@ -353,7 +352,6 @@ impl LocalModelBuilder {
migration_limit: self.migration_limit,
migration_max_seq_len: self.migration_max_seq_len,
self_host_metadata: self.self_host_metadata,
attached_self_host_suffix: None,
})
}
}
Expand All @@ -376,9 +374,6 @@ pub struct LocalModel {
migration_limit: u32,
migration_max_seq_len: Option<u32>,
self_host_metadata: bool,
/// Set by `move_to_self_host` so `clear_self_hosted_artifacts`
/// scopes its unregister to this model's `(slug, suffix)` only.
attached_self_host_suffix: Option<String>,
}

impl LocalModel {
Expand Down Expand Up @@ -512,7 +507,7 @@ impl LocalModel {
);

if self.self_host_metadata {
self.move_to_self_host(endpoint.drt(), model_suffix.as_deref())
self.move_to_self_host(endpoint, model_suffix.as_deref())
.context("move_to_self_host")?;
}

Expand Down Expand Up @@ -555,9 +550,10 @@ impl LocalModel {
/// (recorded as `BASE_SUFFIX` in the registry).
fn move_to_self_host(
&mut self,
drt: &dynamo_runtime::DistributedRuntime,
endpoint: &Endpoint,
Comment thread
nnshah1 marked this conversation as resolved.
Resolved
model_suffix: Option<&str>,
) -> anyhow::Result<()> {
let drt = endpoint.drt();
let Some(base_url) = self_host_base_url(drt)? else {
tracing::warn!(
model_slug = %self.card.slug(),
Expand All @@ -570,6 +566,8 @@ impl LocalModel {
let model_slug = self.card.slug().to_string();
let suffix = model_suffix.unwrap_or(dynamo_runtime::metadata_registry::BASE_SUFFIX);
let registry = drt.metadata_artifacts();
let instance_id = drt.connection_id();
let owner = (instance_id, model_suffix.map(str::to_string));

// Advertise non-typed siblings (preprocessor_config.json,
// special_tokens_map.json, …) so external preprocessors that load
Expand Down Expand Up @@ -621,12 +619,11 @@ impl LocalModel {
let url = url::Url::parse(&format!(
"{base_url}/v1/metadata/{model_slug}/{suffix}/{filename}"
))?;
registry.register(&model_slug, suffix, &filename, absolute);
registry.register(&owner, &model_slug, suffix, &filename, absolute);
cf.move_to_url(url);
rewritten += 1;
}

self.attached_self_host_suffix = Some(suffix.to_string());
tracing::debug!(
model_slug,
suffix,
Expand All @@ -637,15 +634,6 @@ impl LocalModel {
Ok(())
}

/// Idempotent. Call before detach/hot-reload; otherwise the
/// runtime drops the registry entries with the worker process.
pub fn clear_self_hosted_artifacts(&self, drt: &dynamo_runtime::DistributedRuntime) {
if let Some(suffix) = self.attached_self_host_suffix.as_deref() {
drt.metadata_artifacts()
.unregister(self.card.slug().as_ref(), suffix);
}
}

/// Helper associated function to detach a model from an endpoint
///
/// For base models, pass `lora_name = None`.
Expand All @@ -658,8 +646,8 @@ impl LocalModel {
let instance_id = drt.connection_id();
let endpoint_id = endpoint.id();

// Compute model_suffix from lora_name if present
let model_suffix = lora_name.map(|name| Slug::slugify(name).to_string());
let registry_owner = (instance_id, model_suffix.clone());

let instance = DiscoveryInstance::Model {
namespace: endpoint_id.namespace,
Expand All @@ -672,6 +660,8 @@ impl LocalModel {

let discovery = drt.discovery();
discovery.unregister(instance).await?;
drt.metadata_artifacts()
.unregister_for_owner(&registry_owner);

if let Some(lora_name) = lora_name {
tracing::info!(
Expand Down
126 changes: 96 additions & 30 deletions lib/runtime/src/metadata_registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@
//! `suffix` is the LoRA slug (or `"_base"` for non-LoRA). It scopes
//! each registration so detaching a LoRA doesn't unregister the base
//! model's files (or vice versa).
//!
//! Each entry also stores its `Owner = (instance_id, lora_slug)` so
//! `unregister_for_owner` can clean up on detach without the caller
//! threading the model slug. Each `(slug, suffix, filename)` key must
//! have at most one owner — `register` panics on collision with a
//! different owner.

use std::collections::HashMap;
use std::path::PathBuf;
Expand All @@ -24,42 +30,60 @@ use parking_lot::RwLock;
/// `_base` would collide with this sentinel and is not supported.
pub const BASE_SUFFIX: &str = "_base";

/// `(slug, suffix, filename)`.
type Key = (String, String, String);
/// `(instance_id, lora_slug)`. `None` lora_slug = base model.
pub type Owner = (u64, Option<String>);
Comment thread
nnshah1 marked this conversation as resolved.
Resolved

/// Cloning shares the underlying map.
#[derive(Clone, Debug, Default)]
pub struct MetadataArtifactRegistry {
entries: Arc<RwLock<HashMap<Key, PathBuf>>>,
// Key is a single concatenated `{slug}\0{suffix}\0{filename}` string —
// one allocation per lookup instead of a 3-`String` tuple.
entries: Arc<RwLock<HashMap<String, (PathBuf, Owner)>>>,
Comment thread
nnshah1 marked this conversation as resolved.
Resolved
}

fn make_key(slug: &str, suffix: &str, filename: &str) -> String {
let mut s = String::with_capacity(slug.len() + suffix.len() + filename.len() + 2);
s.push_str(slug);
s.push('\0');
s.push_str(suffix);
s.push('\0');
s.push_str(filename);
s
}

impl MetadataArtifactRegistry {
pub fn new() -> Self {
Self {
entries: Arc::new(RwLock::new(HashMap::new())),
}
Self::default()
}

pub fn register(&self, slug: &str, suffix: &str, filename: &str, path: PathBuf) {
/// Panics on owner collision — two `LocalModel` instances attaching the
/// same (slug, suffix, filename) in one process would let detach-#1
/// wipe files detach-#2 still needs. Same-owner re-register is fine
/// (path update).
pub fn register(&self, owner: &Owner, slug: &str, suffix: &str, filename: &str, path: PathBuf) {
let key = make_key(slug, suffix, filename);
let mut entries = self.entries.write();
entries.insert(
(slug.to_string(), suffix.to_string(), filename.to_string()),
path,
);
if let Some((_, prior)) = entries.get(&key) {
assert_eq!(
Comment thread
nnshah1 marked this conversation as resolved.
Resolved
prior, owner,
"metadata-registry collision on {key:?}: prior={prior:?}, new={owner:?}",
);
}
entries.insert(key, (path, owner.clone()));
tracing::debug!(slug, suffix, filename, "registered metadata artifact");
}

pub fn get(&self, slug: &str, suffix: &str, filename: &str) -> Option<PathBuf> {
let entries = self.entries.read();
entries
.get(&(slug.to_string(), suffix.to_string(), filename.to_string()))
.cloned()
self.entries
.read()
.get(&make_key(slug, suffix, filename))
.map(|(p, _)| p.clone())
}

/// Drop entries for a single registration scoped by `(slug, suffix)`.
pub fn unregister(&self, slug: &str, suffix: &str) {
let mut entries = self.entries.write();
entries.retain(|(s, sx, _), _| !(s == slug && sx == suffix));
/// Drop every entry registered by `owner`. No-op if `owner` never
/// registered (e.g. self-host was disabled or skipped).
pub fn unregister_for_owner(&self, owner: &Owner) {
self.entries.write().retain(|_, (_, o)| o != owner);
}

pub fn len(&self) -> usize {
Expand All @@ -75,33 +99,75 @@ impl MetadataArtifactRegistry {
mod tests {
use super::*;

fn base() -> Owner {
(1, None)
}

fn lora(slug: &str) -> Owner {
(1, Some(slug.to_string()))
}

#[test]
fn register_get_roundtrip() {
let reg = MetadataArtifactRegistry::new();
let p = PathBuf::from("/tmp/tokenizer.json");
reg.register("llama-3-8b", "_base", "tokenizer.json", p.clone());
reg.register(&base(), "llama-3-8b", "_base", "tokenizer.json", p.clone());

assert_eq!(reg.get("llama-3-8b", "_base", "tokenizer.json"), Some(p));
assert!(reg.get("llama-3-8b", "_base", "missing.json").is_none());
assert!(reg.get("llama-3-8b", "lora-v1", "tokenizer.json").is_none());
}

#[test]
fn unregister_only_removes_matching_suffix() {
fn unregister_for_owner_clears_only_that_owner() {
let reg = MetadataArtifactRegistry::new();
reg.register("m", "_base", "config.json", PathBuf::from("/m/c"));
reg.register("m", "_base", "tokenizer.json", PathBuf::from("/m/t"));
reg.register("m", "lora-v1", "config.json", PathBuf::from("/m/c"));
let lora_owner = lora("lora-v1");
reg.register(&base(), "m", "_base", "config.json", PathBuf::from("/m/c"));
reg.register(
&base(),
"m",
"_base",
"tokenizer.json",
PathBuf::from("/m/t"),
);
reg.register(
&lora_owner,
"m",
"lora-v1",
"adapter.json",
PathBuf::from("/m/a"),
);

reg.unregister("m", "_base");
reg.unregister_for_owner(&lora_owner);

assert!(reg.get("m", "_base", "config.json").is_none());
assert!(reg.get("m", "_base", "tokenizer.json").is_none());
// LoRA entry on the same slug survives detach of the base.
assert!(reg.get("m", "lora-v1", "adapter.json").is_none());
assert_eq!(
reg.get("m", "lora-v1", "config.json"),
reg.get("m", "_base", "config.json"),
Some(PathBuf::from("/m/c"))
);
assert_eq!(reg.len(), 1);
// Idempotent — second call is a no-op.
reg.unregister_for_owner(&lora_owner);
assert_eq!(reg.len(), 2);
}

#[test]
#[should_panic(expected = "metadata-registry collision")]
fn register_panics_on_owner_collision() {
let reg = MetadataArtifactRegistry::new();
let owner_a = (1, None);
let owner_b = (2, None);
reg.register(&owner_a, "m", "_base", "config.json", PathBuf::from("/a"));
reg.register(&owner_b, "m", "_base", "config.json", PathBuf::from("/b"));
}

#[test]
fn register_same_owner_updates_path() {
let reg = MetadataArtifactRegistry::new();
reg.register(&base(), "m", "_base", "config.json", PathBuf::from("/a"));
reg.register(&base(), "m", "_base", "config.json", PathBuf::from("/b"));
assert_eq!(
reg.get("m", "_base", "config.json"),
Some(PathBuf::from("/b"))
);
}
}
Loading