diff --git a/lib/llm/src/local_model.rs b/lib/llm/src/local_model.rs index 4cf09b04dfd7..310aa5e0affb 100644 --- a/lib/llm/src/local_model.rs +++ b/lib/llm/src/local_model.rs @@ -301,7 +301,6 @@ impl LocalModelBuilder { migration_limit: self.migration_limit, migration_max_seq_len: self.migration_max_seq_len, self_host_metadata: self.self_host_metadata, - attached_self_host_suffix: None, }); } @@ -353,7 +352,6 @@ impl LocalModelBuilder { migration_limit: self.migration_limit, migration_max_seq_len: self.migration_max_seq_len, self_host_metadata: self.self_host_metadata, - attached_self_host_suffix: None, }) } } @@ -376,9 +374,6 @@ pub struct LocalModel { migration_limit: u32, migration_max_seq_len: Option, self_host_metadata: bool, - /// Set by `move_to_self_host` so `clear_self_hosted_artifacts` - /// scopes its unregister to this model's `(slug, suffix)` only. - attached_self_host_suffix: Option, } impl LocalModel { @@ -512,7 +507,7 @@ impl LocalModel { ); if self.self_host_metadata { - self.move_to_self_host(endpoint.drt(), model_suffix.as_deref()) + self.move_to_self_host(endpoint, model_suffix.as_deref()) .context("move_to_self_host")?; } @@ -555,9 +550,13 @@ impl LocalModel { /// (recorded as `BASE_SUFFIX` in the registry). fn move_to_self_host( &mut self, - drt: &dynamo_runtime::DistributedRuntime, + endpoint: &Endpoint, model_suffix: Option<&str>, ) -> anyhow::Result<()> { + let drt = endpoint.drt(); + let namespace = endpoint.component().namespace().name().to_string(); + let component = endpoint.component().name().to_string(); + let endpoint_name = endpoint.name().to_string(); let Some(base_url) = self_host_base_url(drt)? else { tracing::warn!( model_slug = %self.card.slug(), @@ -570,6 +569,8 @@ impl LocalModel { let model_slug = self.card.slug().to_string(); let suffix = model_suffix.unwrap_or(dynamo_runtime::metadata_registry::BASE_SUFFIX); let registry = drt.metadata_artifacts(); + let instance_id = drt.connection_id(); + let owner = (instance_id, model_suffix.map(str::to_string)); // Advertise non-typed siblings (preprocessor_config.json, // special_tokens_map.json, …) so external preprocessors that load @@ -619,14 +620,24 @@ impl LocalModel { }; let url = url::Url::parse(&format!( - "{base_url}/v1/metadata/{model_slug}/{suffix}/{filename}" + "{base_url}/v1/metadata/{namespace}/{component}/{endpoint_name}/{model_slug}/{suffix}/{filename}" ))?; - registry.register(&model_slug, suffix, &filename, absolute); + registry + .register( + &owner, + &namespace, + &component, + &endpoint_name, + &model_slug, + suffix, + &filename, + absolute, + ) + .context("registering metadata artifact")?; cf.move_to_url(url); rewritten += 1; } - self.attached_self_host_suffix = Some(suffix.to_string()); tracing::debug!( model_slug, suffix, @@ -637,15 +648,6 @@ impl LocalModel { Ok(()) } - /// Idempotent. Call before detach/hot-reload; otherwise the - /// runtime drops the registry entries with the worker process. - pub fn clear_self_hosted_artifacts(&self, drt: &dynamo_runtime::DistributedRuntime) { - if let Some(suffix) = self.attached_self_host_suffix.as_deref() { - drt.metadata_artifacts() - .unregister(self.card.slug().as_ref(), suffix); - } - } - /// Helper associated function to detach a model from an endpoint /// /// For base models, pass `lora_name = None`. @@ -658,8 +660,8 @@ impl LocalModel { let instance_id = drt.connection_id(); let endpoint_id = endpoint.id(); - // Compute model_suffix from lora_name if present let model_suffix = lora_name.map(|name| Slug::slugify(name).to_string()); + let registry_owner = (instance_id, model_suffix.clone()); let instance = DiscoveryInstance::Model { namespace: endpoint_id.namespace, @@ -672,6 +674,8 @@ impl LocalModel { let discovery = drt.discovery(); discovery.unregister(instance).await?; + drt.metadata_artifacts() + .unregister_for_owner(®istry_owner); if let Some(lora_name) = lora_name { tracing::info!( diff --git a/lib/runtime/src/metadata_registry.rs b/lib/runtime/src/metadata_registry.rs index 21a8b82834ce..051a41d59cde 100644 --- a/lib/runtime/src/metadata_registry.rs +++ b/lib/runtime/src/metadata_registry.rs @@ -4,62 +4,136 @@ //! Worker-side index from a model metadata file's identity to its //! on-disk path. When a worker self-hosts metadata, it registers each //! file here and rewrites the MDC's `CheckedFile.path` to a -//! `/v1/metadata/{slug}/{suffix}/{filename}` URL on its own -//! `system_status_server`. The route handler reads paths back out by -//! the same key and streams the bytes to the frontend, which +//! `/v1/metadata/{namespace}/{component}/{endpoint}/{slug}/{suffix}/{filename}` +//! URL on its own `system_status_server`. The route handler reads paths +//! back out by the same key and streams the bytes to the frontend, which //! blake3-verifies them against the MDC. //! -//! `suffix` is the LoRA slug (or `"_base"` for non-LoRA). It scopes -//! each registration so detaching a LoRA doesn't unregister the base -//! model's files (or vice versa). +//! The endpoint triple in the key disambiguates multiple `LocalModel` +//! instances coexisting on one DRT (e.g. different roles attaching the +//! same base model). `suffix` is the LoRA slug (or `"_base"` for +//! non-LoRA), scoping each registration so detaching a LoRA doesn't +//! unregister the base model's files. +//! +//! Each entry also stores its `Owner = (instance_id, lora_slug)` so +//! `unregister_for_owner` can clean up on detach without the caller +//! threading the model slug. `register` returns +//! `Err(CollisionError)` on conflict — the caller propagates and the +//! worker fails to start. use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; use parking_lot::RwLock; +use thiserror::Error; /// Sentinel `suffix` for non-LoRA registrations. LoRA suffixes are /// `Slug::slugify` outputs (`[a-z0-9_-]+`); a name that slugifies to /// `_base` would collide with this sentinel and is not supported. pub const BASE_SUFFIX: &str = "_base"; -/// `(slug, suffix, filename)`. -type Key = (String, String, String); +/// `(instance_id, lora_slug)`. `None` lora_slug = base model. +pub type Owner = (u64, Option); + +/// `(namespace, component, endpoint, slug, suffix, filename)`. +type Key = (String, String, String, String, String, String); + +#[allow(clippy::too_many_arguments)] +fn make_key( + namespace: &str, + component: &str, + endpoint: &str, + slug: &str, + suffix: &str, + filename: &str, +) -> Key { + ( + namespace.to_string(), + component.to_string(), + endpoint.to_string(), + slug.to_string(), + suffix.to_string(), + filename.to_string(), + ) +} + +/// Registration collided with a different owner — programmer error. +#[derive(Debug, Error)] +#[error("metadata-registry collision on key {key:?}: prior_owner={prior:?}, new_owner={new:?}")] +pub struct CollisionError { + pub key: Key, + pub prior: Owner, + pub new: Owner, +} /// Cloning shares the underlying map. #[derive(Clone, Debug, Default)] pub struct MetadataArtifactRegistry { - entries: Arc>>, + entries: Arc>>, } impl MetadataArtifactRegistry { pub fn new() -> Self { - Self { - entries: Arc::new(RwLock::new(HashMap::new())), - } + Self::default() } - pub fn register(&self, slug: &str, suffix: &str, filename: &str, path: PathBuf) { + /// Returns `Err(CollisionError)` if a different owner already + /// registered this key. Same-owner re-register updates the path. + #[allow(clippy::too_many_arguments)] + pub fn register( + &self, + owner: &Owner, + namespace: &str, + component: &str, + endpoint: &str, + slug: &str, + suffix: &str, + filename: &str, + path: PathBuf, + ) -> Result<(), Box> { + let key = make_key(namespace, component, endpoint, slug, suffix, filename); let mut entries = self.entries.write(); - entries.insert( - (slug.to_string(), suffix.to_string(), filename.to_string()), - path, + if let Some((_, prior)) = entries.get(&key) + && prior != owner + { + return Err(Box::new(CollisionError { + key, + prior: prior.clone(), + new: owner.clone(), + })); + } + entries.insert(key, (path, owner.clone())); + tracing::debug!( + namespace, + component, + endpoint, + slug, + suffix, + filename, + "registered metadata artifact", ); - tracing::debug!(slug, suffix, filename, "registered metadata artifact"); + Ok(()) } - pub fn get(&self, slug: &str, suffix: &str, filename: &str) -> Option { - let entries = self.entries.read(); - entries - .get(&(slug.to_string(), suffix.to_string(), filename.to_string())) - .cloned() + #[allow(clippy::too_many_arguments)] + pub fn get( + &self, + namespace: &str, + component: &str, + endpoint: &str, + slug: &str, + suffix: &str, + filename: &str, + ) -> Option { + let key = make_key(namespace, component, endpoint, slug, suffix, filename); + self.entries.read().get(&key).map(|(p, _)| p.clone()) } - /// Drop entries for a single registration scoped by `(slug, suffix)`. - pub fn unregister(&self, slug: &str, suffix: &str) { - let mut entries = self.entries.write(); - entries.retain(|(s, sx, _), _| !(s == slug && sx == suffix)); + /// Drop every entry registered by `owner`. No-op if `owner` never + /// registered (e.g. self-host was disabled or skipped). + pub fn unregister_for_owner(&self, owner: &Owner) { + self.entries.write().retain(|_, (_, o)| o != owner); } pub fn len(&self) -> usize { @@ -75,33 +149,202 @@ impl MetadataArtifactRegistry { mod tests { use super::*; + fn base() -> Owner { + (1, None) + } + + fn lora(slug: &str) -> Owner { + (1, Some(slug.to_string())) + } + #[test] fn register_get_roundtrip() { let reg = MetadataArtifactRegistry::new(); let p = PathBuf::from("/tmp/tokenizer.json"); - reg.register("llama-3-8b", "_base", "tokenizer.json", p.clone()); + reg.register( + &base(), + "ns", + "comp", + "ep", + "llama-3-8b", + "_base", + "tokenizer.json", + p.clone(), + ) + .unwrap(); - assert_eq!(reg.get("llama-3-8b", "_base", "tokenizer.json"), Some(p)); - assert!(reg.get("llama-3-8b", "_base", "missing.json").is_none()); - assert!(reg.get("llama-3-8b", "lora-v1", "tokenizer.json").is_none()); + assert_eq!( + reg.get("ns", "comp", "ep", "llama-3-8b", "_base", "tokenizer.json"), + Some(p) + ); + assert!( + reg.get("ns", "comp", "ep", "llama-3-8b", "_base", "missing.json") + .is_none() + ); + assert!( + reg.get( + "ns", + "comp", + "ep", + "llama-3-8b", + "lora-v1", + "tokenizer.json" + ) + .is_none() + ); } #[test] - fn unregister_only_removes_matching_suffix() { + fn unregister_for_owner_clears_only_that_owner() { let reg = MetadataArtifactRegistry::new(); - reg.register("m", "_base", "config.json", PathBuf::from("/m/c")); - reg.register("m", "_base", "tokenizer.json", PathBuf::from("/m/t")); - reg.register("m", "lora-v1", "config.json", PathBuf::from("/m/c")); + let lora_owner = lora("lora-v1"); + reg.register( + &base(), + "ns", + "comp", + "ep", + "m", + "_base", + "config.json", + PathBuf::from("/m/c"), + ) + .unwrap(); + reg.register( + &base(), + "ns", + "comp", + "ep", + "m", + "_base", + "tokenizer.json", + PathBuf::from("/m/t"), + ) + .unwrap(); + reg.register( + &lora_owner, + "ns", + "comp", + "ep", + "m", + "lora-v1", + "adapter.json", + PathBuf::from("/m/a"), + ) + .unwrap(); - reg.unregister("m", "_base"); + reg.unregister_for_owner(&lora_owner); - assert!(reg.get("m", "_base", "config.json").is_none()); - assert!(reg.get("m", "_base", "tokenizer.json").is_none()); - // LoRA entry on the same slug survives detach of the base. + assert!( + reg.get("ns", "comp", "ep", "m", "lora-v1", "adapter.json") + .is_none() + ); assert_eq!( - reg.get("m", "lora-v1", "config.json"), + reg.get("ns", "comp", "ep", "m", "_base", "config.json"), Some(PathBuf::from("/m/c")) ); - assert_eq!(reg.len(), 1); + // Idempotent — second call is a no-op. + reg.unregister_for_owner(&lora_owner); + assert_eq!(reg.len(), 2); + } + + #[test] + fn register_returns_err_on_owner_collision() { + let reg = MetadataArtifactRegistry::new(); + let owner_a = (1, None); + let owner_b = (2, None); + reg.register( + &owner_a, + "ns", + "comp", + "ep", + "m", + "_base", + "config.json", + PathBuf::from("/a"), + ) + .unwrap(); + let err = reg + .register( + &owner_b, + "ns", + "comp", + "ep", + "m", + "_base", + "config.json", + PathBuf::from("/b"), + ) + .unwrap_err(); + assert_eq!(err.prior, owner_a); + assert_eq!(err.new, owner_b); + } + + #[test] + fn register_same_owner_updates_path() { + let reg = MetadataArtifactRegistry::new(); + reg.register( + &base(), + "ns", + "comp", + "ep", + "m", + "_base", + "config.json", + PathBuf::from("/a"), + ) + .unwrap(); + reg.register( + &base(), + "ns", + "comp", + "ep", + "m", + "_base", + "config.json", + PathBuf::from("/b"), + ) + .unwrap(); + assert_eq!( + reg.get("ns", "comp", "ep", "m", "_base", "config.json"), + Some(PathBuf::from("/b")) + ); + } + + #[test] + fn different_endpoints_coexist() { + let reg = MetadataArtifactRegistry::new(); + let owner_a = (1, None); + let owner_b = (2, None); + reg.register( + &owner_a, + "ns", + "comp", + "ep-a", + "m", + "_base", + "config.json", + PathBuf::from("/a"), + ) + .unwrap(); + // Same (slug, suffix, filename) but different endpoint → no collision. + reg.register( + &owner_b, + "ns", + "comp", + "ep-b", + "m", + "_base", + "config.json", + PathBuf::from("/b"), + ) + .unwrap(); + assert_eq!( + reg.get("ns", "comp", "ep-a", "m", "_base", "config.json"), + Some(PathBuf::from("/a")) + ); + assert_eq!( + reg.get("ns", "comp", "ep-b", "m", "_base", "config.json"), + Some(PathBuf::from("/b")) + ); } } diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs index f0c6db7dcc66..6692700d70e8 100644 --- a/lib/runtime/src/system_status_server.rs +++ b/lib/runtime/src/system_status_server.rs @@ -217,10 +217,11 @@ pub async fn spawn_system_status_server( } // Self-hosted MDC files. Always mounted; empty registry → 404. - // Suffix segment scopes per-registration (LoRA slug, or `_base`) - // so detaching one registration doesn't wipe another's entries. + // The endpoint triple disambiguates multi-LocalModel-per-DRT; the + // suffix segment (LoRA slug or `_base`) scopes per-registration so + // detaching one doesn't wipe another's entries. app = app.route( - "/v1/metadata/{model_slug}/{model_suffix}/{*filename}", + "/v1/metadata/{namespace}/{component}/{endpoint}/{model_slug}/{model_suffix}/{*filename}", get({ let state = Arc::clone(&server_state); move |path| metadata_file_handler(State(state), path) @@ -487,20 +488,33 @@ async fn list_loras_handler(State(state): State>) -> impl } } -/// `GET /v1/metadata/{slug}/{suffix}/{filename}` — 404 on miss, -/// 500 on read error, raw bytes on hit. Consumer blake3-verifies. +/// `GET /v1/metadata/{namespace}/{component}/{endpoint}/{slug}/{suffix}/{filename}` +/// — 404 on miss, 500 on read error, raw bytes on hit. Consumer blake3-verifies. async fn metadata_file_handler( State(state): State>, - Path((model_slug, model_suffix, filename)): Path<(String, String, String)>, + Path((namespace, component, endpoint, model_slug, model_suffix, filename)): Path<( + String, + String, + String, + String, + String, + String, + )>, ) -> impl IntoResponse { - let path = match state - .drt() - .metadata_artifacts() - .get(&model_slug, &model_suffix, &filename) - { + let path = match state.drt().metadata_artifacts().get( + &namespace, + &component, + &endpoint, + &model_slug, + &model_suffix, + &filename, + ) { Some(p) => p, None => { tracing::debug!( + namespace, + component, + endpoint, model_slug, model_suffix, filename, @@ -514,6 +528,9 @@ async fn metadata_file_handler( Ok(bytes) => (StatusCode::OK, bytes).into_response(), Err(err) => { tracing::error!( + namespace, + component, + endpoint, model_slug, model_suffix, filename, diff --git a/tests/frontend/test_self_host_metadata.py b/tests/frontend/test_self_host_metadata.py index f391af144434..d5535977e08d 100644 --- a/tests/frontend/test_self_host_metadata.py +++ b/tests/frontend/test_self_host_metadata.py @@ -5,7 +5,8 @@ Single-worker check: opt the mocker into self-host via `DYN_SELF_HOST_METADATA=true`, then `requests.get` the -`/v1/metadata/{slug}/{suffix}/{filename}` route on the worker's own +`/v1/metadata/{namespace}/{component}/{endpoint}/{slug}/{suffix}/{filename}` +route on the worker's own `system_status_server`. No frontend resolution involved — this only exercises the worker producer + HTTP route shipped in PR1 (#8855). @@ -72,8 +73,13 @@ def test_worker_serves_metadata_via_http( slug = _slugify(TEST_MODEL) suffix = "_base" # BASE_SUFFIX in lib/runtime/src/metadata_registry.rs + # MockerWorkerProcess defaults: --component=sample, the rest pick up + # WorkerConfig defaults from lib/runtime — namespace=dynamo, + # endpoint=generate. The route is now keyed on the endpoint triple + # so the URL includes (namespace, component, endpoint). + base = f"http://localhost:{system_port}/v1/metadata/dynamo/sample/generate" - url = f"http://localhost:{system_port}/v1/metadata/{slug}/{suffix}/config.json" + url = f"{base}/{slug}/{suffix}/config.json" response = requests.get(url, timeout=10) assert ( response.status_code == 200 @@ -87,7 +93,7 @@ def test_worker_serves_metadata_via_http( # Unknown filename under the same (slug, suffix) → 404. miss = requests.get( - f"http://localhost:{system_port}/v1/metadata/{slug}/{suffix}/missing.json", + f"{base}/{slug}/{suffix}/missing.json", timeout=5, ) assert miss.status_code == 404, miss.text @@ -96,9 +102,7 @@ def test_worker_serves_metadata_via_http( # Qwen3 HF snapshot (`vocab.json`) must also be served. Doubles # as a symlink-follow check — HF snapshot entries are symlinks # into `blobs/`, so a non-following stat would miss this file. - extra_url = ( - f"http://localhost:{system_port}/v1/metadata/{slug}/{suffix}/vocab.json" - ) + extra_url = f"{base}/{slug}/{suffix}/vocab.json" extra_response = requests.get(extra_url, timeout=10) assert ( extra_response.status_code == 200