Skip to content
7 changes: 6 additions & 1 deletion lib/bindings/python/rust/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ fn lora_name_to_id(lora_name: &str) -> i32 {
/// For LoRA mode, both `lora_name` and `base_model_path` must be provided together.
/// Providing only one of them will result in an error.
#[pyfunction]
#[pyo3(signature = (model_input, model_type, endpoint, model_path, model_name=None, context_length=None, kv_cache_block_size=None, router_config=None, runtime_config=None, user_data=None, custom_template_path=None, media_decoder=None, media_fetcher=None, lora_name=None, base_model_path=None))]
#[pyo3(signature = (model_input, model_type, endpoint, model_path, model_name=None, context_length=None, kv_cache_block_size=None, router_config=None, runtime_config=None, user_data=None, custom_template_path=None, media_decoder=None, media_fetcher=None, lora_name=None, base_model_path=None, self_host_metadata=None))]
#[allow(clippy::too_many_arguments)]
fn register_model<'p>(
py: Python<'p>,
Expand All @@ -283,6 +283,7 @@ fn register_model<'p>(
media_fetcher: Option<MediaFetcher>,
lora_name: Option<&str>,
base_model_path: Option<&str>,
self_host_metadata: Option<bool>,
) -> PyResult<Bound<'p, PyAny>> {
// Validate Prefill model type requirements
if model_type.inner == llm_rs::model_type::ModelType::Prefill
Expand Down Expand Up @@ -406,6 +407,10 @@ fn register_model<'p>(
.custom_template_path(custom_template_path_owned)
.media_decoder(media_decoder.map(|m| m.inner))
.media_fetcher(media_fetcher.map(|m| m.inner));
// Absence falls through to the DYN_SELF_HOST_METADATA env var default.
if let Some(enabled) = self_host_metadata {
builder.self_host_metadata(enabled);
}

let mut local_model = builder.build().await.map_err(to_pyerr)?;

Expand Down
21 changes: 19 additions & 2 deletions lib/llm/src/common/checked_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@ use url::Url;

#[derive(Clone, Debug)]
pub struct CheckedFile {
/// Either a path on local disk or a remote URL (usually nats object store)
/// Either a path on local disk or a remote URL.
Comment thread
grahamking marked this conversation as resolved.
Resolved
path: Either<PathBuf, Url>,

/// Checksum of the contents of path
checksum: Checksum,

/// Size of the file in bytes. `None` for legacy (pre-size) wire format.
size: Option<u64>,
}

#[derive(Debug, Clone, Eq, PartialEq)]
Expand All @@ -48,11 +51,13 @@ impl CheckedFile {
if !path.is_file() {
anyhow::bail!("Not a file: {}", path.display());
}
let size = std::fs::metadata(&path)?.len();
let hash = b3sum(&path)?;

Ok(CheckedFile {
path: Either::Left(path),
checksum: Checksum::blake3(hash),
size: Some(size),
})
}

Expand Down Expand Up @@ -86,6 +91,10 @@ impl CheckedFile {
&self.checksum
}

pub fn size(&self) -> Option<u64> {
self.size
}

/// Does the given file checksum to the same value as this CheckedFile?
pub fn checksum_matches<P: AsRef<Path> + std::fmt::Debug>(&self, disk_file: P) -> bool {
match b3sum(&disk_file) {
Expand Down Expand Up @@ -144,12 +153,16 @@ impl Serialize for CheckedFile {
where
S: Serializer,
{
let mut cf = serializer.serialize_struct("CheckedFile", 2)?;
let n = if self.size.is_some() { 3 } else { 2 };
let mut cf = serializer.serialize_struct("CheckedFile", n)?;
match &self.path {
Either::Left(path) => cf.serialize_field("path", &path)?,
Either::Right(url) => cf.serialize_field("path", &url)?,
};
cf.serialize_field("checksum", &self.checksum)?;
if let Some(size) = self.size {
cf.serialize_field("size", &size)?;
}
cf.end()
}
}
Expand All @@ -159,6 +172,8 @@ impl Serialize for CheckedFile {
struct WireCheckedFile {
path: String,
checksum: Checksum,
#[serde(default)]
size: Option<u64>,
}

// Convert from the temporary struct to CheckedFile with path type logic.
Expand All @@ -169,10 +184,12 @@ impl From<WireCheckedFile> for CheckedFile {
Ok(url) => CheckedFile {
path: Either::Right(url),
checksum: temp.checksum,
size: temp.size,
},
Err(_) => CheckedFile {
path: Either::Left(PathBuf::from(temp.path)),
checksum: temp.checksum,
size: temp.size,
},
}
}
Expand Down
146 changes: 146 additions & 0 deletions lib/llm/src/local_model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,19 @@ const DEFAULT_KV_CACHE_BLOCK_SIZE: u32 = 16;
/// 'pub' because the bindings use it for consistency.
pub const DEFAULT_HTTP_PORT: u16 = 8080;

/// Default for `LocalModelBuilder::self_host_metadata`. Truthy values opt in.
pub const ENV_SELF_HOST_METADATA: &str = "DYN_SELF_HOST_METADATA";

fn env_self_host_metadata_default() -> bool {
match std::env::var(ENV_SELF_HOST_METADATA) {
Ok(v) => matches!(
v.trim().to_lowercase().as_str(),
"1" | "true" | "yes" | "on"
),
Err(_) => false,
}
}

pub struct LocalModelBuilder {
model_path: Option<PathBuf>,
source_path: Option<PathBuf>,
Expand All @@ -52,6 +65,7 @@ pub struct LocalModelBuilder {
is_mocker: bool,
extra_engine_args: Option<PathBuf>,
runtime_config: ModelRuntimeConfig,
self_host_metadata: bool,
user_data: Option<serde_json::Value>,
custom_template_path: Option<PathBuf>,
namespace: Option<String>,
Expand Down Expand Up @@ -81,6 +95,7 @@ impl Default for LocalModelBuilder {
is_mocker: Default::default(),
extra_engine_args: Default::default(),
runtime_config: Default::default(),
self_host_metadata: env_self_host_metadata_default(),
user_data: Default::default(),
custom_template_path: Default::default(),
namespace: Default::default(),
Expand Down Expand Up @@ -142,6 +157,13 @@ impl LocalModelBuilder {
self
}

/// Opt in or out of self-hosting MDC artifacts. Default `false`.
/// Set this at runtime with environment variable DYN_SELF_HOST_METADATA.
pub fn self_host_metadata(&mut self, enabled: bool) -> &mut Self {
self.self_host_metadata = enabled;
self
}

pub fn tls_cert_path(&mut self, p: Option<PathBuf>) -> &mut Self {
self.tls_cert_path = p;
self
Expand Down Expand Up @@ -269,6 +291,7 @@ impl LocalModelBuilder {
namespace_prefix: self.namespace_prefix.clone(),
migration_limit: self.migration_limit,
migration_max_seq_len: self.migration_max_seq_len,
self_host_metadata: self.self_host_metadata,
});
}

Expand Down Expand Up @@ -324,6 +347,7 @@ impl LocalModelBuilder {
namespace_prefix: self.namespace_prefix.clone(),
migration_limit: self.migration_limit,
migration_max_seq_len: self.migration_max_seq_len,
self_host_metadata: self.self_host_metadata,
})
}
}
Expand All @@ -345,6 +369,7 @@ pub struct LocalModel {
namespace_prefix: Option<String>,
migration_limit: u32,
migration_max_seq_len: Option<u32>,
self_host_metadata: bool,
}

impl LocalModel {
Expand Down Expand Up @@ -469,6 +494,11 @@ impl LocalModel {
suffix_for_log
);

if self.self_host_metadata {
self.move_to_self_host(endpoint.drt())
.context("move_to_self_host")?;
}

let source_path = PathBuf::from(self.card.source_path());
if !source_path.exists() {
// The consumers of MDC (frontend) might not have the same local path as us, so
Expand Down Expand Up @@ -501,6 +531,75 @@ impl LocalModel {
Ok(())
}

/// Local-path slots register in the registry and get rewritten to
/// `http://<worker>/v1/metadata/<slug>/<filename>`. URL slots
/// (`hf://`, etc.) are left alone so existing transports keep working.
fn move_to_self_host(
&mut self,
drt: &dynamo_runtime::DistributedRuntime,
) -> anyhow::Result<()> {
let Some(base_url) = self_host_base_url(drt)? else {
tracing::warn!(
model_slug = %self.card.slug(),
"self_host_metadata enabled but system_status_server is not \
running (DYN_SYSTEM_PORT unset); skipping http rewrites — \
set DYN_SYSTEM_PORT to enable",
);
return Ok(());
};
let model_slug = self.card.slug().to_string();
let registry = drt.metadata_artifacts();

let mut rewritten = 0usize;
for cf in self.card.iter_metadata_files_mut() {
let Some(local_path) = cf.path().map(Path::to_path_buf) else {
continue;
};
// Filename from the original path — HF cache symlinks would
// otherwise canonicalize to the LFS SHA1 and break downstream
// lookups by literal name (e.g. `parent.join("generation_config.json")`).
let Some(filename) = local_path
.file_name()
.and_then(|f| f.to_str())
.map(|s| s.to_string())
else {
continue;
};
// Canonicalize so the handler serves the resolved HF blob, not the symlink.
let absolute = match fs::canonicalize(&local_path) {
Ok(p) => p,
Err(err) => {
tracing::warn!(
path = %local_path.display(),
%err,
"failed to canonicalize self-host metadata path; skipping",
);
continue;
}
};

let url = url::Url::parse(&format!("{base_url}/v1/metadata/{model_slug}/{filename}"))?;
registry.register(&model_slug, &filename, absolute);
cf.move_to_url(url);
rewritten += 1;
}

tracing::debug!(
model_slug,
rewritten,
base_url,
"self-hosting model metadata artifacts"
);
Ok(())
}

/// Idempotent. Call before detach/hot-reload; otherwise the
/// runtime drops the registry entries with the worker process.
pub fn clear_self_hosted_artifacts(&self, drt: &dynamo_runtime::DistributedRuntime) {
drt.metadata_artifacts()
.unregister_model(self.card.slug().as_ref());
}
Comment thread
nnshah1 marked this conversation as resolved.
Resolved

/// Helper associated function to detach a model from an endpoint
///
/// For base models, pass `lora_name = None`.
Expand Down Expand Up @@ -550,3 +649,50 @@ fn internal_endpoint(engine: &str) -> EndpointId {
name: "generate".to_string(),
}
}

/// `None` when `system_status_server` isn't running (no `DYN_SYSTEM_PORT`)
/// — lets default-on behavior degrade gracefully without erroring.
pub(crate) fn self_host_base_url(
drt: &dynamo_runtime::DistributedRuntime,
) -> anyhow::Result<Option<String>> {
let Some(info) = drt.system_status_server_info() else {
return Ok(None);
};

let configured = dynamo_runtime::RuntimeConfig::from_settings()
.unwrap_or_default()
.system_host;
let host = match configured.as_str() {
"0.0.0.0" | "::" | "[::]" => {
dynamo_runtime::utils::ip_resolver::get_local_ip_for_advertise()
}
_ => configured,
};

Ok(Some(format!("http://{host}:{}", info.port())))
}

#[cfg(test)]
mod env_self_host_metadata_tests {
use super::*;

#[serial_test::serial]
#[test]
fn env_default_parsing() {
// SAFETY: all env mutations are serialized via serial_test.
unsafe { std::env::remove_var(ENV_SELF_HOST_METADATA) };
assert!(!env_self_host_metadata_default(), "unset → default OFF");

for v in [
"0", "false", "FALSE", "no", "NO", "off", "OFF", "", "garbage",
] {
unsafe { std::env::set_var(ENV_SELF_HOST_METADATA, v) };
assert!(!env_self_host_metadata_default(), "expected OFF for {v:?}");
}
for v in ["1", "true", "TRUE", "yes", "Yes", "on", "ON"] {
unsafe { std::env::set_var(ENV_SELF_HOST_METADATA, v) };
assert!(env_self_host_metadata_default(), "expected ON for {v:?}");
}
unsafe { std::env::remove_var(ENV_SELF_HOST_METADATA) };
}
}
Loading
Loading