Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 7 additions & 11 deletions components/src/dynamo/frontend/sglang_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from dynamo._internal import ModelDeploymentCard
from dynamo.frontend.frontend_args import FrontendConfig
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine, fetch_model
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine
from dynamo.llm.exceptions import InvalidArgument, Unknown

from .sglang_prepost import (
Expand Down Expand Up @@ -623,15 +623,11 @@ async def chat_engine_factory(
)
loop = asyncio.get_running_loop()

# TODO(gh-8749): consume mdc.model_info.path()'s parent (slug_dir)
# instead of re-running fetch_model. The MDC cache already has
# blake3-verified copies; this path duplicates the download.
source_path = mdc.source_path()
if not os.path.exists(source_path):
source_path = await fetch_model(source_path, ignore_weights=True)
# download_config already populated mdc.local_dir(); no refetch.
Comment thread
nnshah1 marked this conversation as resolved.
Resolved
local_dir = mdc.local_dir()

logger.info("Loading SGLang tokenizer from %s", source_path)
tokenizer = _load_tokenizer(source_path, self.trust_remote_code)
logger.info("Loading SGLang tokenizer from %s", local_dir)
tokenizer = _load_tokenizer(local_dir, self.trust_remote_code)

eos_token_id = getattr(tokenizer, "eos_token_id", None)

Expand Down Expand Up @@ -662,13 +658,13 @@ async def chat_engine_factory(
logger.info(
"Creating SGLang preprocess worker pool with %d workers for %s",
preprocess_workers,
source_path,
local_dir,
)
preprocess_pool = ProcessPoolExecutor(
max_workers=preprocess_workers,
initializer=_init_worker,
initargs=(
source_path,
local_dir,
tool_call_parser_name,
reasoning_parser_name,
self.config.exclude_tools_when_tool_choice_none,
Expand Down
14 changes: 5 additions & 9 deletions components/src/dynamo/frontend/vllm_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
from dynamo.common.multimodal.routing_utils import build_mm_routing_info_from_features
from dynamo.common.utils import nvtx_utils as _nvtx
from dynamo.frontend.frontend_args import FrontendConfig
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine, fetch_model
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine

from .prepost import StreamingPostProcessor, preprocess_chat_request
from .utils import (
Expand Down Expand Up @@ -771,12 +771,8 @@ async def chat_engine_factory(
)
loop = asyncio.get_running_loop()

# TODO(gh-8749): consume mdc.model_info.path()'s parent (slug_dir)
# instead of re-running fetch_model. The MDC cache already has
# blake3-verified copies; this path duplicates the download.
source_path = mdc.source_path()
if not os.path.exists(source_path):
source_path = await fetch_model(source_path, ignore_weights=True)
# download_config already populated mdc.local_dir(); no refetch.
Comment thread
nnshah1 marked this conversation as resolved.
Resolved
local_dir = mdc.local_dir()

tokenizer_mode = getattr(self.flags, "tokenizer_mode", None) or "auto"
config_format = getattr(self.flags, "config_format", None) or "auto"
Expand All @@ -785,7 +781,7 @@ async def chat_engine_factory(
enable_auto_tool_choice = getattr(self.flags, "enable_auto_tool_choice", False)

model_config = ModelConfig(
model=source_path,
model=local_dir,
tokenizer_mode=tokenizer_mode,
config_format=config_format,
trust_remote_code=trust_remote_code,
Expand Down Expand Up @@ -820,7 +816,7 @@ async def chat_engine_factory(
# vLLM's renderer skips its AutoProcessor fallback when tools are present,
# so tool calls crash unless tokenizer.chat_template is set; load from disk.
if tokenizer.chat_template is None:
tokenizer.chat_template = resolve_chat_template(source_path)
tokenizer.chat_template = resolve_chat_template(local_dir)

# --chat-template overrides; load_chat_template accepts either a file path
# or an inline Jinja template string.
Expand Down
12 changes: 12 additions & 0 deletions lib/bindings/python/rust/llm/model_card.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

use super::*;
use llm_rs::model_card::ModelDeploymentCard as RsModelDeploymentCard;
use pyo3::exceptions::PyValueError;

#[pyclass]
#[derive(Clone)]
Expand Down Expand Up @@ -35,6 +36,17 @@ impl ModelDeploymentCard {
self.inner.source_path()
}

/// Resolved metadata directory (post-`download_config`).
fn local_dir(&self) -> PyResult<String> {
self.inner
.local_dir()
.into_os_string()
.into_string()
.map_err(|os| {
PyValueError::new_err(format!("MDC local_dir contains non-UTF-8 bytes: {os:?}"))
})
}

fn name(&self) -> &str {
self.inner.name()
}
Expand Down
5 changes: 5 additions & 0 deletions lib/bindings/python/src/dynamo/_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -708,6 +708,11 @@ class ModelDeploymentCard:
"""Return the source path of this deployment card."""
...

def local_dir(self) -> str:
"""Resolved metadata directory (post-`download_config`). Raises
ValueError if the path contains non-UTF-8 bytes."""
...

def runtime_config(self) -> Any:
"""Return the runtime configuration as a dict."""
...
Expand Down
63 changes: 45 additions & 18 deletions lib/llm/src/model_card.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,13 +224,20 @@ fn mdc_blobs_dir() -> anyhow::Result<PathBuf> {
Ok(dir)
}

fn mdc_slug_dir(slug: &Slug, mdcsum: &str) -> anyhow::Result<PathBuf> {
let dir = mdc_cache_root()
/// Per-MDC cache directory: `<root>/by-slug/<slug>/<mdcsum>/`.
/// Pure path computation; use [`mdc_local_dir`] when you need the
/// directory created.
fn mdc_local_path(slug: &Slug, mdcsum: &str) -> PathBuf {
mdc_cache_root()
.join("by-slug")
.join(slug.to_string())
.join(mdcsum);
.join(mdcsum)
}

fn mdc_local_dir(slug: &Slug, mdcsum: &str) -> anyhow::Result<PathBuf> {
let dir = mdc_local_path(slug, mdcsum);
std::fs::create_dir_all(&dir)
.with_context(|| format!("creating MDC slug dir {}", dir.display()))?;
.with_context(|| format!("creating MDC local dir {}", dir.display()))?;
Ok(dir)
}

Expand Down Expand Up @@ -350,9 +357,9 @@ fn file_uri_parent(uri: &str) -> Option<PathBuf> {
parent.is_dir().then(|| parent.to_path_buf())
}

/// Symlink non-weight files from `snapshot_dir` into `slug_dir`. Picks up
/// Symlink non-weight files from `snapshot_dir` into `local_dir`. Picks up
/// `preprocessor_config.json` and other sibling files that
/// `from_pretrained(slug_dir)` consumers need.
/// `from_pretrained(local_dir)` consumers need.
///
/// Names in `typed_filenames` are owned by the resolve loop's typed-slot
/// pass — never overwritten. Every other harvested sibling is re-linked
Expand All @@ -361,7 +368,7 @@ fn file_uri_parent(uri: &str) -> Option<PathBuf> {
/// cover harvested files).
fn harvest_siblings(
snapshot_dir: &Path,
slug_dir: &Path,
local_dir: &Path,
typed_filenames: &std::collections::HashSet<String>,
) -> anyhow::Result<()> {
let entries = match std::fs::read_dir(snapshot_dir) {
Expand All @@ -388,7 +395,7 @@ fn harvest_siblings(
if typed_filenames.contains(&name) {
continue;
}
let dst = slug_dir.join(&name);
let dst = local_dir.join(&name);
// Resolve through the canonical target so a downstream
// `canonicalize` lands on a stable blob path rather than
// chasing snapshot-dir symlinks. `symlink_force` is idempotent
Expand All @@ -399,7 +406,7 @@ fn harvest_siblings(
tracing::debug!(
file = %name,
target = %target.display(),
"harvested sibling into slug_dir",
"harvested sibling into local_dir",
);
}
Ok(())
Expand Down Expand Up @@ -837,6 +844,14 @@ impl ModelDeploymentCard {
Ok(serde_json::to_string(self)?)
}

/// Per-MDC resolve directory. After `download_config` runs, every
/// typed slot + harvested sibling is symlinked here for
/// `from_pretrained(local_dir)` consumers. Pure path — does not
/// create the directory; the resolve pipeline owns that.
pub fn local_dir(&self) -> PathBuf {
mdc_local_path(&self.slug, self.mdcsum())
}

pub fn mdcsum(&self) -> &str {
self.checksum
.get_or_init(|| {
Expand Down Expand Up @@ -870,7 +885,7 @@ impl ModelDeploymentCard {
// (a) workers with identical siblings produce the same
// mdcsum regardless of `read_dir` order, and (b) the same
// bytes under different filenames don't collide — otherwise
// the frontend cache could serve a slug_dir missing siblings.
// the frontend cache could serve a local_dir missing siblings.
let mut extras: Vec<(&str, &str)> = self
.extra_files
.iter()
Expand Down Expand Up @@ -1229,7 +1244,7 @@ impl ModelDeploymentCard {
let source = self.source_path().to_string();
let mdcsum = self.mdcsum().to_string();
let blobs = mdc_blobs_dir()?;
let slug_dir = mdc_slug_dir(&self.slug, &mdcsum)?;
let local_dir = mdc_local_dir(&self.slug, &mdcsum)?;

let entries: Vec<(String, CheckedFile)> = self
.iter_metadata_files()
Expand Down Expand Up @@ -1271,7 +1286,7 @@ impl ModelDeploymentCard {
let blob = blobs.join(blake3_hex);
tracing::debug!(filename = %filename, uri = %uri, blake3 = %blake3_hex, "resolving");
resolve_uri(&client, uri, expected, &blob, &hf_snapshots).await?;
symlink_force(&blob, &slug_dir.join(&filename))?;
symlink_force(&blob, &local_dir.join(&filename))?;
}
tracing::debug!(
display_name = %self.display_name,
Expand All @@ -1295,13 +1310,13 @@ impl ModelDeploymentCard {
}
}
for snap in &snapshot_dirs {
harvest_siblings(snap, &slug_dir, &typed_filenames)?;
harvest_siblings(snap, &local_dir, &typed_filenames)?;
}

// Pass 3: rewrite cf.path to the cache symlink so downstream
// tokenizer/config loaders read from a verified location.
for (cf, _) in self.iter_metadata_files_mut() {
cf.update_dir(&slug_dir);
cf.update_dir(&local_dir);
}
Ok(())
}
Expand Down Expand Up @@ -1949,7 +1964,7 @@ fn check_valid_local_repo_path(path: impl AsRef<Path>) -> Result<()> {

#[cfg(test)]
mod tests {
use super::HFConfig;
use super::{HFConfig, ModelDeploymentCard};
use std::collections::HashSet;
use std::path::{Path, PathBuf};

Expand Down Expand Up @@ -2152,7 +2167,7 @@ mod tests {

// Sibling harvest: TinyLlama_v1.1 fixture ships
// `special_tokens_map.json` and `tokenizer.model` outside the
// typed slots — both must land in slug_dir for
// typed slots — both must land in local_dir for
// `from_pretrained()` to see a complete model dir.
assert!(snap.join("special_tokens_map.json").exists());
assert!(snap.join("tokenizer.model").exists());
Expand Down Expand Up @@ -2180,7 +2195,7 @@ mod tests {

/// Two MDCs with `extra_files` that share bytes but differ in basename
/// must produce distinct mdcsums — otherwise the frontend cache would
/// alias them and a slug_dir built from one worker's harvest would be
/// alias them and a local_dir built from one worker's harvest would be
/// reused for another worker that needs a differently-named sibling.
#[test]
fn mdcsum_extras_distinguish_basename_at_equal_checksum() {
Expand Down Expand Up @@ -2236,6 +2251,18 @@ mod tests {
);
}

#[test]
fn local_dir_computes_expected_path() {
// Sentinel for cache-layout drift: the public `local_dir()` must
// stay in lockstep with `mdc_local_path`. Resolve-pipeline
// integration is covered by the vllm/sglang serve tests.
let card = ModelDeploymentCard::with_name_only("Qwen/Qwen3-0.6B");
assert_eq!(
card.local_dir(),
super::mdc_local_path(card.slug(), card.mdcsum())
);
}

/// Rung 4: when worker path is missing and `--model-path` doesn't
/// supply the basename, synthesize hf://; if `is_custom`, error
/// with a clear operator-action message instead (HF doesn't host
Expand Down Expand Up @@ -2348,7 +2375,7 @@ mod tests {
let snap = tempfile::tempdir()?;
let slug = tempfile::tempdir()?;

// Typed slot: blob in the dynamo cache; slug_dir links to it.
// Typed slot: blob in the dynamo cache; local_dir links to it.
let typed_blob = blob_dir.path().join("config-blob");
std::fs::write(&typed_blob, b"typed-slot-content")?;
super::symlink_force(&typed_blob, &slug.path().join("config.json"))?;
Expand Down
Loading