Skip to content

Commit 654a1a4

Browse files
committed
fix: resolve overlinking false positives for staging outputs
When a package output inherits from a staging cache, the staging cache's host dependencies aren't installed in the prefix during overlinking checks. This means libraries like libz.so.1 can't be attributed to zlib, even though zlib is a run dependency via inherited run_exports. Fix: at staging build time, capture a LibraryNameMap (filename to package mapping) from PrefixInfo while conda-meta still exists. Store it in the staging cache metadata, thread it to BuildOutput, and use it as a fallback in perform_linking_checks when resolve_libraries can't find the library on disk. Fixes #2186 Supersedes #2210 test: add e2e test for staging overlinking check
1 parent 809e112 commit 654a1a4

File tree

9 files changed

+213
-8
lines changed

9 files changed

+213
-8
lines changed

crates/rattler_build_core/src/build.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,12 @@ pub async fn run_build(
136136
// This will build or restore staging caches and return their dependencies/sources if inherited
137137
let staging_result = output.process_staging_caches(tool_configuration).await?;
138138

139-
// If we inherit from a staging cache, store its dependencies and sources
140-
if let Some((deps, sources)) = staging_result {
139+
// If we inherit from a staging cache, store its dependencies, sources, and
140+
// library name map for overlinking checks
141+
if let Some((deps, sources, library_name_map)) = staging_result {
141142
output.finalized_cache_dependencies = Some(deps);
142143
output.finalized_cache_sources = Some(sources);
144+
output.staging_library_name_map = Some(library_name_map);
143145
}
144146

145147
// Fetch sources for this output

crates/rattler_build_core/src/post_process/checks.rs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -528,6 +528,7 @@ pub fn perform_linking_checks(
528528
let system_libs = find_system_libs(output)?;
529529

530530
let prefix_info = PrefixInfo::from_prefix(output.prefix())?;
531+
let staging_lib_map = output.staging_library_name_map.as_ref();
531532

532533
let host_dso_packages = host_run_export_dso_packages(output, &prefix_info.package_to_nature);
533534
tracing::trace!("Host run_export DSO packages: {host_dso_packages:#?}",);
@@ -644,6 +645,26 @@ pub fn perform_linking_checks(
644645
);
645646
}
646647

648+
// Fallback: if the library couldn't be resolved on disk (e.g. from
649+
// a staging cache whose host deps are not installed), try to match
650+
// it by filename against the cached library name map.
651+
if let Some(lib_map) = staging_lib_map
652+
&& let Some(providing_package) = lib_map.find_package(lib)
653+
&& run_dependency_names.contains(&providing_package)
654+
{
655+
tracing::debug!(
656+
"Library {lib:?} matched to '{}' via staging library name map",
657+
providing_package.as_normalized()
658+
);
659+
link_info.linked_packages.push(LinkedPackage {
660+
name: lib.to_path_buf(),
661+
link_origin: LinkOrigin::ForeignPackage(
662+
providing_package.as_normalized().to_string(),
663+
),
664+
});
665+
continue;
666+
}
667+
647668
// Check if the library is one of the system libraries (i.e. comes from sysroot).
648669
if system_libs.allow.is_match(lib) && !system_libs.deny.is_match(lib) {
649670
link_info.linked_packages.push(LinkedPackage {

crates/rattler_build_core/src/post_process/package_nature.rs

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,72 @@ impl PrefixInfo {
215215
}
216216
}
217217

218+
/// A mapping from shared library filenames to the package that provides them.
219+
///
220+
/// This is used as a fallback during overlinking checks when the staging
221+
/// cache's host dependencies are not physically installed in the prefix.
222+
/// Instead of requiring files on disk, this allows name-based attribution
223+
/// of libraries to packages.
224+
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
225+
pub struct LibraryNameMap {
226+
/// Maps library filenames (e.g. "libz.so.1", "libz.1.dylib") to the
227+
/// package name that provides them.
228+
pub library_to_package: HashMap<String, PackageName>,
229+
}
230+
231+
impl LibraryNameMap {
232+
/// Build a `LibraryNameMap` from a `PrefixInfo` by extracting the
233+
/// filenames of all files that look like shared objects.
234+
pub(crate) fn from_prefix_info(prefix_info: &PrefixInfo) -> Self {
235+
let mut library_to_package = HashMap::new();
236+
237+
for (path, package_name) in &prefix_info.path_to_package {
238+
if is_dso(&path.path)
239+
&& let Some(file_name) = path.path.file_name()
240+
{
241+
library_to_package.insert(
242+
file_name.to_string_lossy().to_string(),
243+
package_name.clone(),
244+
);
245+
}
246+
}
247+
248+
Self { library_to_package }
249+
}
250+
251+
/// Look up a library path by extracting its filename and checking the map.
252+
/// Returns the package name if found.
253+
///
254+
/// Handles various library path forms:
255+
/// - Plain filenames: `libz.so.1`
256+
/// - macOS @rpath references: `@rpath/libz.1.dylib`
257+
/// - Full or relative paths: `lib/libz.so.1`
258+
pub fn find_package(&self, library: &Path) -> Option<PackageName> {
259+
let path_str = library.to_string_lossy();
260+
261+
// Strip @rpath/ or @loader_path/ prefixes (macOS)
262+
let stripped = path_str
263+
.strip_prefix("@rpath/")
264+
.or_else(|| path_str.strip_prefix("@loader_path/"))
265+
.unwrap_or(&path_str);
266+
267+
// Try the stripped path directly (handles plain filenames)
268+
if let Some(pkg) = self.library_to_package.get(stripped) {
269+
return Some(pkg.clone());
270+
}
271+
272+
// Try just the filename component
273+
let file_name = Path::new(stripped).file_name()?.to_string_lossy();
274+
275+
self.library_to_package.get(file_name.as_ref()).cloned()
276+
}
277+
278+
/// Returns true if the map is empty.
279+
pub fn is_empty(&self) -> bool {
280+
self.library_to_package.is_empty()
281+
}
282+
}
283+
218284
#[cfg(test)]
219285
mod tests {
220286
use super::*;

crates/rattler_build_core/src/staging.rs

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ use crate::{
2222
env_vars,
2323
metadata::{Output, build_reindexed_channels},
2424
packaging::Files,
25+
post_process::package_nature::{LibraryNameMap, PrefixInfo},
2526
render::resolved_dependencies::{
2627
FinalizedDependencies, RunExportsDownload, install_environments, resolve_dependencies,
2728
},
@@ -71,6 +72,13 @@ pub struct StagingCacheMetadata {
7172

7273
/// The variant configuration that was used
7374
pub variant: BTreeMap<NormalizedKey, Variable>,
75+
76+
/// Mapping from shared library filenames to the package that provides them.
77+
/// Captured at staging build time while conda-meta is still present, used
78+
/// during overlinking checks as a fallback when these packages are not
79+
/// physically installed in the host prefix.
80+
#[serde(default)]
81+
pub library_name_map: LibraryNameMap,
7482
}
7583

7684
impl Output {
@@ -140,7 +148,8 @@ impl Output {
140148
/// 2. If yes, restore the cached files to the prefix
141149
/// 3. If no, build the staging cache and save it
142150
///
143-
/// Returns the finalized dependencies and sources from the staging cache
151+
/// Returns the finalized dependencies, sources, and library name map from
152+
/// the staging cache.
144153
pub async fn build_or_restore_staging_cache(
145154
&self,
146155
staging: &StagingCache,
@@ -149,6 +158,7 @@ impl Output {
149158
(
150159
FinalizedDependencies,
151160
Vec<rattler_build_recipe::stage1::Source>,
161+
LibraryNameMap,
152162
),
153163
miette::Error,
154164
> {
@@ -215,6 +225,7 @@ impl Output {
215225
(
216226
FinalizedDependencies,
217227
Vec<rattler_build_recipe::stage1::Source>,
228+
LibraryNameMap,
218229
),
219230
miette::Error,
220231
> {
@@ -252,6 +263,13 @@ impl Output {
252263
.await
253264
.into_diagnostic()?;
254265

266+
// Capture the library name map while conda-meta still exists in the
267+
// prefix. This maps shared library filenames to their providing
268+
// packages so overlinking checks can attribute libraries even after
269+
// the staging cache's host deps are no longer installed.
270+
let prefix_info = PrefixInfo::from_prefix(self.prefix()).into_diagnostic()?;
271+
let library_name_map = LibraryNameMap::from_prefix_info(&prefix_info);
272+
255273
// Run the build script
256274
let target_platform = self.build_configuration.target_platform;
257275
let mut env_vars = env_vars::vars(self, "BUILD");
@@ -374,6 +392,7 @@ impl Output {
374392
work_dir_files: copied_work_dir.copied_paths().to_vec(),
375393
prefix: self.prefix().to_path_buf(),
376394
variant: staging.used_variant.clone(),
395+
library_name_map: library_name_map.clone(),
377396
};
378397

379398
let metadata_json = serde_json::to_string_pretty(&metadata).into_diagnostic()?;
@@ -385,7 +404,7 @@ impl Output {
385404
metadata.work_dir_files.len()
386405
);
387406

388-
Ok((finalized_dependencies, finalized_sources))
407+
Ok((finalized_dependencies, finalized_sources, library_name_map))
389408
}
390409

391410
/// Restore a staging cache from disk
@@ -397,6 +416,7 @@ impl Output {
397416
(
398417
FinalizedDependencies,
399418
Vec<rattler_build_recipe::stage1::Source>,
419+
LibraryNameMap,
400420
),
401421
miette::Error,
402422
> {
@@ -436,7 +456,11 @@ impl Output {
436456
metadata.name
437457
);
438458

439-
Ok((metadata.finalized_dependencies, metadata.finalized_sources))
459+
Ok((
460+
metadata.finalized_dependencies,
461+
metadata.finalized_sources,
462+
metadata.library_name_map,
463+
))
440464
}
441465

442466
/// Process all staging caches for this output
@@ -451,6 +475,7 @@ impl Output {
451475
Option<(
452476
FinalizedDependencies,
453477
Vec<rattler_build_recipe::stage1::Source>,
478+
LibraryNameMap,
454479
)>,
455480
miette::Error,
456481
> {
@@ -467,7 +492,7 @@ impl Output {
467492
"Building or restoring staging cache: {}",
468493
staging_cache.name
469494
);
470-
let (_deps, _sources) = self
495+
let (_deps, _sources, _lib_map) = self
471496
.build_or_restore_staging_cache(staging_cache, tool_configuration)
472497
.await?;
473498
}
@@ -488,11 +513,11 @@ impl Output {
488513
})?;
489514

490515
// Get or build the cache
491-
let (deps, sources) = self
516+
let (deps, sources, lib_map) = self
492517
.build_or_restore_staging_cache(staging, tool_configuration)
493518
.await?;
494519

495-
Ok(Some((deps, sources)))
520+
Ok(Some((deps, sources, lib_map)))
496521
} else {
497522
Ok(None)
498523
}

crates/rattler_build_core/src/types/build_output.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use std::{
2020

2121
use crate::{
2222
console_utils::github_integration_enabled,
23+
post_process::package_nature::LibraryNameMap,
2324
render::resolved_dependencies::FinalizedDependencies,
2425
system_tools::SystemTools,
2526
types::{BuildConfiguration, BuildSummary, PlatformWithVirtualPackages},
@@ -60,6 +61,12 @@ pub struct BuildOutput {
6061
/// that created this artifact
6162
#[serde(skip_serializing_if = "Option::is_none")]
6263
pub extra_meta: Option<BTreeMap<String, Value>>,
64+
65+
/// Library name map from the staging cache, used as a fallback during
66+
/// overlinking checks when the staging cache's host dependencies are not
67+
/// physically installed in the host prefix.
68+
#[serde(default, skip_serializing_if = "Option::is_none")]
69+
pub staging_library_name_map: Option<LibraryNameMap>,
6370
}
6471

6572
impl BuildOutput {

py-rattler-build/rust/src/build.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ pub(crate) fn output_from_rendered_variant(
196196
finalized_sources: None,
197197
finalized_cache_dependencies: None,
198198
finalized_cache_sources: None,
199+
staging_library_name_map: None,
199200
build_summary: Arc::new(Mutex::new(BuildSummary::default())),
200201
system_tools: SystemTools::new("rattler-build", env!("CARGO_PKG_VERSION")),
201202
extra_meta: None,

src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,7 @@ pub async fn get_build_output(
553553
finalized_sources: None,
554554
finalized_cache_dependencies: None,
555555
finalized_cache_sources: None,
556+
staging_library_name_map: None,
556557
system_tools: SystemTools::new("rattler-build", env!("CARGO_PKG_VERSION")),
557558
build_summary: Arc::new(Mutex::new(BuildSummary::default())),
558559
extra_meta: Some(
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Test case: Overlinking check with staging outputs
2+
#
3+
# When a package inherits from a staging cache, the staging cache's host
4+
# dependencies (like zlib) are not installed in the prefix during the
5+
# overlinking check. The staging library name map must be used as a fallback
6+
# to resolve libraries to their providing packages.
7+
8+
schema_version: 1
9+
10+
recipe:
11+
name: staging-overlinking-test
12+
version: "1.0.0"
13+
14+
build:
15+
number: 0
16+
dynamic_linking:
17+
overlinking_behavior: error
18+
missing_dso_allowlist:
19+
- "libc*"
20+
21+
outputs:
22+
- staging:
23+
name: compile-stage
24+
requirements:
25+
build:
26+
- ${{ compiler('c') }}
27+
host:
28+
- zlib
29+
build:
30+
script:
31+
- if: unix
32+
then: |
33+
cat > test_zlib.c << 'EOF'
34+
#include <zlib.h>
35+
#include <stdio.h>
36+
int main() {
37+
printf("zlib version: %s\n", zlibVersion());
38+
return 0;
39+
}
40+
EOF
41+
mkdir -p $PREFIX/bin
42+
$CC $CFLAGS $LDFLAGS test_zlib.c -lz -o $PREFIX/bin/test_zlib
43+
- if: win
44+
then: |
45+
echo int main() { return 0; } > test_zlib.c
46+
mkdir %PREFIX%\bin
47+
cl test_zlib.c /Fe%PREFIX%\bin\test_zlib.exe
48+
49+
- package:
50+
name: staging-overlinking-test
51+
inherit: compile-stage
52+
build:
53+
files:
54+
- bin/**

test/end-to-end/test_staging.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -614,6 +614,34 @@ def test_staging_run_exports_ignore_by_name(
614614
)
615615

616616

617+
def test_staging_overlinking(
618+
rattler_build: RattlerBuild, recipes: Path, tmp_path: Path
619+
):
620+
"""Test that overlinking checks pass for staging outputs via library name map.
621+
622+
When a package inherits from a staging cache, the staging cache's host
623+
dependencies (e.g. zlib) are not installed in the host prefix during the
624+
overlinking check of the inheriting package. The fix captures a library name
625+
map at staging build time and uses it as a fallback in the overlinking check.
626+
627+
This test compiles a small C program that links against libz, packages it
628+
via a staging output with overlinking_behavior: error, and verifies the
629+
build succeeds (i.e. libz.so.1 is correctly attributed to zlib via the
630+
staging library name map).
631+
"""
632+
rattler_build.build(
633+
recipes / "staging/staging-overlinking",
634+
tmp_path,
635+
extra_args=["--experimental"],
636+
)
637+
638+
pkg = get_extracted_package(tmp_path, "staging-overlinking-test")
639+
if platform.system() == "Windows":
640+
assert (pkg / "bin/test_zlib.exe").exists()
641+
else:
642+
assert (pkg / "bin/test_zlib").exists()
643+
644+
617645
if __name__ == "__main__":
618646
# Allow running individual tests
619647
pytest.main([__file__, "-v"])

0 commit comments

Comments
 (0)