Always build libraries into the same location

Previously Cargo would compile a library into a different location depending on
whether it was the "root crate" or not. In the ongoing saga of reducing Cargo's
reliance on the idea of a "root crate" this PR is the next step. With workspaces
the root crate of a compliation changes all the time, so the output needs to be
the same whether a crate is at the root or not.

Fixing this inconsistence in turn fixes bugs like #2855 and #2897 which arise
due to this discrepancy. Additionally, Cargo will no longer recompile a library
when it's used as a "root crate" or not.

This is fixed by taking a few steps:

* Everything is now compiled into the `deps` directory, regardless of whether
  it's a root output or not.
* If a "root crate" is being compiled, then the relevant outputs are hard-linked
  up one level to where they are today. This means that your binaries, dylibs,
  staticlibs, etc, will all show up where they used to.
* The `-C metadata` flag is always omitted for path dependencies now. These
  dependencies are always path dependencies and already all have unique crate
  names. Additionally, they're the only crates in the DAG without metadata, so
  there's no need to provide additional metadata. This in turn means that none
  of the file names of the generated crates are mangled.

Closes #2855

Author: alexcrichton

src/cargo/ops/cargo_clean.rs

@@ -73,7 +73,7 @@ pub fn clean(ws: &Workspace, opts: &CleanOptions) -> CargoResult<()> {
     try!(cx.probe_target_info(&units));
 
     for unit in units.iter() {
-        let layout = cx.layout(&unit.pkg, unit.kind);
+        let layout = cx.layout(unit);
         try!(rm_rf(&layout.proxy().fingerprint(&unit.pkg)));
         try!(rm_rf(&layout.build(&unit.pkg)));
 

src/cargo/ops/cargo_rustc/context.rs

@@ -96,7 +96,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
 
     /// Prepare this context, ensuring that all filesystem directories are in
     /// place.
-    pub fn prepare(&mut self, root: &Package) -> CargoResult<()> {
+    pub fn prepare(&mut self) -> CargoResult<()> {
         let _p = profile::start("preparing layout");
 
         try!(self.host.prepare().chain_error(|| {
@@ -111,10 +111,9 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             None => {}
         }
 
-        self.compilation.root_output =
-                self.layout(root, Kind::Target).proxy().dest().to_path_buf();
-        self.compilation.deps_output =
-                self.layout(root, Kind::Target).proxy().deps().to_path_buf();
+        let layout = self.target.as_ref().unwrap_or(&self.host);
+        self.compilation.root_output = layout.dest().to_path_buf();
+        self.compilation.deps_output = layout.deps().to_path_buf();
         Ok(())
     }
 
@@ -237,23 +236,28 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
     }
 
     /// Returns the appropriate directory layout for either a plugin or not.
-    pub fn layout(&self, pkg: &Package, kind: Kind) -> LayoutProxy {
-        let primary = pkg.package_id() == self.resolve.root();
-        match kind {
+    pub fn layout(&self, unit: &Unit) -> LayoutProxy {
+        let primary = unit.pkg.package_id() == self.resolve.root();
+        match unit.kind {
             Kind::Host => LayoutProxy::new(&self.host, primary),
             Kind::Target => LayoutProxy::new(self.target.as_ref()
                                                  .unwrap_or(&self.host),
                                              primary),
         }
     }
 
+    /// Returns the path for plugin/dylib dependencies
+    pub fn host_dylib_path(&self) -> &Path {
+        self.host.deps()
+    }
+
     /// Returns the appropriate output directory for the specified package and
     /// target.
     pub fn out_dir(&self, unit: &Unit) -> PathBuf {
         if unit.profile.doc {
-            self.layout(unit.pkg, unit.kind).doc_root()
+            self.layout(unit).doc_root()
         } else {
-            self.layout(unit.pkg, unit.kind).out_dir(unit.pkg, unit.target)
+            self.layout(unit).out_dir(unit)
         }
     }
 
@@ -289,7 +293,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             let mut metadata = unit.pkg.generate_metadata();
             metadata.mix(&format!("bin-{}", unit.target.name()));
             Some(metadata)
-        } else if unit.pkg.package_id() == self.resolve.root() &&
+        } else if unit.pkg.package_id().source_id().is_path() &&
                   !unit.profile.test {
             // If we're not building a unit test then the root package never
             // needs any metadata as it's guaranteed to not conflict with any

src/cargo/ops/cargo_rustc/custom_build.rs

@@ -87,9 +87,10 @@ pub fn prepare<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>)
 
 fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>)
                         -> CargoResult<(Work, Work)> {
+    let host_unit = Unit { kind: Kind::Host, ..*unit };
     let (script_output, build_output) = {
-        (cx.layout(unit.pkg, Kind::Host).build(unit.pkg),
-         cx.layout(unit.pkg, unit.kind).build_out(unit.pkg))
+        (cx.layout(&host_unit).build(unit.pkg),
+         cx.layout(unit).build_out(unit.pkg))
     };
 
     // Building the command to execute
@@ -161,8 +162,8 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>)
     };
     cx.build_explicit_deps.insert(*unit, (output_file.clone(), rerun_if_changed));
 
-    try!(fs::create_dir_all(&cx.layout(unit.pkg, Kind::Host).build(unit.pkg)));
-    try!(fs::create_dir_all(&cx.layout(unit.pkg, unit.kind).build(unit.pkg)));
+    try!(fs::create_dir_all(&cx.layout(&host_unit).build(unit.pkg)));
+    try!(fs::create_dir_all(&cx.layout(unit).build(unit.pkg)));
 
     // Prepare the unit of "dirty work" which will actually run the custom build
     // command.

src/cargo/ops/cargo_rustc/fingerprint.rs

@@ -497,7 +497,7 @@ pub fn prepare_init(cx: &mut Context, unit: &Unit) -> CargoResult<()> {
 
 /// Return the (old, new) location for fingerprints for a package
 pub fn dir(cx: &Context, unit: &Unit) -> PathBuf {
-    cx.layout(unit.pkg, unit.kind).proxy().fingerprint(unit.pkg)
+    cx.layout(unit).proxy().fingerprint(unit.pkg)
 }
 
 /// Returns the (old, new) location for the dep info file of a target.

src/cargo/ops/cargo_rustc/layout.rs

@@ -49,9 +49,10 @@ use std::fs;
 use std::io;
 use std::path::{PathBuf, Path};
 
-use core::{Package, Target, Workspace};
+use core::{Package, Workspace};
 use util::{Config, FileLock, CargoResult, Filesystem};
 use util::hex::short_hash;
+use super::Unit;
 
 pub struct Layout {
     root: PathBuf,
@@ -165,11 +166,13 @@ impl<'a> LayoutProxy<'a> {
 
     pub fn proxy(&self) -> &'a Layout { self.root }
 
-    pub fn out_dir(&self, pkg: &Package, target: &Target) -> PathBuf {
-        if target.is_custom_build() {
-            self.build(pkg)
-        } else if target.is_example() {
+    pub fn out_dir(&self, unit: &Unit) -> PathBuf {
+        if unit.target.is_custom_build() {
+            self.build(unit.pkg)
+        } else if unit.target.is_example() {
             self.examples().to_path_buf()
+        } else if unit.target.is_lib() {
+            self.deps().to_path_buf()
         } else {
             self.root().to_path_buf()
         }

src/cargo/ops/cargo_rustc/mod.rs

@@ -87,7 +87,7 @@ pub fn compile_targets<'a, 'cfg: 'a>(ws: &Workspace<'cfg>,
 
     let mut queue = JobQueue::new(&cx);
 
-    try!(cx.prepare(root));
+    try!(cx.prepare());
     try!(cx.probe_target_info(&units));
     try!(custom_build::build_map(&mut cx, &units));
 
@@ -104,7 +104,7 @@ pub fn compile_targets<'a, 'cfg: 'a>(ws: &Workspace<'cfg>,
     try!(queue.execute(&mut cx));
 
     for unit in units.iter() {
-        let out_dir = cx.layout(unit.pkg, unit.kind).build_out(unit.pkg)
+        let out_dir = cx.layout(unit).build_out(unit.pkg)
                         .display().to_string();
         cx.compilation.extra_env.entry(unit.pkg.package_id().clone())
           .or_insert(Vec::new())
@@ -228,6 +228,7 @@ fn rustc(cx: &mut Context, unit: &Unit) -> CargoResult<Work> {
     let do_rename = unit.target.allows_underscores() && !unit.profile.test;
     let real_name = unit.target.name().to_string();
     let crate_name = unit.target.crate_name();
+    let move_outputs_up = unit.pkg.package_id() == cx.resolve.root();
 
     let rustc_dep_info_loc = if do_rename {
         root.join(&crate_name)
@@ -271,7 +272,7 @@ fn rustc(cx: &mut Context, unit: &Unit) -> CargoResult<Work> {
             let src = dst.with_file_name(dst.file_name().unwrap()
                                             .to_str().unwrap()
                                             .replace(&real_name, &crate_name));
-            if !has_custom_args || fs::metadata(&src).is_ok() {
+            if !has_custom_args || src.exists() {
                 try!(fs::rename(&src, &dst).chain_error(|| {
                     internal(format!("could not rename crate {:?}", src))
                 }));
@@ -286,6 +287,40 @@ fn rustc(cx: &mut Context, unit: &Unit) -> CargoResult<Work> {
             try!(fingerprint::append_current_dir(&dep_info_loc, &cwd));
         }
 
+        // If we're a "root crate", e.g. the target of this compilation, then we
+        // hard link our outputs out of the `deps` directory into the directory
+        // above. This means that `cargo build` will produce binaries in
+        // `target/debug` which one probably expects.
+        if move_outputs_up {
+            for &(ref filename, _linkable) in filenames.iter() {
+                let src = root.join(filename);
+                // This may have been a `cargo rustc` command which changes the
+                // output, so the source may not actually exist.
+                if !src.exists() {
+                    continue
+                }
+
+                // We currently only lift files up from the `deps` directory. If
+                // it was compiled into something like `example/` or `doc/` then
+                // we don't want to link it up.
+                let src_dir = src.parent().unwrap();
+                if !src_dir.ends_with("deps") {
+                    continue
+                }
+                let dst = src_dir.parent().unwrap()
+                                 .join(src.file_name().unwrap());
+                if dst.exists() {
+                    try!(fs::remove_file(&dst).chain_error(|| {
+                        human(format!("failed to remove: {}", dst.display()))
+                    }));
+                }
+                try!(fs::hard_link(&src, &dst).chain_error(|| {
+                    human(format!("failed to link `{}` to `{}`",
+                                  src.display(), dst.display()))
+                }));
+            }
+        }
+
         Ok(())
     }));
 
@@ -390,8 +425,7 @@ fn rustdoc(cx: &mut Context, unit: &Unit) -> CargoResult<Work> {
     try!(build_deps_args(&mut rustdoc, cx, unit));
 
     if unit.pkg.has_custom_build() {
-        rustdoc.env("OUT_DIR", &cx.layout(unit.pkg, unit.kind)
-                                  .build_out(unit.pkg));
+        rustdoc.env("OUT_DIR", &cx.layout(unit).build_out(unit.pkg));
     }
 
     rustdoc.args(&try!(cx.rustdocflags_args(unit)));
@@ -552,12 +586,7 @@ fn build_plugin_args(cmd: &mut CommandPrototype, cx: &Context, unit: &Unit) {
 
 fn build_deps_args(cmd: &mut CommandPrototype, cx: &Context, unit: &Unit)
                    -> CargoResult<()> {
-    let layout = cx.layout(unit.pkg, unit.kind);
-    cmd.arg("-L").arg(&{
-        let mut root = OsString::from("dependency=");
-        root.push(layout.root());
-        root
-    });
+    let layout = cx.layout(unit);
     cmd.arg("-L").arg(&{
         let mut deps = OsString::from("dependency=");
         deps.push(layout.deps());
@@ -569,7 +598,7 @@ fn build_deps_args(cmd: &mut CommandPrototype, cx: &Context, unit: &Unit)
     }
 
     for unit in try!(cx.dep_targets(unit)).iter() {
-        if unit.target.linkable() {
+        if unit.target.linkable() && !unit.profile.doc {
             try!(link_to(cmd, cx, unit));
         }
     }
@@ -578,16 +607,14 @@ fn build_deps_args(cmd: &mut CommandPrototype, cx: &Context, unit: &Unit)
 
     fn link_to(cmd: &mut CommandPrototype, cx: &Context, unit: &Unit)
                -> CargoResult<()> {
-        let layout = cx.layout(unit.pkg, unit.kind);
-
         for (filename, linkable) in try!(cx.target_filenames(unit)) {
             if !linkable {
                 continue
             }
             let mut v = OsString::new();
             v.push(&unit.target.crate_name());
             v.push("=");
-            v.push(layout.root());
+            v.push(cx.out_dir(unit));
             v.push(&path::MAIN_SEPARATOR.to_string());
             v.push(&filename);
             cmd.arg("--extern").arg(&v);
@@ -600,9 +627,8 @@ pub fn process(cmd: CommandType, pkg: &Package,
                cx: &Context) -> CargoResult<CommandPrototype> {
     // When invoking a tool, we need the *host* deps directory in the dynamic
     // library search path for plugins and such which have dynamic dependencies.
-    let layout = cx.layout(pkg, Kind::Host);
     let mut search_path = util::dylib_path();
-    search_path.push(layout.deps().to_path_buf());
+    search_path.push(cx.host_dylib_path().to_path_buf());
 
     // We want to use the same environment and such as normal processes, but we
     // want to override the dylib search path with the one we just calculated.

src/cargo/ops/cargo_test.rs

@@ -134,7 +134,7 @@ fn run_doc_tests(options: &TestOptions,
             p.arg("--test").arg(lib)
              .arg("--crate-name").arg(&crate_name);
 
-            for &rust_dep in &[&compilation.deps_output, &compilation.root_output] {
+            for &rust_dep in &[&compilation.deps_output] {
                 let mut arg = OsString::from("dependency=");
                 arg.push(rust_dep);
                 p.arg("-L").arg(arg);

tests/build-lib.rs

@@ -10,9 +10,8 @@ fn verbose_output_for_lib(p: &ProjectBuilder) -> String {
     format!("\
 [COMPILING] {name} v{version} ({url})
 [RUNNING] `rustc src{sep}lib.rs --crate-name {name} --crate-type lib -g \
-        --out-dir {dir}{sep}target{sep}debug \
+        --out-dir [..] \
         --emit=dep-info,link \
-        -L dependency={dir}{sep}target{sep}debug \
         -L dependency={dir}{sep}target{sep}debug{sep}deps`
 [FINISHED] debug [unoptimized + debuginfo] target(s) in [..]
 ", sep = SEP,

tests/build-script.rs

@@ -766,19 +766,17 @@ fn build_cmd_with_a_build_cmd() {
 [RUNNING] `rustc a[..]build.rs [..] --extern b=[..]`
 [RUNNING] `[..]a-[..]build-script-build[..]`
 [RUNNING] `rustc [..]lib.rs --crate-name a --crate-type lib -g \
-    -C metadata=[..] -C extra-filename=-[..] \
     --out-dir [..]target[..]deps --emit=dep-info,link \
-    -L [..]target[..]deps -L [..]target[..]deps`
+    -L [..]target[..]deps`
 [COMPILING] foo v0.5.0 (file://[..])
 [RUNNING] `rustc build.rs --crate-name build_script_build --crate-type bin \
-    -g \
-    --out-dir [..]build[..]foo-[..] --emit=dep-info,link \
-    -L [..]target[..]debug -L [..]target[..]deps \
-    --extern a=[..]liba-[..].rlib`
+    -g --out-dir [..] --emit=dep-info,link \
+    -L [..]target[..]deps \
+    --extern a=[..]liba[..].rlib`
 [RUNNING] `[..]foo-[..]build-script-build[..]`
 [RUNNING] `rustc [..]lib.rs --crate-name foo --crate-type lib -g \
-    --out-dir [..]target[..]debug --emit=dep-info,link \
-    -L [..]target[..]debug -L [..]target[..]deps`
+    --out-dir [..] --emit=dep-info,link \
+    -L [..]target[..]deps`
 [FINISHED] debug [unoptimized + debuginfo] target(s) in [..]
 "));
 }
@@ -1175,7 +1173,7 @@ fn build_script_with_dynamic_native_dependency() {
 
             fn main() {
                 let src = PathBuf::from(env::var("SRC").unwrap());
-                println!("cargo:rustc-link-search={}/target/debug",
+                println!("cargo:rustc-link-search={}/target/debug/deps",
                          src.display());
             }
         "#)