Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 77 additions & 12 deletions crates/turborepo-lib/src/run/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ use std::{

use chrono::Local;
use tracing::Instrument;
use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, RelativeUnixPathBuf};
use turbopath::{
AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPath, RelativeUnixPathBuf,
};
use turborepo_analytics::{start_analytics, AnalyticsHandle};
use turborepo_api_client::{APIAuth, APIClient, CacheClient, SharedHttpClient};
use turborepo_cache::{AsyncCache, CacheScmState, LazyScmState};
Expand Down Expand Up @@ -284,21 +286,39 @@ impl RunBuilder {
}
}

fn all_package_prefixes(pkg_dep_graph: &PackageGraph) -> Vec<RelativeUnixPathBuf> {
fn package_prefix_for_repo_index(
repo_root: &AbsoluteSystemPath,
index_root: &AbsoluteSystemPath,
package_dir: &AnchoredSystemPath,
) -> Result<RelativeUnixPathBuf, Error> {
let full_package_dir = repo_root.resolve(package_dir);
Ok(index_root.anchor(&full_package_dir)?.to_unix())
}

fn all_package_prefixes(
pkg_dep_graph: &PackageGraph,
scm: &SCM,
) -> Result<Vec<RelativeUnixPathBuf>, Error> {
let repo_root = pkg_dep_graph.repo_root();
let index_root = scm.git_root().unwrap_or(repo_root);
let mut prefixes = pkg_dep_graph
.packages()
.filter_map(|(name, _)| pkg_dep_graph.package_dir(name))
.map(|package_dir| package_dir.to_unix())
.collect::<Vec<_>>();
.map(|package_dir| {
Self::package_prefix_for_repo_index(repo_root, index_root, package_dir)
})
.collect::<Result<Vec<_>, _>>()?;

prefixes.extend(
pkg_dep_graph
.root_internal_package_dependencies_paths()
.into_iter()
.map(|package_dir| package_dir.to_unix()),
);
let root_dependency_prefixes = pkg_dep_graph
.root_internal_package_dependencies_paths()
.into_iter()
.map(|package_dir| {
Self::package_prefix_for_repo_index(repo_root, index_root, package_dir)
})
.collect::<Result<Vec<_>, _>>()?;
prefixes.extend(root_dependency_prefixes);

prefixes
Ok(prefixes)
}

/// Resolve the set of packages that should participate in this run.
Expand Down Expand Up @@ -493,11 +513,11 @@ impl RunBuilder {
// parallel walk for untracked file discovery. This replaces the
// subprocess approach (ls-tree + diff-index + ls-files race) which
// burned ~500ms of CPU on background threads.
let all_prefixes = Self::all_package_prefixes(&pkg_dep_graph);
let scm = scm_task
.instrument(tracing::info_span!("scm_task_await"))
.await
.expect("detecting scm panicked");
let all_prefixes = Self::all_package_prefixes(&pkg_dep_graph, &scm)?;
let repo_index_task = if all_prefixes.is_empty() {
None
} else {
Expand Down Expand Up @@ -1052,6 +1072,51 @@ fn hosts_match(url1: &str, url2: &str) -> bool {
}
}

#[cfg(test)]
mod package_prefix_tests {
use super::*;

#[test]
fn repo_index_prefixes_are_git_root_relative_for_nested_turbo_root() {
let tmp = tempfile::tempdir().unwrap();
let git_root = AbsoluteSystemPathBuf::try_from(tmp.path()).unwrap();
let repo_root = git_root.join_component("downloaded-app");

let root_package = AnchoredSystemPath::new("").unwrap();
let workspace_package = AnchoredSystemPath::new("packages/web").unwrap();

assert_eq!(
RunBuilder::package_prefix_for_repo_index(&repo_root, &git_root, root_package).unwrap(),
RelativeUnixPathBuf::new("downloaded-app").unwrap()
);
assert_eq!(
RunBuilder::package_prefix_for_repo_index(&repo_root, &git_root, workspace_package)
.unwrap(),
RelativeUnixPathBuf::new("downloaded-app/packages/web").unwrap()
);
}

#[test]
fn repo_index_prefixes_stay_repo_relative_when_git_root_matches_turbo_root() {
let tmp = tempfile::tempdir().unwrap();
let repo_root = AbsoluteSystemPathBuf::try_from(tmp.path()).unwrap();

let root_package = AnchoredSystemPath::new("").unwrap();
let workspace_package = AnchoredSystemPath::new("packages/web").unwrap();

assert_eq!(
RunBuilder::package_prefix_for_repo_index(&repo_root, &repo_root, root_package)
.unwrap(),
RelativeUnixPathBuf::new("").unwrap()
);
assert_eq!(
RunBuilder::package_prefix_for_repo_index(&repo_root, &repo_root, workspace_package)
.unwrap(),
RelativeUnixPathBuf::new("packages/web").unwrap()
);
}
}

#[cfg(test)]
mod hosts_match_tests {
use super::*;
Expand Down
5 changes: 5 additions & 0 deletions crates/turborepo/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,11 @@ still warming the client before the first network request in the common case.
actually needs for hashing and augments that tracked index with untracked
files only for those prefixes

Those prefixes are relative to the repo index root, which is usually the Git
root. This matters when the Turbo root is nested inside a larger Git repository:
the root package should scope to the nested Turbo directory, not request an
untracked walk of the entire parent repository.

This keeps the cheap tracked-index work overlapped with other startup work while
avoiding a repo-wide untracked walk when only a subset of packages will be
hashed.
Expand Down
Loading