Skip to content

Commit a8fb349

Browse files
authored
Merge pull request #7461 from Turbo87/lazy-repo
worker: Clone index repository in the background
2 parents 3cbd5ee + 7ed4297 commit a8fb349

File tree

3 files changed

+59
-21
lines changed

3 files changed

+59
-21
lines changed

src/bin/background-worker.rs

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,14 @@ use crates_io::worker::swirl::Runner;
2424
use crates_io::worker::{Environment, RunnerExt};
2525
use crates_io::{db, ssh};
2626
use crates_io_env_vars::{var, var_parsed};
27-
use crates_io_index::{Repository, RepositoryConfig};
27+
use crates_io_index::RepositoryConfig;
2828
use diesel::r2d2;
2929
use diesel::r2d2::ConnectionManager;
3030
use reqwest::blocking::Client;
3131
use secrecy::ExposeSecret;
3232
use std::sync::Arc;
3333
use std::thread::sleep;
34-
use std::time::{Duration, Instant};
34+
use std::time::Duration;
3535

3636
fn main() -> anyhow::Result<()> {
3737
let _sentry = crates_io::sentry::init();
@@ -59,18 +59,11 @@ fn main() -> anyhow::Result<()> {
5959

6060
let job_start_timeout = var_parsed("BACKGROUND_JOB_TIMEOUT")?.unwrap_or(30);
6161

62-
info!("Cloning index");
63-
6462
if var("HEROKU")?.is_some() {
6563
ssh::write_known_hosts_file().unwrap();
6664
}
6765

68-
let clone_start = Instant::now();
6966
let repository_config = RepositoryConfig::from_environment()?;
70-
let repository = Repository::open(&repository_config).expect("Failed to clone index");
71-
72-
let clone_duration = clone_start.elapsed();
73-
info!(duration = ?clone_duration, "Index cloned");
7467

7568
let cloudfront = CloudFront::from_environment();
7669
let fastly = Fastly::from_environment();
@@ -81,10 +74,19 @@ fn main() -> anyhow::Result<()> {
8174
.build()
8275
.expect("Couldn't build client");
8376

84-
let environment = Environment::new(repository, client, cloudfront, fastly, storage);
77+
let environment = Environment::new(repository_config, client, cloudfront, fastly, storage);
8578

8679
let environment = Arc::new(environment);
8780

81+
std::thread::spawn({
82+
let environment = environment.clone();
83+
move || {
84+
if let Err(err) = environment.lock_index() {
85+
warn!(%err, "Failed to clone index");
86+
};
87+
}
88+
});
89+
8890
let connection_pool = r2d2::Pool::builder()
8991
.max_size(10)
9092
.min_idle(Some(0))

src/tests/util/test_app.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use crates_io::worker::{Environment, RunnerExt};
1111
use crates_io::{App, Emails, Env};
1212
use crates_io_env_vars::required_var;
1313
use crates_io_index::testing::UpstreamIndex;
14-
use crates_io_index::{Credentials, Repository as WorkerRepository, RepositoryConfig};
14+
use crates_io_index::{Credentials, RepositoryConfig};
1515
use crates_io_test_db::TestDatabase;
1616
use diesel::PgConnection;
1717
use futures_util::TryStreamExt;
@@ -252,9 +252,8 @@ impl TestAppBuilder {
252252
index_location: UpstreamIndex::url(),
253253
credentials: Credentials::Missing,
254254
};
255-
let index = WorkerRepository::open(&repository_config).expect("Could not clone index");
256255
let environment = Environment::new(
257-
index,
256+
repository_config,
258257
app.http_client().clone(),
259258
None,
260259
None,

src/worker/environment.rs

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,15 @@ use crate::cloudfront::CloudFront;
22
use crate::fastly::Fastly;
33
use crate::storage::Storage;
44
use crate::worker::swirl::PerformError;
5-
use crates_io_index::Repository;
5+
use crates_io_index::{Repository, RepositoryConfig};
66
use reqwest::blocking::Client;
7+
use std::ops::{Deref, DerefMut};
78
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
9+
use std::time::Instant;
810

911
pub struct Environment {
10-
index: Mutex<Repository>,
12+
repository_config: RepositoryConfig,
13+
repository: Mutex<Option<Repository>>,
1114
http_client: Client,
1215
cloudfront: Option<CloudFront>,
1316
fastly: Option<Fastly>,
@@ -16,14 +19,15 @@ pub struct Environment {
1619

1720
impl Environment {
1821
pub fn new(
19-
index: Repository,
22+
repository_config: RepositoryConfig,
2023
http_client: Client,
2124
cloudfront: Option<CloudFront>,
2225
fastly: Option<Fastly>,
2326
storage: Arc<Storage>,
2427
) -> Self {
2528
Self {
26-
index: Mutex::new(index),
29+
repository_config,
30+
repository: Mutex::new(None),
2731
http_client,
2832
cloudfront,
2933
fastly,
@@ -32,10 +36,25 @@ impl Environment {
3236
}
3337

3438
#[instrument(skip_all)]
35-
pub fn lock_index(&self) -> Result<MutexGuard<'_, Repository>, PerformError> {
36-
let repo = self.index.lock().unwrap_or_else(PoisonError::into_inner);
37-
repo.reset_head()?;
38-
Ok(repo)
39+
pub fn lock_index(&self) -> Result<RepositoryLock<'_>, PerformError> {
40+
let mut repo = self
41+
.repository
42+
.lock()
43+
.unwrap_or_else(PoisonError::into_inner);
44+
45+
if repo.is_none() {
46+
info!("Cloning index");
47+
let clone_start = Instant::now();
48+
49+
*repo = Some(Repository::open(&self.repository_config)?);
50+
51+
let clone_duration = clone_start.elapsed();
52+
info!(duration = ?clone_duration, "Index cloned");
53+
}
54+
55+
let repo_lock = RepositoryLock { repo };
56+
repo_lock.reset_head()?;
57+
Ok(repo_lock)
3958
}
4059

4160
/// Returns a client for making HTTP requests to upload crate files.
@@ -51,3 +70,21 @@ impl Environment {
5170
self.fastly.as_ref()
5271
}
5372
}
73+
74+
pub struct RepositoryLock<'a> {
75+
repo: MutexGuard<'a, Option<Repository>>,
76+
}
77+
78+
impl Deref for RepositoryLock<'_> {
79+
type Target = Repository;
80+
81+
fn deref(&self) -> &Self::Target {
82+
self.repo.as_ref().unwrap()
83+
}
84+
}
85+
86+
impl DerefMut for RepositoryLock<'_> {
87+
fn deref_mut(&mut self) -> &mut Self::Target {
88+
self.repo.as_mut().unwrap()
89+
}
90+
}

0 commit comments

Comments
 (0)