Skip to content

Commit 37deb2b

Browse files
arlosiarlosi
committed
Respond to review comments
1 parent 5e527ba commit 37deb2b

File tree

7 files changed

+99
-49
lines changed

7 files changed

+99
-49
lines changed

‎.env.sample‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,12 @@ export TEST_DATABASE_URL=
3434
# export S3_REGION=
3535

3636
# Credentials for uploading index metadata to S3. You can leave these commented
37-
# out if you're not publishing to s3 from your crates.io instance.
38-
# export S3_INDEX_BUCKET
39-
# export S3_INDEX_ACCESS_KEY
40-
# export S3_INDEX_SECRET_KEY
37+
# out if you're not publishing index metadata to s3 from your crates.io instance.
38+
# export S3_INDEX_BUCKET=
39+
# export S3_INDEX_ACCESS_KEY=
40+
# export S3_INDEX_SECRET_KEY=
4141
# not needed if the S3 bucket is in US standard
42-
# export S3_INDEX_REGION
42+
# export S3_INDEX_REGION=
4343

4444
# Upstream location of the registry index. Background jobs will push to
4545
# this URL. The default points to a local index for development.

‎cargo-registry-index/lib.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ impl Repository {
317317
/// the given crate name for usage in URLs (i.e. with `/` separator).
318318
///
319319
/// see <https://doc.rust-lang.org/cargo/reference/registries.html#index-format>
320-
pub fn relative_index_file_as_string(name: &str) -> String {
320+
pub fn relative_index_file_for_url(name: &str) -> String {
321321
let name = name.to_lowercase();
322322
Self::relative_index_file_helper(&name).join("/")
323323
}
@@ -358,7 +358,8 @@ impl Repository {
358358
self.push("refs/heads/master")
359359
}
360360

361-
/// Gets a list of files that have been modified since a given `starting_commit` (use `starting_commit = None` for a list of all files).
361+
/// Gets a list of files that have been modified since a given `starting_commit`
362+
/// (use `starting_commit = None` for a list of all files).
362363
pub fn get_files_modified_since(
363364
&self,
364365
starting_commit: Option<&str>,
@@ -385,12 +386,14 @@ impl Repository {
385386
.repository
386387
.find_commit(self.head_oid()?)?
387388
.as_object()
388-
.peel_to_tree()?;
389+
.peel_to_tree()
390+
.context("failed to find tree for HEAD")?;
389391
let diff = self
390392
.repository
391393
.diff_tree_to_tree(starting_commit.as_ref(), Some(&head), None)
392394
.context("failed to run diff")?;
393-
let files = diff.deltas()
395+
let files = diff
396+
.deltas()
394397
.map(|delta| delta.new_file())
395398
.filter(|file| file.exists())
396399
.map(|file| file.path().unwrap().to_path_buf())

‎src/admin/render_readmes.rs‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use crate::{
22
config, db,
33
models::Version,
44
schema::{crates, readme_renderings, versions},
5-
uploaders::Uploader,
5+
uploaders::{UploadBucket, Uploader},
66
};
77
use anyhow::{anyhow, Context};
88
use std::{io::Read, path::Path, sync::Arc, thread};
@@ -14,6 +14,7 @@ use flate2::read::GzDecoder;
1414
use reqwest::{blocking::Client, header};
1515
use tar::{self, Archive};
1616

17+
const CACHE_CONTROL_README: &str = "public,max-age=604800";
1718
const USER_AGENT: &str = "crates-admin";
1819

1920
#[derive(clap::Parser, Debug)]
@@ -110,10 +111,26 @@ pub fn run(opts: Opts) -> anyhow::Result<()> {
110111
let handle = thread::spawn::<_, anyhow::Result<()>>(move || {
111112
println!("[{}-{}] Rendering README...", krate_name, version.num);
112113
let readme = get_readme(base_config.uploader(), &client, &version, &krate_name)?;
114+
let content_length = readme.len() as u64;
115+
let content = std::io::Cursor::new(readme);
116+
let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
117+
let mut extra_headers = header::HeaderMap::new();
118+
extra_headers.insert(
119+
header::CACHE_CONTROL,
120+
header::HeaderValue::from_static(CACHE_CONTROL_README),
121+
);
113122

114123
base_config
115124
.uploader()
116-
.upload_readme(&client, &krate_name, &version.num, readme)
125+
.upload(
126+
&client,
127+
&readme_path,
128+
content,
129+
content_length,
130+
"text/html",
131+
extra_headers,
132+
UploadBucket::Default,
133+
)
117134
.context("Failed to upload rendered README file to S3")?;
118135

119136
Ok(())

‎src/admin/upload_index.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::time::{Duration, Instant};
2+
13
use crate::admin::dialoguer;
24
use cargo_registry_index::{Repository, RepositoryConfig};
35
use reqwest::blocking::Client;
@@ -31,7 +33,8 @@ pub fn run(opts: Opts) -> anyhow::Result<()> {
3133
return Ok(());
3234
}
3335

34-
for file in files {
36+
let mut progress_update_time = Instant::now();
37+
for (i, file) in files.iter().enumerate() {
3538
let crate_name = file.file_name().unwrap().to_str().unwrap();
3639
let path = repo.index_file(crate_name);
3740
if !path.exists() {
@@ -40,6 +43,13 @@ pub fn run(opts: Opts) -> anyhow::Result<()> {
4043
}
4144
let contents = std::fs::read_to_string(&path)?;
4245
uploader.upload_index(&client, crate_name, contents)?;
46+
47+
// Print a progress update every 10 seconds.
48+
let now = Instant::now();
49+
if now - progress_update_time > Duration::from_secs(10) {
50+
progress_update_time = now;
51+
println!("uploading {}/{}", i, files.len());
52+
}
4353
}
4454

4555
println!(

‎src/config/base.rs‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -83,15 +83,15 @@ impl Base {
8383
// sniff/record it, but everywhere else we use https
8484
"http",
8585
)),
86-
index_bucket: Box::new(s3::Bucket::new(
86+
index_bucket: Some(Box::new(s3::Bucket::new(
8787
String::from("alexcrichton-test"),
8888
None,
8989
dotenv::var("S3_INDEX_ACCESS_KEY").unwrap_or_default(),
9090
dotenv::var("S3_INDEX_SECRET_KEY").unwrap_or_default(),
9191
// When testing we route all API traffic over HTTP so we can
9292
// sniff/record it, but everywhere else we use https
9393
"http",
94-
)),
94+
))),
9595
cdn: None,
9696
};
9797
Self {
@@ -105,6 +105,16 @@ impl Base {
105105
}
106106

107107
fn s3_panic_if_missing_keys() -> Uploader {
108+
let index_bucket = match dotenv::var("S3_INDEX_BUCKET") {
109+
Ok(name) => Some(Box::new(s3::Bucket::new(
110+
name,
111+
dotenv::var("S3_INDEX_REGION").ok(),
112+
env("S3_INDEX_ACCESS_KEY"),
113+
env("S3_INDEX_SECRET_KEY"),
114+
"https",
115+
))),
116+
Err(_) => None,
117+
};
108118
Uploader::S3 {
109119
bucket: Box::new(s3::Bucket::new(
110120
env("S3_BUCKET"),
@@ -113,18 +123,22 @@ impl Base {
113123
env("S3_SECRET_KEY"),
114124
"https",
115125
)),
116-
index_bucket: Box::new(s3::Bucket::new(
117-
env("S3_INDEX_BUCKET"),
118-
dotenv::var("S3_INDEX_REGION").ok(),
119-
env("S3_INDEX_ACCESS_KEY"),
120-
env("S3_INDEX_SECRET_KEY"),
121-
"https",
122-
)),
126+
index_bucket,
123127
cdn: dotenv::var("S3_CDN").ok(),
124128
}
125129
}
126130

127131
fn s3_maybe_read_only() -> Uploader {
132+
let index_bucket = match dotenv::var("S3_INDEX_BUCKET") {
133+
Ok(name) => Some(Box::new(s3::Bucket::new(
134+
name,
135+
dotenv::var("S3_INDEX_REGION").ok(),
136+
dotenv::var("S3_INDEX_ACCESS_KEY").unwrap_or_default(),
137+
dotenv::var("S3_INDEX_SECRET_KEY").unwrap_or_default(),
138+
"https",
139+
))),
140+
Err(_) => None,
141+
};
128142
Uploader::S3 {
129143
bucket: Box::new(s3::Bucket::new(
130144
env("S3_BUCKET"),
@@ -133,13 +147,7 @@ impl Base {
133147
dotenv::var("S3_SECRET_KEY").unwrap_or_default(),
134148
"https",
135149
)),
136-
index_bucket: Box::new(s3::Bucket::new(
137-
env("S3_INDEX_BUCKET"),
138-
dotenv::var("S3_INDEX_REGION").ok(),
139-
dotenv::var("S3_INDEX_ACCESS_KEY").unwrap_or_default(),
140-
dotenv::var("S3_INDEX_SECRET_KEY").unwrap_or_default(),
141-
"https",
142-
)),
150+
index_bucket,
143151
cdn: dotenv::var("S3_CDN").ok(),
144152
}
145153
}

‎src/uploaders.rs‎

Lines changed: 32 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use crate::util::errors::{internal, AppResult};
55

66
use std::env;
77
use std::fs::{self, File};
8-
use std::io::{Cursor, SeekFrom};
8+
use std::io::{Cursor, Seek, SeekFrom};
99

1010
use crate::models::Crate;
1111

@@ -19,7 +19,7 @@ pub enum Uploader {
1919
/// For test usage with `TestApp::with_proxy()`, the recording proxy is used.
2020
S3 {
2121
bucket: Box<s3::Bucket>,
22-
index_bucket: Box<s3::Bucket>,
22+
index_bucket: Option<Box<s3::Bucket>>,
2323
cdn: Option<String>,
2424
},
2525

@@ -88,7 +88,7 @@ impl Uploader {
8888

8989
/// Returns the internal path of an uploaded crate's index file.
9090
fn index_path(name: &str) -> String {
91-
let path = cargo_registry_index::Repository::relative_index_file_as_string(name);
91+
let path = cargo_registry_index::Repository::relative_index_file_for_url(name);
9292
format!("index/{}", path)
9393
}
9494

@@ -100,11 +100,12 @@ impl Uploader {
100100
///
101101
/// This function can panic on an `Self::Local` during development.
102102
/// Production and tests use `Self::S3` which should not panic.
103-
pub fn upload<R: std::io::Read + std::io::Seek + Send + 'static>(
103+
pub fn upload<R: std::io::Read + Send + 'static>(
104104
&self,
105105
client: &Client,
106106
path: &str,
107107
mut content: R,
108+
content_length: u64,
108109
content_type: &str,
109110
extra_headers: header::HeaderMap,
110111
upload_bucket: UploadBucket,
@@ -116,20 +117,21 @@ impl Uploader {
116117
..
117118
} => {
118119
let bucket = match upload_bucket {
119-
UploadBucket::Index => index_bucket,
120-
UploadBucket::Default => bucket,
120+
UploadBucket::Default => Some(bucket),
121+
UploadBucket::Index => index_bucket.as_ref(),
121122
};
122-
let content_length = content.seek(SeekFrom::End(0))?;
123-
content.seek(SeekFrom::Start(0))?;
124-
125-
bucket.put(
126-
client,
127-
path,
128-
content,
129-
content_length,
130-
content_type,
131-
extra_headers,
132-
)?;
123+
124+
if let Some(bucket) = bucket {
125+
bucket.put(
126+
client,
127+
path,
128+
content,
129+
content_length,
130+
content_type,
131+
extra_headers,
132+
)?;
133+
}
134+
133135
Ok(Some(String::from(path)))
134136
}
135137
Uploader::Local => {
@@ -152,11 +154,13 @@ impl Uploader {
152154
..
153155
} => {
154156
let bucket = match upload_bucket {
155-
UploadBucket::Index => index_bucket,
156-
UploadBucket::Default => bucket,
157+
UploadBucket::Default => Some(bucket),
158+
UploadBucket::Index => index_bucket.as_ref(),
157159
};
158160

159-
bucket.delete(client, path)?;
161+
if let Some(bucket) = bucket {
162+
bucket.delete(client, path)?;
163+
}
160164
}
161165
Uploader::Local => {
162166
let filename = env::current_dir().unwrap().join("local_uploads").join(path);
@@ -175,6 +179,7 @@ impl Uploader {
175179
vers: &semver::Version,
176180
) -> AppResult<()> {
177181
let path = Uploader::crate_path(&krate.name, &vers.to_string());
182+
let content_length = body.len() as u64;
178183
let content = Cursor::new(body);
179184
let mut extra_headers = header::HeaderMap::new();
180185
extra_headers.insert(
@@ -185,6 +190,7 @@ impl Uploader {
185190
http_client,
186191
&path,
187192
content,
193+
content_length,
188194
"application/x-tar",
189195
extra_headers,
190196
UploadBucket::Default,
@@ -201,6 +207,7 @@ impl Uploader {
201207
readme: String,
202208
) -> Result<()> {
203209
let path = Uploader::readme_path(crate_name, vers);
210+
let content_length = readme.len() as u64;
204211
let content = Cursor::new(readme);
205212
let mut extra_headers = header::HeaderMap::new();
206213
extra_headers.insert(
@@ -211,6 +218,7 @@ impl Uploader {
211218
http_client,
212219
&path,
213220
content,
221+
content_length,
214222
"text/html",
215223
extra_headers,
216224
UploadBucket::Default,
@@ -225,7 +233,9 @@ impl Uploader {
225233
index: String,
226234
) -> Result<()> {
227235
let path = Uploader::index_path(crate_name);
228-
let content = Cursor::new(index);
236+
let mut content = Cursor::new(index);
237+
let content_length = content.seek(SeekFrom::End(0))?;
238+
content.seek(SeekFrom::Start(0))?;
229239
let mut extra_headers = header::HeaderMap::new();
230240
extra_headers.insert(
231241
header::CACHE_CONTROL,
@@ -235,6 +245,7 @@ impl Uploader {
235245
http_client,
236246
&path,
237247
content,
248+
content_length,
238249
"text/plain",
239250
extra_headers,
240251
UploadBucket::Index,

‎src/worker/dump_db.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ impl DumpTarball {
200200
&client,
201201
target_name,
202202
tarfile,
203+
content_length,
203204
"application/gzip",
204205
header::HeaderMap::new(),
205206
UploadBucket::Default,

0 commit comments

Comments
 (0)