Prototype: Public database dumps

Cargo.toml

@@ -83,12 +83,14 @@ tokio = "0.1"
 hyper = "0.12"
 ctrlc = { version = "3.0", features = ["termination"] }
 indexmap = "1.0.2"
+handlebars = "2.0.1"
 
 [dev-dependencies]
 conduit-test = "0.8"
 hyper-tls = "0.3"
 lazy_static = "1.0"
 tokio-core = "0.1"
+diesel_migrations = { version = "1.3.0", features = ["postgres"] }
 
 [build-dependencies]
 dotenv = "0.11"

app/router.js

@@ -46,6 +46,7 @@ Router.map(function() {
   this.route('category-slugs', { path: 'category_slugs' });
   this.route('team', { path: '/teams/:team_id' });
   this.route('policies');
+  this.route('data-access');
   this.route('confirm', { path: '/confirm/:email_token' });
 
   this.route('catch-all', { path: '*path' });

app/templates/data-access.hbs

@@ -0,0 +1,34 @@
+<div id='crates-heading'>
+  {{svg-jar 'circle-with-i'}}
+  <h1>Accessing the Crates.io Data</h1>
+</div>
+
+<p>
+  There are several ways of accessing the Crates.io data. You should try the
+  options in the order listed.
+</p>
+
+<ol>
+  <li>
+    <b>
+      The <a href='https://github.com/rust-lang/crates.io-index'>crates.io index</a>.
+    </b>
+    This git repository is updated by crates.io, and it is used
+    by Cargo to speed up local dependency resolution. It contains the majority
+    of the data exposed by crates.io and is cheap to clone and get updates.
+  </li>
+  <li>
+    <b>The database dumps (experimental).</b> The dump contains all information
+    exposed by the API in a single download. It is updated every 24 hours.
+    The latest dump is available at the address
+    <a href='https://static.crates.io/db-dump.tar.gz'>https://static.crates.io/db-dump.tar.gz</a>.
+    Information on using the dump is contained in the tarball.
+  </li>
+  <li>
+    <b>Crawl the crates.io API.</b> This should be used as a last resort, and
+    doing so is subject to our {{#link-to 'policies'}}crawling policy{{/link-to}}.
+    If the index and the database dumps do not satisfy your needs, we're happy to
+    discuss solutions that don't require you to crawl the registry.
+    You can email us at <a href="mailto:[email protected]">[email protected]</a>.
+  </li>
+</ol>

app/templates/policies.hbs

@@ -112,15 +112,8 @@
 <h2 id='crawlers'><a href='#crawlers'>Crawlers</a></h2>
 
 <p>
-  Before resorting to crawling crates.io, you should first see if you are able to
-  gather the information you need from the
-  <a href='https://github.com/rust-lang/crates.io-index'>crates.io index</a>,
-  which is a public git repository containing the majority
-  of the information availble through our API.
-
-  If the index does not have the information you need, we're also happy to
-  discuss solutions to your needs that don't require you to crawl the registry.
-  You can email us at <a href="mailto:[email protected]">[email protected]</a>.
+  Before resorting to crawling crates.io, please read
+  {{#link-to 'data-access'}}Accessing the Crates.io Data{{/link-to}}.
 </p>
 
 <p>

migrations/2017-10-08-193512_category_trees/up.sql

@@ -1,5 +1,4 @@
--- Your SQL goes here
-CREATE EXTENSION ltree;
+CREATE EXTENSION IF NOT EXISTS ltree;
 
 -- Create the new column which will represent our category tree.
 -- Fill it with values from `slug` column and then set to non-null

migrations/2019-05-14-165316_index_crate_name_for_like/up.sql

@@ -1,2 +1,2 @@
-CREATE EXTENSION pg_trgm;
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
 CREATE INDEX index_crates_name_tgrm ON crates USING gin (canon_crate_name(name) gin_trgm_ops);

src/bin/enqueue-job.rs

@@ -1,17 +1,29 @@
-use cargo_registry::util::{CargoError, CargoResult};
-use cargo_registry::{db, tasks};
-use std::env::args;
-use swirl::Job;
+use cargo_registry::util::{human, CargoError, CargoResult};
+use cargo_registry::{db, env, tasks};
+use diesel::PgConnection;
 
 fn main() -> CargoResult<()> {
     let conn = db::connect_now()?;
+    let mut args = std::env::args().skip(1);
+    match &*args.next().unwrap_or_default() {
+        "update_downloads" => tasks::update_downloads().enqueue(&conn),
+        "dump_db" => {
+            let database_url = args.next().unwrap_or_else(|| env("DATABASE_URL"));
+            let target_name = args
+                .next()
+                .unwrap_or_else(|| String::from("db-dump.tar.gz"));
+            tasks::dump_db(database_url, target_name).enqueue(&conn)
+        }
+        other => Err(human(&format!("Unrecognized job type `{}`", other))),
+    }
+}
 
-    match &*args().nth(1).unwrap_or_default() {
-        "update_downloads" => tasks::update_downloads()
-            .enqueue(&conn)
-            .map_err(|e| CargoError::from_std_error(e))?,
-        other => panic!("Unrecognized job type `{}`", other),
-    };
-
-    Ok(())
+/// Helper to map the `PerformError` returned by `swirl::Job::enqueue()` to a
+/// `CargoError`. Can be removed once `map_err()` isn't needed any more.
+trait Enqueue: swirl::Job {
+    fn enqueue(self, conn: &PgConnection) -> CargoResult<()> {
+        <Self as swirl::Job>::enqueue(self, conn).map_err(|e| CargoError::from_std_error(e))
+    }
 }
+
+impl<J: swirl::Job> Enqueue for J {}

src/tasks.rs

@@ -1,3 +1,5 @@
+pub mod dump_db;
 mod update_downloads;
 
+pub use dump_db::dump_db;
 pub use update_downloads::update_downloads;