Detect changes using crates.io index repository

This assures that no matter how often we poll, we will always
see all changes since the last time we fetched.

Fixes #89

Author: Byron

Cargo.lock

@@ -13,6 +13,8 @@ rustc-serialize = "0.3"
 regex = "0.1"
 postgres = { version = "0.11", features = [ "time", "rustc-serialize" ] }
 clap = "2.14"
+crates-index-diff = "2.0.0"
+git2 = "0.4"
 time = "0.1"
 hyper = "0.9"
 semver = "0.5"

Cargo.toml

@@ -2,52 +2,24 @@
 
 
 use super::DocBuilder;
-use rustc_serialize::json::{Json, Array};
-use hyper;
-use hyper::header::{Accept, qitem};
 use db::connect_db;
 use errors::*;
+use crates_index_diff::{ChangeKind, Index};
 
 
 impl DocBuilder {
     /// Updates crates.io-index repository and adds new crates into build queue
     pub fn get_new_crates(&mut self) -> Result<()> {
         try!(self.load_database_cache());
-
-        let body = {
-            use std::io::Read;
-            let client = hyper::Client::new();
-            let mut res = try!(client.get("https://crates.io/summary")
-                .header(Accept(vec![qitem("application/json".parse().unwrap())]))
-                .send());
-            let mut body = String::new();
-            try!(res.read_to_string(&mut body));
-            body
-        };
-
-        let json = try!(Json::from_str(&body));
-
-        let crates = {
-            let mut crates: Vec<(String, String)> = Vec::new();
-            for section in ["just_updated", "new_crates"].iter() {
-                match json.as_object()
-                    .and_then(|o| o.get(&section[..]))
-                    .and_then(|j| j.as_array())
-                    .map(get_crates_from_array) {
-                    Some(mut c) => crates.append(c.as_mut()),
-                    None => continue,
-                }
-            }
-            crates
-        };
-
         let conn = try!(connect_db());
-        for (name, version) in crates {
-            if self.db_cache.contains(&format!("{}-{}", name, version)[..]) {
+        let index = try!(Index::from_path_or_cloned(&self.options.crates_io_index_path));
+        let changes = try!(index.fetch_changes());
+        for krate in changes.iter().filter(|k| k.kind != ChangeKind::Yanked) {
+            if self.db_cache.contains(&format!("{}-{}", krate.name, krate.version)) {
                 continue;
             }
-            let _ = conn.execute("INSERT INTO queue (name, version) VALUES ($1, $2)",
-                                 &[&name, &version]);
+            conn.execute("INSERT INTO queue (name, version) VALUES ($1, $2)",
+                         &[&krate.name, &krate.version]).ok();
         }
 
         Ok(())
@@ -80,33 +52,6 @@ impl DocBuilder {
     }
 }
 
-
-/// Returns Vec<CRATE_NAME, CRATE_VERSION> from a summary array
-fn get_crates_from_array<'a>(crates: &'a Array) -> Vec<(String, String)> {
-    let mut crates_vec: Vec<(String, String)> = Vec::new();
-    for crte in crates {
-        let name = match crte.as_object()
-            .and_then(|o| o.get("id"))
-            .and_then(|i| i.as_string())
-            .map(|s| s.to_owned()) {
-            Some(s) => s,
-            None => continue,
-        };
-        let version = match crte.as_object()
-            .and_then(|o| o.get("max_version"))
-            .and_then(|v| v.as_string())
-            .map(|s| s.to_owned()) {
-            Some(s) => s,
-            None => continue,
-        };
-        crates_vec.push((name, version));
-    }
-    crates_vec
-}
-
-
-
-
 #[cfg(test)]
 mod test {
     extern crate env_logger;

src/docbuilder/queue.rs

@@ -6,6 +6,7 @@ use postgres;
 use cargo;
 use hyper;
 use magic::MagicError;
+use git2;
 
 
 error_chain! {
@@ -22,6 +23,7 @@ error_chain! {
         postgres::error::ConnectError, PostgresConnectError;
         postgres::error::Error, PostgresError;
         hyper::Error, HyperError;
+        git2::Error, Git2Error;
         MagicError, MagicError;
         Box<cargo::CargoError>, CargoError;
     }

src/errors.rs

@@ -25,6 +25,8 @@ extern crate url;
 extern crate params;
 extern crate libc;
 extern crate badge;
+extern crate crates_index_diff;
+extern crate git2;
 
 pub use self::docbuilder::DocBuilder;
 pub use self::docbuilder::ChrootBuilderResult;