Merge branch 'build-queue-crates-io'

Author: onur

Cargo.lock

@@ -18,7 +18,6 @@ hyper = "0.9"
 semver = "0.5"
 slug = "=0.1.1"
 env_logger = "0.3"
-git2 = "0.4"
 magic = "0.12"
 iron = "0.3.1"
 router = "0.1.1"

Cargo.toml

@@ -2,109 +2,55 @@
 
 
 use super::DocBuilder;
-use rustc_serialize::json::Json;
-use git2;
+use rustc_serialize::json::{Json, Array};
+use hyper;
 use db::connect_db;
 use errors::*;
 
 
 impl DocBuilder {
     /// Updates crates.io-index repository and adds new crates into build queue
-    pub fn get_new_crates(&self) -> Result<()> {
-
-        let repo = try!(git2::Repository::open(&self.options.crates_io_index_path));
-
-        let old_tree = try!(self.head_tree(&repo));
-        try!(self.update_repo(&repo));
-        let new_tree = try!(self.head_tree(&repo));
-
-        let diff = try!(repo.diff_tree_to_tree(Some(&old_tree), Some(&new_tree), None));
-        let conn = try!(connect_db());
-        let mut line_n = 0;
-
-        try!(diff.print(git2::DiffFormat::Patch, |_, hunk, diffline| -> bool {
-            let line = String::from_utf8_lossy(diffline.content()).into_owned();
-            // crate strings starts with '{'
-            // skip if line is not a crate string
-            if line.chars().nth(0) != Some('{') {
-                line_n = 0;
-                return true;
-            }
-
-            line_n += 1;
-
-            if match hunk {
-                Some(hunk) => hunk.new_lines() != line_n,
-                None => true,
-            } {
-                return true;
-            }
-
-            let json = match Json::from_str(&line[..]) {
-                Ok(j) => j,
-                Err(err) => {
-                    error!("Failed to parse crate string: {}", err);
-                    // just continue even if we get an error for a crate
-                    return true;
+    pub fn get_new_crates(&mut self) -> Result<()> {
+        try!(self.load_database_cache());
+
+        let body = {
+            use std::io::Read;
+            let client = hyper::Client::new();
+            let mut res = try!(client.get("https://crates.io/summary").send());
+            let mut body = String::new();
+            try!(res.read_to_string(&mut body));
+            body
+        };
+
+        let json = try!(Json::from_str(&body));
+
+        let crates = {
+            let mut crates: Vec<(String, String)> = Vec::new();
+            for section in ["just_updated", "new_crates"].iter() {
+                match json.as_object()
+                    .and_then(|o| o.get(&section[..]))
+                    .and_then(|j| j.as_array())
+                    .map(get_crates_from_array) {
+                    Some(mut c) => crates.append(c.as_mut()),
+                    None => continue,
                 }
-            };
-
-            // check if a crate is yanked just in case
-            if json.as_object()
-                .and_then(|obj| obj.get("yanked"))
-                .and_then(|y| y.as_boolean())
-                .unwrap_or(true) {
-                return true;
             }
+            crates
+        };
 
-            if let Some((krate, version)) = json.as_object()
-                .map(|obj| {
-                    (obj.get("name")
-                        .and_then(|n| n.as_string()),
-                     obj.get("vers")
-                        .and_then(|n| n.as_string()))
-                }) {
-
-                // Skip again if we can't get crate name and version
-                if krate.is_none() || version.is_none() {
-                    return true;
-                }
-
-                let _ = conn.execute("INSERT INTO queue (name, version) VALUES ($1, $2)",
-                                     &[&krate.unwrap(), &version.unwrap()]);
+        let conn = try!(connect_db());
+        for (name, version) in crates {
+            if self.db_cache.contains(&format!("{}-{}", name, version)[..]) {
+                continue;
             }
-
-            true
-        }));
-
-        Ok(())
-    }
-
-
-    fn update_repo(&self, repo: &git2::Repository) -> Result<()> {
-        let mut remote = try!(repo.find_remote("origin"));
-        try!(remote.fetch(&["refs/heads/*:refs/remotes/origin/*"], None, None));
-
-        // checkout master
-        try!(repo.refname_to_id("refs/remotes/origin/master")
-            .and_then(|oid| repo.find_object(oid, None))
-            .and_then(|object| repo.reset(&object, git2::ResetType::Hard, None)));
+            let _ = conn.execute("INSERT INTO queue (name, version) VALUES ($1, $2)",
+                                 &[&name, &version]);
+        }
 
         Ok(())
     }
 
 
-    fn head_tree<'a>(&'a self, repo: &'a git2::Repository) -> Result<git2::Tree> {
-        repo.head()
-            .ok()
-            .and_then(|head| head.target())
-            .ok_or(git2::Error::from_str("HEAD SHA1 not found"))
-            .and_then(|oid| repo.find_commit(oid))
-            .and_then(|commit| commit.tree())
-            .or_else(|e| Err(ErrorKind::Git2Error(e).into()))
-    }
-
-
     /// Builds packages from queue
     pub fn build_packages_queue(&mut self) -> Result<()> {
         let conn = try!(connect_db());
@@ -117,7 +63,7 @@ impl DocBuilder {
             match self.build_package(&name[..], &version[..]) {
                 Ok(_) => {
                     let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
-                },
+                }
                 Err(e) => {
                     error!("Failed to build package {}-{} from queue: {}",
                            name,
@@ -132,6 +78,29 @@ impl DocBuilder {
 }
 
 
+/// Returns Vec<CRATE_NAME, CRATE_VERSION> from a summary array
+fn get_crates_from_array<'a>(crates: &'a Array) -> Vec<(String, String)> {
+    let mut crates_vec: Vec<(String, String)> = Vec::new();
+    for crte in crates {
+        let name = match crte.as_object()
+            .and_then(|o| o.get("id"))
+            .and_then(|i| i.as_string())
+            .map(|s| s.to_owned()) {
+            Some(s) => s,
+            None => continue,
+        };
+        let version = match crte.as_object()
+            .and_then(|o| o.get("max_version"))
+            .and_then(|v| v.as_string())
+            .map(|s| s.to_owned()) {
+            Some(s) => s,
+            None => continue,
+        };
+        crates_vec.push((name, version));
+    }
+    crates_vec
+}
+
 
 
 
@@ -146,8 +115,12 @@ mod test {
     fn test_get_new_crates() {
         let _ = env_logger::init();
         let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
-        let docbuilder = DocBuilder::new(options);
-        assert!(docbuilder.get_new_crates().is_ok());
+        let mut docbuilder = DocBuilder::new(options);
+        let res = docbuilder.get_new_crates();
+        if res.is_err() {
+            error!("{:?}", res);
+        }
+        assert!(res.is_ok());
     }
 
 

src/docbuilder/queue.rs

@@ -5,7 +5,6 @@ use rustc_serialize::json;
 use postgres;
 use cargo;
 use hyper;
-use git2;
 use magic::MagicError;
 
 
@@ -23,7 +22,6 @@ error_chain! {
         postgres::error::ConnectError, PostgresConnectError;
         postgres::error::Error, PostgresError;
         hyper::Error, HyperError;
-        git2::Error, Git2Error;
         MagicError, MagicError;
         Box<cargo::CargoError>, CargoError;
     }

src/errors.rs

@@ -13,7 +13,6 @@ extern crate hyper;
 extern crate time;
 extern crate semver;
 extern crate slug;
-extern crate git2;
 extern crate magic;
 extern crate iron;
 extern crate router;

src/lib.rs

@@ -45,24 +45,23 @@ pub fn start_daemon() {
     }
 
 
-    // check new crates every 15 minutes
+    // check new crates every 5 minutes
     // FIXME: why not just check new crates and build them if there is any?
     thread::spawn(move || {
         loop {
-            thread::sleep(Duration::from_secs(900));
+            thread::sleep(Duration::from_secs(300));
             debug!("Checking new crates");
-            let doc_builder = DocBuilder::new(opts());
-            if let Err(e) = doc_builder.get_new_crates() {
+            if let Err(e) = DocBuilder::new(opts()).get_new_crates() {
                 error!("Failed to get new crates: {}", e);
             }
         }
     });
 
 
-    // build crates in que every 12 minutes
+    // build crates in que every 5 minutes
     thread::spawn(move || {
         loop {
-            thread::sleep(Duration::from_secs(720));
+            thread::sleep(Duration::from_secs(300));
 
             let mut opts = opts();
             opts.skip_if_exists = true;