Add sitemap

Author: onur

src/web/mod.rs

@@ -43,6 +43,7 @@ mod pool;
 mod file;
 mod builds;
 mod error;
+mod sitemap;
 
 use std::env;
 use std::error::Error;
@@ -96,6 +97,8 @@ impl CratesfyiHandler {
         router.get("/about", |_: &mut Request| {
             page::Page::new(false).title("About Docs.rs").to_resp("about")
         });
+        router.get("/robots.txt", sitemap::robots_txt_handler);
+        router.get("/sitemap.xml", sitemap::sitemap_handler);
         router.get("/releases", releases::releases_handler);
         router.get("/releases/feed", releases::releases_feed_handler);
         router.get("/releases/recent/:page", releases::releases_handler);

src/web/sitemap.rs

@@ -0,0 +1,33 @@
+
+
+use iron::prelude::*;
+use iron::headers::ContentType;
+use super::page::Page;
+use super::pool::Pool;
+use time;
+
+
+pub fn sitemap_handler(req: &mut Request) -> IronResult<Response> {
+    let conn = extension!(req, Pool);
+    let mut releases: Vec<(String, String)> = Vec::new();
+    for row in &conn.query("SELECT DISTINCT ON (crates.name)
+                                   crates.name,
+                                   releases.release_time
+                            FROM crates
+                            INNER JOIN releases ON releases.crate_id = crates.id
+                            WHERE rustdoc_status = true",
+               &[])
+        .unwrap() {
+        releases.push((row.get(0), format!("{}", time::at(row.get(1)).rfc3339())));
+    }
+    let mut resp = ctry!(Page::new(releases).to_resp("sitemap"));
+    resp.headers.set(ContentType("application/xml".parse().unwrap()));
+    Ok(resp)
+}
+
+
+pub fn robots_txt_handler(_: &mut Request) -> IronResult<Response> {
+    let mut resp = Response::with("Sitemap: https://docs.rs/sitemap.xml");
+    resp.headers.set(ContentType("text/plain".parse().unwrap()));
+    Ok(resp)
+}

templates/sitemap.hbs

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  {{#each content}}
+  <url>
+    <loc>https://docs.rs/{{this.[0]}}</loc>
+    <lastmod>{{this.[1]}}</lastmod>
+  </url>
+  {{/each}}
+</urlset>