Merge pull request #365 from Turbo87/downloads-without-placeholders

crates.io: Add support for download URLs without placeholders

Author: jdno

terragrunt/modules/crates-io/cloudfront-functions/static-router.js

@@ -7,5 +7,15 @@ function handler(event) {
         request.uri = request.uri.replace("+", "%2B");
     }
 
+    // cargo versions before 1.24 don't support placeholders in the `dl` field
+    // of the index, so we need to rewrite the download URL to point to the
+    // crate file instead.
+    var match = request.uri.match(/^\/crates\/([^\/]+)\/([^\/]+)\/download$/);
+    if (match) {
+        var crate = match[1];
+        var version = match[2];
+        request.uri = `/crates/${crate}/${crate}-${version}.crate`;
+    }
+
     return request;
 }

terragrunt/modules/crates-io/compute-static/Cargo.lock

@@ -19,6 +19,8 @@ derive_builder = "0.12.0"
 fastly = "0.9.1"
 log = "0.4.17"
 log-fastly = "0.9.1"
+once_cell = "1.18.0"
+regex = "1.7.1"
 serde = { version = "1.0.152", features = ["derive"] }
 serde_json = "1.0.92"
 time = { version = "0.3.17", features = ["serde-human-readable"] }

terragrunt/modules/crates-io/compute-static/Cargo.toml

@@ -2,6 +2,8 @@ use fastly::http::{Method, StatusCode};
 use fastly::{Error, Request, Response};
 use log::{info, warn, LevelFilter};
 use log_fastly::Logger;
+use once_cell::sync::Lazy;
+use regex::Regex;
 use serde_json::json;
 use time::OffsetDateTime;
 
@@ -74,6 +76,7 @@ fn handle_request(config: &Config, mut request: Request) -> Result<Response, Err
 
     set_ttl(config, &mut request);
     rewrite_urls_with_plus_character(&mut request);
+    rewrite_download_urls(&mut request);
 
     // Database dump is too big to cache on Fastly
     if request.get_url_str().ends_with("db-dump.tar.gz") {
@@ -127,6 +130,28 @@ fn rewrite_urls_with_plus_character(request: &mut Request) {
     }
 }
 
+/// Rewrite `/crates/{crate}/{version}/download` URLs to
+/// `/crates/{crate}/{crate}-{version}.crate`
+///
+/// cargo versions before 1.24 don't support placeholders in the `dl` field
+/// of the index, so we need to rewrite the download URL to point to the
+/// crate file instead.
+fn rewrite_download_urls(request: &mut Request) {
+    static RE: Lazy<Regex> = Lazy::new(|| {
+        Regex::new(r"^/crates/(?P<crate>[^/]+)/(?P<version>[^/]+)/download$").unwrap()
+    });
+
+    let url = request.get_url_mut();
+    let path = url.path();
+
+    if let Some(captures) = RE.captures(path) {
+        let krate = captures.name("crate").unwrap().as_str();
+        let version = captures.name("version").unwrap().as_str();
+        let new_path = format!("/crates/{krate}/{krate}-{version}.crate");
+        url.set_path(&new_path);
+    }
+}
+
 /// Redirect request to CloudFront
 ///
 /// As of early 2023, certain files are too large to be served through Fastly. One of those is the
@@ -199,3 +224,34 @@ fn build_and_send_log(log_line: LogLineV1Builder, config: &Config) {
         }
     };
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_rewrite_download_urls() {
+        fn test(url: &str, expected: &str) {
+            let mut request = Request::get(url);
+            rewrite_download_urls(&mut request);
+            assert_eq!(request.get_url_str(), expected);
+        }
+
+        test(
+            "https://static.crates.io/unrelated",
+            "https://static.crates.io/unrelated",
+        );
+        test(
+            "https://static.crates.io/crates/serde/serde-1.0.0.crate",
+            "https://static.crates.io/crates/serde/serde-1.0.0.crate",
+        );
+        test(
+            "https://static.crates.io/crates/serde/1.0.0/download",
+            "https://static.crates.io/crates/serde/serde-1.0.0.crate",
+        );
+        test(
+            "https://static.crates.io/crates/serde/1.0.0-alpha.1+foo-bar/download",
+            "https://static.crates.io/crates/serde/serde-1.0.0-alpha.1+foo-bar.crate",
+        );
+    }
+}