diff --git a/site/src/job_queue.rs b/site/src/job_queue.rs index 7ed8ad133..b359e80ba 100644 --- a/site/src/job_queue.rs +++ b/site/src/job_queue.rs @@ -1,10 +1,15 @@ -use std::{str::FromStr, sync::Arc}; +use std::{ + path::Path, + str::FromStr, + sync::{Arc, LazyLock}, +}; use crate::load::{partition_in_place, SiteCtxt}; -use chrono::Utc; +use chrono::{DateTime, NaiveDate, Utc}; use database::{BenchmarkRequest, BenchmarkRequestStatus, BenchmarkRequestType}; use hashbrown::HashSet; use parking_lot::RwLock; +use regex::Regex; use tokio::time::{self, Duration}; /// Store the latest master commits or do nothing if all of them are @@ -36,6 +41,94 @@ async fn create_benchmark_request_master_commits( Ok(()) } +/// Parses strings in the following formats extracting the Date & release tag +/// `static.rust-lang.org/dist/2016-05-24/channel-rust-1.9.0.toml` +/// `static.rust-lang.org/dist/2016-05-31/channel-rust-nightly.toml` +/// `static.rust-lang.org/dist/2016-06-01/channel-rust-beta.toml` +/// `static.rust-lang.org/dist/2025-06-26/channel-rust-1.89-beta.toml` +/// `static.rust-lang.org/dist/2025-06-26/channel-rust-1.89.0-beta.toml` +/// `static.rust-lang.org/dist/2025-06-26/channel-rust-1.89.0-beta.2.toml` +fn parse_release_string(url: &str) -> Option<(String, DateTime)> { + static VERSION_RE: LazyLock = LazyLock::new(|| Regex::new(r"(\d+\.\d+\.\d+)").unwrap()); + + // Grab ".../YYYY-MM-DD/FILE.toml" components with Path helpers. + let file = Path::new(url).file_name().and_then(|n| n.to_str())?; + + let date_str = Path::new(url) + .parent() + .and_then(Path::file_name) + .and_then(|n| n.to_str())?; + + // No other beta releases are recognized as toolchains. + // + // We also have names like this: + // + // * channel-rust-1.75-beta.toml + // * channel-rust-1.75.0-beta.toml + // * channel-rust-1.75.0-beta.1.toml + // + // Which should get ignored for now, they're not consumable via rustup yet. + if file.contains("beta") && file != "channel-rust-beta.toml" { + return None; + } + + // Parse the YYYY-MM-DD segment and stamp it with *current* UTC time. + if let Ok(naive) = NaiveDate::parse_from_str(date_str, "%Y-%m-%d") { + let published = naive + .and_hms_opt(0, 0, 0) + .expect("valid HMS") + .and_local_timezone(Utc) + .single() + .unwrap(); + + // Special-case the rolling beta channel. + if file == "channel-rust-beta.toml" { + return Some((format!("beta-{date_str}"), published)); + } + + // Otherwise pull out a semver like "1.70.0" and return it. + if let Some(cap) = VERSION_RE.captures(file).and_then(|m| m.get(1)) { + return Some((cap.as_str().to_owned(), published)); + } + } + + None +} + +/// Store the latest release commits or do nothing if all of them are +/// already in the database +async fn create_benchmark_request_releases( + conn: &dyn database::pool::Connection, +) -> anyhow::Result<()> { + let releases: String = reqwest::get("https://static.rust-lang.org/manifests.txt") + .await? + .text() + .await?; + // TODO; delete at some point in the future + let cutoff: chrono::DateTime = chrono::DateTime::from_str("2025-06-01T00:00:00.000Z")?; + + let releases: Vec<_> = releases + .lines() + .rev() + .filter_map(parse_release_string) + .take(20) + .collect(); + + for (name, date_time) in releases { + if date_time >= cutoff { + let release_request = BenchmarkRequest::create_release( + &name, + date_time, + BenchmarkRequestStatus::ArtifactsReady, + "", + "", + ); + conn.insert_benchmark_request(&release_request).await; + } + } + Ok(()) +} + /// Sorts try and master requests that are in the `ArtifactsReady` status. /// Doesn't consider in-progress requests or release artifacts. fn sort_benchmark_requests(done: &HashSet, request_queue: &mut [BenchmarkRequest]) { @@ -186,6 +279,8 @@ async fn cron_enqueue_jobs(site_ctxt: &Arc) -> anyhow::Result<()> { let mut conn = site_ctxt.conn().await; // Put the master commits into the `benchmark_requests` queue create_benchmark_request_master_commits(site_ctxt, &*conn).await?; + // Put the releases into the `benchmark_requests` queue + create_benchmark_request_releases(&*conn).await?; enqueue_next_job(&mut *conn).await?; Ok(()) } @@ -213,10 +308,21 @@ pub async fn cron_main(site_ctxt: Arc>>>, seconds: u #[cfg(test)] mod tests { use super::*; - use chrono::{Datelike, Duration, TimeZone, Utc}; - + use chrono::{Datelike, Duration, NaiveDate, TimeZone, Utc}; use database::tests::run_postgres_test; + /// Helper: unwrap the Option, panic otherwise. + fn tag(url: &str) -> String { + parse_release_string(url) + .expect("Some") // Option<_> + .0 // take the tag + } + + /// Helper: unwrap the DateTime and keep only the YYYY-MM-DD part + fn day(url: &str) -> NaiveDate { + parse_release_string(url).expect("Some").1.date_naive() + } + fn days_ago(day_str: &str) -> chrono::DateTime { // Walk backwards until the first non-digit, then slice let days = day_str @@ -602,4 +708,55 @@ mod tests { }) .await; } + + #[test] + fn parses_stable_versions() { + assert_eq!( + tag("static.rust-lang.org/dist/2016-05-24/channel-rust-1.9.0.toml"), + "1.9.0" + ); + assert_eq!( + day("static.rust-lang.org/dist/2016-05-24/channel-rust-1.9.0.toml"), + NaiveDate::from_ymd_opt(2016, 5, 24).unwrap() + ); + + assert_eq!( + tag("static.rust-lang.org/dist/2025-06-26/channel-rust-1.88.0.toml"), + "1.88.0" + ); + assert_eq!( + day("static.rust-lang.org/dist/2025-06-26/channel-rust-1.88.0.toml"), + NaiveDate::from_ymd_opt(2025, 6, 26).unwrap() + ); + } + + #[test] + fn parses_plain_beta_channel() { + let want = "beta-2016-06-01"; + let url = "static.rust-lang.org/dist/2016-06-01/channel-rust-beta.toml"; + + assert_eq!(tag(url), want); + assert_eq!(day(url), NaiveDate::from_ymd_opt(2016, 6, 1).unwrap()); + } + + #[test] + fn skips_unconsumable_channels() { + // nightly never returns Anything + assert!(parse_release_string( + "static.rust-lang.org/dist/2016-05-31/channel-rust-nightly.toml" + ) + .is_none()); + + // versioned-beta artefacts are skipped too + for should_ignore in [ + "static.rust-lang.org/dist/2025-06-26/channel-rust-1.89-beta.toml", + "static.rust-lang.org/dist/2025-06-26/channel-rust-1.89.0-beta.toml", + "static.rust-lang.org/dist/2025-06-26/channel-rust-1.89.0-beta.2.toml", + ] { + assert!( + parse_release_string(should_ignore).is_none(), + "{should_ignore} should be ignored" + ); + } + } }