diff --git a/src/branching.rs b/src/branching.rs new file mode 100644 index 0000000..151b1e0 --- /dev/null +++ b/src/branching.rs @@ -0,0 +1,72 @@ +use crate::Context; + +impl Context { + /// Let $stable, $beta, $master be the tips of each branch (when starting). + /// + /// * Set stable to $beta. + /// * Set beta to $master (ish, look for the version bump). + /// * Create a rust-lang/cargo branch for the appropriate beta commit. + /// * Post a PR against the newly created beta branch bump src/ci/channel to `beta`. + pub fn do_branching(&mut self) -> anyhow::Result<()> { + let mut github = if let Some(github) = self.config.github() { + github + } else { + eprintln!("Skipping branching -- github credentials not configured"); + return Ok(()); + }; + let mut token = github.token("rust-lang/rust")?; + let prebump_sha = token.last_commit_for_file("src/version")?; + let beta_sha = token.get_ref("heads/beta")?; + + let stable_version = token.read_file(Some("stable"), "src/version")?; + let beta_version = token.read_file(Some("beta"), "src/version")?; + let future_beta_version = token.read_file(Some(&prebump_sha), "src/version")?; + + // Check that we've not already promoted. Rather than trying to assert + // +1 version numbers, we instead have a simpler check that all the + // versions are unique -- before promotion we should have: + // + // * stable @ 1.61.0 + // * beta @ 1.62.0 + // * prebump @ 1.63.0 + // + // and after promotion we will have (if we were to read the files again): + // + // * stable @ 1.62.0 + // * beta @ 1.63.0 + // * prebump @ 1.63.0 + // + // In this state, if we try to promote again, we want to bail out. The + // stable == beta check isn't as useful, but still nice to have. + if stable_version.content()? == beta_version.content()? { + anyhow::bail!( + "Stable and beta have the same version: {}; refusing to promote branches.", + stable_version.content()?.trim() + ); + } + if beta_version.content()? == future_beta_version.content()? { + anyhow::bail!( + "Beta and pre-bump master ({}) have the same version: {}; refusing to promote branches.", + prebump_sha, + beta_version.content()?.trim() + ); + } + + // No need to disable branch protection, as the promote-release app is + // specifically authorized to force-push to these branches. + token.update_ref("heads/stable", &beta_sha, true)?; + token.update_ref("heads/beta", &prebump_sha, true)?; + + let cargo_sha = token + .read_file(Some(&prebump_sha), "src/tools/cargo")? + .submodule_sha() + .to_owned(); + + let mut github = self.config.github().unwrap(); + let mut token = github.token("rust-lang/cargo")?; + let new_beta = future_beta_version.content()?.trim().to_owned(); + token.create_ref(&dbg!(format!("heads/rust-{}", new_beta)), &cargo_sha)?; + + Ok(()) + } +} diff --git a/src/config.rs b/src/config.rs index af6443d..c7586af 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,3 +1,4 @@ +use crate::discourse::Discourse; use crate::github::Github; use crate::Context; use anyhow::{Context as _, Error}; @@ -46,7 +47,40 @@ impl std::fmt::Display for Channel { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum Action { + /// This is the default action, what we'll do if the environment variable + /// isn't set. It takes the configured channel and pushes artifacts into the + /// appropriate buckets, taking care of other helper tasks along the way. + PromoteRelease, + + /// This promotes the branches up a single release: + /// + /// Let $stable, $beta, $master be the tips of each branch (when starting). + /// + /// * Set stable to $beta. + /// * Set beta to $master (ish, look for the version bump). + /// * Create a rust-lang/cargo branch for the appropriate beta commit. + /// * Post a PR against the newly created beta branch bump src/ci/channel to `beta`. + PromoteBranches, +} + +impl FromStr for Action { + type Err = Error; + + fn from_str(input: &str) -> Result { + match input { + "promote-release" => Ok(Action::PromoteRelease), + "promote-branches" => Ok(Action::PromoteBranches), + _ => anyhow::bail!("unknown channel: {}", input), + } + } +} + pub(crate) struct Config { + /// This is the action we're expecting to take. + pub(crate) action: Action, + /// The channel we're currently releasing. pub(crate) channel: Channel, /// CloudFront distribution ID for doc.rust-lang.org. @@ -113,6 +147,27 @@ pub(crate) struct Config { /// Should be a org/repo code, e.g., rust-lang/rust. pub(crate) rustc_tag_repository: Option, + /// Where to publish new blog PRs. + /// + /// We create a new PR announcing releases in this repository; currently we + /// don't automatically merge it (but that might change in the future). + /// + /// Should be a org/repo code, e.g., rust-lang/blog.rust-lang.org. + pub(crate) blog_repository: Option, + + /// The expected release date, for the blog post announcing dev-static + /// releases. Expected to be in YYYY-MM-DD format. + /// + /// This is used to produce the expected release date in blog posts and to + /// generate the release notes URL (targeting stable branch on + /// rust-lang/rust). + pub(crate) scheduled_release_date: Option, + + /// These are Discourse configurations for where to post dev-static + /// announcements. Currently we only post dev release announcements. + pub(crate) discourse_api_key: Option, + pub(crate) discourse_api_user: Option, + /// This is a github app private key, used for the release steps which /// require action on GitHub (e.g., kicking off a new thanks GHA build, /// opening pull requests against the blog for dev releases, promoting @@ -131,6 +186,7 @@ pub(crate) struct Config { impl Config { pub(crate) fn from_env() -> Result { Ok(Self { + action: default_env("ACTION", Action::PromoteRelease)?, bypass_startup_checks: bool_env("BYPASS_STARTUP_CHECKS")?, channel: require_env("CHANNEL")?, cloudfront_doc_id: require_env("CLOUDFRONT_DOC_ID")?, @@ -151,6 +207,10 @@ impl Config { upload_dir: require_env("UPLOAD_DIR")?, wip_recompress: bool_env("WIP_RECOMPRESS")?, rustc_tag_repository: maybe_env("RUSTC_TAG_REPOSITORY")?, + blog_repository: maybe_env("BLOG_REPOSITORY")?, + scheduled_release_date: maybe_env("BLOG_SCHEDULED_RELEASE_DATE")?, + discourse_api_user: maybe_env("DISCOURSE_API_USER")?, + discourse_api_key: maybe_env("DISCOURSE_API_KEY")?, github_app_key: maybe_env("GITHUB_APP_KEY")?, github_app_id: maybe_env("GITHUB_APP_ID")?, }) @@ -163,6 +223,70 @@ impl Config { None } } + pub(crate) fn discourse(&self) -> Option { + if let (Some(key), Some(user)) = (&self.discourse_api_key, &self.discourse_api_user) { + Some(Discourse::new( + "https://internals.rust-lang.org".to_owned(), + user.clone(), + key.clone(), + )) + } else { + None + } + } + + pub(crate) fn blog_contents( + &self, + release: &str, + archive_date: &str, + for_blog: bool, + internals_url: Option<&str>, + ) -> Option { + let scheduled_release_date = self.scheduled_release_date?; + let release_notes_url = format!( + "https://github.com/rust-lang/rust/blob/stable/RELEASES.md#version-{}-{}", + release.replace('.', ""), + scheduled_release_date.format("%Y-%m-%d"), + ); + let human_date = scheduled_release_date.format("%B %d"); + let internals = internals_url + .map(|url| format!("You can leave feedback on the [internals thread]({url}).")) + .unwrap_or_default(); + let prefix = if for_blog { + format!( + r#"--- +layout: post +title: "{} pre-release testing" +author: Release automation +team: The Release Team +---{}"#, + release, "\n\n", + ) + } else { + String::new() + }; + Some(format!( + "{prefix}The {release} pre-release is ready for testing. The release is scheduled for +{human_date}. [Release notes can be found here.][relnotes] + +You can try it out locally by running: + +```plain +RUSTUP_DIST_SERVER=https://dev-static.rust-lang.org rustup update stable +``` + +The index is . + +{internals} + +The release team is also thinking about changes to our pre-release process: +we'd love your feedback [on this GitHub issue][feedback]. + +[relnotes]: {release_notes_url} +[feedback]: https://github.com/rust-lang/release-team/issues/16 + " + )) + } } fn maybe_env(name: &str) -> Result, Error> diff --git a/src/curl_helper.rs b/src/curl_helper.rs new file mode 100644 index 0000000..30052ec --- /dev/null +++ b/src/curl_helper.rs @@ -0,0 +1,68 @@ +use anyhow::Context; +use curl::easy::Easy; + +pub trait BodyExt { + fn with_body(&mut self, body: S) -> Request<'_, S>; + fn without_body(&mut self) -> Request<'_, ()>; +} + +impl BodyExt for Easy { + fn with_body(&mut self, body: S) -> Request<'_, S> { + Request { + body: Some(body), + client: self, + } + } + fn without_body(&mut self) -> Request<'_, ()> { + Request { + body: None, + client: self, + } + } +} + +pub struct Request<'a, S> { + body: Option, + client: &'a mut Easy, +} + +impl Request<'_, S> { + pub fn send_with_response(self) -> anyhow::Result { + use std::io::Read; + let mut response = Vec::new(); + let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); + { + let mut transfer = self.client.transfer(); + // The unwrap in the read_function is basically guaranteed to not + // happen: reading into a slice can't fail. We can't use `?` since the + // return type inside transfer isn't compatible with io::Error. + if let Some(mut body) = body.as_deref() { + transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; + } + transfer.write_function(|new_data| { + response.extend_from_slice(new_data); + Ok(new_data.len()) + })?; + transfer.perform()?; + } + serde_json::from_slice(&response) + .with_context(|| format!("{}", String::from_utf8_lossy(&response))) + } + + pub fn send(self) -> anyhow::Result<()> { + use std::io::Read; + let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); + { + let mut transfer = self.client.transfer(); + // The unwrap in the read_function is basically guaranteed to not + // happen: reading into a slice can't fail. We can't use `?` since the + // return type inside transfer isn't compatible with io::Error. + if let Some(mut body) = body.as_deref() { + transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; + } + transfer.perform()?; + } + + Ok(()) + } +} diff --git a/src/discourse.rs b/src/discourse.rs new file mode 100644 index 0000000..bd2b9c1 --- /dev/null +++ b/src/discourse.rs @@ -0,0 +1,69 @@ +use crate::curl_helper::BodyExt; +use curl::easy::Easy; + +pub struct Discourse { + root: String, + api_key: String, + api_username: String, + client: Easy, +} + +impl Discourse { + pub fn new(root: String, api_username: String, api_key: String) -> Discourse { + Discourse { + root, + api_key, + api_username, + client: Easy::new(), + } + } + + fn start_new_request(&mut self) -> anyhow::Result<()> { + self.client.reset(); + self.client.useragent("rust-lang/promote-release")?; + let mut headers = curl::easy::List::new(); + headers.append(&format!("Api-Key: {}", self.api_key))?; + headers.append(&format!("Api-Username: {}", self.api_username))?; + headers.append("Content-Type: application/json")?; + self.client.http_headers(headers)?; + Ok(()) + } + + /// Returns a URL to the topic + pub fn create_topic( + &mut self, + category: u32, + title: &str, + body: &str, + ) -> anyhow::Result { + #[derive(serde::Serialize)] + struct Request<'a> { + title: &'a str, + #[serde(rename = "raw")] + body: &'a str, + category: u32, + archetype: &'a str, + } + #[derive(serde::Deserialize)] + struct Response { + topic_id: u32, + topic_slug: String, + } + self.start_new_request()?; + self.client.post(true)?; + self.client.url(&format!("{}/posts.json", self.root))?; + let resp = self + .client + .with_body(Request { + title, + body, + category, + archetype: "regular", + }) + .send_with_response::()?; + Ok(format!( + "{}/t/{}/{}", + self.root, resp.topic_slug, resp.topic_id + )) + } +} diff --git a/src/github.rs b/src/github.rs index de7ce80..87d8bf2 100644 --- a/src/github.rs +++ b/src/github.rs @@ -1,4 +1,4 @@ -use anyhow::Context; +use crate::curl_helper::BodyExt; use curl::easy::Easy; use rsa::pkcs1::DecodeRsaPrivateKey; use sha2::Digest; @@ -138,13 +138,6 @@ impl RepositoryClient<'_> { email: &'a str, } - #[derive(serde::Serialize)] - struct CreateRefInternal<'a> { - #[serde(rename = "ref")] - ref_: &'a str, - sha: &'a str, - } - #[derive(serde::Deserialize)] struct CreatedTag { sha: String, @@ -170,6 +163,40 @@ impl RepositoryClient<'_> { }) .send_with_response::()?; + self.create_ref(&format!("refs/tags/{}", tag.tag_name), &created.sha)?; + + Ok(()) + } + + /// Returns the SHA of the tip of this ref, if it exists. + pub(crate) fn get_ref(&mut self, name: &str) -> anyhow::Result { + // This mostly exists to make sure the request is successful rather than + // really checking the created ref (which we already know). + #[derive(serde::Deserialize)] + struct Reference { + object: Object, + } + #[derive(serde::Deserialize)] + struct Object { + sha: String, + } + + self.start_new_request()?; + self.github.client.get(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repository}/git/ref/{name}", + repository = self.repo, + ))?; + Ok(self + .github + .client + .without_body() + .send_with_response::()? + .object + .sha) + } + + pub(crate) fn create_ref(&mut self, name: &str, sha: &str) -> anyhow::Result<()> { // This mostly exists to make sure the request is successful rather than // really checking the created ref (which we already know). #[derive(serde::Deserialize)] @@ -178,6 +205,13 @@ impl RepositoryClient<'_> { #[allow(unused)] ref_: String, } + #[derive(serde::Serialize)] + struct CreateRefInternal<'a> { + #[serde(rename = "ref")] + name: &'a str, + sha: &'a str, + } + self.start_new_request()?; self.github.client.post(true)?; self.github.client.url(&format!( @@ -186,15 +220,44 @@ impl RepositoryClient<'_> { ))?; self.github .client - .with_body(CreateRefInternal { - ref_: &format!("refs/tags/{}", tag.tag_name), - sha: &created.sha, - }) + .with_body(CreateRefInternal { name, sha }) .send_with_response::()?; Ok(()) } + pub(crate) fn update_ref(&mut self, name: &str, sha: &str, force: bool) -> anyhow::Result<()> { + // This mostly exists to make sure the request is successful rather than + // really checking the created ref (which we already know). + #[derive(serde::Deserialize)] + struct CreatedRef { + #[serde(rename = "ref")] + #[allow(unused)] + ref_: String, + } + #[derive(serde::Serialize)] + struct UpdateRefInternal<'a> { + sha: &'a str, + force: bool, + } + + self.start_new_request()?; + // We want curl to read the request body, so configure POST. + self.github.client.post(true)?; + // However, the actual request should be a PATCH request. + self.github.client.custom_request("PATCH")?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repository}/git/refs/{name}", + repository = self.repo, + ))?; + self.github + .client + .with_body(UpdateRefInternal { sha, force }) + .send_with_response::()?; + + Ok(()) + } + pub(crate) fn workflow_dispatch(&mut self, workflow: &str, branch: &str) -> anyhow::Result<()> { #[derive(serde::Serialize)] struct Request<'a> { @@ -215,79 +278,141 @@ impl RepositoryClient<'_> { Ok(()) } -} - -#[derive(Copy, Clone)] -pub(crate) struct CreateTag<'a> { - pub(crate) commit: &'a str, - pub(crate) tag_name: &'a str, - pub(crate) message: &'a str, - pub(crate) tagger_name: &'a str, - pub(crate) tagger_email: &'a str, -} -trait BodyExt { - fn with_body(&mut self, body: S) -> Request<'_, S>; - fn without_body(&mut self) -> Request<'_, ()>; -} + /// Note that this API *will* fail if the file already exists in this + /// branch; we don't update existing files. + pub(crate) fn create_file( + &mut self, + branch: &str, + path: &str, + content: &str, + ) -> anyhow::Result<()> { + #[derive(serde::Serialize)] + struct Request<'a> { + message: &'a str, + content: &'a str, + branch: &'a str, + } + self.start_new_request()?; + self.github.client.put(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repository}/contents/{path}", + repository = self.repo, + ))?; + self.github + .client + .with_body(Request { + branch, + message: "Creating file via promote-release automation", + content: &base64::encode(&content), + }) + .send()?; + Ok(()) + } -impl BodyExt for Easy { - fn with_body(&mut self, body: S) -> Request<'_, S> { - Request { - body: Some(body), - client: self, + pub(crate) fn create_pr( + &mut self, + base: &str, + head: &str, + title: &str, + body: &str, + ) -> anyhow::Result<()> { + #[derive(serde::Serialize)] + struct Request<'a> { + head: &'a str, + base: &'a str, + title: &'a str, + body: &'a str, } + self.start_new_request()?; + self.github.client.post(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repository}/pulls", + repository = self.repo, + ))?; + self.github + .client + .with_body(Request { + base, + head, + title, + body, + }) + .send()?; + Ok(()) } - fn without_body(&mut self) -> Request<'_, ()> { - Request { - body: None, - client: self, + + /// Returns the last commit (SHA) on a repository's default branch which changed + /// the passed path. + pub(crate) fn last_commit_for_file(&mut self, path: &str) -> anyhow::Result { + #[derive(serde::Deserialize)] + struct CommitData { + sha: String, + } + self.start_new_request()?; + self.github.client.get(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repo}/commits?path={path}", + repo = self.repo + ))?; + let mut commits = self + .github + .client + .without_body() + .send_with_response::>()?; + if commits.is_empty() { + anyhow::bail!("No commits for path {:?}", path); } + Ok(commits.remove(0).sha) + } + + /// Returns the contents of the file + pub(crate) fn read_file(&mut self, sha: Option<&str>, path: &str) -> anyhow::Result { + self.start_new_request()?; + self.github.client.get(true)?; + self.github.client.url(&format!( + "https://api.github.com/repos/{repo}/contents/{path}{maybe_ref}", + repo = self.repo, + maybe_ref = sha.map(|s| format!("?ref={}", s)).unwrap_or_default() + ))?; + self.github + .client + .without_body() + .send_with_response::() } } -struct Request<'a, S> { - body: Option, - client: &'a mut Easy, +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub(crate) enum GitFile { + File { encoding: String, content: String }, + Submodule { sha: String }, } -impl Request<'_, S> { - fn send_with_response(self) -> anyhow::Result { - use std::io::Read; - let mut response = Vec::new(); - let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); - { - let mut transfer = self.client.transfer(); - // The unwrap in the read_function is basically guaranteed to not - // happen: reading into a slice can't fail. We can't use `?` since the - // return type inside transfer isn't compatible with io::Error. - if let Some(mut body) = body.as_deref() { - transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; - } - transfer.write_function(|new_data| { - response.extend_from_slice(new_data); - Ok(new_data.len()) - })?; - transfer.perform()?; +impl GitFile { + pub(crate) fn submodule_sha(&self) -> &str { + if let GitFile::Submodule { sha } = self { + sha + } else { + panic!("{:?} not a submodule", self); } - serde_json::from_slice(&response) - .with_context(|| format!("{}", String::from_utf8_lossy(&response))) } - fn send(self) -> anyhow::Result<()> { - use std::io::Read; - let body = self.body.map(|body| serde_json::to_vec(&body).unwrap()); - { - let mut transfer = self.client.transfer(); - // The unwrap in the read_function is basically guaranteed to not - // happen: reading into a slice can't fail. We can't use `?` since the - // return type inside transfer isn't compatible with io::Error. - if let Some(mut body) = body.as_deref() { - transfer.read_function(move |dest| Ok(body.read(dest).unwrap()))?; - } - transfer.perform()?; + pub(crate) fn content(&self) -> anyhow::Result { + if let GitFile::File { encoding, content } = self { + assert_eq!(encoding, "base64"); + Ok(String::from_utf8(base64::decode(&content.trim())?)?) + } else { + panic!("content() on {:?}", self); } - - Ok(()) } } + +#[derive(Copy, Clone)] +pub(crate) struct CreateTag<'a> { + pub(crate) commit: &'a str, + pub(crate) tag_name: &'a str, + pub(crate) message: &'a str, + pub(crate) tagger_name: &'a str, + pub(crate) tagger_email: &'a str, +} diff --git a/src/main.rs b/src/main.rs index 53cd3ca..73b76d3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,10 @@ #![allow(clippy::rc_buffer)] +mod branching; mod build_manifest; mod config; +mod curl_helper; +mod discourse; mod github; mod sign; mod smoke_test; @@ -28,6 +31,8 @@ use crate::config::{Channel, Config}; const TARGET: &str = env!("TARGET"); +const BLOG_PRIMARY_BRANCH: &str = "master"; + struct Context { work: PathBuf, handle: Easy, @@ -67,8 +72,10 @@ impl Context { fn run(&mut self) -> Result<(), Error> { let _lock = self.lock()?; - self.do_release()?; - + match self.config.action { + config::Action::PromoteRelease => self.do_release()?, + config::Action::PromoteBranches => self.do_branching()?, + } Ok(()) } @@ -223,6 +230,11 @@ impl Context { // around. let _ = fs::remove_dir_all(&self.dl_dir()); + // This opens a PR and starts an internals thread announcing a + // stable dev-release (we distinguish dev by the presence of metadata + // which lets us know where to create and what to put in the blog). + self.open_blog()?; + // We do this last, since it triggers triagebot posting the GitHub // release announcement (and since this is not actually really // important). @@ -672,6 +684,79 @@ impl Context { Ok(()) } + fn open_blog(&mut self) -> Result<(), Error> { + // We rely on the blog variables not being set in production to disable + // blogging on the actual release date. + if self.config.channel != Channel::Stable { + eprintln!("Skipping blogging -- not on stable"); + return Ok(()); + } + + let mut github = if let Some(github) = self.config.github() { + github + } else { + eprintln!("Skipping blogging - GitHub credentials not configured"); + return Ok(()); + }; + let mut discourse = if let Some(discourse) = self.config.discourse() { + discourse + } else { + eprintln!("Skipping blogging - Discourse credentials not configured"); + return Ok(()); + }; + let repository_for_blog = if let Some(repo) = &self.config.blog_repository { + repo.as_str() + } else { + eprintln!("Skipping blogging - blog repository not configured"); + return Ok(()); + }; + + let version = self.current_version.as_ref().expect("has current version"); + let internals_contents = + if let Some(contents) = self.config.blog_contents(version, &self.date, false, None) { + contents + } else { + eprintln!("Skipping internals - insufficient information to create blog post"); + return Ok(()); + }; + + let announcements_category = 18; + let internals_url = discourse.create_topic( + announcements_category, + &format!("Rust {} pre-release testing", version), + &internals_contents, + )?; + let blog_contents = if let Some(contents) = + self.config + .blog_contents(version, &self.date, true, Some(&internals_url)) + { + contents + } else { + eprintln!("Skipping blogging - insufficient information to create blog post"); + return Ok(()); + }; + + // Create a new branch so that we don't need to worry about the file + // already existing. In practice this *could* collide, but after merging + // a PR branches should get deleted, so it's very unlikely. + let name = format!("automation-{:x}", rand::random::()); + let mut token = github.token(repository_for_blog)?; + let master_sha = token.get_ref(&format!("heads/{BLOG_PRIMARY_BRANCH}"))?; + token.create_ref(&format!("refs/heads/{name}"), &master_sha)?; + token.create_file( + &name, + &format!( + "posts/inside-rust/{}-{}-prerelease.md", + chrono::Utc::today().format("%Y-%m-%d"), + version, + ), + &blog_contents, + )?; + token.create_pr(BLOG_PRIMARY_BRANCH, &name, "Pre-release announcement", "")?; + + Ok(()) + } + fn dl_dir(&self) -> PathBuf { self.work.join("dl") }