release: change release process

Consolidate version information in the Cargo.toml file.

Also moving forward we will bump version following a release by using
-dev suffixes.

Add a new bump-version.sh tool to help with this task

Resolves GitHub issue #3057

Signed-off-by: Pablo Barbáchano <[email protected]>

Author: pb8o

build.rs

@@ -1,25 +1,10 @@
 // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-use std::process::Command;
-
 // this build script is called on en every `devtool build`,
 // embedding the FIRECRACKER_VERSION directly in the resulting binary
 fn main() {
-    let firecracker_version = Command::new("git")
-        .args(["describe", "--dirty"])
-        .output()
-        .ok()
-        .and_then(|output| {
-            if output.status.success() {
-                return Some(output.stdout);
-            }
-            None
-        })
-        .and_then(|version_bytes| String::from_utf8(version_bytes).ok())
-        .map(|version_string| version_string.trim_start_matches('v').to_string())
-        .unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_string());
-
+    let firecracker_version = env!("CARGO_PKG_VERSION").to_string();
     println!(
         "cargo:rustc-env=FIRECRACKER_VERSION={}",
         firecracker_version

tests/conftest.py

@@ -463,9 +463,9 @@ def firecracker_id(fc):
 
 def firecracker_artifacts(*args, **kwargs):
     """Return all supported firecracker binaries."""
-    max_version = [int(x) for x in get_firecracker_version_from_toml().split(".")]
+    version = get_firecracker_version_from_toml()
     # until the next minor version (but not including)
-    max_version[1] += 1
+    max_version = (version.major, version.minor + 1, 0)
     params = {
         "min_version": "1.2.0",
         "max_version_open": ".".join(str(x) for x in max_version),

tests/framework/utils.py

@@ -17,6 +17,7 @@
 from pathlib import Path
 from typing import Dict
 
+import packaging.version
 import psutil
 from retry import retry
 from retry.api import retry_call
@@ -607,11 +608,9 @@ def get_firecracker_version_from_toml():
     the code has not been released.
     """
     cmd = "cd ../src/firecracker && cargo pkgid | cut -d# -f2 | cut -d: -f2"
-
-    rc, stdout, _ = run_cmd(cmd)
-    assert rc == 0
-
-    return stdout
+    rc, stdout, stderr = run_cmd(cmd)
+    assert rc == 0, stderr
+    return packaging.version.parse(stdout)
 
 
 def compare_versions(first, second):

tests/integration_tests/functional/test_api.py

@@ -7,17 +7,17 @@
 import os
 import platform
 import resource
-import subprocess
 import time
 
+import packaging.version
 import pytest
 
-import framework.utils_cpuid as utils
 import host_tools.drive as drive_tools
 import host_tools.network as net_tools
+from framework import utils_cpuid
 from framework.artifacts import NetIfaceConfig, SnapshotType
 from framework.builder import MicrovmBuilder, SnapshotBuilder
-from framework.utils import is_io_uring_supported
+from framework.utils import get_firecracker_version_from_toml, is_io_uring_supported
 
 MEM_LIMIT = 1000000000
 
@@ -469,7 +469,7 @@ def test_api_machine_config(test_microvm_with_api):
         assert test_microvm.api_session.is_status_no_content(response.status_code)
 
     response = test_microvm.actions.put(action_type="InstanceStart")
-    if utils.get_cpu_vendor() == utils.CpuVendor.AMD:
+    if utils_cpuid.get_cpu_vendor() == utils_cpuid.CpuVendor.AMD:
         # We shouldn't be able to apply Intel templates on AMD hosts
         fail_msg = "CPU vendor mismatched between actual CPU and CPU template"
         assert test_microvm.api_session.is_status_bad_request(response.status_code)
@@ -1008,20 +1008,14 @@ def test_api_version(test_microvm_with_api):
     # Validate VM version post-boot is the same as pre-boot.
     assert preboot_response.json() == postboot_response.json()
 
-    # Check that the version is the same as `git describe --dirty`.
-    # Abbreviated to post-tag commit metadata
-    out = subprocess.check_output(["git", "describe", "--dirty"]).decode()
-
-    # Skip the "v" at the start
-    tag_version = out[1:].strip()
-
-    # Git tag should match FC API version
-    assert (
-        tag_version == preboot_response.json()["firecracker_version"]
-    ), "Expected [{}], Actual [{}]".format(
-        preboot_response.json()["firecracker_version"], tag_version
+    cargo_version = get_firecracker_version_from_toml()
+    api_version = packaging.version.parse(
+        preboot_response.json()["firecracker_version"]
     )
 
+    # Cargo version should match FC API version
+    assert cargo_version == api_version
+
 
 def test_api_vsock(bin_cloner_path):
     """
@@ -1163,7 +1157,7 @@ def test_get_full_config_after_restoring_snapshot(bin_cloner_path):
     test_microvm = vm_instance.vm
     root_disk = vm_instance.disks[0]
     ssh_key = vm_instance.ssh_key
-    cpu_vendor = utils.get_cpu_vendor()
+    cpu_vendor = utils_cpuid.get_cpu_vendor()
 
     setup_cfg = {}
 
@@ -1175,10 +1169,10 @@ def test_get_full_config_after_restoring_snapshot(bin_cloner_path):
         "track_dirty_pages": False,
     }
 
-    if cpu_vendor == utils.CpuVendor.ARM:
+    if cpu_vendor == utils_cpuid.CpuVendor.ARM:
         setup_cfg["machine-config"]["smt"] = False
 
-    if cpu_vendor == utils.CpuVendor.INTEL:
+    if cpu_vendor == utils_cpuid.CpuVendor.INTEL:
         setup_cfg["machine-config"]["cpu_template"] = "C3"
 
     test_microvm.machine_cfg.patch(**setup_cfg["machine-config"])

tests/integration_tests/functional/test_snapshot_version.py

@@ -136,8 +136,7 @@ def test_snapshot_current_version(bin_cloner_path):
 
     version = get_firecracker_version_from_toml()
     # normalize to a snapshot version
-    (major, minor, _) = version.split(".", maxsplit=3)
-    target_version = f"{major}.{minor}.0"
+    target_version = f"{version.major}.{version.minor}.0"
     # Create a snapshot builder from a microvm.
     snapshot_builder = SnapshotBuilder(vm)
     disks = [vm_instance.disks[0].local_path()]

tools/bump-version.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -eu -o pipefail
+shopt -s lastpipe
+
+FC_TOOLS_DIR=$(dirname $(realpath $0))
+source "$FC_TOOLS_DIR/functions"
+FC_ROOT_DIR=$FC_TOOLS_DIR/..
+
+
+if [ $# -ne 1 ]; then
+    cat <<EOF
+$0 <version>
+
+    Example: $0 1.4.0-dev
+
+    Bump Firecracker release version:
+    1. Updates Cargo.toml / Cargo.lock
+    2. Runs 'cargo update'
+EOF
+    exit 1
+fi
+version=$1
+
+
+function check_snapshot_version {
+    local version=$1
+    local snap_version=$(echo $version |cut -f-2 -d. |tr . _)
+    if ! grep -s FC_V${snap_version}_SNAP_VERSION src/vmm/src/version_map.rs; then
+       die "I couldn't find FC_V${snap_version}_SNAP_VERSION in src/vmm/src/version_map.rs"
+    fi
+}
+
+check_snapshot_version "$version"
+
+
+# Get current version from the swagger spec.
+prev_ver=$(get_swagger_version)
+
+say "Updating from $prev_ver to $version ..."
+# Update version in files.
+files_to_change=(
+    "$FC_ROOT_DIR/src/api_server/swagger/firecracker.yaml"
+    "$FC_ROOT_DIR/src/firecracker/Cargo.toml"
+    "$FC_ROOT_DIR/src/jailer/Cargo.toml"
+    "$FC_ROOT_DIR/src/rebase-snap/Cargo.toml"
+    "$FC_ROOT_DIR/src/seccompiler/Cargo.toml"
+    "$FC_ROOT_DIR/src/cpu-template-helper/Cargo.toml"
+)
+say "Updating source files:"
+for file in "${files_to_change[@]}"; do
+    say "- $file"
+    if [[ "$file" =~ .+\.toml$ ]]; then
+        # For TOML
+        sed -i "s/^version = \"$prev_ver\"/version = \"$version\"/" "$file"
+    elif [[ "$file" =~ .+\.yaml$ ]]; then
+        # For YAML
+        sed -i "s/version: $prev_ver/version: $version/" "$file"
+    else
+        echo "ERROR: Unrecognized file '$file'"
+        exit 1
+    fi
+done
+
+# Run `cargo check` to update firecracker and jailer versions in all
+# `Cargo.lock`.
+# NOTE: This will break if it finds paths with spaces in them
+find . -path ./build -prune -o -name Cargo.lock -print |while read -r cargo_lock; do
+    say "Updating $cargo_lock ..."
+    (cd "$(dirname "$cargo_lock")"; cargo check; cargo update)
+done

tools/release-notes.sh

@@ -9,34 +9,28 @@ FC_TOOLS_DIR=$(dirname $(realpath $0))
 FC_ROOT_DIR=$FC_TOOLS_DIR/..
 
 
-if [ $# -ne 2 ]; then
+if [ $# -ne 1 ]; then
     cat <<EOF
 Compose the text for a new release using the information in the changelog,
 between the two specified releases.
 
-$0 <previous_version> <version>
+$0 <version>
 
-    Example: $0 1.1.1 1.1.2
+    Example: $0 1.1.2
 EOF
     exit 1;
 fi
 
-prev_ver="$1"
-curr_ver="$2"
+curr_ver="$1"
 changelog="$FC_ROOT_DIR/CHANGELOG.md"
 
-if [[ ! $prev_ver < $curr_ver ]]; then
-    echo "$prev_ver >= $curr_ver. Did you switch the argument order?"
-    exit 1
-fi
-
 # Patterns for the sections in the changelog corresponding to the versions.
-pat_curr="^##\s\[$curr_ver\]"
-pat_prev="^##\s\[$prev_ver\]"
+pat_curr="/^##\s\[$curr_ver\]/"
+pat_prev="/^##\s\[.+]/ && NR>3"
 # Extract the section enclosed between the 2 headers and strip off the first
 # 2 and last 2 lines (one is blank and one contains the header `## [A.B.C]`).
 # Then, replace `-` with `*` and remove section headers.
-sed "/$pat_curr/,/$pat_prev/!d" "$changelog" \
+awk "$pat_curr,$pat_prev" "$changelog" \
     | sed '1,2d;$d' \
     | sed "s/^-/*/g" \
     | sed "s/^###\s//g"

tools/release-prepare.sh

@@ -6,14 +6,6 @@
 set -eu -o pipefail
 shopt -s lastpipe
 
-function check_snapshot_version {
-    local version=$1
-    local snap_version=$(echo $version |cut -f-2 -d. |tr . _)
-    if ! grep -s FC_V${snap_version}_SNAP_VERSION src/vmm/src/version_map.rs; then
-       die "I couldn't find FC_V${snap_version}_SNAP_VERSION in src/vmm/src/version_map.rs"
-    fi
-}
-
 FC_TOOLS_DIR=$(dirname $(realpath $0))
 source "$FC_TOOLS_DIR/functions"
 FC_ROOT_DIR=$FC_TOOLS_DIR/..
@@ -26,18 +18,16 @@ $0 <version>
 
     Prepare a new Firecracker release:
     1. Update the version number
-    2. Update Crate dependencies
-    3. Generate CREDITS.md and CHANGELOG.md
-    4. Commit the result
-    5. Create a link to PR the changes
+    2. Generate CREDITS.md and CHANGELOG.md
+    3. Commit the result
+    4. Create a link to PR the changes
 EOF
     exit 1
 fi
 version=$1
 validate_version "$version"
 
 check_local_branch_is_release_branch
-check_snapshot_version "$version"
 
 # Create GitHub PR link
 ORIGIN_URL=$(git config --get remote.origin.url)
@@ -55,49 +45,19 @@ if [ "$PATCH" -gt 0 ]; then
 fi
 PR_URL="https://github.com/firecracker-microvm/$REPO/compare/$TARGET_BRANCH...$GH_USER:$REPO:$LOCAL_BRANCH?expand=1"
 
-# Get current version from the swagger spec.
-prev_ver=$(get_swagger_version)
-
-say "Updating from $prev_ver to $version ..."
-# Update version in files.
-files_to_change=(
-    "$FC_ROOT_DIR/src/api_server/swagger/firecracker.yaml"
-    "$FC_ROOT_DIR/src/cpu-template-helper/Cargo.toml"
-    "$FC_ROOT_DIR/src/firecracker/Cargo.toml"
-    "$FC_ROOT_DIR/src/jailer/Cargo.toml"
-    "$FC_ROOT_DIR/src/rebase-snap/Cargo.toml"
-    "$FC_ROOT_DIR/src/seccompiler/Cargo.toml"
-)
-say "Updating source files:"
-for file in "${files_to_change[@]}"; do
-    say "- $file"
-    # Dirty hack to make this work on both macOS/BSD and Linux.
-    # FIXME This is very hacky and can unintentionally bump other versions, so
-    # only do the replacement *once*.
-    sed -i "s/$prev_ver/$version/" "$file"
-done
-
-CHANGED=()
-# Run `cargo check` to update firecracker and jailer versions in all
-# `Cargo.lock`.
-# NOTE: This will break if it finds paths with spaces in them
-find . -path ./build -prune -o -name Cargo.lock -print |while read -r cargo_lock; do
-    say "Updating $cargo_lock ..."
-    (cd "$(dirname "$cargo_lock")"; cargo check)
-    CHANGED+=("$cargo_lock")
-done
+# Update version
+$FC_TOOLS_DIR/bump-version.sh "$version"
 
 # Update credits.
 say "Updating credits..."
 $FC_TOOLS_DIR/update-credits.sh
-CHANGED+=(CREDITS.md)
 
 # Update changelog.
 say "Updating changelog..."
 sed -i "s/\[Unreleased\]/\[$version\]/g" "$FC_ROOT_DIR/CHANGELOG.md"
-CHANGED+=(CHANGELOG.md)
 
-git add "${files_to_change[@]}" "${CHANGED[@]}"
+# Add all changed files
+git add -u
 git commit -s -m "chore: release v$version"
 
 
@@ -123,7 +83,7 @@ $(pp-li 1. Check the changes made to the repo:)
 
 $(pp-li 2. Preview the release notes)
 
-   $(pp-code ./tools/release-notes.sh "$prev_ver" "$version")
+   $(pp-code ./tools/release-notes.sh "$version")
 
 $(pp-li 3. If you want to undo the changes, run)
 
@@ -135,5 +95,5 @@ $(pp-li 4. Review and merge this change)
    $PR_URL
 
 $(pp-li 5. Once it is reviewed and merged, run the tag script)
-   $(pp-code ./tools/release-tag.sh $prev_ver $version)
+   $(pp-code ./tools/release-tag.sh $version)
 EOF