Adapt upgrade.sh scripts to new versioning (#717)

Follow-up to #701

Author: mlafeldt

crates/libduckdb-sys/upgrade.sh

@@ -1,21 +1,35 @@
 #!/bin/bash
 
-set -e
-
-# NOTE: This script is currently broken and needs updating for the new
-# duckdb-rs versioning scheme before it can be used reliably.
+set -e -o pipefail
 
 SCRIPT=$(realpath "$0")
 SCRIPT_DIR=$(dirname "$SCRIPT")
+WORKSPACE_DIR=$(realpath "$SCRIPT_DIR/../..")
 
 echo "$SCRIPT_DIR"
 cd "$SCRIPT_DIR"
+
+current_workspace_version() {
+    grep '^version = "' "$WORKSPACE_DIR/Cargo.toml" | head -n1 | sed -E 's/version = "([^"]+)"/\1/'
+}
+
+crate_version_to_duckdb_version() {
+    local CRATE_VERSION=$1
+    local ENCODED
+    ENCODED=$(echo "$CRATE_VERSION" | cut -d. -f2)
+    local DUCKDB_MAJOR=$((ENCODED / 10000))
+    local DUCKDB_MINOR=$(((ENCODED / 100) % 100))
+    local DUCKDB_PATCH=$((ENCODED % 100))
+    printf 'v%s.%s.%s' "$DUCKDB_MAJOR" "$DUCKDB_MINOR" "$DUCKDB_PATCH"
+}
+
 cargo clean
 mkdir -p "$SCRIPT_DIR/../../target" "$SCRIPT_DIR/duckdb"
 export DUCKDB_LIB_DIR="$SCRIPT_DIR/duckdb"
 
 # Download and extract amalgamation
-DUCKDB_VERSION=v1.5.0
+DUCKDB_VERSION=${1:-$(crate_version_to_duckdb_version "$(current_workspace_version)")}
+DUCKDB_VERSION="v${DUCKDB_VERSION#v}"
 git submodule update --init --checkout
 cd "$SCRIPT_DIR/duckdb-sources"
 git fetch
@@ -29,6 +43,10 @@ rm -f "$SCRIPT_DIR/src/bindgen_bundled_version_loadable.rs"
 find "$SCRIPT_DIR/../../target" -type f -name bindgen.rs -exec rm {} \;
 cargo build --features "extensions-full buildtime_bindgen loadable-extension"
 find "$SCRIPT_DIR/../../target" -type f -name bindgen.rs -exec cp {} "$SCRIPT_DIR/src/bindgen_bundled_version_loadable.rs" \;
+if [ ! -f "$SCRIPT_DIR/src/bindgen_bundled_version_loadable.rs" ]; then
+    echo "ERROR: bindgen_bundled_version_loadable.rs was not regenerated" >&2
+    exit 1
+fi
 
 # Sanity checks
 # FIXME: how to test this here?
@@ -39,6 +57,10 @@ rm -f "$SCRIPT_DIR/src/bindgen_bundled_version.rs"
 find "$SCRIPT_DIR/../../target" -type f -name bindgen.rs -exec rm {} \;
 cargo build --features "extensions-full buildtime_bindgen"
 find "$SCRIPT_DIR/../../target" -type f -name bindgen.rs -exec cp {} "$SCRIPT_DIR/src/bindgen_bundled_version.rs" \;
+if [ ! -f "$SCRIPT_DIR/src/bindgen_bundled_version.rs" ]; then
+    echo "ERROR: bindgen_bundled_version.rs was not regenerated" >&2
+    exit 1
+fi
 
 # Sanity checks
 cd "$SCRIPT_DIR/.."

upgrade.sh

@@ -1,44 +1,73 @@
 #!/bin/bash
 
-set -e
+set -e -o pipefail
 
-# NOTE: This script is currently broken and needs updating for the new
-# duckdb-rs versioning scheme before it can be used reliably.
+sed_inplace() {
+    if sed --version 2>/dev/null | grep -q GNU; then
+        sed -i "$@"
+    else
+        sed -i '' "$@"
+    fi
+}
 
-if sed --version 2>/dev/null | grep -q GNU; then
-  SED_INPLACE="sed -i"
-else
-  SED_INPLACE="sed -i ''"
-fi
+get_latest_release() {
+    local REPO=$1
+    gh api "repos/$REPO/releases/latest" --jq '.tag_name'
+}
 
-## How to run
-##   `./upgrade.sh`
+current_workspace_version() {
+    grep '^version = "' Cargo.toml | head -n1 | sed -E 's/version = "([^"]+)"/\1/'
+}
 
-# https://gist.github.com/lukechilds/a83e1d7127b78fef38c2914c4ececc3c
-# Usage
-# $ get_latest_release "duckdb/duckdb"
-get_latest_release() {
-    curl -fSs "https://api.github.com/repos/$1/releases/latest" | # Get latest release from GitHub api
-      grep '"tag_name":' |                                            # Get tag line
-      sed -E 's/.*"v([^"]+)".*/\1/'                                   # Pluck JSON value
+crate_version_to_duckdb_version() {
+    local CRATE_VERSION=$1
+    local ENCODED
+    ENCODED=$(echo "$CRATE_VERSION" | cut -d. -f2)
+    local DUCKDB_MAJOR=$((ENCODED / 10000))
+    local DUCKDB_MINOR=$(((ENCODED / 100) % 100))
+    local DUCKDB_PATCH=$((ENCODED % 100))
+    printf 'v%s.%s.%s' "$DUCKDB_MAJOR" "$DUCKDB_MINOR" "$DUCKDB_PATCH"
 }
 
-duckdb_version=$(get_latest_release "duckdb/duckdb")
-duckdb_rs_version=$(get_latest_release "duckdb/duckdb-rs")
+duckdb_version_to_crate_version() {
+    local DUCKDB_VERSION=${1#v}
+    local CRATE_MAJOR=$2
+    local DUCKDB_MAJOR DUCKDB_MINOR DUCKDB_PATCH
+    IFS=. read -r DUCKDB_MAJOR DUCKDB_MINOR DUCKDB_PATCH <<<"$DUCKDB_VERSION"
+    printf '%s.%d%02d%02d.0' "$CRATE_MAJOR" "$DUCKDB_MAJOR" "$DUCKDB_MINOR" "$DUCKDB_PATCH"
+}
+
+DUCKDB_VERSION=${1:-$(get_latest_release "duckdb/duckdb")}
+DUCKDB_VERSION="v${DUCKDB_VERSION#v}"
+CURRENT_CRATE_VERSION=$(current_workspace_version)
+CURRENT_DUCKDB_VERSION=$(crate_version_to_duckdb_version "$CURRENT_CRATE_VERSION")
+CRATE_MAJOR=$(echo "$CURRENT_CRATE_VERSION" | cut -d. -f1)
+TARGET_CRATE_VERSION=$(duckdb_version_to_crate_version "$DUCKDB_VERSION" "$CRATE_MAJOR")
+CURRENT_CRATE_VERSION_PATTERN=${CURRENT_CRATE_VERSION//./\\.}
+CURRENT_DUCKDB_VERSION_PATTERN=${CURRENT_DUCKDB_VERSION//./\\.}
 
-if [ $duckdb_version = $duckdb_rs_version ]; then
-    echo "Already update to date, latest version is $duckdb_version"
+if [ "$DUCKDB_VERSION" = "$CURRENT_DUCKDB_VERSION" ]; then
+    echo "Already up to date, latest DuckDB version is $DUCKDB_VERSION and workspace version is $CURRENT_CRATE_VERSION"
     exit 0
 fi
 
-echo "Start to upgrade from $duckdb_rs_version to $duckdb_version"
+echo "Start to upgrade DuckDB from $CURRENT_DUCKDB_VERSION to $DUCKDB_VERSION"
+echo "Update crate version from $CURRENT_CRATE_VERSION to $TARGET_CRATE_VERSION"
 
-$SED_INPLACE "s/$duckdb_rs_version/$duckdb_version/g" \
+sed_inplace "s!$CURRENT_CRATE_VERSION_PATTERN!$TARGET_CRATE_VERSION!g" \
     Cargo.toml \
     crates/duckdb/Cargo.toml \
-    crates/libduckdb-sys/upgrade.sh \
     crates/libduckdb-sys/Cargo.toml \
-    .github/workflows/rust.yaml \
-    README.md
+    crates/duckdb-loadable-macros/Cargo.toml
+
+sed_inplace "s!$CURRENT_DUCKDB_VERSION_PATTERN!$DUCKDB_VERSION!g" \
+    .github/workflows/rust.yaml
+
+# Update README: only Cargo.toml examples and download URLs, not prose/history.
+sed_inplace "/version = \"/s!$CURRENT_CRATE_VERSION_PATTERN!$TARGET_CRATE_VERSION!g" README.md
+sed_inplace "/releases\/download/s!$CURRENT_DUCKDB_VERSION_PATTERN!$DUCKDB_VERSION!g" README.md
+
+# Let Cargo rewrite Cargo.lock from the updated manifests instead of editing it as text.
+cargo metadata --format-version 1 >/dev/null
 
-exec ./crates/libduckdb-sys/upgrade.sh
+exec ./crates/libduckdb-sys/upgrade.sh "$DUCKDB_VERSION"