|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +# Copyright 2025 The Cockroach Authors. |
| 4 | +# |
| 5 | +# Use of this software is governed by the CockroachDB Software License |
| 6 | +# included in the /LICENSE file. |
| 7 | + |
| 8 | +# This script schedules daily maintenance commands on a 2‐week cycle. |
| 9 | +# Optional parameters OLD_RELEASE and NEW_RELEASE can be provided. |
| 10 | +# Usage: ./mixed_version.sh <CLUSTER> [OLD_RELEASE=v24.3.8] [NEW_RELEASE=v25.2.0-alpha.2] |
| 11 | + |
| 12 | +# Schedule of commands: |
| 13 | +# |
| 14 | +# Week 1: |
| 15 | +# - Tuesday: |
| 16 | +# - Wipe the cluster |
| 17 | +# - Full cluster restart with OLD_RELEASE |
| 18 | +# - Set cluster.preserve_downgrade_option to OLD_RELEASE |
| 19 | +# - upgrade nodes 1-3 to NEW_RELEASE (50% nodes) |
| 20 | +# - Start tpcc_init_cct_tpcc.sh and roachtest_operations_run.sh |
| 21 | +# - Thursday: |
| 22 | +# - Deploy NEW_RELEASE to the entire cluster (not finalized) |
| 23 | +# - Friday: |
| 24 | +# - Revert all nodes to OLD_RELEASE |
| 25 | +# - upgrade nodes 1-2 to NEW_RELEASE (33% nodes) |
| 26 | +# |
| 27 | +# Week 2: |
| 28 | +# - Monday: |
| 29 | +# - Upgrade nodes 3-5 to NEW_RELEASE (80% nodes as 1,2 were already upgraded) |
| 30 | +# - Friday: |
| 31 | +# - Reset cluster.preserve_downgrade_option |
| 32 | +# - Upgrade node 6 to NEW_RELEASE (100% nodes as 1-5 were already upgraded) |
| 33 | + |
| 34 | +CLUSTER="$1" |
| 35 | +if [ -z "$CLUSTER" ]; then |
| 36 | + echo "Usage: $0 <CLUSTER> [OLD_RELEASE] [NEW_RELEASE]" |
| 37 | + exit 1 |
| 38 | +fi |
| 39 | + |
| 40 | +/home/ubuntu/drtprod sync |
| 41 | + |
| 42 | +# Set optional release versions |
| 43 | +OLD_RELEASE="${2:-v24.3.8}" |
| 44 | +NEW_RELEASE="${3:-v25.2.0-alpha.2}" |
| 45 | + |
| 46 | +# Get today's day of week (1 for Monday, ... 7 for Sunday) and date |
| 47 | +day_of_week=$(date +%u) |
| 48 | +today=$(date +%F) |
| 49 | +cycle_file="/home/ubuntu/.cycle_info.txt" |
| 50 | + |
| 51 | +if [ -f "$cycle_file" ]; then |
| 52 | + read saved_cycle saved_day < "$cycle_file" |
| 53 | +else |
| 54 | + # Initialize cycle_week to 0 if no previous info exists |
| 55 | + saved_cycle=0 |
| 56 | + saved_day=$today |
| 57 | +fi |
| 58 | + |
| 59 | +# On Monday, if this is the first run of today, flip the cycle week |
| 60 | +if [ "$day_of_week" -eq 1 ] && [ "$saved_day" != "$today" ]; then |
| 61 | + cycle_week=$((1 - saved_cycle)) |
| 62 | +else |
| 63 | + cycle_week=$saved_cycle |
| 64 | +fi |
| 65 | + |
| 66 | +# Save the cycle week and today's date for persistence |
| 67 | +echo "$cycle_week $today" > "$cycle_file" |
| 68 | + |
| 69 | +# Use an array to store multiple commands |
| 70 | +cmds=() |
| 71 | + |
| 72 | +if [ "$day_of_week" -eq 1 ] && [ "$cycle_week" -eq 1 ]; then |
| 73 | + # Week 2 - Monday |
| 74 | + cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:3-5 release $NEW_RELEASE") |
| 75 | +elif [ "$day_of_week" -eq 2 ] && [ "$cycle_week" -eq 0 ]; then |
| 76 | + # Tuesday in Week 1 only |
| 77 | + cmds+=("sudo systemctl stop tpcc_run_cct_tpcc") |
| 78 | + cmds+=("sudo systemctl stop roachtest_ops") |
| 79 | + cmds+=("/home/ubuntu/drtprod stop $CLUSTER") |
| 80 | + cmds+=("/home/ubuntu/drtprod wipe $CLUSTER") |
| 81 | + cmds+=("/home/ubuntu/drtprod stage $CLUSTER release $OLD_RELEASE") |
| 82 | + cmds+=("/home/ubuntu/drtprod start $CLUSTER --binary ./cockroach --args=--wal-failover=among-stores --enable-fluent-sink=true --restart=false --sql-port=26257 --store-count=4") |
| 83 | + version=$(echo "$OLD_RELEASE" | sed -E 's/^v([0-9]+\.[0-9]+)\..*/\1/') |
| 84 | + cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e \"SET CLUSTER SETTING cluster.preserve_downgrade_option ='$version'\"") |
| 85 | + cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-3 release $NEW_RELEASE") |
| 86 | + cmds+=("rm -rf /home/ubuntu/certs") |
| 87 | + cmds+=("/home/ubuntu/drtprod get $CLUSTER:1 certs /home/ubuntu/certs") |
| 88 | + cmds+=("chmod 600 /home/ubuntu/certs/*") |
| 89 | + cmds+=("/home/ubuntu/tpcc_init_cct_tpcc.sh") |
| 90 | + cmds+=("sudo systemd-run --unit tpcc_run_cct_tpcc --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/tpcc_run_cct_tpcc.sh") |
| 91 | + cmds+=("sleep 30") |
| 92 | + # Note that roachtest_operations_run.sh needs to be setup manually for the first time. |
| 93 | + cmds+=("sudo systemd-run --unit roachtest_ops --same-dir --uid $(id -u) --gid $(id -g) bash /home/ubuntu/roachtest_operations_run.sh") |
| 94 | +elif [ "$day_of_week" -eq 4 ] && [ "$cycle_week" -eq 0 ]; then |
| 95 | + # Thursday in Week 1 only |
| 96 | + cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:4-6 release $NEW_RELEASE") |
| 97 | +elif [ "$day_of_week" -eq 5 ]; then |
| 98 | + # Friday for both Weeks |
| 99 | + if [ "$cycle_week" -eq 0 ]; then |
| 100 | + # Week 1 friday commands |
| 101 | + cmds+=("/home/ubuntu/drtprod deploy $CLUSTER release $OLD_RELEASE") |
| 102 | + cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:1-2 release $NEW_RELEASE") |
| 103 | + else |
| 104 | + # Week 2 friday commands |
| 105 | + cmds+=("/home/ubuntu/drtprod sql $CLUSTER:1 -- -e 'RESET CLUSTER SETTING cluster.preserve_downgrade_option'") |
| 106 | + cmds+=("/home/ubuntu/drtprod deploy $CLUSTER:6 release $NEW_RELEASE") |
| 107 | + fi |
| 108 | +fi |
| 109 | + |
| 110 | +# Always check the status of the cluster |
| 111 | +cmds+=("/home/ubuntu/drtprod status $CLUSTER") |
| 112 | + |
| 113 | +if [ ${#cmds[@]} -gt 0 ]; then |
| 114 | + for cmd in "${cmds[@]}"; do |
| 115 | + echo "Executing: $cmd" |
| 116 | + if ! eval "$cmd"; then |
| 117 | + echo "Error executing: $cmd" >&2 |
| 118 | + exit 1 |
| 119 | + fi |
| 120 | + done |
| 121 | +else |
| 122 | + echo "No scheduled command for today." |
| 123 | +fi |
0 commit comments