Skip to content

refactor: improve readability and structure of release script #3196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Swaraj-SG wants to merge 2 commits into
base: main
Choose a base branch
from
+136 −104
Open

refactor: improve readability and structure of release script #3196

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
009567e
Refactor automq_release.py for improved readability
Swaraj-SG Feb 1, 2026
77613d4
refactor: Refactor automq_release.py for improved readability
Swaraj-SG Feb 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
240 changes: 136 additions & 104 deletions automq_release.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# !/usr/bin/python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#!/usr/bin/env python3

import re
import subprocess
Expand All @@ -22,178 +37,195 @@
import time
from pathlib import Path
import argparse
from shutil import which

main_branch = "develop"
MAIN_BRANCH = "develop"


def fail(msg):
print(msg)
def fail(message: str) -> None:
"""Print error message and exit."""
print(message)
sys.exit(1)


def print_output(output):
if output is None or len(output) == 0:
def print_output(output: str | None) -> None:
"""Pretty print command output."""
if not output:
return
for line in output.split('\n'):
for line in output.split("\n"):
print(">", line)


def cmd(action, cmd_arg, *args, **kwargs):
if isinstance(cmd_arg, str) and not kwargs.get("shell", False):
cmd_arg = cmd_arg.split()
def run_cmd(action: str, command, *args, **kwargs):
"""
Run a shell command with logging, retries, and optional failure allowance.
"""
if isinstance(command, str) and not kwargs.get("shell", False):
command = command.split()

allow_failure = kwargs.pop("allow_failure", False)
num_retries = kwargs.pop("num_retries", 0)
retries_left = kwargs.pop("num_retries", 0)

stdin_log = ""
if "stdin" in kwargs and isinstance(kwargs["stdin"], str):
stdin_log = "--> " + kwargs["stdin"]
stdin = tempfile.TemporaryFile()
stdin.write(kwargs["stdin"].encode('utf-8'))
stdin.seek(0)
kwargs["stdin"] = stdin
stdin_file = tempfile.TemporaryFile()
stdin_file.write(kwargs["stdin"].encode("utf-8"))
stdin_file.seek(0)
kwargs["stdin"] = stdin_file

print(action, command, stdin_log)

print(action, cmd_arg, stdin_log)
try:
output = subprocess.check_output(cmd_arg, *args, stderr=subprocess.STDOUT, **kwargs)
print_output(output.decode('utf-8'))
except subprocess.CalledProcessError as e:
print_output(e.output.decode('utf-8'))

if num_retries > 0:
kwargs['num_retries'] = num_retries - 1
kwargs['allow_failure'] = allow_failure
print("Retrying... %d remaining retries" % (num_retries - 1))
time.sleep(4. / (num_retries + 1)) # e.g., if retries=3, sleep for 1s, 1.3s, 2s
return cmd(action, cmd_arg, *args, **kwargs)
output = subprocess.check_output(command, *args, stderr=subprocess.STDOUT, **kwargs)
print_output(output.decode("utf-8"))

except subprocess.CalledProcessError as error:
print_output(error.output.decode("utf-8"))

if retries_left > 0:
kwargs["num_retries"] = retries_left - 1
kwargs["allow_failure"] = allow_failure
print(f"Retrying... {retries_left - 1} remaining retries")
time.sleep(4.0 / (retries_left + 1))
return run_cmd(action, command, *args, **kwargs)

if allow_failure:
return

print("*************************************************")
print("*" * 49)
print("*** First command failure occurred here. ***")
print("*************************************************")
print("*" * 49)
fail("")


def cmd_output(cmd, *args, **kwargs):
if isinstance(cmd, str):
cmd = cmd.split()
return subprocess.check_output(cmd, *args, stderr=subprocess.STDOUT, **kwargs).decode('utf-8')

def get_cmd_output(command, *args, **kwargs) -> str:
"""Run command and return decoded output."""
if isinstance(command, str):
command = command.split()
return subprocess.check_output(command, *args, stderr=subprocess.STDOUT, **kwargs).decode("utf-8")

def replace(path, pattern, replacement):
updated = []
with open(path, 'r') as f:
for line in f:
updated.append((replacement + '\n') if line.startswith(pattern) else line)

with open(path, 'w') as f:
for line in updated:
f.write(line)
def replace_line_start(path: str, pattern: str, replacement: str) -> None:
"""Replace lines that start with a specific pattern."""
updated_lines = []
with open(path, "r") as file:
for line in file:
updated_lines.append((replacement + "\n") if line.startswith(pattern) else line)

with open(path, "w") as file:
file.writelines(updated_lines)

def regexReplace(path, pattern, replacement):
updated = []
with open(path, 'r') as f:
for line in f:
updated.append(re.sub(pattern, replacement, line))

with open(path, 'w') as f:
for line in updated:
f.write(line)
def regex_replace(path: str, pattern: str, replacement: str) -> None:
"""Regex-based replacement for each line in a file."""
updated_lines = []
with open(path, "r") as file:
for line in file:
updated_lines.append(re.sub(pattern, replacement, line))

with open(path, "w") as file:
file.writelines(updated_lines)

def is_tool(name):
"""Check whether `name` is on PATH and marked as executable."""

# from whichcraft import which
from shutil import which

def is_tool_available(name: str) -> bool:
"""Check whether a tool exists in system PATH."""
return which(name) is not None


def check_tools(tools):
"""Check whether tools are available in the system"""
def check_required_tools(tools: list[str]) -> None:
"""Ensure required CLI tools are installed."""
for tool in tools:
if not is_tool(tool):
raise Exception("%s is not installed" % tool)
if not is_tool_available(tool):
raise Exception(f"{tool} is not installed")


def is_valid_kafka_tag(tag):
"""Check whether the tag is a valid kafka tag"""
pattern = r'^\d+\.\d+\.\d+(-rc\d+)?$'
return re.match(pattern, tag) is not None
def is_valid_kafka_tag(tag: str) -> bool:
"""Validate Kafka release tag format."""
return re.match(r"^\d+\.\d+\.\d+(-rc\d+)?$", tag) is not None


def is_valid_s3stream_tag(tag):
"""Check whether the tag is a valid s3stream tag"""
pattern = r'^\d+\.\d+\.\d+-s3stream(-rc\d+)?$'
return re.match(pattern, tag) is not None
def is_valid_s3stream_tag(tag: str) -> bool:
"""Validate s3stream release tag format."""
return re.match(r"^\d+\.\d+\.\d+-s3stream(-rc\d+)?$", tag) is not None


def get_project_path():
# Path object for the current file
current_file = Path(__file__)
# Get the directory of the current file
return current_file.parent
def get_project_path() -> Path:
"""Return project root path."""
return Path(__file__).parent


def do_release(tag, s3stream_tag):
def create_release(tag: str, s3stream_tag: str) -> str:
"""Create release branch, update versions, and push tags."""
new_branch = f"release-{tag}-{int(time.time())}"
cmd(f"Checking out to release branch", f'git checkout --quiet -b {new_branch}')

run_cmd("Checking out to release branch", f"git checkout --quiet -b {new_branch}")

print("Updating docker compose")
regexReplace("docker/docker-compose.yaml", "image: automqinc/automq:.*$", f"image: automqinc/automq:{tag}")
replace("gradle/dependencies.gradle", ' branch = "main"', f' require "{s3stream_tag}"')
regex_replace("docker/docker-compose.yaml", r"image: automqinc/automq:.*$", f"image: automqinc/automq:{tag}")
replace_line_start("gradle/dependencies.gradle", ' branch = "main"', f' require "{s3stream_tag}"')

cmd("Committing changes", ["git", "commit", "--all", "--gpg-sign", "--signoff", "--message", f"ci: bump version to {tag}"])
cmd("Pushing changes", ["git", "push", "--quiet", "origin", new_branch])
run_cmd("Committing changes", ["git", "commit", "--all", "--gpg-sign", "--signoff",
"--message", f"ci: bump version to {tag}"])
run_cmd("Pushing changes", ["git", "push", "--quiet", "origin", new_branch])

cmd("Tagging %s" % tag, ["git", "tag", "--sign", "--annotate", tag, "--message", f"release {tag}"])
cmd("Pushing tag %s" % tag, ["git", "push", "--quiet", "origin", tag])
run_cmd(f"Tagging {tag}", ["git", "tag", "--sign", "--annotate", tag, "--message", f"release {tag}"])
run_cmd(f"Pushing tag {tag}", ["git", "push", "--quiet", "origin", tag])

return new_branch


def tag_exists(tag):
maybe_tag = cmd_output(f'git tag --list {tag}').strip()
return len(maybe_tag) > 0
def tag_exists_locally(tag: str) -> bool:
"""Check if a Git tag exists locally."""
return len(get_cmd_output(f"git tag --list {tag}").strip()) > 0


def pre_release_checks(tag: str, s3stream_tag: str) -> None:
"""Run all validations before starting release."""
check_required_tools(["git"])

run_cmd("Fetching latest code", "git fetch origin")
run_cmd("Checking workspace clean", "git diff-index --quiet HEAD --")

current_branch = get_cmd_output("git rev-parse --abbrev-ref HEAD").strip()
if current_branch != MAIN_BRANCH:
fail(f"You must run this script on the {MAIN_BRANCH} branch. current: {current_branch}")

run_cmd(f"Checking branch {MAIN_BRANCH} up-to-date",
f"git diff origin/{MAIN_BRANCH}...{MAIN_BRANCH} --quiet")

run_cmd(f"Checking tag {s3stream_tag} exists",
f"git ls-remote --quiet --exit-code --tags git@github.com:AutoMQ/automq-for-rocketmq.git {s3stream_tag}")

run_cmd(f"Checking tag {tag} not exists in remote",
f"! git ls-remote --quiet --exit-code --tags origin {tag}", shell=True)

def check_before_started(tag, s3stream_tag):
check_tools(["git"])
cmd("Fetching latest code", 'git fetch origin')
cmd("Checking workspace clean", 'git diff-index --quiet HEAD --')
starting_branch = cmd_output('git rev-parse --abbrev-ref HEAD').strip()
if starting_branch != main_branch:
fail(f"You must run this script on the {main_branch} branch. current: {starting_branch}")
cmd(f"Checking branch {main_branch} up-to-date", f'git diff origin/{main_branch}...{main_branch} --quiet')
cmd(f"Checking tag {s3stream_tag} exists", f'git ls-remote --quiet --exit-code --tags git@github.com:AutoMQ/automq-for-rocketmq.git {s3stream_tag}')
cmd(f"Checking tag {tag} not exists in remote", f'! git ls-remote --quiet --exit-code --tags origin {tag}', shell=True)
print(f"Checking tag {tag} not exists in local")
if tag_exists(tag):
if tag_exists_locally(tag):
fail(f"Tag {tag} already exists. Please delete it before running this script.")


if __name__ == '__main__':
parser = argparse.ArgumentParser(description='release AutoMQ Kafka')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="release AutoMQ Kafka")
parser.add_argument("--kafka-tag", required=True, type=str,
help="release tag, which should be in the format of 'x.y.z' or 'x.y.z-rcN', e.g., 1.2.3 or 11.22.33-rc44")
help="Format: x.y.z or x.y.z-rcN")
parser.add_argument("--s3stream-tag", required=True, type=str,
help="related s3stream tag, which should have been pushed to the AutoMQ/automq-for-rocketmq repository,\
and should be in the format of 'x.y.z-s3stream' or 'x.y.z-s3stream-rcN', e.g., 1.2.3-s3stream or 11.22.33-s3stream-rc44")
help="Format: x.y.z-s3stream or x.y.z-s3stream-rcN")

# get tag from command line
args = parser.parse_args()
tag = args.kafka_tag
s3stream_tag = args.s3stream_tag

print(f"=== try to release {tag} ===")

if not is_valid_kafka_tag(tag):
fail(f"Invalid tag {tag}")
if not is_valid_s3stream_tag(s3stream_tag):
fail(f"Invalid s3stream tag {s3stream_tag}")
check_before_started(tag, s3stream_tag)
branch = do_release(tag, s3stream_tag)

pre_release_checks(tag, s3stream_tag)
branch = create_release(tag, s3stream_tag)

print(f"=== release {tag} done ===")
print(f"Please create a PR to merge release branch to {main_branch} visiting:")
print(f" https://github.com/AutoMQ/automq-for-kafka/pull/new/{main_branch}...{branch}")
print(f"Please create a PR to merge release branch to {MAIN_BRANCH} visiting:")
print(f" https://github.com/AutoMQ/automq-for-kafka/pull/new/{MAIN_BRANCH}...{branch}")