Skip to content

Commit 46162b5

Browse files
AkrogAkrog
committed
Hack: Testing PR helpers
This patch adds some scripts to facilitate the testing of PRs with dependencies. Four scripts are introduced: - hack/cleandeps.sh: Cleans up contents of go.work - hack/showdeps.py: Show the dependencies of a PR based on its commit messages contents (Depends-On:) - hack/setdeps.py: Replaces modules in the go workspace with local directories, PRs, remote branches, etc. - hack/checkout_pr.sh: Leverages the other scripts to do the checkout of a PR and replace the dependencies as needed.
1 parent 88a40bc commit 46162b5

File tree

6 files changed

+280
-0
lines changed

6 files changed

+280
-0
lines changed

‎go.work‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
go 1.18
2+
3+
use (
4+
.
5+
./api
6+
)

‎hack/README.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,31 @@
11
# Hacking
22

3+
### Testing PRs
4+
5+
To facilitate testing PRs we have the `hack/checkout_pr.sh` script that will
6+
checkout the specified cinder PR into local branch pr#, then recursively check
7+
the dependencies using the PRs commit messages and then replace modules in our
8+
go workspace to use those dependencies.
9+
10+
Example to get the PR#65 from cinder-operator:
11+
12+
```sh
13+
$ hack/checkout_pr.sh 65
14+
Cleaning go.work
15+
Fetching PR 65 on upstream/pr65
16+
Checking PR dependecies
17+
Setting dependencies: lib-common=88
18+
Source for lib-common PR#88 is github.com/fmount/lib-common@extra_volumes
19+
Checking the go mod version for branch @extra_volumes
20+
go work edit -replace github.com/openstack-k8s-operators/lib-common/modules/common=github.com/fmount/lib-common/modules/[email protected]
21+
go work edit -replace github.com/openstack-k8s-operators/lib-common/modules/database=github.com/fmount/lib-common/modules/[email protected]
22+
go work edit -replace github.com/openstack-k8s-operators/lib-common/modules/storage=github.com/fmount/lib-common/modules/[email protected]
23+
```
24+
25+
This script leverages scripts `hack/cleandeps.py` that cleans existing
26+
dependencies, `hack/showdeps.py` that shows dependencies for a given PR, and
27+
`hack/setdeps.py` that sets the go workspace replaces.
28+
329
### Ceph cluster
430

531
As describe in the [Getting Started Guide](../README.md#getting-started), the

‎hack/checkout_pr.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
LOCATION=$(realpath `dirname -- $BASH_SOURCE[0]`)
5+
6+
if [ $# -eq 0 ]; then
7+
echo "Error, missing arguments: $0 <PR#> [<remote_name> [<local_branch>]]"
8+
exit 1
9+
fi
10+
11+
PR=$1
12+
REMOTE=${2:-upstream}
13+
BRANCH=${3:-pr$1}
14+
15+
$LOCATION/cleandeps.sh
16+
17+
# Get the code
18+
echo Fetching PR $PR on $REMOTE/$BRANCH
19+
git fetch $REMOTE pull/$PR/head:$BRANCH
20+
git checkout $BRANCH
21+
22+
# Get dependencies
23+
echo Checking PR dependecies
24+
deps=`$LOCATION/showdeps.py $PR`
25+
if [[ -n "$deps" ]]; then
26+
echo Setting dependencies: $deps
27+
$LOCATION/setdeps.py $deps
28+
else
29+
echo PR has no dependencies
30+
fi

‎hack/cleandeps.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env bash
2+
LOCATION=$(realpath `dirname -- $BASH_SOURCE[0]`)
3+
echo Cleaning go.work
4+
echo -e "go 1.18\n\nuse (\n .\n ./api\n)\n" > $LOCATION/../go.work

‎hack/setdeps.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env python3
2+
# Script to replace go module dependencies for Cinder:
3+
# Supports 4 different formats:
4+
# - By PR numbers:
5+
# ./setdeps.py lib-common=88 openstack-operator=38
6+
# - By local directory where the local dir is already with the right code:
7+
# ./setdeps.py lib-common=../lib-common \
8+
# openstack-operator=../openstack-operator
9+
# - By upstream repository
10+
# ./setdeps.py \
11+
# lib-common=https://github.com/fmount/lib-common/@extra_volumes \
12+
# openstack-operator=https://github.com/fmount/openstack-operator/@extra_volumes
13+
# - By short upstream repository:
14+
# ./setdeps.py lib-common=fmount/lib-common@extra_volumes \
15+
# openstack-operator=fmount/openstack-operator@extra_volumes
16+
import json
17+
import os
18+
import re
19+
import subprocess
20+
import sys
21+
import urllib.request
22+
23+
GH_API_URL = 'https://api.github.com/repos/openstack-k8s-operators'
24+
25+
26+
def get_modules(repo, requires):
27+
"""Return a list of tuples of (repository, module) for a specific repo."""
28+
result = []
29+
for require in requires:
30+
try:
31+
index = require.index('/' + repo)
32+
33+
module_index = index + 1 + len(repo)
34+
url = require[:module_index]
35+
module = require[module_index:]
36+
if module[0] == '/':
37+
result.append((url, module))
38+
except ValueError:
39+
continue
40+
return result
41+
42+
43+
def get_requires():
44+
"""Get all requires modules from go.mod"""
45+
# Get location of the go.mod file
46+
result = subprocess.run('go env GOMOD', shell=True, check=True,
47+
capture_output=True)
48+
go_mod_path = result.stdout.strip()
49+
re_res = re.search(r"^require \(\n(.*?)\n\)",
50+
open(go_mod_path).read(),
51+
re.MULTILINE | re.DOTALL)
52+
if not re_res:
53+
raise Exception('Error parsing go.mod')
54+
lines = re_res.group(1).split('\n')
55+
return [line.strip().split()[0] for line in lines if line]
56+
57+
58+
def main(args):
59+
source_is = None
60+
61+
requires = get_requires()
62+
for arg in args:
63+
repo = arg[0]
64+
go_module_url = f'github.com/openstack-k8s-operators/{repo}'
65+
66+
source_version = ''
67+
68+
try:
69+
pr = int(arg[1])
70+
source_is = 'PR'
71+
except ValueError:
72+
src_path = arg[1]
73+
if os.path.exists(src_path):
74+
source_is = 'LOCAL'
75+
else:
76+
source_is = 'REPO'
77+
# Build url if we where just provided a partial repo
78+
# eg: fpantano/lib-common@extravol
79+
if not src_path.startswith('http'):
80+
src_path = 'github.com/' + src_path
81+
82+
if source_is == 'PR':
83+
api_url = f'{GH_API_URL}/{repo}/pulls/{pr}'
84+
85+
contents_str = urllib.request.urlopen(api_url).read()
86+
contents = json.loads(contents_str)
87+
source_version = '@' + contents['head']['ref']
88+
89+
src_path = 'github.com/' + contents['head']['repo']['full_name']
90+
print(f'Source for {repo} PR#{pr} is {src_path}{source_version}')
91+
92+
elif source_is == 'LOCAL':
93+
src_path = os.path.abspath(src_path)
94+
print(f'Source repo for {repo} is {src_path}')
95+
elif source_is == 'REPO': # is a repo
96+
if '@' in src_path:
97+
src_path, source_version = src_path.split('@')
98+
source_version = '@' + source_version
99+
print(f'Source repo for {repo} is {src_path}')
100+
101+
if source_version:
102+
print(f'Checking the go mod version for branch {source_version}')
103+
sha = contents['head']['sha'][:12]
104+
result = subprocess.run(f'go list -m -json {src_path}@{sha}',
105+
shell=True, check=True,
106+
capture_output=True)
107+
source_version = '@' + json.loads(result.stdout)['Version']
108+
109+
modules = get_modules(repo, requires)
110+
for go_module_url, module in modules:
111+
cmd = (f'go work edit -replace {go_module_url}{module}='
112+
f'{src_path}{module}{source_version}')
113+
print(cmd)
114+
os.system(cmd)
115+
print()
116+
117+
118+
if __name__ == '__main__':
119+
if len(sys.argv) < 2:
120+
print('Error, missing arguments')
121+
exit(1)
122+
123+
args = []
124+
for arg in sys.argv[1:]:
125+
args.append(arg.split('='))
126+
127+
main(args)

‎hack/showdeps.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env python3
2+
# Script to get dependencies recursively for a PR.
3+
# PR message and commit messages must use Depends-On in one of these 3 formats:
4+
# - Depends-On: lib-common=88
5+
# - Depends-On: openstack-k8s-operators/lib-common#88
6+
# - Depends-On: https://github.com/openstack-k8s-operators/lib-common/88
7+
# Output is consumable directly by the setdeps.py script
8+
# $ ./showdeps PR# [repo]
9+
# eg:
10+
# $ ./showdeps 65
11+
# lib-common=88
12+
#
13+
# $ ./showdeps 65 cinder-operator
14+
# lib-common=88
15+
16+
import json
17+
import re
18+
import sys
19+
import urllib.request
20+
21+
GH_API_URL = 'https://api.github.com/repos/openstack-k8s-operators'
22+
REPO = 'cinder-operator'
23+
24+
25+
def get_gh_json(repo, pr, ending=''):
26+
api_url = f'{GH_API_URL}/{repo}/pulls/{pr}{ending}'
27+
contents_str = urllib.request.urlopen(api_url).read()
28+
return json.loads(contents_str)
29+
30+
31+
def find_dependencies(text):
32+
result = []
33+
depends = re.findall(r"\n\s*Depends-On:\s*(\S+)\s*\n", text, re.IGNORECASE)
34+
for dep in depends:
35+
# lib-common=88
36+
if '=' in dep:
37+
res = dep.split('=')
38+
# openstack-k8s-operators/lib-common#88
39+
elif '#' in dep:
40+
res = dep.rsplit('#', 1)
41+
res[0] = res[0].rsplit('/', 1)[1]
42+
# https://github.com/openstack-k8s-operators/lib-common/88
43+
else:
44+
r = dep.rsplit('/', 3)
45+
if len(r) < 4:
46+
sys.stderr.write(f'Wrong Depends-On on: {dep}\n')
47+
continue
48+
res = r[1], r[3]
49+
result.append(res)
50+
return result
51+
52+
53+
def get_dependencies(repo, pr):
54+
contents = get_gh_json(repo, pr)
55+
pr_message = contents['body']
56+
# Initialize to the dependencies found in the PR message
57+
result = find_dependencies(pr_message)
58+
59+
# Find additional dependencies in commit messages
60+
contents = get_gh_json(repo, pr, '/commits')
61+
for commit in contents:
62+
message = commit['commit']['message']
63+
deps = find_dependencies(message)
64+
if deps:
65+
result.extend(deps)
66+
return result
67+
68+
69+
def main(args):
70+
pr = int(args[1])
71+
repo = args[2] if len(args) > 2 else REPO
72+
dependencies = []
73+
to_check = [(repo, pr)]
74+
while to_check:
75+
repo, pr = to_check.pop()
76+
new_deps = get_dependencies(repo, pr)
77+
dependencies.extend(new_deps)
78+
to_check.extend(new_deps)
79+
80+
if dependencies:
81+
# Convert to str and remove duplicated dependencies
82+
deps_str = set(f'{dep[0]}={dep[1]}' for dep in dependencies)
83+
print(' '.join(deps_str))
84+
85+
86+
if __name__ == '__main__':
87+
main(sys.argv)

0 commit comments

Comments
 (0)