Skip to content

Commit e6566e1

Browse files
AkrogAkrog
committed
Hack: Testing PR helpers
This patch adds some scripts to facilitate the testing of PRs with dependencies. Four scripts are introduced: - hack/cleandeps.sh: Cleans up contents of go.work - hack/showdeps.py: Show the dependencies of a PR based on its commit messages contents (Depends-On:) - hack/setdeps.py: Replaces modules in the go workspace with local directories, PRs, remote branches, etc. - hack/checkout_pr.sh: Leverages the other scripts to do the checkout of a PR and replace the dependencies as needed.
1 parent 88a40bc commit e6566e1

File tree

6 files changed

+290
-0
lines changed

6 files changed

+290
-0
lines changed

‎go.work‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
go 1.18
2+
3+
use (
4+
.
5+
./api
6+
)

‎hack/README.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,31 @@
11
# Hacking
22

3+
### Testing PRs
4+
5+
To facilitate testing PRs we have the `hack/checkout_pr.sh` script that will
6+
checkout the specified cinder PR into local branch pr#, then recursively check
7+
the dependencies using the PRs commit messages and then replace modules in our
8+
go workspace to use those dependencies.
9+
10+
Example to get the PR#65 from cinder-operator:
11+
12+
```sh
13+
$ hack/checkout_pr.sh 65
14+
Cleaning go.work
15+
Fetching PR 65 on upstream/pr65
16+
Checking PR dependecies
17+
Setting dependencies: lib-common=88
18+
Source for lib-common PR#88 is github.com/fmount/lib-common@extra_volumes
19+
Checking the go mod version for branch @extra_volumes
20+
go work edit -replace github.com/openstack-k8s-operators/lib-common/modules/common=github.com/fmount/lib-common/modules/[email protected]
21+
go work edit -replace github.com/openstack-k8s-operators/lib-common/modules/database=github.com/fmount/lib-common/modules/[email protected]
22+
go work edit -replace github.com/openstack-k8s-operators/lib-common/modules/storage=github.com/fmount/lib-common/modules/[email protected]
23+
```
24+
25+
This script leverages scripts `hack/cleandeps.py` that cleans existing
26+
dependencies, `hack/showdeps.py` that shows dependencies for a given PR, and
27+
`hack/setdeps.py` that sets the go workspace replaces.
28+
329
### Ceph cluster
430

531
As describe in the [Getting Started Guide](../README.md#getting-started), the

‎hack/checkout_pr.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
LOCATION=$(realpath `dirname -- $BASH_SOURCE[0]`)
5+
6+
if [ $# -eq 0 ]; then
7+
echo "Error, missing arguments: $0 <PR#> [<remote_name> [<local_branch>]]"
8+
exit 1
9+
fi
10+
11+
PR=$1
12+
REMOTE=${2:-upstream}
13+
BRANCH=${3:-pr$1}
14+
15+
$LOCATION/cleandeps.sh
16+
17+
# Get the code
18+
echo Fetching PR $PR on $REMOTE/$BRANCH
19+
git fetch $REMOTE pull/$PR/head:$BRANCH
20+
git checkout $BRANCH
21+
22+
# Get dependencies
23+
echo Checking PR dependecies
24+
deps=`$LOCATION/showdeps.py $PR`
25+
if [[ -n "$deps" ]]; then
26+
echo Setting dependencies: $deps
27+
$LOCATION/setdeps.py $deps
28+
else
29+
echo PR has no dependencies
30+
fi

‎hack/cleandeps.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env bash
2+
LOCATION=$(realpath `dirname -- $BASH_SOURCE[0]`)
3+
echo Cleaning go.work
4+
echo -e "go 1.18\n\nuse (\n\t.\n\t./api\n)" > $LOCATION/../go.work

‎hack/setdeps.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env python3
2+
# Script to replace go module dependencies for Cinder:
3+
# Supports 4 different formats:
4+
# - By PR numbers:
5+
# ./setdeps.py lib-common=88 openstack-operator=38
6+
# - By local directory where the local dir is already with the right code:
7+
# ./setdeps.py lib-common=../lib-common \
8+
# openstack-operator=../openstack-operator
9+
# - By upstream repository
10+
# ./setdeps.py \
11+
# lib-common=https://github.com/fmount/lib-common/@extra_volumes \
12+
# openstack-operator=https://github.com/fmount/openstack-operator/@extra_volumes
13+
# - By short upstream repository:
14+
# ./setdeps.py lib-common=fmount/lib-common@extra_volumes \
15+
# openstack-operator=fmount/openstack-operator@extra_volumes
16+
import json
17+
import os
18+
import re
19+
import subprocess
20+
import sys
21+
import urllib.request
22+
23+
GH_API_URL = 'https://api.github.com/repos/openstack-k8s-operators'
24+
25+
26+
def get_modules(repo, requires):
27+
"""Return a list of tuples of (repository, module) for a specific repo."""
28+
result = []
29+
for require in requires:
30+
try:
31+
index = require.index('/' + repo)
32+
33+
module_index = index + 1 + len(repo)
34+
url = require[:module_index]
35+
module = require[module_index:]
36+
if module[0] == '/':
37+
result.append((url, module))
38+
except ValueError:
39+
continue
40+
return result
41+
42+
43+
def get_requires():
44+
"""Get all requires modules from go.mod"""
45+
# Get location of the go.mod file
46+
result = subprocess.run('go env GOMOD', shell=True, check=True,
47+
capture_output=True)
48+
go_mod_path = result.stdout.strip()
49+
re_res = re.search(r"^require \(\n(.*?)\n\)",
50+
open(go_mod_path).read(),
51+
re.MULTILINE | re.DOTALL)
52+
if not re_res:
53+
raise Exception('Error parsing go.mod')
54+
lines = re_res.group(1).split('\n')
55+
return [line.strip().split()[0] for line in lines if line]
56+
57+
58+
def main(args):
59+
source_is = None
60+
61+
requires = get_requires()
62+
for arg in args:
63+
repo = arg[0]
64+
go_module_url = f'github.com/openstack-k8s-operators/{repo}'
65+
66+
source_version = ''
67+
68+
try:
69+
pr = int(arg[1])
70+
source_is = 'PR'
71+
except ValueError:
72+
src_path = arg[1]
73+
if os.path.exists(src_path):
74+
source_is = 'LOCAL'
75+
else:
76+
source_is = 'REPO'
77+
# Build url if we where just provided a partial repo
78+
# eg: fpantano/lib-common@extravol
79+
if not src_path.startswith('http'):
80+
src_path = 'github.com/' + src_path
81+
82+
if source_is == 'PR':
83+
api_url = f'{GH_API_URL}/{repo}/pulls/{pr}'
84+
85+
contents_str = urllib.request.urlopen(api_url).read()
86+
contents = json.loads(contents_str)
87+
source_version = '@' + contents['head']['ref']
88+
89+
src_path = 'github.com/' + contents['head']['repo']['full_name']
90+
print(f'Source for {repo} PR#{pr} is {src_path}{source_version}')
91+
92+
elif source_is == 'LOCAL':
93+
src_path = os.path.abspath(src_path)
94+
print(f'Source repo for {repo} is {src_path}')
95+
elif source_is == 'REPO': # is a repo
96+
if '@' in src_path:
97+
src_path, source_version = src_path.split('@')
98+
source_version = '@' + source_version
99+
print(f'Source repo for {repo} is {src_path}')
100+
101+
if source_version:
102+
print(f'Checking the go mod version for branch {source_version}')
103+
sha = contents['head']['sha'][:12]
104+
result = subprocess.run(f'go list -m -json {src_path}@{sha}',
105+
shell=True, check=True,
106+
capture_output=True)
107+
source_version = '@' + json.loads(result.stdout)['Version']
108+
109+
modules = get_modules(repo, requires)
110+
for go_module_url, module in modules:
111+
cmd = (f'go work edit -replace {go_module_url}{module}='
112+
f'{src_path}{module}{source_version}')
113+
print(cmd)
114+
os.system(cmd)
115+
print()
116+
117+
118+
if __name__ == '__main__':
119+
if len(sys.argv) < 2:
120+
print('Error, missing arguments')
121+
exit(1)
122+
123+
args = []
124+
for arg in sys.argv[1:]:
125+
args.append(arg.split('='))
126+
127+
main(args)

‎hack/showdeps.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env python3
2+
# Script to get dependencies recursively for a PR.
3+
# PR message and commit messages must use Depends-On in one of these 4 formats:
4+
# - Depends-On: lib-common=88
5+
# - Depends-On: openstack-k8s-operators/lib-common#88
6+
# - Depends-On: lib-common#88
7+
# - Depends-On: https://github.com/openstack-k8s-operators/lib-common/88
8+
# Output is consumable directly by the setdeps.py script
9+
# $ ./showdeps PR# [repo]
10+
# eg:
11+
# $ ./showdeps 65
12+
# lib-common=88
13+
#
14+
# $ ./showdeps 65 cinder-operator
15+
# lib-common=88
16+
17+
import json
18+
import re
19+
import sys
20+
import urllib.request
21+
22+
GH_API_URL = 'https://api.github.com/repos/openstack-k8s-operators'
23+
REPO = 'cinder-operator'
24+
RECURSIVE = False
25+
26+
27+
def get_gh_json(repo, pr, ending=''):
28+
api_url = f'{GH_API_URL}/{repo}/pulls/{pr}{ending}'
29+
contents_str = urllib.request.urlopen(api_url).read()
30+
return json.loads(contents_str)
31+
32+
33+
def find_dependencies(text):
34+
result = []
35+
depends = re.findall(r"\n\s*Depends-On:\s*(\S+)\s*?", text, re.IGNORECASE)
36+
for dep in depends:
37+
# lib-common=88
38+
if '=' in dep:
39+
res = dep.split('=')
40+
# openstack-k8s-operators/lib-common#88
41+
# lib-common#88
42+
elif '#' in dep:
43+
res = dep.rsplit('#', 1)
44+
res[0] = res[0].rsplit('/', 1)[-1]
45+
# https://github.com/openstack-k8s-operators/lib-common/88
46+
else:
47+
r = dep.rsplit('/', 3)
48+
if len(r) < 4:
49+
sys.stderr.write(f'Wrong Depends-On on: {dep}\n')
50+
continue
51+
res = r[1], r[3]
52+
result.append(res)
53+
return result
54+
55+
56+
def get_dependencies(repo, pr):
57+
contents = get_gh_json(repo, pr)
58+
pr_message = contents['body']
59+
# Initialize to the dependencies found in the PR message
60+
result = find_dependencies(pr_message)
61+
62+
# Find additional dependencies in commit messages
63+
contents = get_gh_json(repo, pr, '/commits')
64+
for commit in contents:
65+
message = commit['commit']['message']
66+
deps = find_dependencies(message)
67+
if deps:
68+
result.extend(deps)
69+
return result
70+
71+
72+
def main(args):
73+
pr = args[1]
74+
repo = args[2] if len(args) > 2 else REPO
75+
dependencies = []
76+
to_check = [(repo, pr)]
77+
checked = []
78+
while to_check:
79+
check = to_check.pop()
80+
# Detect circular references
81+
if check in checked:
82+
continue
83+
new_deps = get_dependencies(*check)
84+
dependencies.extend(new_deps)
85+
if not RECURSIVE:
86+
break
87+
to_check.extend(new_deps)
88+
checked.append(check)
89+
90+
if dependencies:
91+
# Convert to str and remove duplicated dependencies
92+
deps_str = set(f'{dep[0]}={dep[1]}' for dep in dependencies)
93+
print(' '.join(deps_str))
94+
95+
96+
if __name__ == '__main__':
97+
main(sys.argv)

0 commit comments

Comments
 (0)