Skip to content

Commit bef9550

Browse files
author
Anton Benkevich
committed
Definitions Update
Add gh token Add defintions automatic release pipeline Add documentation
1 parent dfb6866 commit bef9550

File tree

8 files changed

+334
-15
lines changed

8 files changed

+334
-15
lines changed

.travis.yml

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,55 @@
1+
env:
2+
matrix:
3+
- GITHUB_REPO=alertlogic/alertlogic-sdk-definitions
4+
global:
5+
- secure: BXki2OvWA47YSLeKnAyNENgMNz148IFl+8ohpV7liXHORW/rkbUE3MYCqNJgT+zNqOJDhGSO/fiAb8mOxjBNjeWgqhxu8H+KqOBKU9B0XpagqM1fqpAn2duKxb51ULNKQvjRcsn/tEXqyIe2ipO0G7P8VJQzuHvrVEqVWDXMbbJAj2s1ZAfgzzFRiI7dDweWrPwLNwK9kMxjauTEpaAVuhFCGl2ga6nG8a26ORxecxvELXwEcKBGW7gJWZhpk+Y1Fhd6YeJKNmvok0XthqLJNaRV2JRErzyYpaLnnw+FgTpfSF2taREge4wlDzYbKRP4yVlC3IFn2QLq77kff/NsPck5n6M7B+PIzKrsu5bwCU3Apx1y78pstLTVVNb0xCLh61JGn8Va/vfExX1lpDMd42+9a2RTALEXHcgvdF+gx/KL8Hwa4HsmOaAJqWmfivdQilLzvdbxmnu0f+lsDnwsJy5FciBLjdZ/VTmphWOTMzopMHJdl7dNzFafwDTTAVaS0FA7EoHdZAm1m/4C5iZ+B+85SFCQxt1/Ot3/8ntTub0OPep98S1RNEqGqjeIXosh3KKdT8gp27kIcuqj3iQvsqFGMVeDqQqij/RUkizX1z0CoX8M4Pbi9BQpM7g8aNGqTJtLPaYbccURy56UarkbK7mG63tWTW4pba0vyfSwTg0=
16
language: python
2-
python:
3-
- 3.8
4-
- 3.7
5-
install: pip install -U tox-travis
6-
script: tox
7-
deploy:
8-
provider: pypi
9-
distributions: sdist bdist_wheel
10-
skip_cleanup: true
11-
user: __token__
12-
password:
13-
secure: eJMWcJY8inSzxPccLTlEN5nWzeDa2zzovgYX7HB+oeoNGEMSu+XS6HV1ILARHyDJovhPlkgZplFcJxjRZITBsjFAatpFeBSTqeV1/qkXs4rSY4wYF71x7dkBRQxEuAjsH9v1qzHvE9JXdzdvy/z/U36Dh1HXLhW9uYQ+Rfo0k/+VR6sAByMHAEFWCibt3UbrMm3l1j8X/95Z0h5+xJODnTc4QCiNK/HYhdzq5MU415CLHo1BLFlP6GyWK+Mi+bQu2cZnU+qnKlSYjmfKJYtL3tJ97TtXNRgvhiS2pgi+qldfNv8kaj9trkUJgsn6QaTGl93CEMthMCcVyYPd7qDjpN2If5NrWBqiTVid/g2HO33WleFRQjp9UF6eRsDozecSSVGefVZc6Aorl8usL9Kf0nKFHGm9P2lbjHofiJ23la8W43CT1gOSxUFs4u8v9AFh4aCI1v6TwdlVX7/1EbcsFsOFVXBEuE6RB0ZGbSQcafcObdxbg8wNJsDBpz/JVFicWdfTZ1C4LEZkAsuwwHvLlfwYX51984fr5493h7CNmWfHziMXTp68YvZsoQ8OCVDEAerYMf6eeWBXhZWDJLUUBPl89u7pVnQrKq9YyZCsUJSfrq6pQ4+3bsiLgwp4Zyho4XS8w8Y1bXbjX/6jFAK5tXMexKkg19jLd/V0ef6eEsY=
14-
on:
15-
tags: true
16-
repo: alertlogic/alertlogic-sdk-definitions
7+
jobs:
8+
include:
9+
- stage: test
10+
name: 'Run python tests '
1711
python: 3.8
12+
install: pip install -U tox-travis
13+
script: tox
14+
- python: 3.7
15+
install: pip install -U tox-travis
16+
script: tox
17+
- stage: merge-automated-prs
18+
name: "(Pipeline Stage 1)Merge pull request produced by automation"
19+
if: "(fork = false) AND (head_branch =~ /^auto-update-\\d*/) AND (type = pull_request)"
20+
python: 3.8
21+
script: BRANCHES_TO_MERGE_REGEX='^auto-update-' BRANCH_TO_MERGE_INTO=master scripts/travis-automerge.sh
22+
- stage: auto_release
23+
name: "(Pipeline Stage 2)Automatically release definitions update"
24+
if: branch = master AND type = push
25+
python: 3.8
26+
install: pip install -U packaging requests
27+
script: scripts/create_release.py --repo $GITHUB_REPO -re "^Definitions Update.*" --create_release
28+
- stage: github_release
29+
if: type = push AND tag =~ /^v\d*\.\d*\.\d*$/
30+
name: "(Pipeline Stage 3)Create release containing just definitions"
31+
script: zip -rj AlertLogicOpenAPIDefinitions.zip alsdkdefs/apis
32+
deploy:
33+
provider: releases
34+
api_key: "$GITHUB_SECRET_TOKEN"
35+
skip_cleanup: true
36+
file: AlertLogicOpenAPIDefinitions.zip
37+
on:
38+
tags: true
39+
- stage: deploy
40+
name: "(Pipeline Stage 4)Deploy to PyPI"
41+
if: type = push AND tag =~ /^v\d*\.\d*\.\d*$/
42+
python: 3.8
43+
install: pip install -U tox-travis
44+
script: tox
45+
deploy:
46+
provider: pypi
47+
distributions: sdist bdist_wheel
48+
skip_cleanup: true
49+
user: __token__
50+
password:
51+
secure: eJMWcJY8inSzxPccLTlEN5nWzeDa2zzovgYX7HB+oeoNGEMSu+XS6HV1ILARHyDJovhPlkgZplFcJxjRZITBsjFAatpFeBSTqeV1/qkXs4rSY4wYF71x7dkBRQxEuAjsH9v1qzHvE9JXdzdvy/z/U36Dh1HXLhW9uYQ+Rfo0k/+VR6sAByMHAEFWCibt3UbrMm3l1j8X/95Z0h5+xJODnTc4QCiNK/HYhdzq5MU415CLHo1BLFlP6GyWK+Mi+bQu2cZnU+qnKlSYjmfKJYtL3tJ97TtXNRgvhiS2pgi+qldfNv8kaj9trkUJgsn6QaTGl93CEMthMCcVyYPd7qDjpN2If5NrWBqiTVid/g2HO33WleFRQjp9UF6eRsDozecSSVGefVZc6Aorl8usL9Kf0nKFHGm9P2lbjHofiJ23la8W43CT1gOSxUFs4u8v9AFh4aCI1v6TwdlVX7/1EbcsFsOFVXBEuE6RB0ZGbSQcafcObdxbg8wNJsDBpz/JVFicWdfTZ1C4LEZkAsuwwHvLlfwYX51984fr5493h7CNmWfHziMXTp68YvZsoQ8OCVDEAerYMf6eeWBXhZWDJLUUBPl89u7pVnQrKq9YyZCsUJSfrq6pQ4+3bsiLgwp4Zyho4XS8w8Y1bXbjX/6jFAK5tXMexKkg19jLd/V0ef6eEsY=
52+
on:
53+
tags: true
54+
repo: "$GITHUB_REPO"
55+
python: 3.8

MANIFEST.in

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
include LICENSE
2+
include README.md
3+
exclude scripts/

README.md

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,40 @@
11
# alertlogic-sdk-definitions
22
Alert Logic APIs definitions
3+
4+
[![Build Status](https://travis-ci.com/alertlogic/alertlogic-sdk-definitions.svg?branch=master)](https://travis-ci.com/alertlogic/alertlogic-sdk-definitions)
5+
6+
Repository contains static definitions of Alert Logic APIs, used for documentation generation,
7+
[SDK](https://github.com/alertlogic/alertlogic-sdk-python) and [CLI](https://github.com/alertlogic/alcli).
8+
9+
### Usage
10+
11+
#### Install
12+
`pip install alertlogic-sdk-definitions`
13+
14+
For the one who doesn't require python code, GitHub releases are produced
15+
containing an archive with OpenAPI definitions only, see
16+
[here](https://github.com/alertlogic/alertlogic-sdk-definitions/releases)
17+
18+
#### Test
19+
`python -m unittest`
20+
21+
#### Use
22+
23+
List available service definitions:
24+
```
25+
>>> import alsdkdefs
26+
>>> alsdkdefs.list_services()
27+
['aefr', 'aerta', 'aetag', 'aetuner', 'aims', 'assets_query', 'credentials', 'deployments', 'ingest', 'iris', 'policies', 'search', 'themis']
28+
```
29+
30+
Get path to a service definitions paths:
31+
```
32+
>>> import alsdkdefs
33+
>>> alsdkdefs.get_service_defs("aerta")
34+
['/usr/local/lib/python3.8/site-packages/alsdkdefs/apis/aerta/aerta.v1.yaml']
35+
```
36+
37+
### Development
38+
39+
Please submit a PR. Please note that API definitions are updated automatically and any changes to it will be overwritten, see:
40+
[automatic update process](doc/automatic_releases.md)

doc/CI.puml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
@startuml
2+
(*) --> "service API defintion update detected"
3+
--> "Automated PR is created"
4+
If "PR change test passed" then
5+
--> [Yes] "PR automatically merged"
6+
--> "master branch is tagged with new version \nif commit is detected to be made by automation"
7+
--> "package is deployed to PyPi"
8+
--> "github release is created"
9+
-> (*)
10+
else
11+
--> "repo maintainer notified"
12+
--> "probem resolved"
13+
->(*)
14+
Endif
15+
@enduml

doc/automatic_releases.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
## Automatic microservices API releases
2+
3+
Alert Logic microservices code is maintained privately, OpenAPI definitions source code
4+
is kept along with the code of the microservices.
5+
Automated process is keeping current repository up-to-date:
6+
7+
![CI process](images/CI.png)
8+
9+
Definitions release pipeline is broken onto stages:
10+
- (Stage 0)Automation creates a pull request with definitions change
11+
- (Stage 1)Merge pull request produced by automation
12+
- (Stage 2)Automatically release definitions update(calculate new version and tag changes)
13+
- (Stage 3)Create release containing just definitions
14+
- (Stage 4)Deploy to PyPI
15+
16+
Please note, that automation only changes X.Y.<micro> version, major and minor would be changed manually.

doc/images/CI.png

35 KB
Loading

scripts/create_release.py

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
#!/usr/bin/env python3
2+
import requests
3+
from packaging import version
4+
from argparse import ArgumentParser
5+
import re
6+
import os
7+
import datetime
8+
import json
9+
from functools import reduce
10+
11+
12+
def make_auth_header(token):
13+
return {"Authorization": f"token {token}"}
14+
15+
16+
def list_github_tags(token, repo):
17+
return requests.get(f"https://api.github.com/repos/{repo}/tags", headers=make_auth_header(token)).json()
18+
19+
20+
def list_version_tags(github_tags):
21+
return list(filter(lambda v: isinstance(v, version.Version), map(lambda t: version.parse(t['name']), github_tags)))
22+
23+
24+
def reduce_tag(acc, tag):
25+
acc[version.parse(tag['name'])] = tag
26+
return acc
27+
28+
29+
def make_tags_search_hash(github_tags):
30+
return reduce(reduce_tag, github_tags, {})
31+
32+
33+
def get_latest_version(parsed_tags):
34+
sorted_tags = sorted(parsed_tags, reverse=True)
35+
if sorted_tags:
36+
return sorted_tags[0]
37+
else:
38+
return version.parse("0.0.0")
39+
40+
41+
def get_branch_commit_sha(token, repo, branch):
42+
url = f"https://api.github.com/repos/{repo}/branches/{branch}"
43+
return requests.get(url, headers=make_auth_header(token)).json()['commit']['sha']
44+
45+
46+
def get_branch_commit_message(token, repo, branch):
47+
url = f"https://api.github.com/repos/{repo}/branches/{branch}"
48+
return requests.get(url, headers=make_auth_header(token)).json()['commit']['commit']['message']
49+
50+
51+
def create_lightweight_tag(token, repo, tag_obj):
52+
url = f"https://api.github.com/repos/{repo}/git/refs"
53+
r = requests.post(url, headers=make_auth_header(token), json=tag_obj)
54+
# meh, just roughly
55+
if 201 <= r.status_code < 300:
56+
return True
57+
else:
58+
print(
59+
f"Failed to create tag {json.dumps(tag_obj, indent=4)}, because {r.status_code} {json.dumps(r.json(), indent=4)}")
60+
return False
61+
62+
63+
def create_annotated_tag(token, repo, tag_obj):
64+
url = f"https://api.github.com/repos/{repo}/git/tags"
65+
r = requests.post(url, headers=make_auth_header(token), json=tag_obj)
66+
# meh, just roughly
67+
if 201 <= r.status_code < 300:
68+
return True
69+
else:
70+
print(
71+
f"Failed to create tag {json.dumps(tag_obj, indent=4)}, because {r.status_code} {json.dumps(r.json(), indent=4)}")
72+
return False
73+
74+
75+
def make_lightweight_tag_object(version, sha):
76+
return {
77+
"ref": f"refs/tags/v{version}",
78+
"sha": sha
79+
}
80+
81+
82+
def make_annotated_tag_object(version, tag_message, sha, tagger='CI Bot', email='travis@travis'):
83+
date = datetime.datetime.now().astimezone().replace(microsecond=0).isoformat()
84+
return {
85+
"tag": f"v{version}",
86+
"message": tag_message,
87+
"object": sha,
88+
"type": "commit",
89+
"tagger": {
90+
"name": tagger,
91+
"email": email,
92+
"date": date
93+
}
94+
}
95+
96+
97+
if __name__ == "__main__":
98+
parser = ArgumentParser(description="Calculates and returns new version based on the version tags of the "
99+
"specified repo, only PEP440 versions are taken into account. "
100+
"If micro version is not passed, "
101+
"version is incremented by --increment, default is 1")
102+
parser.add_argument("-t", "--token", dest="token",
103+
help="github api token, if not set taken from GITHUB_SECRET_TOKEN",
104+
default=os.getenv('GITHUB_SECRET_TOKEN'))
105+
parser.add_argument("-c", "--create_release", action="store_true", default=False,
106+
dest="do_release", help="create calculated new release")
107+
parser.add_argument("-b", "--branch", dest="branch", help="branch to tag", default="master")
108+
parser.add_argument("-r", "--repo", dest="repo", help="github repo", required=True)
109+
parser.add_argument("-re", "--commit_message_regex", dest="regex", default=".*",
110+
help="commit regex to be tagged, if not set any commit on given branch will be tagged")
111+
parser.add_argument("-m", "--micro", dest="micro", type=int, help="new micro version <major>.<minor>.<micro>")
112+
parser.add_argument("-i", "--increment", dest="increment", help="increment minor version by this number",
113+
default=1, type=int)
114+
options = parser.parse_args()
115+
sec_token = options.token
116+
if not sec_token:
117+
print("Secret token is not set neither by parameter nor by environment variable")
118+
exit(1)
119+
repo = options.repo
120+
do_release = options.do_release
121+
newmicro = options.micro
122+
increment = options.increment
123+
branch = options.branch
124+
regex = options.regex
125+
tags = list_github_tags(sec_token, repo)
126+
tag_search = make_tags_search_hash(tags)
127+
parsed_tags = list_version_tags(tags)
128+
latest = get_latest_version(parsed_tags)
129+
latest_tag_sha = tag_search[latest]['commit']['sha']
130+
ma = latest.major
131+
mi = latest.minor
132+
mic = latest.micro
133+
if newmicro:
134+
if mic > newmicro:
135+
print(f"latest micro version {ma}.{mi}.{mic} is bigger than provided, provided {newmicro}")
136+
exit(1)
137+
else:
138+
newrel_version = f"{ma}.{mi}.{newmicro}"
139+
else:
140+
newrel_version = f"{ma}.{mi}.{mic + 1}"
141+
new_commit_sha = get_branch_commit_sha(sec_token, repo, branch)
142+
commit_message = get_branch_commit_message(sec_token, repo, branch)
143+
tag_anno_obj = make_annotated_tag_object(newrel_version, commit_message, new_commit_sha)
144+
tag_ref_obj = make_lightweight_tag_object(newrel_version, new_commit_sha)
145+
create_log = f"Will create tag \n{json.dumps(tag_anno_obj, indent=4)} \n" \
146+
f" new commit {new_commit_sha}, tag {newrel_version}\n " \
147+
f"old commit {latest_tag_sha}, tag {str(latest)}"
148+
if re.match(regex, commit_message):
149+
print(f"Commit message {commit_message} matched {regex}, proceeding to release {newrel_version}")
150+
if latest_tag_sha == new_commit_sha:
151+
print(f"Release aborted release {latest} already created for {new_commit_sha}")
152+
else:
153+
if do_release:
154+
print(create_log)
155+
create_annotated_tag(sec_token, repo, tag_anno_obj)
156+
create_lightweight_tag(sec_token, repo, tag_ref_obj)
157+
else:
158+
print(create_log)
159+
print("Release aborted, specify -c to actually do release")
160+
else:
161+
print(f"Commit message {commit_message} don't match {regex}, aborted release")

scripts/travis-automerge.sh

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash -e
2+
3+
# Inspired by https://github.com/cdown/travis-automerge
4+
# See https://github.com/cdown/travis-automerge/blob/90cc2a65a887d0a80f78c098892f29d458b29813/LICENSE
5+
6+
: "${BRANCHES_TO_MERGE_REGEX?}" "${BRANCH_TO_MERGE_INTO?}"
7+
: "${GITHUB_SECRET_TOKEN?}" "${GITHUB_REPO?}"
8+
9+
export GIT_COMMITTER_EMAIL='[email protected]'
10+
export GIT_COMMITTER_NAME='CI bot'
11+
12+
if [ "$TRAVIS_REPO_SLUG" != "$GITHUB_REPO" ]; then
13+
printf "PR repo %s is not current %s, exiting\\n" \
14+
"$TRAVIS_REPO_SLUG" "$TRAVIS_REPO_SLUG" >&2
15+
exit 0
16+
fi
17+
18+
if ! grep -q "$BRANCHES_TO_MERGE_REGEX" <<< "$TRAVIS_PULL_REQUEST_BRANCH"; then
19+
printf "Current branch %s doesn't match regex %s, exiting\\n" \
20+
"$TRAVIS_PULL_REQUEST_BRANCH" "$BRANCHES_TO_MERGE_REGEX" >&2
21+
exit 0
22+
fi
23+
24+
# Since Travis does a partial checkout, we need to get the whole thing
25+
repo_temp=$(mktemp -d)
26+
git clone "https://github.com/$GITHUB_REPO" "$repo_temp"
27+
28+
# shellcheck disable=SC2164
29+
cd "$repo_temp"
30+
31+
printf 'Fetching branch %s to merge to %s\n' "$TRAVIS_PULL_REQUEST_BRANCH" "$BRANCH_TO_MERGE_INTO" >&2
32+
git fetch origin $TRAVIS_PULL_REQUEST_BRANCH:$TRAVIS_PULL_REQUEST_BRANCH
33+
34+
printf 'Checking out %s\n' "$BRANCH_TO_MERGE_INTO" >&2
35+
git checkout "$BRANCH_TO_MERGE_INTO"
36+
37+
MERGE_COMMIT_MESSAGE=`git show-branch --no-name $TRAVIS_PULL_REQUEST_BRANCH`
38+
39+
printf 'Merging %s using "%s" message for if merge commit\n' "$TRAVIS_PULL_REQUEST_BRANCH" "MERGE_COMMIT_MESSAGE" >&2
40+
git merge "$TRAVIS_PULL_REQUEST_BRANCH" -m '$MERGE_COMMIT_MESSAGE'
41+
42+
printf 'Pushing to %s\n' "$GITHUB_REPO" >&2
43+
44+
push_uri="https://$GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO"
45+
46+
# Redirect to /dev/null to avoid secret leakage
47+
git push "$push_uri" "$BRANCH_TO_MERGE_INTO" >/dev/null 2>&1
48+
git push "$push_uri" :"$TRAVIS_PULL_REQUEST_BRANCH" >/dev/null 2>&1

0 commit comments

Comments
 (0)