Skip to content

Commit 53ea0c0

Browse files
BigCat20196RAY-316msyyc
authored
Add auto-close and auto-run-pipline (Azure#20516)
* release_iseus_status_auto_reply * issue_aoto_close_revert * Update main.py * Update main.py * Update update_issue_body.py * Update reply_generator.py * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update update_issue_body.py * Update update_issue_body.py * Update main.py * Update reply_generator.py * Update main.py * Update update_issue_body.py * Update main.py * Update scripts/release_issue_status/update_issue_body.py * Update update_issue_body.py * Update main.py * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update scripts/release_issue_status/main.py * Update main.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/update_issue_body.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/update_issue_body.py * Update reply_generator.py * Update main.py * Update scripts/release_issue_status/update_issue_body.py Co-authored-by: msyyc <[email protected]> * Update scripts/release_issue_status/update_issue_body.py Co-authored-by: msyyc <[email protected]> * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update main.py * Update update_issue_body.py * Update main.py * Update scripts/release_issue_status/main.py * Update main.py * Update reply_generator.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update update_issue_body.py * Update main.py * Update update_issue_body.py * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update update_issue_body.py * Update update_issue_body.py * Update update_issue_body.py * Update update_issue_body.py * Update update_issue_body.py * Update main.py * Update main.py * Update release_issue_status.yml for Azure Pipelines * Update main.py * Update reply_generator.py * Add files via upload * Update reply_generator.py * Update update_issue_body.py * Update reply_generator.py * Update auto_pipeline_run.py * Update auto_pipeline_run.py * add auto-close * Update release_issue_status.yml for Azure Pipelines * Update auto_close.py * Update main.py * Update release_issue_status.yml for Azure Pipelines * Update auto_pipeline_run.py * fix bug * Update main.py * Update auto_close.py * Update auto_close.py * Update main.py * Update reply_generator.py * Update main.py * Update main.py * Update auto_close.py * Update main.py * Update auto-close * Update auto_pipeline_run.py * Update update_issue_body.py * Update auto_pipeline_run.py * Update release_issue_status.yml for Azure Pipelines * Update auto_pipeline_run.py * Update release_issue_status.yml for Azure Pipelines * Update auto_pipeline_run.py * Update auto_pipeline_run.py * Add pipeline link * Update pipeline link * Update auto_pipeline_run.py * Update release_issue_status.yml for Azure Pipelines * Update auto_pipeline_run.py * Update requirement.txt * Update auto_pipeline_run.py * Add get_python_pipeline * Update auto_pipeline_run.py * test * test * add outputfolder * add label * Update main.py * Update get_python_pipeline.py * Update main.py * Update auto_pipeline_run.py Co-authored-by: Zed <[email protected]> Co-authored-by: Zed Lei <[email protected]> Co-authored-by: msyyc <[email protected]>
1 parent acfb309 commit 53ea0c0

File tree

8 files changed

+195
-34
lines changed

8 files changed

+195
-34
lines changed

‎scripts/release_issue_status/auto_close.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import datetime
2+
3+
import requests
4+
from bs4 import BeautifulSoup
5+
6+
7+
def auto_close_issue(sdk_repo, item):
8+
issue_number, package_name = item.issue_object.number, item.package
9+
issue_info = sdk_repo.get_issue(number=issue_number)
10+
issue_author = issue_info.user.login
11+
last_comment = list(issue_info.get_comments())[-1]
12+
last_comment_date = last_comment.created_at
13+
last_version, last_time = get_last_released_date(package_name)
14+
if last_time and last_time > last_comment_date:
15+
comment = f'Hi @{issue_author}, pypi link: https://pypi.org/project/{package_name}/{last_version}/'
16+
issue_info.create_comment(body=comment)
17+
issue_info.edit(state='closed')
18+
item.labels.append('auto-closed')
19+
item.issue_object.set_labels(*item.labels)
20+
print(f"issue number：{issue_number} has been closed!")
21+
22+
23+
def get_last_released_date(package_name):
24+
pypi_link = f'https://pypi.org/project/{package_name}/#history'
25+
res = requests.get(pypi_link)
26+
soup = BeautifulSoup(res.text, 'html.parser')
27+
# find top div from <div class="release-timeline">
28+
try:
29+
package_info = soup.select('div[class="release-timeline"]')[0].find_all('div')[0]
30+
last_version_mix = package_info.find_all('p', class_="release__version")[0].contents[0]
31+
except IndexError as e:
32+
return '', ''
33+
last_version = last_version_mix.replace(' ', '').replace('\n', '')
34+
last_version_date_str = package_info.time.attrs['datetime'].split('+')[0]
35+
last_version_date = datetime.datetime.strptime(last_version_date_str, '%Y-%m-%dT%H:%M:%S')
36+
return last_version, last_version_date

‎scripts/release_issue_status/auto_pipeline_run.py

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import json
2+
import os
3+
import re
4+
from msrest.authentication import BasicAuthentication
5+
from azure.devops.v6_0.pipelines.pipelines_client import PipelinesClient
6+
from azure.devops.v6_0.pipelines import models
7+
import requests
8+
9+
10+
def run_pipeline(issue_link, sdk_issue_object, pipeline_url):
11+
paramaters = {
12+
"stages_to_skip": [],
13+
"resources": {
14+
"repositories": {
15+
"self": {
16+
"refName": "refs/heads/main"
17+
}
18+
}
19+
},
20+
"variables": {
21+
"BASE_BRANCH": {
22+
"value": f"{sdk_issue_object.head.label}",
23+
"isSecret": False
24+
},
25+
"ISSUE_LINK": {
26+
"value": f"{issue_link}",
27+
"isSecret": False
28+
},
29+
"PIPELINE_LINK": {
30+
"value": f"{pipeline_url}",
31+
"isSecret": False
32+
}
33+
}
34+
}
35+
# Fill in with your personal access token and org URL
36+
personal_access_token = os.getenv('PIPELINE_TOKEN')
37+
organization_url = 'https://dev.azure.com/azure-sdk'
38+
39+
# Create a connection to the org
40+
credentials = BasicAuthentication('', personal_access_token)
41+
run_parameters = models.RunPipelineParameters(**paramaters)
42+
client = PipelinesClient(base_url=organization_url, creds=credentials)
43+
result = client.run_pipeline(project='internal',pipeline_id=2500,run_parameters=run_parameters)
44+
if result.state == 'inProgress':
45+
return True
46+
else:
47+
return False

‎scripts/release_issue_status/get_python_pipeline.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import os
2+
import re
3+
4+
from msrest.authentication import BasicAuthentication
5+
from azure.devops.v6_0.pipelines.pipelines_client import PipelinesClient
6+
7+
8+
def get_python_pipelines():
9+
python_piplines = {}
10+
pipeline_client = PipelinesClient(base_url='https://dev.azure.com/azure-sdk',
11+
creds=BasicAuthentication('', os.getenv('PIPELINE_TOKEN')))
12+
pipelines = pipeline_client.list_pipelines(project='internal')
13+
for pipeline in pipelines:
14+
if re.findall('^python - \w*$', pipeline.name):
15+
key = pipeline.name.replace('python - ', '')
16+
python_piplines[key] = pipeline.id
17+
return python_piplines
18+
19+
20+
def get_pipeline_url(python_piplines, output_folder):
21+
definitionId = python_piplines.get(output_folder)
22+
if definitionId:
23+
pipeline_url = 'https://dev.azure.com/azure-sdk/internal/_build?definitionId={}'.format(definitionId)
24+
else:
25+
print('Cannot find definitionId, Do not display pipeline_url')
26+
pipeline_url = ''
27+
return pipeline_url

‎scripts/release_issue_status/main.py

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
11
import time
22
import os
33
import re
4-
from github import Github
54
from datetime import date, datetime
65
import subprocess as sp
6+
import traceback
7+
8+
from github import Github
79
from azure.storage.blob import BlobClient
10+
811
import reply_generator as rg
9-
from update_issue_body import update_issue_body, find_readme_link
10-
import traceback
12+
from update_issue_body import update_issue_body, find_readme_and_output_folder
13+
from auto_close import auto_close_issue
14+
from get_python_pipeline import get_python_pipelines, get_pipeline_url
15+
1116

1217
_NULL = ' '
1318
_FILE_OUT = 'release_issue_status.csv'
@@ -121,32 +126,41 @@ def _latest_comment_time(comments, delay_from_create_date):
121126
return delay_from_create_date if not q else int((time.time() - q[-1][0]) / 3600 / 24)
122127

123128

124-
def auto_reply(item, sdk_repo, rest_repo, duplicated_issue):
129+
def auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines):
125130
print("==========new issue number: {}".format(item.issue_object.number))
131+
if 'Configured' in item.labels:
132+
item.labels.remove('Configured')
133+
126134
if 'auto-link' not in item.labels:
135+
item.labels.append('auto-link')
136+
item.issue_object.set_labels(*item.labels)
127137
try:
128-
package_name, readme_link = update_issue_body(sdk_repo, rest_repo, item.issue_object.number)
138+
package_name, readme_link, output_folder = update_issue_body(request_repo, rest_repo, item.issue_object.number)
129139
print("pkname, readme", package_name, readme_link)
130140
item.package = package_name
131141
key = ('Python', item.package)
132142
duplicated_issue[key] = duplicated_issue.get(key, 0) + 1
133143
except Exception as e:
134144
item.bot_advice = 'failed to modify the body of the new issue. Please modify manually'
135145
item.labels.append('attention')
146+
item.issue_object.set_labels(*item.labels)
136147
print(e)
137148
raise
138-
item.labels.append('auto-link')
139-
item.issue_object.set_labels(*item.labels)
140149
else:
141150
try:
142-
readme_link = find_readme_link(sdk_repo, item.issue_object.number)
151+
readme_link, output_folder = find_readme_and_output_folder(request_repo, rest_repo, item.issue_object.number)
143152
except Exception as e:
144153
print('Issue: {} updates body failed'.format(item.issue_object.number))
145154
item.bot_advice = 'failed to find Readme link, Please check !!'
146155
item.labels.append('attention')
156+
item.issue_object.set_labels(*item.labels)
147157
raise
148158
try:
149-
reply = rg.begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link)
159+
print("*********************")
160+
print(python_piplines)
161+
pipeline_url = get_pipeline_url(python_piplines, output_folder)
162+
rg.begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link,
163+
sdk_repo=sdk_repo, pipeline_url=pipeline_url)
150164
except Exception as e:
151165
item.bot_advice = 'auto reply failed, Please intervene manually !!'
152166
print('Error from auto reply ========================')
@@ -158,14 +172,18 @@ def auto_reply(item, sdk_repo, rest_repo, duplicated_issue):
158172
def main():
159173
# get latest issue status
160174
g = Github(os.getenv('TOKEN')) # please fill user_token
161-
sdk_repo = g.get_repo('Azure/sdk-release-request')
162-
rest_repo = g.get_repo('Azure/azure-rest-api-specs')
163-
label1 = sdk_repo.get_label('ManagementPlane')
164-
open_issues = sdk_repo.get_issues(state='open', labels=[label1])
175+
request_repo = g.get_repo('Azure/sdk-release-request')
176+
rest_repo = g.get_repo('Azure/azure-rest-api-specs')
177+
sdk_repo = g.get_repo('Azure/azure-sdk-for-python')
178+
label1 = request_repo.get_label('ManagementPlane')
179+
open_issues = request_repo.get_issues(state='open', labels=[label1])
165180
issue_status = []
166181
issue_status_python = []
167182
duplicated_issue = dict()
168183
start_time = time.time()
184+
# get pipeline definitionid
185+
python_piplines = get_python_pipelines()
186+
169187
for item in open_issues:
170188
if not item.number:
171189
continue
@@ -204,16 +222,22 @@ def main():
204222
for item in issue_status:
205223
if item.status == 'release':
206224
item.bot_advice = 'better to release asap.'
207-
elif item.comment_num == 0 and 'Python' in item.labels:
225+
elif (item.comment_num == 0 or 'Configured' in item.labels) and 'Python' in item.labels:
208226
item.bot_advice = 'new issue and better to confirm quickly.'
209227
try:
210-
auto_reply(item, sdk_repo, rest_repo, duplicated_issue)
228+
auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines)
211229
except Exception as e:
212230
continue
213231
elif not item.author_latest_comment in _PYTHON_SDK_ADMINISTRATORS:
214232
item.bot_advice = 'new comment for author.'
215233
elif item.delay_from_latest_update >= 7:
216234
item.bot_advice = 'delay for a long time and better to handle now.'
235+
if item.comment_num > 1 and item.language == 'Python':
236+
try:
237+
auto_close_issue(request_repo, item)
238+
except Exception as e:
239+
item.bot_advice = 'auto-close failed, please check!'
240+
print(f"=====issue: {item.issue_object.number}, {e}")
217241

218242
if item.days_from_latest_commit >= 30 and item.language == 'Python' and '30days attention' not in item.labels:
219243
item.labels.append('30days attention')
@@ -243,10 +267,10 @@ def main():
243267
print_check('git push -f origin HEAD')
244268

245269
# upload to storage account(it is created in advance)
246-
blob = BlobClient.from_connection_string(conn_str=os.getenv('CONN_STR'), container_name=os.getenv('FILE'),
247-
blob_name=_FILE_OUT)
248-
with open(_FILE_OUT, 'rb') as data:
249-
blob.upload_blob(data, overwrite=True)
270+
# blob = BlobClient.from_connection_string(conn_str=os.getenv('CONN_STR'), container_name=os.getenv('FILE'),
271+
# blob_name=_FILE_OUT)
272+
# with open(_FILE_OUT, 'rb') as data:
273+
# blob.upload_blob(data, overwrite=True)
250274

251275

252276
if __name__ == '__main__':

‎scripts/release_issue_status/release_issue_status.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ jobs:
3636
export CONN_STR=$(ENV_CONN_STR)
3737
export FILE=$(ENV_FILE)
3838
export TOKEN=$(USR_TOKEN)
39+
export HEADERS=$(PIPELINE_HEADERS)
40+
export URL=$(PIPELINE_URL)
41+
export PIPELINE_TOKEN = $(PIPELINE_TOKEN)
3942
4043
# create virtual env
4144
python -m venv venv-sdk

‎scripts/release_issue_status/reply_generator.py

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import auto_pipeline_run as apr
12
import re
23

34
issue_object_rg = None
@@ -7,11 +8,10 @@ def weather_change_readme(rest_repo, link_dict, labels):
78
# to see whether need change readme
89
contents = str(rest_repo.get_contents(link_dict['readme_path']).decoded_content)
910
pattern_tag = re.compile(r'tag: package-[\w+-.]+')
10-
package_tag = pattern_tag.search(contents).group()
11-
package_tag = package_tag.split(':')[1].strip()
11+
package_tag = pattern_tag.findall(contents)
1212
readme_python_contents = str(rest_repo.get_contents(link_dict['readme_python_path']).decoded_content)
1313
whether_multi_api = 'multi-api' in readme_python_contents
14-
whether_same_tag = package_tag == link_dict['readme_tag']
14+
whether_same_tag = link_dict['readme_tag'] in package_tag
1515
whether_change_readme = not whether_same_tag or whether_multi_api and not 'MultiAPI' in labels
1616
return whether_change_readme
1717

@@ -28,7 +28,7 @@ def get_links(readme_link):
2828
resource_manager = pattern_resource_manager.search(readme_link).group()
2929
link_dict['readme_path'] = readme_path
3030
link_dict['readme_python_path'] = readme_path[:readme_path.rfind('/')] + '/readme.python.md'
31-
link_dict['readme_tag'] = readme_tag
31+
link_dict['readme_tag'] = 'tag: ' + readme_tag
3232
link_dict['resource_manager'] = resource_manager
3333
return link_dict
3434

@@ -76,13 +76,16 @@ def swagger_generator_parse(context, latest_pr_number):
7676
python_track2_info = re.search(pattern_python_track2, python).group()
7777
track2_info_model = '<details open><summary><b> python-track2</b></summary>{} </details>'.format(
7878
python_track2_info)
79+
pattern_sdk_changes = re.compile('/azure-sdk-for-python/pull/\d*">Release SDK Changes</a>', re.DOTALL)
80+
sdk_link = re.search(pattern_sdk_changes, python_track2_info).group()
81+
sdk_link_number = re.search(re.compile('[0-9]+'), sdk_link).group()
7982
info_model = 'hi @{} Please check the package whether works well and the changelog info is as below:\n' \
8083
'{}\n{}\n' \
8184
'\n* (The version of the package is only a temporary version for testing)\n' \
8285
'\nhttps://github.com/Azure/azure-rest-api-specs/pull/{}\n' \
8386
.format(issue_object_rg.user.login, track1_info_model, track2_info_model, str(latest_pr_number))
8487

85-
return info_model
88+
return info_model, sdk_link_number
8689

8790

8891
def reply_owner(reply_content):
@@ -95,16 +98,24 @@ def add_label(label_name, labels):
9598
issue_object_rg.set_labels(*labels)
9699

97100

98-
def begin_reply_generate(item, rest_repo, readme_link):
101+
def begin_reply_generate(item, rest_repo, readme_link, sdk_repo, pipeline_url):
99102
global issue_object_rg
100103
issue_object_rg = item.issue_object
101104
link_dict = get_links(readme_link)
102105
labels = item.labels
103106
whether_change_readme = weather_change_readme(rest_repo, link_dict, labels)
104107

105108
if not whether_change_readme:
106-
latest_pr_number = get_latest_pr_from_readme(rest_repo,link_dict)
107-
reply_content = latest_pr_parse(rest_repo, latest_pr_number)
109+
latest_pr_number = get_latest_pr_from_readme(rest_repo, link_dict)
110+
reply_content, sdk_link_number = latest_pr_parse(rest_repo, latest_pr_number)
111+
run_pipeline = apr.run_pipeline(issue_link=issue_object_rg.html_url,
112+
sdk_issue_object=sdk_repo.get_pull(int(sdk_link_number)),
113+
pipeline_url=pipeline_url
114+
)
115+
if run_pipeline:
116+
print(f'{issue_object_rg.number} run pipeline successfully')
117+
else:
118+
print(f'{issue_object_rg.number} run pipeline fail')
108119
reply_owner(reply_content)
109120
add_label('auto-ask-check', labels)
110121
else:

‎scripts/release_issue_status/requirement.txt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
11
PyGithub
22
datetime
3-
azure.storage.blob==12.8.1
3+
requests
4+
bs4
5+
azure.storage.blob==12.8.1
6+
azure-devops
7+
msrest

‎scripts/release_issue_status/update_issue_body.py

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def update_issue_body(sdk_repo, rest_repo, issue_number):
2020
link = link.split(']')[0]
2121
link = link.replace('[', "").replace(']', "").replace('(', "").replace(')', "")
2222

23-
package_name, readme_link = get_pkname_and_readme_link(rest_repo, link)
23+
package_name, readme_link, output_folder = get_pkname_and_readme_link(rest_repo, link)
2424

2525
# Check readme tag format
2626
if 'package' not in readme_tag:
@@ -36,7 +36,7 @@ def update_issue_body(sdk_repo, rest_repo, issue_number):
3636
issue_body_up += raw + '\n'
3737

3838
issue_info.edit(body=issue_body_up)
39-
return package_name, readme_link
39+
return package_name, readme_link, output_folder
4040

4141

4242
def get_pkname_and_readme_link(rest_repo, link):
@@ -46,6 +46,7 @@ def get_pkname_and_readme_link(rest_repo, link):
4646
commit_sha = link.split('commit/')[-1]
4747
commit = rest_repo.get_commit(commit_sha)
4848
link = commit.files[0].blob_url
49+
link = re.sub('blob/(.*?)/specification', 'blob/main/specification', link)
4950

5051
# if link is a pr, it can get both pakeage name and readme link.
5152
if 'pull' in link:
@@ -79,18 +80,26 @@ def get_pkname_and_readme_link(rest_repo, link):
7980
readme_link_part = '/specification' + readme_link.split('/specification')[-1]
8081
readme_contents = str(rest_repo.get_contents(readme_link_part).decoded_content)
8182
pk_name = re.findall(r'package-name: (.*?)\\n', readme_contents)[0]
83+
out_folder = re.findall(r'\$\(python-sdks-folder\)/(.*?)/azure-', readme_contents)[0]
8284
readme_link = readme_link.replace('python.', '')
8385

84-
return pk_name, readme_link
86+
return pk_name, readme_link, out_folder
8587

8688

87-
def find_readme_link(sdk_repo, issue_number):
89+
def find_readme_and_output_folder(sdk_repo, rest_repo, issue_number):
8890
# Get Issue Number
8991
issue_info = sdk_repo.get_issue(number=issue_number)
9092
issue_body = issue_info.body
9193
issue_body_list = issue_body.split("\n")
9294
for row in issue_body_list:
9395
if 'resource-manager' in row:
94-
readme_link = row + '/readme.md'
95-
return readme_link
96+
readme_link = '{}/readme.md'.format(row.strip("\r"))
97+
# Get output folder from readme.python.md
98+
readme_python_link = readme_link.split('/resource-manager')[0] + '/resource-manager/readme.python.md'
99+
readme_python_link_part = '/specification' + readme_python_link.split('/specification')[-1]
100+
readme_contents = str(rest_repo.get_contents(readme_python_link_part).decoded_content)
101+
output_folder = re.findall(r'\$\(python-sdks-folder\)/(.*?)/azure-', readme_contents)[0]
102+
103+
return readme_link, output_folder
96104
raise Exception('Not find readme link,please check')
105+

0 commit comments

Comments
 (0)