[tool]Release issue status statistics tool (Azure#19617)

* release issue status

* fix directory

* debug

* debug

* update git repo

* add storage account

* delete surplus git commit

* add env var

* debug

* code format

Author: msyyc

scripts/release_issue_status/main.py

@@ -0,0 +1,153 @@
+import time
+import os
+import re
+from github import Github
+from datetime import date, datetime
+import subprocess as sp
+from azure.storage.blob import BlobClient
+
+_NULL = ' '
+_FILE_OUT = 'release_issue_status.csv'
+
+
+def my_print(cmd):
+    print('==' + cmd + ' ==\n')
+
+
+def print_check(cmd):
+    my_print(cmd)
+    sp.check_call(cmd, shell=True)
+
+
+class IssueStatus:
+    link = _NULL
+    author = _NULL
+    package = _NULL
+    create_date = 0.0
+    delay_from_create_date = 0
+    latest_update = 0.0
+    delay_from_latest_update = 0
+    status = 'confirm'
+    bot_advice = _NULL
+    comment_num = 0
+    language = _NULL
+    author_latest_comment = _NULL
+
+    def output(self):
+        return '{},{},{},{},{},{},{},{},{},{}\n'.format(self.language, self.link, self.author,
+                                                        self.package,
+                                                        str(date.fromtimestamp(self.create_date)),
+                                                        self.delay_from_create_date,
+                                                        str(date.fromtimestamp(self.latest_update)),
+                                                        self.delay_from_latest_update,
+                                                        self.status, self.bot_advice)
+
+
+def _extract(str_list, key_word):
+    for item in str_list:
+        if re.fullmatch(key_word, item):
+            return item.strip()
+    return _NULL
+
+
+def _time_format_transform(time_gmt):
+    return str(datetime.strptime(time_gmt, '%a, %d %b %Y %H:%M:%S GMT'))
+
+
+def _judge_status(labels):
+    for label in labels:
+        if label.name == 'release':
+            return 'release'
+    return 'confirm'
+
+
+def _extract_language(labels):
+    language = {'Python', 'JS', 'Go', 'Java', 'Ruby'}
+    label_set = {label.name for label in labels}
+    intersect = language.intersection(label_set)
+    return _NULL if not intersect else intersect.pop()
+
+
+def _key_select(item):
+    return item.package
+
+
+def _extract_author_latest_comment(comments):
+    q = [(comment.updated_at.timestamp(), comment.user.login) for comment in comments]
+    q.sort()
+
+    return _NULL if not q else q[-1][1]
+
+
+def main():
+    # get latest issue status
+    g = Github(os.getenv('TOKEN'))  # please fill user_token
+    repo = g.get_repo('Azure/sdk-release-request')
+    label1 = repo.get_label('ManagementPlane')
+    open_issues = repo.get_issues(state='open', labels=[label1])
+    issue_status = []
+    duplicated_issue = dict()
+    start_time = time.time()
+    for item in open_issues:
+        if not item.number:
+            continue
+        issue = IssueStatus()
+        issue.link = f'https://github.com/Azure/sdk-release-request/issues/{item.number}'
+        issue.author = item.user.login
+        issue.package = _extract(item.body.split('\n'), 'azure-.*')
+        issue.create_date = item.created_at.timestamp()
+        issue.delay_from_create_date = int((time.time() - item.created_at.timestamp()) / 3600 / 24)
+        issue.latest_update = item.updated_at.timestamp()
+        issue.delay_from_latest_update = int((time.time() - item.updated_at.timestamp()) / 3600 / 24)
+        issue.status = _judge_status(item.labels)
+        issue.comment_num = item.comments
+        issue.language = _extract_language(item.labels)
+        issue.author_latest_comment = _extract_author_latest_comment(item.get_comments())
+
+        issue_status.append(issue)
+        key = (issue.language, issue.package)
+        duplicated_issue[key] = duplicated_issue.get(key, 0) + 1
+
+    my_print('Have cost {} seconds'.format(int(time.time() - start_time)))
+
+    # rule1: if status is 'release', need to release asap
+    # rule2: if latest comment is from author, need response asap
+    # rule3: if comment num is 0, it is new issue, better to deal with it asap
+    # rule4: if delay from latest update is over 7 days, better to deal with it soon.
+    # rule5: if delay from created date is over 30 days, better to close.
+    for item in issue_status:
+        if item.status == 'release':
+            item.bot_advice = 'better to release asap.'
+        elif item.author == item.author_latest_comment:
+            item.bot_advice = 'new comment for author.'
+        elif item.comment_num == 0:
+            item.bot_advice = 'new issue and better to confirm quickly.'
+        elif item.delay_from_latest_update >= 7:
+            item.bot_advice = 'delay for a long time and better to handle now.'
+        elif item.delay_from_create_date >= 30:
+            item.bot_advice = 'delay for a month and better to close.'
+
+        # judge whether there is duplicated issue for same package
+        if item.package != _NULL and duplicated_issue.get((item.language, item.package)) > 1:
+            item.bot_advice = f'Warning:There is duplicated issue for {item.package}. ' + item.bot_advice
+
+    # output result
+    with open(_FILE_OUT, 'w') as file_out:
+        file_out.write('language,issue,author,package,created date,delay from created date,latest update time,'
+                       'delay from latest update,status,bot advice\n')
+        file_out.writelines([item.output() for item in sorted(issue_status, key=_key_select)])
+
+    # commit to github
+    print_check('git add .')
+    print_check('git commit -m \"update excel\"')
+    print_check('git push -f origin HEAD')
+
+    # upload to storage account(it is created in advance)
+    blob = BlobClient.from_connection_string(conn_str=os.getenv('CONN_STR'), container_name=os.getenv('FILE'),
+                                             blob_name=_FILE_OUT)
+    with open(_FILE_OUT, 'rb') as data:
+        blob.upload_blob(data, overwrite=True)
+
+
+if __name__ == '__main__':
+    main()

scripts/release_issue_status/release_issue_status.yml

@@ -0,0 +1,50 @@
+# Release status statistics
+
+name: ReleaseIssueStatus
+
+trigger:
+  branches:
+    exclude:
+    - '*'
+
+
+jobs:
+- job: ReleaseIssueStatus
+  displayName: ReleaseIssueStatus Python 3.8
+  timeoutInMinutes: 30
+  strategy:
+    maxParallel: 3
+  pool:
+    vmImage: 'ubuntu-20.04'
+  steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.8'
+        addToPath: true
+        architecture: 'x64'
+    - bash: |
+        script_path=$(pwd)/scripts/release_issue_status
+        cd ..
+        git config --global user.email "ReleaseIssueStatus"
+        git config --global user.name "ReleaseIssueStatus"
+
+        # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx)
+        mkdir file-storage
+        git clone ${REPO:0:8}$(USR_NAME):$(USR_TOKEN)@${REPO:8} $(pwd)/file-storage
+
+        # import env variable
+        export CONN_STR=$(ENV_CONN_STR)
+        export FILE=$(ENV_FILE)
+        export TOKEN=$(USR_TOKEN)
+
+        # create virtual env
+        python -m venv venv-sdk
+        source venv-sdk/bin/activate
+        pip install -r $script_path/requirement.txt
+
+        # checkout the target branch
+        cd file-storage
+        git checkout release-issue-status
+
+        # run
+        python $script_path/main.py

scripts/release_issue_status/requirement.txt

@@ -0,0 +1,3 @@
+PyGithub
+datetime
+azure.storage.blob==12.8.1