[Release sdk status] count sdk folder (Azure#23783)

* add parameter

* trigger swagger pipeline and sheduled trigger

* avoid duplicated trigger

* debug

* debug

* debug

* debug

* debug

* optmization

* special case

* special case

* optimize track2 ga version rule

* delete upload code

* testproxy adaptation

* new timeoutlimit

* add package size

* format

* fix

* add log

* fix

* fix git commit error

* add statistics for sdk folder

* optimize title

Author: msyyc

scripts/release_sdk_status/main.py

@@ -5,9 +5,7 @@
 from lxml import etree
 import lxml.html
 import subprocess as sp
-from azure.storage.blob import BlobClient
-from pathlib import Path
-from typing import List
+from typing import List, Dict
 from github import Github
 from github.Repository import Repository
 import time
@@ -16,10 +14,10 @@
 
 from util import add_certificate
 
-SERVICE_TEST_PATH = {}
 MAIN_REPO_SWAGGER = 'https://github.com/Azure/azure-rest-api-specs/tree/main'
 PR_URL = 'https://github.com/Azure/azure-rest-api-specs/pull/'
 FAILED_RESULT = []
+SKIP_TEXT = '-, -, -, -\n'
 
 def my_print(cmd):
     print('== ' + cmd + ' ==\n')
@@ -120,14 +118,15 @@ def output_package_size(self) -> str:
         else:
             return self.package_size
 
-    def write_to_list(self):
+    def write_to_list(self, sdk_folder: str) -> str:
         response = self.project_html("/#history")
         if 199 < response.status_code < 400:
             self.get_release_dict(response)
             self.bot_analysis()
-            return '{package_name},{pypi_link},{track1_latest_version},{track1_latest_release_date},' \
+            return '{sdk_folder}/{package_name},{pypi_link},{track1_latest_version},{track1_latest_release_date},' \
                    '{track1_ga_version},{track2_latest_version},{track2_latest_release_date},{track2_ga_version},' \
                    '{cli_version},{track_config},{bot},{readme_link},{multiapi},{whl_size},'.format(
+                        sdk_folder=sdk_folder.split('/')[0],
                         package_name=self._package_name,
                         pypi_link=self.pypi_link,
                         track1_latest_version=self.track1_latest_version,
@@ -144,7 +143,7 @@ def write_to_list(self):
                         whl_size=self.output_package_size())
         else:
             self.pypi_link = 'NA'
-        return
+        return ''
 
     def find_track1_ga_version(self, versions: List[str]) -> None:
         if '1.0.0' in versions:
@@ -210,35 +209,32 @@ def bot_analysis(self):
             self.bot_warning += 'Need to add track2 config.'
 
 
-def sdk_info_from_pypi(sdk_info, cli_dependency):
+def sdk_info_from_pypi(sdk_info: List[Dict[str, str]], cli_dependency):
     all_sdk_status = []
     add_certificate(str(Path('../venv-sdk/lib/python3.8/site-packages/certifi/cacert.pem')))
     for package in sdk_info:
-        if ',' in package:
-            package = package.split(',')
-            sdk_name = package[0].strip()
-            if sdk_name in cli_dependency.keys():
-                cli_version = cli_dependency[sdk_name]
-            else:
-                cli_version = 'NA'
-            track_config = package[1].strip()
-            readme_link = package[2].strip()
-            rm_link = package[3].strip()
-            multi_api = package[4].strip()
-            pypi_ins = PyPIClient(package_name=sdk_name, track_config=track_config,
-                                  readme_link=readme_link, rm_link=rm_link, cli_version=cli_version, multi_api=multi_api)
-            text_to_write = pypi_ins.write_to_list()
-            if pypi_ins.pypi_link != 'NA':
-                if sdk_name == 'azure-mgmt-resource':
-                    test_result = run_playback_test('resources')
-                else:
-                    try:
-                        service_name = [k for k, v in SERVICE_TEST_PATH.items() if sdk_name in v][0]
-                        test_result = run_playback_test(service_name)
-                    except:
-                        print(f'[Error] fail to play back test recordings: {sdk_name}')
-                text_to_write += test_result
-                all_sdk_status.append(text_to_write)
+        sdk_name = package['package_name']
+        if sdk_name in cli_dependency.keys():
+            cli_version = cli_dependency[sdk_name]
+        else:
+            cli_version = 'NA'
+        track_config = package['track_config']
+        readme_link = package['readme_python_path']
+        rm_link = package['readme_html']
+        multi_api = package['multi_api']
+        pypi_ins = PyPIClient(package_name=sdk_name, track_config=track_config,
+                              readme_link=readme_link, rm_link=rm_link, cli_version=cli_version, multi_api=multi_api)
+        sdk_folder = package['sdk_folder']
+        text_to_write = pypi_ins.write_to_list(sdk_folder)
+        service_name = package['service_name']
+        if pypi_ins.pypi_link != 'NA':
+            test_result = SKIP_TEXT
+            try:
+                test_result = run_playback_test(service_name, sdk_folder)
+            except:
+                print(f'[Error] fail to play back test recordings: {sdk_name}')
+            text_to_write += test_result
+            all_sdk_status.append(text_to_write)
 
     my_print(f'total pypi package kinds: {len(all_sdk_status)}')
     return all_sdk_status
@@ -264,12 +260,12 @@ def get_test_result(txt_path):
     return f'{coverage}, {passed}, {failed}, {skipped}\n'
 
 
-def run_playback_test(service_name):
+def run_playback_test(service_name: str, sdk_folder: str):
     if os.getenv('SKIP_COVERAGE') in ('true', 'yes'):
-        return '-, -, -, -\n'
+        return SKIP_TEXT
 
     # eg: coverage_path='$(pwd)/sdk-repo/sdk/containerregistry/azure-mgmt-containerregistry/azure/mgmt/containerregistry/'
-    coverage_path = ''.join([os.getenv('SDK_REPO'), '/sdk/', SERVICE_TEST_PATH[service_name]])
+    coverage_path = ''.join([os.getenv('SDK_REPO'), '/sdk/', sdk_folder])
     service_path = coverage_path.split('/azure/mgmt')[0]
     test_path = service_path + '/tests'
     if os.path.exists(test_path):
@@ -295,12 +291,12 @@ def run_playback_test(service_name):
         if os.path.exists(service_path+'/result.txt'):
             return get_test_result(service_path+'/result.txt')
 
-    return '-, -, -, -\n'
+    return SKIP_TEXT
 
 
 def write_to_csv(sdk_status_list, csv_name):
     with open(csv_name, 'w') as file_out:
-        file_out.write('package name,'
+        file_out.write('foler/package name,'
                        'pypi link,'
                        'latest track1 version,'
                        'latest track1 release date,'
@@ -360,17 +356,16 @@ def read_file(file_name):
     return content
 
 
-def find_test_path(line: str, service_name: str) -> bool:
+def find_test_path(line: str) -> str:
     line = line.strip('\n') + '\n'
     try:
-        SERVICE_TEST_PATH[service_name] = re.findall('output-folder: \$\(python-sdks-folder\)/(.*?)\n', line)[0]
-        return True
+        return re.findall('output-folder: \$\(python-sdks-folder\)/(.*?)\n', line)[0]
     except:
         FAILED_RESULT.append('[Fail to find sdk path] ' + line)
-        return False
+        return ''
 
 
-def sdk_info_from_swagger():
+def sdk_info_from_swagger() -> List[Dict[str, str]]:
     sdk_name_re = re.compile(r'azure-mgmt-[a-z]+-*([a-z])+')
     sdk_folder_re = re.compile('output-folder: \$\(python-sdks-folder\)/')
     resource_manager = []
@@ -379,7 +374,7 @@ def sdk_info_from_swagger():
     readme_folders = glob.glob(target_file_pattern)
     my_print(f'total readme folders: {len(readme_folders)}')
     for folder in readme_folders:
-        found_sdk_folder = False
+        sdk_folder = ''
         multi_api = ''
         linux_folder = Path(folder).as_posix()
         service_name = re.findall(r'specification/(.*?)/resource-manager/', linux_folder)[0]
@@ -395,16 +390,16 @@ def sdk_info_from_swagger():
                 track_config += 1
             if readme_python == 'NA' and sdk_name_re.search(line) is not None and package_name == '':
                 package_name = sdk_name_re.search(line).group()
-            if sdk_folder_re.search(line) and not found_sdk_folder:
-                found_sdk_folder = find_test_path(line, service_name)
+            if sdk_folder_re.search(line) and not sdk_folder:
+                sdk_folder = find_test_path(line)
 
         if readme_python != 'NA':
             readme_python_text = read_file(readme_python)
             for text in readme_python_text:
                 if sdk_name_re.search(text) is not None:
                     package_name = sdk_name_re.search(text).group()
-                if sdk_folder_re.search(text) and not found_sdk_folder:
-                    found_sdk_folder = find_test_path(text, service_name)
+                if sdk_folder_re.search(text) and not sdk_folder:
+                    sdk_folder = find_test_path(text)
                 if 'batch:' in text and multi_api == '':
                     multi_api = 'fake'
                     print(f'*********{service_name} is fake ')
@@ -416,11 +411,15 @@ def sdk_info_from_swagger():
         readme_html = folder.replace(SWAGGER_FOLDER, MAIN_REPO_SWAGGER)
         readme_html = Path(readme_html).as_posix()
         if package_name != '':
-            resource_manager.append('{},{},{},{},{}\n'.format(package_name,
-                                                              track_config,
-                                                              readme_python,
-                                                              readme_html,
-                                                              multi_api))
+            resource_manager.append({
+                'package_name': package_name.strip(),
+                'track_config': track_config,
+                'readme_python_path': readme_python.strip(),
+                'readme_html': readme_html.strip(),
+                'multi_api': multi_api.strip(),
+                'sdk_folder': sdk_folder.strip(),  # eg: resources/azure-mgmt-resource/azure/mgmt/resource
+                'service_name': service_name,
+            })
         my_print(f'{folder} : {package_name}')
 
     my_print(f'total package kinds: {len(resource_manager)}')

scripts/release_sdk_status/requirement.txt

@@ -4,7 +4,6 @@ idna==2.10
 lxml==4.6.5
 requests==2.25.1
 urllib3==1.26.6
-azure.storage.blob==12.8.1
 pytest==6.2.5
 pytest-cov==3.0.0
 PyGithub==1.55