[AutoPR-agrifood]autogenerated

.vscode/cspell.json

@@ -196,7 +196,7 @@
     "livekvtestcertrec",
     "livekvtestgetcertperfcert",
     "livekvtestgetsecretperfsecret",
-    "livekvtestlistperfsecret",    
+    "livekvtestlistperfsecret",
     "livekvtestdecryptperfkey",
     "livekvtestgetkeyperfkey",
     "livekvtestsignperfkey",
@@ -242,6 +242,7 @@
     "PSECRET",
     "pygobject",
     "parameterizing",
+    "pytyped",
     "pytz",
     "pywin",
     "pyversion",

doc/dev/how_to_request_a_feature_in_sdk.md

@@ -0,0 +1,26 @@
+How to Request a Feature in SDK
+======
+
+This article aims to provide a guide for customers to request a feature in Azure Python SDK.
+
+Python SDK is automatically generated based on rest API, so we generally do not recommend modifying SDK code manually. If you need a new function, but the SDK does not provide it, you need to open an issue in the [rest API](https://github.com/Azure/azure-rest-api-specs/issues) to describe clearly the feature you want.
+
+Then, if the feature is adopted by the service team and the relevant rest API is updated, we will regenerate the SDK and release it after approved.
+
+
+The overall workflow is:
+1. Swagger and service ready
+2. Azure Python SDK release (PyPI)
+
+
+```
+ ---------      ------------ 
+|         |    |            | 
+| Service | -> | Python SDK | 
+|         |    |            |  
+ ---------      ------------
+```
+
+This is the way that Azure Python SDK works with the Service team. Swagger PR should be merged before requesting azure Python SDK support.
+
+Feel free to contact Azure CLI team at any time through any channels. We are passionate to build the world-class cloud product.

eng/common/TestResources/New-TestResources.ps1

@@ -375,7 +375,7 @@ try {
             $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16)
             Log "Generated base name '$BaseName' for CI build"
         } else {
-            $BaseName = GetBaseName $UserName $ServiceDirectory
+            $BaseName = GetBaseName $UserName (GetServiceLeafDirectoryName $ServiceDirectory)
             Log "BaseName was not set. Using default base name '$BaseName'"
         }
     }
@@ -520,7 +520,7 @@ try {
         $ResourceGroupName
     } elseif ($CI) {
         # Format the resource group name based on resource group naming recommendations and limitations.
-        "rg-{0}-$BaseName" -f ($serviceName -replace '[\\\/:]', '-').Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-')
+        "rg-{0}-$BaseName" -f ($serviceName -replace '[\.\\\/:]', '-').ToLowerInvariant().Substring(0, [Math]::Min($serviceName.Length, 90 - $BaseName.Length - 4)).Trim('-')
     } else {
         "rg-$BaseName"
     }

eng/common/TestResources/README.md

@@ -77,6 +77,49 @@ setx KEYVAULT_SKU ${env:KEYVAULT_SKU}
 setx AZURE_KEYVAULT_URL ${env:AZURE_KEYVAULT_URL}
 ```
 
+### Pre- and Post- Scripts
+
+Sometimes creating test resources requires either some work to be done prior to or after the main test-resources.json script is executed.
+For these scenarios a `test-resources-pre.ps1` or `test-resources-post.ps1`, respectively, can be created in the same folder as the `test-resources.json` file.
+
+For example, it may be necessary to create artifacts prior to provisioning the actual resource, such as a certificate.
+Typically the created artifact will need to be passed to `test-resources.json` to be used in the ARM template or as output (or both).
+
+Below is an example of how `$templateFileParameters` can be used to pass data from the `pre-` script to `test-resources.json`.
+
+**Snippet from `test-resources-pre.ps1`**
+```powershell
+$cert = New-X509Certificate2 -SubjectName '[email protected], CN=Azure SDK, OU=Azure SDK, O=Microsoft, L=Frisco, S=TX, C=US' -ValidDays 3652
+# Create new entries in $templateFileParameters
+$templateFileParameters['ConfidentialLedgerPrincipalPEM'] = Format-X509Certificate2 -Certificate $cert
+$templateFileParameters['ConfidentialLedgerPrincipalPEMPK'] = Format-X509Certificate2 -Type Pkcs8 -Certificate $cert
+```
+
+**Snippet from the corresponding `test-resources.json`.**
+
+Note that the values present in `$templateFileParameters` will map to parameters of the same name.
+```json
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "_comment": "Other required parameters would go here... (this is not part of the actual test-resources.json)",
+    "ConfidentialLedgerPrincipalPEM": {
+      "type": "string",
+      "metadata": {
+        "description": "The certificate to configure as a certBasedSecurityPrincipal."
+      }
+    },
+    "ConfidentialLedgerPrincipalPEMPK": {
+      "type": "string",
+      "metadata": {
+        "description": "The certificate to configure as a certBasedSecurityPrincipal."
+      }
+    }
+  },
+}
+```
+
 ### Cleaning up Resources
 
 By default, resource groups are tagged with a `DeleteAfter` value and date according to the default or specified

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -1,4 +1,5 @@
 function BuildServiceDirectoryPrefix([string]$serviceName) {
+    $serviceName = $serviceName -replace '[\./\\]', '_'
     return $serviceName.ToUpperInvariant() + "_"
 }
 
@@ -17,8 +18,8 @@ function GetUserName() {
 
 function GetBaseName([string]$user, [string]$serviceDirectoryName) {
     # Handle service directories in nested directories, e.g. `data/aztables`
-    $serviceDirectorySafeName = $serviceDirectoryName -replace '[/\\]', ''
-    return "$user$serviceDirectorySafeName"
+    $serviceDirectorySafeName = $serviceDirectoryName -replace '[\./\\]', ''
+    return "$user$serviceDirectorySafeName".ToLowerInvariant()
 }
 
 function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$value, [array]$allowedValues = @())

eng/common/TestResources/remove-test-resources.yml

@@ -34,5 +34,5 @@ steps:
         -Force `
         -Verbose
     displayName: Remove test resources
-    condition: eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true')
+    condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true'))
     continueOnError: true

eng/tox/tox.ini

@@ -330,7 +330,7 @@ deps =
   {[base]deps}
 commands =
     # install API stub generator
-    {envbindir}/python -m pip install api-stub-generator --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
+    {envbindir}/python -m pip install api-stub-generator==0.3.1 --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
     {envbindir}/python -m pip freeze
     {envbindir}/python {toxinidir}/../../../eng/tox/run_apistubgen.py -t {toxinidir} -w {envtmpdir} {posargs}
 

scripts/check_change_log/README.md

@@ -0,0 +1,25 @@
+# Overview
+This script will generate change_log by comparing latest two versions of mgmt package on pypi. It can help us quickly verify whether the changes to the `change_log` tool are correct 
+
+
+
+# preprequisites
+- [Python 3.6](https://www.python.org/downloads/windows/) or later is required
+- [docker desktop](https://www.docker.com/get-started/)
+
+# How to use this script
+1.Use gitbash to create a new branch to work in.
+
+2.Please make sure your docker is running on your computer.
+
+3.Open your powershell and step in path where the script is in
+
+4.Install the dependency
+```bash
+pip install -r requirements.txt
+```
+
+5.Run command in your powershell:
+```
+python main.py
+```

scripts/check_change_log/main.py

@@ -0,0 +1,121 @@
+# coding=utf-8
+import glob
+import os
+import subprocess as sp
+import time
+from pypi_tools.pypi import PyPIClient
+from pathlib import Path
+
+error_packages_info = {}
+
+
+def find_report_name(result):
+    signal_api_pattern = 'written to'
+    multi_api_pattern = 'merged_report'
+    for line in result:
+        idx = line.find(signal_api_pattern)
+        idx1 = line.find(multi_api_pattern)
+        if idx > 0 and idx1 > 0:
+            return "_/" + line[idx + len(signal_api_pattern):].replace(" ", "")
+
+    for line in result:
+        idx = line.find(signal_api_pattern)
+        if idx > 0:
+            return "_/" + line[idx + len(signal_api_pattern):].replace(" ", "")
+
+    return ''
+
+
+def create_folder(name):
+    if not os.path.exists(name):
+        os.mkdir(name)
+
+
+def write_txt(folder, text_name, content, older_version, last_version):
+    file_path = str(Path(f"{folder}/{text_name}_{older_version}_{last_version}.txt"))
+    with open(file=file_path, mode="w", encoding="utf-8") as file:
+        file.write(content)
+    print(f"{text_name} has been created successfully")
+
+
+def create_code_report(cmd, service_name):
+    process = sp.Popen(
+        cmd,
+        stderr=sp.STDOUT,
+        stdout=sp.PIPE,
+        universal_newlines=True,
+        cwd=None,
+        shell=False,
+        env=None,
+        encoding="utf-8",
+    )
+    output_buffer = [line.rstrip() for line in process.stdout]
+    process.wait()
+    if process.returncode:
+        error_packages_info[service_name] = '\n'.join(output_buffer[-min(len(output_buffer), 10):])
+        raise Exception(f'fail to create code report of {service_name}')
+    return output_buffer
+
+
+if __name__ == '__main__':
+    # get sdk path
+    env = Path.cwd()
+    docker_path = env.parent.parent
+    docker_cmd = 'docker exec -it Change_log /bin/bash -c'
+
+    # create docker env in sdk path
+    sp.call(fr"docker create -it --rm -h Change_log --name Change_log -v {docker_path}:/_ l601306339/autorest")
+    sp.call("docker start Change_log")
+
+    # install azure tools
+    sp.call(f'{docker_cmd}  "python _/scripts/dev_setup.py -p azure-core"  ')
+
+    # get all azure-mgmt-package paths
+    in_files = glob.glob(str(Path(f'{docker_path}/sdk/*/azure-mgmt-*')))
+    for i in in_files:
+        path = Path(i)
+        service_name = path.parts[-1]
+
+        # get package version in pypi
+        client = PyPIClient()
+        versions = [str(v) for v in client.get_ordered_versions(service_name)]
+        if len(versions) >= 2:
+            older_version = versions[-2]
+            last_version = versions[-1]
+
+            # generate code_report
+            cmd_last_version = fr'{docker_cmd} "cd _/ && python -m packaging_tools.code_report  {service_name} --version={last_version}"'
+            cmd_older_version = fr'{docker_cmd} "cd _/ && python -m packaging_tools.code_report {service_name} --version={older_version}"'
+            try:
+                last_code_report_info = create_code_report(cmd_last_version, service_name)
+                older_code_report_info = create_code_report(cmd_older_version, service_name)
+
+                # get code_report path
+                route_last_version = find_report_name(last_code_report_info)
+                route_older_version = find_report_name(older_code_report_info)
+
+                # use change_log on these two code_reports
+                result = sp.check_output(
+                    f'{docker_cmd} "python -m packaging_tools.change_log {route_older_version} {route_last_version}"',
+                    shell=True, universal_newlines=True)
+            except sp.CalledProcessError as e:
+                print(f'error happened for {service_name} during changelog generation')
+                error_packages_info[service_name] = str(e)
+            except Exception as e:
+                print(f'error happened for {service_name} during code report generation: {e}')
+            else:
+                output_message = result.split("\n")
+                result_text = "\n".join(output_message[1:])
+
+                # write a txt to save change_log
+                change_log_folder_path = str(env / "Change_Log")
+                create_folder(change_log_folder_path)
+                write_txt(change_log_folder_path, service_name, result_text, older_version, last_version)
+
+    if error_packages_info:
+        for k, v in error_packages_info.items():
+            print(f'== {k} encountered an error, info: {v}')
+
+    # exit and stop docker
+    sp.call(f'{docker_cmd} "exit"')
+    sp.call(f'docker stop Change_log')

scripts/check_change_log/requirements.txt

@@ -0,0 +1 @@
+-e ../../tools/azure-sdk-tools

scripts/quickstart_tooling_dpg/dev_requirements.txt

@@ -1 +1 @@
-jinja2
+jinja2==3.1.2

scripts/quickstart_tooling_dpg/main.py

@@ -5,7 +5,7 @@
 from jinja2 import Environment, FileSystemLoader
 from subprocess import check_call
 import time
-from typing import Any
+from typing import Any, Dict
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -106,6 +106,11 @@ def build_package(**kwargs) -> None:
     _LOGGER.info("Build complete: %s", package_name)
 
 
+def validate_params(**kwargs):
+    if not kwargs.get("security_scope") and not kwargs.get("security_header_name"):
+        raise Exception('At least one of "security-scope" and "security-header-name" is needed')
+
+
 def main(**kwargs):
     build_package(**kwargs)
 
@@ -132,8 +137,14 @@ def main(**kwargs):
     parser.add_argument(
         "--security-scope", "-c",
         dest="security_scope",
-        required=True,
-        help="Each service for data plan should have specific scopes",
+        required=False,
+        help="If authentication is AADToken, this param is necessary",
+    )
+    parser.add_argument(
+        "--security-header-name",
+        dest="security_header_name",
+        required=False,
+        help="If authentication is api key, this param is necessary",
     )
     parser.add_argument(
         "--package-name", "-p",
@@ -160,4 +171,5 @@ def main(**kwargs):
     main_logger.setLevel(logging.INFO)
 
     parameters = vars(args)
+    validate_params(**parameters)
     main(**parameters)

scripts/quickstart_tooling_dpg/template/swagger_README.md

@@ -11,6 +11,11 @@ no-namespace-folders: true
 title: {{ client_name }}
 version-tolerant: true
 package-version: 1.0.0b1
+{%- if security_scope %}
 security: AADToken
 security-scopes: {{ security_scope }}
+{%- elif security_header_name %}
+security: AzureKey
+security-header-name: {{ security_header_name }}
+{%- endif %}
 ```

sdk/agrifood/azure-mgmt-agrifood/_meta.json

@@ -1,11 +1,11 @@
 {
-  "autorest": "3.4.2",
+  "autorest": "3.7.2",
   "use": [
-    "@autorest/python@5.8.0",
-    "@autorest/[email protected].2"
+    "@autorest/python@5.13.0",
+    "@autorest/[email protected].3"
   ],
-  "commit": "5ee4aa771330a0120416c066386e6d86693978b4",
+  "commit": "e77640c0064f23db07babee0043c90e71fb03baa",
   "repository_url": "https://github.com/Azure/azure-rest-api-specs",
-  "autorest_command": "autorest specification/agfood/resource-manager/readme.md --multiapi --python --python-mode=update --python-sdks-folder=/home/vsts/work/1/s/azure-sdk-for-python/sdk --track2 --use=@autorest/python@5.8.0 --use=@autorest/[email protected].2 --version=3.4.2",
-  "readme": "specification/agfood/resource-manager/readme.md"
+  "autorest_command": "autorest agrifood/resource-manager/readme.md --multiapi --python --python-sdks-folder=/sdk-repo/sdk --python3-only --use=@autorest/python@5.13.0 --use=@autorest/[email protected].3 --version=3.7.2",
+  "readme": "agrifood/resource-manager/readme.md"
 }

sdk/agrifood/azure-mgmt-agrifood/azure/mgmt/agrifood/__init__.py

@@ -12,8 +12,7 @@
 __version__ = VERSION
 __all__ = ['AzureAgriFoodRPService']
 
-try:
-    from ._patch import patch_sdk  # type: ignore
-    patch_sdk()
-except ImportError:
-    pass
+# `._patch.py` is used for handwritten extensions to the generated code
+# Example: https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/customize_code/how-to-patch-sdk-code.md
+from ._patch import patch_sdk
+patch_sdk()