Update Python SDK versions, add type hints (#305)

* Updated CI image to windows-2019

* Update Python SDK versions

* Fix linting warnings and added type annotations

Author: dpwatrous

Python/Batch/common/helpers.py

@@ -4,6 +4,5 @@ batchaccountkey=
 batchserviceurl=
 
 [Storage]
-storageaccountname=
+storageaccounturl=
 storageaccountkey=
-storageaccountsuffix=core.windows.net

Python/Batch/configuration.cfg

@@ -1,2 +1,2 @@
-azure-batch==8.0.0
-azure-storage-blob==1.3.1
+azure-batch==11.0.0
+azure-storage-blob==12.8.1

Python/Batch/requirements.txt

@@ -1,4 +1,4 @@
 [DEFAULT]
 shoulddeletejob=true
-poolvmsize=STANDARD_D1_V2
+poolvmsize=STANDARD_DS1_V2
 poolvmcount=1

Python/Batch/sample1_helloworld.cfg

@@ -24,38 +24,50 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-from __future__ import print_function
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+"""
+Submit a simple 'hello world' task
+"""
+
 import datetime
 import os
+from configparser import ConfigParser
 
-import azure.batch._batch_service_client as batch
-import azure.batch.batch_auth as batchauth
+from azure.batch import BatchServiceClient
+from azure.batch.batch_auth import SharedKeyCredentials
 import azure.batch.models as batchmodels
 
 import common.helpers
 
 
-def submit_job_and_add_task(batch_client, job_id, vm_size, vm_count):
+def submit_job_and_add_task(
+    batch_client: BatchServiceClient,
+    job_id: str,
+    vm_size: str,
+    node_count: int
+):
     """Submits a job to the Azure Batch service and adds a simple task.
 
     :param batch_client: The batch client to use.
-    :type batch_client: `batchserviceclient.BatchServiceClient`
-    :param str job_id: The id of the job to create.
+    :param job_id: The id of the job to create.
+    :param vm_size: The VM size to use.
+    :param node_count: The number of dedicated nodes to start.
     """
 
-    cloud_service_config = batchmodels.CloudServiceConfiguration(
-        os_family='5')
+    vm_config = batchmodels.VirtualMachineConfiguration(
+        image_reference=batchmodels.ImageReference(
+            publisher="canonical",
+            offer="ubuntuserver",
+            sku="18.04-lts"
+        ),
+        node_agent_sku_id="batch.node.ubuntu 18.04"
+    )
     pool_info = batchmodels.PoolInformation(
         auto_pool_specification=batchmodels.AutoPoolSpecification(
             auto_pool_id_prefix="HelloWorld",
             pool=batchmodels.PoolSpecification(
                 vm_size=vm_size,
-                target_dedicated_nodes=vm_count,
-                cloud_service_configuration=cloud_service_config),
+                target_dedicated_nodes=node_count,
+                virtual_machine_configuration=vm_config),
             keep_alive=False,
             pool_lifetime_option=batchmodels.PoolLifetimeOption.job))
 
@@ -66,19 +78,17 @@ def submit_job_and_add_task(batch_client, job_id, vm_size, vm_count):
     task = batchmodels.TaskAddParameter(
         id="HelloWorld",
         command_line=common.helpers.wrap_commands_in_shell(
-            'windows', ['echo Hello world from the Batch Hello world sample!'])
+            'linux', ['echo Hello world from the Batch Hello world sample!'])
     )
 
     batch_client.task.add(job_id=job.id, task=task)
 
 
-def execute_sample(global_config, sample_config):
+def execute_sample(global_config: ConfigParser, sample_config: ConfigParser):
     """Executes the sample with the specified configurations.
 
     :param global_config: The global configuration to use.
-    :type global_config: `configparser.ConfigParser`
     :param sample_config: The sample specific configuration to use.
-    :type sample_config: `configparser.ConfigParser`
     """
     # Set up the configuration
     batch_account_key = global_config.get('Batch', 'batchaccountkey')
@@ -99,11 +109,11 @@ def execute_sample(global_config, sample_config):
     common.helpers.print_configuration(global_config)
     common.helpers.print_configuration(sample_config)
 
-    credentials = batchauth.SharedKeyCredentials(
+    credentials = SharedKeyCredentials(
         batch_account_name,
         batch_account_key)
 
-    batch_client = batch.BatchServiceClient(
+    batch_client = BatchServiceClient(
         credentials,
         batch_url=batch_service_url)
 
@@ -134,11 +144,11 @@ def execute_sample(global_config, sample_config):
 
 
 if __name__ == '__main__':
-    global_config = configparser.ConfigParser()
-    global_config.read(common.helpers._SAMPLES_CONFIG_FILE_NAME)
+    global_cfg = ConfigParser()
+    global_cfg.read(common.helpers.SAMPLES_CONFIG_FILE_NAME)
 
-    sample_config = configparser.ConfigParser()
-    sample_config.read(
+    sample_cfg = ConfigParser()
+    sample_cfg.read(
         os.path.splitext(os.path.basename(__file__))[0] + '.cfg')
 
-    execute_sample(global_config, sample_config)
+    execute_sample(global_cfg, sample_cfg)

Python/Batch/sample1_helloworld.py

@@ -2,5 +2,5 @@
 shoulddeletecontainer=true
 shoulddeletejob=true
 shoulddeletepool=true
-poolvmsize=STANDARD_D1_V2
+poolvmsize=STANDARD_DS1_V2
 poolvmcount=1

Python/Batch/sample2_pools_and_resourcefiles.cfg

@@ -24,17 +24,19 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-from __future__ import print_function
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+"""
+Create a pool and submit a task which use resource files.
+"""
+
 import datetime
 import os
+from configparser import ConfigParser
+
+from azure.core.exceptions import ResourceExistsError
 
-import azure.storage.blob as azureblob
-import azure.batch._batch_service_client as batch
-import azure.batch.batch_auth as batchauth
+from azure.storage.blob import BlobServiceClient
+from azure.batch import BatchServiceClient
+from azure.batch.batch_auth import SharedKeyCredentials
 import azure.batch.models as batchmodels
 
 import common.helpers
@@ -44,28 +46,32 @@
 _SIMPLE_TASK_PATH = os.path.join('resources', 'simple_task.py')
 
 
-def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count):
+def create_pool(
+        batch_client: BatchServiceClient,
+        blob_service_client: BlobServiceClient,
+        pool_id: str,
+        vm_size: str,
+        vm_count: int):
     """Creates an Azure Batch pool with the specified id.
 
     :param batch_client: The batch client to use.
-    :type batch_client: `batchserviceclient.BatchServiceClient`
-    :param block_blob_client: The storage block blob client to use.
-    :type block_blob_client: `azure.storage.blob.BlockBlobService`
-    :param str pool_id: The id of the pool to create.
-    :param str vm_size: vm size (sku)
-    :param int vm_count: number of vms to allocate
+    :param blob_service_client: The storage block blob client to use.
+    :param pool_id: The id of the pool to create.
+    :param vm_size: vm size (sku)
+    :param vm_count: number of vms to allocate
     """
     # pick the latest supported 16.04 sku for UbuntuServer
     sku_to_use, image_ref_to_use = \
         common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
-            batch_client, 'Canonical', 'UbuntuServer', '16.04')
+            batch_client, 'canonical', 'ubuntuserver', '18.04')
 
-    block_blob_client.create_container(
-        _CONTAINER_NAME,
-        fail_on_exist=False)
+    try:
+        blob_service_client.create_container(_CONTAINER_NAME)
+    except ResourceExistsError:
+        pass
 
     sas_url = common.helpers.upload_blob_and_create_sas(
-        block_blob_client,
+        blob_service_client,
         _CONTAINER_NAME,
         _SIMPLE_TASK_NAME,
         _SIMPLE_TASK_PATH,
@@ -87,29 +93,32 @@ def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count):
     common.helpers.create_pool_if_not_exist(batch_client, pool)
 
 
-def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id):
+def submit_job_and_add_task(
+        batch_client: BatchServiceClient,
+        blob_service_client: BlobServiceClient,
+        job_id: str,
+        pool_id: str):
     """Submits a job to the Azure Batch service and adds
     a task that runs a python script.
 
     :param batch_client: The batch client to use.
-    :type batch_client: `batchserviceclient.BatchServiceClient`
-    :param block_blob_client: The storage block blob client to use.
-    :type block_blob_client: `azure.storage.blob.BlockBlobService`
-    :param str job_id: The id of the job to create.
-    :param str pool_id: The id of the pool to use.
+    :param blob_service_client: The storage block blob client to use.
+    :param job_id: The id of the job to create.
+    :param pool_id: The id of the pool to use.
     """
     job = batchmodels.JobAddParameter(
         id=job_id,
         pool_info=batchmodels.PoolInformation(pool_id=pool_id))
 
     batch_client.job.add(job)
 
-    block_blob_client.create_container(
-        _CONTAINER_NAME,
-        fail_on_exist=False)
+    try:
+        blob_service_client.create_container(_CONTAINER_NAME)
+    except ResourceExistsError:
+        pass
 
     sas_url = common.helpers.upload_blob_and_create_sas(
-        block_blob_client,
+        blob_service_client,
         _CONTAINER_NAME,
         _SIMPLE_TASK_NAME,
         _SIMPLE_TASK_PATH,
@@ -125,24 +134,19 @@ def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id):
     batch_client.task.add(job_id=job.id, task=task)
 
 
-def execute_sample(global_config, sample_config):
+def execute_sample(global_config: ConfigParser, sample_config: ConfigParser):
     """Executes the sample with the specified configurations.
 
     :param global_config: The global configuration to use.
-    :type global_config: `configparser.ConfigParser`
     :param sample_config: The sample specific configuration to use.
-    :type sample_config: `configparser.ConfigParser`
     """
     # Set up the configuration
     batch_account_key = global_config.get('Batch', 'batchaccountkey')
     batch_account_name = global_config.get('Batch', 'batchaccountname')
     batch_service_url = global_config.get('Batch', 'batchserviceurl')
 
+    storage_account_url = global_config.get('Storage', 'storageaccounturl')
     storage_account_key = global_config.get('Storage', 'storageaccountkey')
-    storage_account_name = global_config.get('Storage', 'storageaccountname')
-    storage_account_suffix = global_config.get(
-        'Storage',
-        'storageaccountsuffix')
 
     should_delete_container = sample_config.getboolean(
         'DEFAULT',
@@ -164,35 +168,35 @@ def execute_sample(global_config, sample_config):
     common.helpers.print_configuration(global_config)
     common.helpers.print_configuration(sample_config)
 
-    credentials = batchauth.SharedKeyCredentials(
+    credentials = SharedKeyCredentials(
         batch_account_name,
         batch_account_key)
-    batch_client = batch.BatchServiceClient(
+
+    batch_client = BatchServiceClient(
         credentials,
         batch_url=batch_service_url)
 
     # Retry 5 times -- default is 3
     batch_client.config.retry_policy.retries = 5
 
-    block_blob_client = azureblob.BlockBlobService(
-        account_name=storage_account_name,
-        account_key=storage_account_key,
-        endpoint_suffix=storage_account_suffix)
+    blob_service_client = BlobServiceClient(
+        account_url=storage_account_url,
+        credential=storage_account_key)
 
     job_id = common.helpers.generate_unique_resource_name(
         "PoolsAndResourceFilesJob")
     pool_id = "PoolsAndResourceFilesPool"
     try:
         create_pool(
             batch_client,
-            block_blob_client,
+            blob_service_client,
             pool_id,
             pool_vm_size,
             pool_vm_count)
 
         submit_job_and_add_task(
             batch_client,
-            block_blob_client,
+            blob_service_client,
             job_id, pool_id)
 
         common.helpers.wait_for_tasks_to_complete(
@@ -207,9 +211,7 @@ def execute_sample(global_config, sample_config):
     finally:
         # clean up
         if should_delete_container:
-            block_blob_client.delete_container(
-                _CONTAINER_NAME,
-                fail_not_exist=False)
+            blob_service_client.delete_container(_CONTAINER_NAME)
         if should_delete_job:
             print("Deleting job: ", job_id)
             batch_client.job.delete(job_id)
@@ -219,11 +221,11 @@ def execute_sample(global_config, sample_config):
 
 
 if __name__ == '__main__':
-    global_config = configparser.ConfigParser()
-    global_config.read(common.helpers._SAMPLES_CONFIG_FILE_NAME)
+    global_cfg = ConfigParser()
+    global_cfg.read(common.helpers.SAMPLES_CONFIG_FILE_NAME)
 
-    sample_config = configparser.ConfigParser()
-    sample_config.read(
+    sample_cfg = ConfigParser()
+    sample_cfg.read(
         os.path.splitext(os.path.basename(__file__))[0] + '.cfg')
 
-    execute_sample(global_config, sample_config)
+    execute_sample(global_cfg, sample_cfg)

Python/Batch/sample2_pools_and_resourcefiles.py

@@ -3,5 +3,5 @@ shoulddeletecontainer=true
 shoulddeletejob=true
 shoulddeletepool=true
 shoulddeletecert=true
-poolvmsize=STANDARD_D1_V2
+poolvmsize=STANDARD_DS1_V2
 poolvmcount=1