Convert Blob Source to HttpUrl when expanding template (#99)

* Update bad model

* minor changes

* Move blobSource to resourceFile to template parsing

* Add to job templates as well

* Move to parse json level

* pylint

* update tests

* remove redundant return statement

Author: bgklein

HISTORY.rst

@@ -3,6 +3,12 @@
 SDK Release History
 ===================
 
+5.0.3 (2019-02-19)
+------------------
+
+* Fix bug where blobSource was no longer an attribute of ExtendedResourceFile
+* Improve test coverage
+
 5.0.2 (2019-02-15)
 ------------------
 

azext/batch/_file_utils.py

@@ -12,15 +12,12 @@
 import copy
 import pathlib
 
-from logging import getLogger
 from six.moves.urllib.parse import urlsplit  # pylint: disable=import-error,relative-import
 from six.moves.urllib.parse import quote  # pylint: disable=import-error,no-name-in-module,relative-import
 
 from azure.storage.blob import BlobPermissions, BlockBlobService
 from . import models
 
-logger = getLogger(__name__)
-
 def construct_sas_url(blob, uri):
     """Make up blob URL with container URL"""
     newuri = copy.copy(uri)
@@ -330,16 +327,6 @@ def resolve_resource_file(self, resource_file):
                                  'also have \'file_path\' attribute')
             return [resource_file]
 
-        if resource_file.blob_source:
-            if not resource_file.file_path:
-                raise ValueError('Malformed ResourceFile: \'blobSource\' must '
-                                 'also have \'file_path\' attribute')
-            resource_file.http_url = resource_file.blob_source
-            logger.warning('BlobSource has been updated to HttpUrl to reflect new '
-                           'functionality of accepting any http url instead of just storage '
-                           'blobs. Please update your templates to reflect this')
-            return [resource_file]
-
         if resource_file.storage_container_url or resource_file.auto_storage_container_name:
             return [resource_file]
 

azext/batch/_template_utils.py

@@ -9,6 +9,7 @@
 import copy
 import itertools
 import json
+from logging import getLogger
 import re
 from msrest.serialization import Model
 try:
@@ -20,7 +21,7 @@
 from . import _pool_utils as pool_utils
 from . import models
 
-
+logger = getLogger(__name__)
 try:
     _UNICODE_TYPE = unicode
 except NameError:
@@ -1184,3 +1185,26 @@ def validate_json_object(json_obj, obj):
         else:
             inner_type = getattr(obj, key_attr_map[item.lower()])
             validate_json_object(json_obj[item], inner_type)
+
+
+def convert_blob_source_to_http_url(obj):
+    if isinstance(obj, dict):
+        for key in obj:
+            if key in ['resourceFiles', 'commonResourceFiles']:
+                obj[key] = \
+                    [_convert_blob_source_to_http_url(resource_file) for resource_file in obj[key]]
+            obj[key] = convert_blob_source_to_http_url(obj[key])
+    return obj
+
+
+def _convert_blob_source_to_http_url(resource_file):
+    if 'blobSource' in resource_file:
+        if 'filePath' not in resource_file:
+            raise ValueError('Malformed ResourceFile: \'blobSource\' must '
+                             'also have \'file_path\' attribute')
+        resource_file['httpUrl'] = resource_file.pop('blobSource', None)
+        logger.warning('BlobSource has been updated to HttpUrl to reflect new '
+                       'functionality of accepting any http url instead of just'
+                       ' storage blobs. Please update your templates to'
+                       ' reflect this.')
+    return resource_file

azext/batch/operations/job_operations.py

@@ -74,6 +74,7 @@ def jobparameter_from_json(json_data):
          ExtendedJobParameter or a JobTemplate
         """
         result = 'JobTemplate' if json_data.get('properties') else 'ExtendedJobParameter'
+        json_data = templates.convert_blob_source_to_http_url(json_data)
         try:
             if result == 'JobTemplate':
                 if 'apiVersion' in json_data:

azext/batch/operations/pool_operations.py

@@ -54,6 +54,7 @@ def poolparameter_from_json(json_data):
          ExtendedPoolParameter or a PoolTemplate.
         """
         result = 'PoolTemplate' if json_data.get('properties') else 'ExtendedPoolParameter'
+        json_data = templates.convert_blob_source_to_http_url(json_data)
         try:
             if result == 'PoolTemplate':
                 if 'apiVersion' in json_data:

azext/batch/version.py

@@ -3,4 +3,4 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-VERSION = "5.0.2"
+VERSION = "5.0.3"

tests/data/batch.pool.simple.resourcefile-legacy.json

@@ -0,0 +1,36 @@
+{
+  "pool": {
+    "id": "blobsource1",
+    "displayName": "Blender Ubuntu standard pool",
+    "vmSize": "Standard_D1_v2",
+    "virtualMachineConfiguration": {
+      "imageReference": {
+        "publisher": "Canonical",
+        "offer" : "UbuntuServer",
+        "sku": "16.04.0-LTS",
+        "version": "latest"
+      },
+      "nodeAgentSKUId": "batch.node.ubuntu 16.04"
+    },
+    "targetDedicatedNodes": "1",
+    "targetLowPriorityNodes": "0",
+    "enableAutoScale": false,
+    "startTask": {
+      "commandLine": "sleep 1",
+      "waitForSuccess": true,
+      "maxTaskRetryCount": 0,
+      "userIdentity": {
+        "autoUser": {
+          "scope": "pool",
+          "elevationLevel": "admin"
+        }
+      },
+      "resourceFiles": [
+        {
+          "blobSource": "https://raw.githubusercontent.com/Azure/BatchExplorer-data/master/ncj/blender/scripts/setup-linux-pool.sh",
+          "filePath": "setup-linux-pool.sh"
+        }
+      ]
+    }
+  }
+}