Tag for 2.8.0rc2 release

- Fix regression with private docker image pulls
- Resolves #103
- Resolves #105

Author: alfpark

CHANGELOG.md

@@ -2,13 +2,20 @@
 
 ## [Unreleased]
 
+## [2.8.0rc2] - 2017-06-30
 ### Added
 - Support Mac OS X and Windows Subsystem for Linux installations via
-`install.sh`
+`install.sh` (#101)
 - Guide for Windows Subsystem for Linux installations
+- Automated Nvidia driver install for NV-series
 
 ### Changed
 - Drop unsupported designations for Mac OS X and Windows
+- Update Docker engine to 17.06 for Ubuntu, Debian, CentOS and 17.04 for
+OpenSUSE
+
+### Fixed
+- Regression in private registry image pulls
 
 ## [2.8.0rc1] - 2017-06-27
 ### Added
@@ -710,7 +717,8 @@ transfer is disabled
 #### Added
 - Initial release
 
-[Unreleased]: https://github.com/Azure/batch-shipyard/compare/2.8.0rc1...HEAD
+[Unreleased]: https://github.com/Azure/batch-shipyard/compare/2.8.0rc2...HEAD
+[2.8.0rc2]: https://github.com/Azure/batch-shipyard/compare/2.8.0rc1...2.8.0rc2
 [2.8.0rc1]: https://github.com/Azure/batch-shipyard/compare/2.8.0b1...2.8.0rc1
 [2.8.0b1]: https://github.com/Azure/batch-shipyard/compare/2.7.0...2.8.0b1
 [2.7.0]: https://github.com/Azure/batch-shipyard/compare/2.7.0rc1...2.7.0

cascade/cascade.py

@@ -429,7 +429,7 @@ def _pull(self, image: str) -> tuple:
         if proc.returncode == 0 and not _pub:
             # tag image to remove registry ip
             subprocess.check_call(
-                'docker tag {} {}'.format(_REGISTRY, src, image),
+                'docker tag {} {}'.format(src, image),
                 shell=True)
         return proc.returncode, stdout, stderr
 

convoy/batch.py

@@ -1925,15 +1925,25 @@ def generate_docker_login_settings(config, for_ssh=False):
                 '-p $DOCKER_LOGIN_PASSWORD')
     # transform env and cmd into single command for ssh
     if for_ssh and len(cmd) > 0:
+        srv = None
+        for ev in env:
+            if ev.name == 'DOCKER_LOGIN_PASSWORD':
+                pw = ev.value
+            elif ev.name == 'DOCKER_LOGIN_USERNAME':
+                user = ev.value
+            elif ev.name == 'DOCKER_LOGIN_SERVER':
+                srv = ev.value
         key = '${}'.format('DOCKER_LOGIN_PASSWORD')
         if encrypt:
-            pw = cmd[0][22:].replace(key, env[1].value)
+            pw = cmd[0][22:].replace(key, pw)
             cmd = cmd[1].replace(key, pw)
         else:
-            pw = env[1].value
             cmd = cmd[0].replace(key, pw)
         key = '${}'.format('DOCKER_LOGIN_USERNAME')
-        cmd = cmd.replace(key, env[0].value)
+        cmd = cmd.replace(key, user)
+        if util.is_not_empty(srv):
+            key = '${}'.format('DOCKER_LOGIN_SERVER')
+            cmd = cmd.replace(key, srv)
         if encrypt:
             key = 'openssl'
             if key in cmd:

convoy/fleet.py

@@ -94,12 +94,19 @@
         'sha256': (
             '59e37f570ba5f3d7148028e96684d77f347d49a54e3722189782fc9b17d201c0'
         ),
+        'target': 'resources/nvidia-driver.run'
+    },
+    'visualization': {
+        'url': 'https://go.microsoft.com/fwlink/?linkid=849941',
+        'sha256': (
+            'f5e39c9abf6d48d9883cd61d8fec8c67f05c9d6a7cc8b450af0efa790fbbd1a7'
+        ),
+        'target': 'resources/nvidia-driver-grid.run'
     },
     'license': (
         'http://www.nvidia.com/content/DriverDownload-March2009'
         '/licence.php?lang=us'
     ),
-    'target': 'resources/nvidia-driver.run'
 }
 _NODEPREP_FILE = (
     'shipyard_nodeprep.sh',
@@ -289,14 +296,8 @@ def _setup_nvidia_driver_package(blob_client, config, vm_size):
     :rtype: pathlib.Path
     :return: package path
     """
-    if settings.is_gpu_compute_pool(vm_size):
-        gpu_type = 'compute'
-    elif settings.is_gpu_visualization_pool(vm_size):
-        gpu_type = 'visualization'
-        raise RuntimeError(
-            ('pool consisting of {} nodes require gpu driver '
-             'configuration').format(vm_size))
-    pkg = pathlib.Path(_ROOT_PATH, _NVIDIA_DRIVER['target'])
+    gpu_type = settings.get_gpu_type_from_vm_size(vm_size)
+    pkg = pathlib.Path(_ROOT_PATH, _NVIDIA_DRIVER[gpu_type]['target'])
     # check to see if package is downloaded
     if (not pkg.exists() or
             util.compute_sha256_for_file(pkg, False) !=
@@ -314,14 +315,14 @@ def _setup_nvidia_driver_package(blob_client, config, vm_size):
             logger.info('NVIDIA Software License accepted')
         # download driver
         logger.debug('downloading NVIDIA driver to {}'.format(
-            _NVIDIA_DRIVER['target']))
+            _NVIDIA_DRIVER[gpu_type]['target']))
         response = requests.get(_NVIDIA_DRIVER[gpu_type]['url'], stream=True)
         with pkg.open('wb') as f:
             for chunk in response.iter_content(chunk_size=_REQUEST_CHUNK_SIZE):
                 if chunk:
                     f.write(chunk)
         logger.debug('wrote {} bytes to {}'.format(
-            pkg.stat().st_size, _NVIDIA_DRIVER['target']))
+            pkg.stat().st_size, _NVIDIA_DRIVER[gpu_type]['target']))
         # check sha256
         if (util.compute_sha256_for_file(pkg, False) !=
                 _NVIDIA_DRIVER[gpu_type]['sha256']):
@@ -833,7 +834,9 @@ def _add_pool(
                 blob_client, config, pool_settings.vm_size)
             _rflist.append((gpu_driver.name, gpu_driver))
         else:
-            gpu_driver = pathlib.Path(_NVIDIA_DRIVER['target'])
+            gpu_type = settings.get_gpu_type_from_vm_size(
+                pool_settings.vm_size)
+            gpu_driver = pathlib.Path(_NVIDIA_DRIVER[gpu_type]['target'])
         gpupkg = _setup_nvidia_docker_package(blob_client, config)
         _rflist.append((gpupkg.name, gpupkg))
         gpu_env = '{}:{}:{}'.format(

convoy/settings.py

@@ -388,6 +388,21 @@ def is_gpu_visualization_pool(vm_size):
     return False
 
 
+def get_gpu_type_from_vm_size(vm_size):
+    # type: (str) -> str
+    """Get GPU type as string
+    :param str vm_size: vm size
+    :rtype: str
+    :return: compute for gpgpu and visualization for viz
+    """
+    if is_gpu_compute_pool(vm_size):
+        return 'compute'
+    elif is_gpu_visualization_pool(vm_size):
+        return 'visualization'
+    else:
+        return None
+
+
 def is_rdma_pool(vm_size):
     # type: (str) -> bool
     """Check if pool is IB/RDMA capable

convoy/version.py

@@ -22,4 +22,4 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-__version__ = '2.8.0rc1'
+__version__ = '2.8.0rc2'

docs/11-batch-shipyard-configuration-credentials.md

@@ -215,6 +215,6 @@ or private repositories on Docker Hub.
       Azure KeyVault.
 
 ## Full template
-An full template of a credentials file can be found
+A full template of a credentials file can be found
 [here](../config\_templates/credentials.json). Note that this template cannot
 be used as-is and must be modified to fit your scenario.

docs/12-batch-shipyard-configuration-global.md

@@ -449,6 +449,6 @@ one or all of `data_volumes` and `shared_data_volumes` if you do not require
 this functionality.
 
 ## Full template
-An full template of a credentials file can be found
+A full template of a credentials file can be found
 [here](../config\_templates/config.json). Note that this template cannot
 be used as-is and must be modified to fit your scenario.

docs/13-batch-shipyard-configuration-pool.md

@@ -255,15 +255,16 @@ on how to generate an SSH keypair for use with Batch Shipyard.
     [HPN patches](https://www.psc.edu/index.php/using-joomla/extensions/templates/atomic/636-hpn-ssh)
     to be swapped with the standard distribution OpenSSH server. This is not
     supported on all Linux distributions and may be force disabled.
-* (required for `STANDARD_NV` instances, optional for `STANDARD_NC` instances)
-`gpu` property defines additional information for NVIDIA GPU-enabled VMs:
+* (optional) `gpu` property defines additional information for NVIDIA
+GPU-enabled VMs. If not specified, Batch Shipyard will automatically download
+the driver for the `vm_size` specified.
   * `nvidia_driver` property contains the following required members:
     * `source` is the source url to download the driver.
 * (optional) `additional_node_prep_commands` is an array of additional commands
 to execute on the compute node host as part of node preparation. This can
 be empty or omitted.
 
 ## Full template
-An full template of a credentials file can be found
+A full template of a credentials file can be found
 [here](../config\_templates/pool.json). Note that this template cannot
 be used as-is and must be modified to fit your scenario.

docs/14-batch-shipyard-configuration-jobs.md

@@ -465,6 +465,6 @@ transferred again. This object currently supports `azure_batch` and
     This property may be null.
 
 ## Full template
-An full template of a credentials file can be found
+A full template of a credentials file can be found
 [here](../config\_templates/jobs.json). Note that this template cannot
 be used as-is and must be modified to fit your scenario.