Skip to content
This repository was archived by the owner on Mar 20, 2023. It is now read-only.

Commit 005bb77

Browse files
alfparkalfpark
committed
Ubuntu 20.04 Support
- Resolves #363 - Add CentOS 7.8 support - Remove CentOS 8 support - Add Datacenter-Core-20H2 - Unblock -gs, -g2, -gensecond support - Update Nvidia Tesla driver to 470.57.02 and CUDA 11.4 - Update Nvidia Grid driver to 470.63.01 and CUDA 11.4 - Update LIS driver to 3.4.5
1 parent f50a153 commit 005bb77

File tree

5 files changed

+105
-109
lines changed

5 files changed

+105
-109
lines changed

‎convoy/fleet.py‎

Lines changed: 27 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -62,23 +62,23 @@
6262
'compute': {
6363
'url': (
6464
'https://us.download.nvidia.com/tesla/'
65-
'460.73.01/NVIDIA-Linux-x86_64-460.73.01.run'
65+
'470.57.02/NVIDIA-Linux-x86_64-470.57.02.run'
6666
),
6767
'sha256': (
68-
'11b1c918de26799e9ee3dc5db13d8630922b6aa602b9af3fbbd11a9a8aab1e88'
68+
'55d7ae104827faa79e975321fe2b60f9dd42fbff65642053443c0e56fdb4c47d'
6969
),
7070
'target': 'nvidia-driver-compute.run'
7171
},
7272
'viz': {
7373
# https://aka.ms/nvgrid-linux
7474
# https://go.microsoft.com/fwlink/?linkid=874272
7575
'url': (
76-
'https://download.microsoft.com/download/9/5/c/'
77-
'95c667ff-ab95-4c56-89e0-e13e9a76782d/'
78-
'NVIDIA-Linux-x86_64-460.32.03-grid-azure.run'
76+
'https://download.microsoft.com/download/b/d/d/'
77+
'bdd729ee-5003-4427-ace4-a7b9172b2e29/'
78+
'NVIDIA-Linux-x86_64-470.63.01-grid-azure.run'
7979
),
8080
'sha256': (
81-
'6b539691e3b4a3b92d907ce3a002d03f4564420ff7f0c13b11826f140fb6c44e'
81+
'71852dc93f8c28b289db8b6ac8d41fae6139d63bb16b4f4afc91dc8b2418cdd6'
8282
),
8383
'target': 'nvidia-driver-grid.run'
8484
},
@@ -105,10 +105,10 @@
105105
'url': (
106106
'http://download.microsoft.com/download/6/8/F/'
107107
'68FE11B8-FAA4-4F8D-8C7D-74DA7F2CFC8C/'
108-
'lis-rpms-4.3.4.x86_64.tar.gz'
108+
'lis-rpms-4.3.5.x86_64.tar.gz'
109109
),
110110
'sha256': (
111-
'5317176c536c6f013b5090c20e1a6045df155ac927afbcd1fb5059e49d19bc2b'
111+
'2fcfd68473fc3a46c97688acf6702a966561d02ec7b08c6b9ca668d54c83f20d'
112112
),
113113
'target': 'lis.tar.gz',
114114
'intermediate': 'lis_compact.tar',
@@ -430,7 +430,8 @@ def _download_file(desc, pkg, dldict):
430430
:param dict dldict: download dict
431431
"""
432432
logger.debug('downloading {} to {}'.format(desc, dldict['target']))
433-
response = requests.get(dldict['url'], stream=True)
433+
response = requests.get(
434+
dldict['url'], stream=True, headers={'User-Agent': 'Mozilla/5.0'})
434435
with pkg.open('wb') as f:
435436
for chunk in response.iter_content(chunk_size=_REQUEST_CHUNK_SIZE):
436437
if chunk:
@@ -2635,20 +2636,18 @@ def _adjust_settings_for_pool_creation(config):
26352636
allowed = False
26362637
shipyard_container_required = True
26372638
if publisher == 'microsoft-azure-batch':
2638-
if offer == 'centos-container':
2639-
allowed = True
2640-
elif offer == 'centos-container-rdma':
2641-
allowed = True
2642-
elif offer == 'ubuntu-server-container':
2643-
allowed = True
2644-
elif offer == 'ubuntu-server-container-rdma':
2639+
if (offer == 'centos-container' or
2640+
offer == 'centos-container-rdma' or
2641+
offer == 'ubuntu-server-container' or
2642+
offer == 'ubuntu-server-container-rdma'):
26452643
allowed = True
26462644
elif publisher == 'canonical':
26472645
if offer == 'ubuntuserver':
2648-
if sku == '16.04-lts':
2646+
if sku == '18.04-lts':
26492647
allowed = True
26502648
shipyard_container_required = False
2651-
elif sku == '18.04-lts':
2649+
elif offer == '0001-com-ubuntu-server-focal':
2650+
if sku == '20_04-lts':
26522651
allowed = True
26532652
shipyard_container_required = False
26542653
elif publisher == 'debian':
@@ -2657,20 +2656,16 @@ def _adjust_settings_for_pool_creation(config):
26572656
allowed = True
26582657
elif publisher == 'openlogic':
26592658
if offer.startswith('centos'):
2660-
if sku >= '7':
2659+
if sku >= '7' and sku < '8':
26612660
allowed = True
26622661
elif publisher == 'microsoftwindowsserver':
26632662
if offer == 'windowsserver':
2664-
if (sku == '2016-datacenter-with-containers' or
2665-
sku == '2019-datacenter-with-containers' or
2666-
sku == '2019-datacenter-with-containers-smalldisk' or
2667-
sku == '2019-datacenter-core-with-containers' or
2668-
sku == '2019-datacenter-core-with-containers-smalldisk'):
2669-
allowed = True
2670-
elif offer == 'windowsserversemiannual':
2671-
if (sku == 'datacenter-core-1709-with-containers-smalldisk' or
2672-
sku == 'datacenter-core-1803-with-containers-smalldisk' or
2673-
sku == 'datacenter-core-1809-with-containers-smalldisk'):
2663+
if (sku.startswith('2016-datacenter-with-containers') or
2664+
sku.startswith('2019-datacenter-with-containers') or
2665+
sku.startswith('2022-datacenter-with-containers') or
2666+
sku.startswith('2019-datacenter-core-with-containers') or
2667+
sku.startswith('2022-datacenter-core-with-containers') or
2668+
sku.startswith('datacenter-core-20h2-with-containers')):
26742669
allowed = True
26752670
if (util.is_not_empty(node_agent) and
26762671
node_agent.lower().startswith('batch.node.ubuntu')):
@@ -2868,7 +2863,9 @@ def _adjust_settings_for_pool_creation(config):
28682863
if is_windows:
28692864
raise ValueError(
28702865
'Cannot install kata_containers runtime on Windows')
2871-
if not ((publisher == 'canonical' and offer == 'ubuntuserver') or
2866+
if not ((publisher == 'canonical' and
2867+
(offer == 'ubuntuserver' or
2868+
offer.startswith('0001-com-ubuntu-server'))) or
28722869
(publisher == 'openlogic' and
28732870
offer.startswith('centos')) or
28742871
publisher == 'microsoft-azure-batch'):

‎convoy/settings.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -815,9 +815,11 @@ def gpu_configuration_check(config, vm_size=None):
815815
sku = pool_sku(config, lower=True)
816816
if publisher == 'microsoft-azure-batch':
817817
return True
818-
elif (publisher == 'canonical' and offer == 'ubuntuserver' and
819-
sku > '16.04'):
820-
return True
818+
elif publisher == 'canonical':
819+
if offer == 'ubuntuserver' and sku == '18.04':
820+
return True
821+
elif offer.startswith('0001-com-ubuntu-server') and sku >= '20_04':
822+
return True
821823
elif publisher == 'openlogic':
822824
if offer == 'centos-hpc' and sku >= '7.3':
823825
return True
@@ -844,7 +846,9 @@ def is_lis_install_required(config, vm_size=None):
844846
publisher = pool_publisher(config, lower=True)
845847
offer = pool_offer(config, lower=True)
846848
sku = pool_sku(config, lower=True)
847-
if publisher == 'openlogic' and offer == 'centos' and sku > '7.3':
849+
# current lis (4.3.5) does not support 7.8+
850+
if (publisher == 'openlogic' and offer == 'centos' and
851+
sku > '7.3' and sku <= '7.7'):
848852
return True
849853
return False
850854

@@ -982,7 +986,8 @@ def temp_disk_mountpoint(config, offer=None):
982986
offer = '!ubuntu'
983987
else:
984988
offer = offer.lower()
985-
if offer.startswith('ubuntu'):
989+
if (offer.startswith('ubuntu') or
990+
offer.startswith('0001-com-ubuntu-server')):
986991
return '/mnt'
987992
elif offer.startswith('windows'):
988993
return 'D:\\batch'
@@ -1130,21 +1135,22 @@ def _populate_pool_vm_configuration(config):
11301135
if not vm_config.native and _kv_read(conf, 'native', default=False):
11311136
vm_size = _pool_vm_size(config)
11321137
if (vm_config.publisher == 'canonical' and
1133-
vm_config.offer == 'ubuntuserver' and
1134-
vm_config.sku == '16.04-lts'):
1138+
vm_config.offer == '0001-com-ubuntu-server-focal' and
1139+
vm_config.sku == '20_04-lts'):
11351140
vm_config = PoolVmPlatformImageSettings(
11361141
publisher='microsoft-azure-batch',
11371142
offer='ubuntu-server-container{}'.format(
11381143
'-rdma' if is_rdma_pool(vm_size) else ''),
1139-
sku=vm_config.sku.replace('.', '-'),
1144+
sku=vm_config.sku.replace('_', '-'),
11401145
version='latest',
11411146
native=True,
11421147
license_type=None,
11431148
)
11441149
elif (vm_config.publisher == 'openlogic' and
11451150
vm_config.offer.startswith('centos') and
11461151
(vm_config.sku == '7.4' or vm_config.sku == '7.5' or
1147-
vm_config.sku == '7.6' or vm_config.sku == '7.7')):
1152+
vm_config.sku == '7.6' or vm_config.sku == '7.7' or
1153+
vm_config.sku == '7_8')):
11481154
vm_config = PoolVmPlatformImageSettings(
11491155
publisher='microsoft-azure-batch',
11501156
offer='centos-container{}'.format(

‎convoy/slurm.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,8 @@ def _apply_slurm_config_to_batch_pools(
9393
'Cannot create a Slurm partition {} on a Windows '
9494
'pool {}'.format(partname, pool.id))
9595
elif (na_sku != 'batch.node.centos 7' and
96-
na_sku != 'batch.node.ubuntu 16.04' and
97-
na_sku != 'batch.node.ubuntu 18.04'):
96+
na_sku != 'batch.node.ubuntu 18.04' and
97+
na_sku != 'batch.node.ubuntu 20.04'):
9898
raise RuntimeError(
9999
'Cannot create a Slurm partition {} on pool {} with node '
100100
'agent sku id {}'.format(partname, pool.id, na_sku))

‎docs/25-batch-shipyard-platform-image-support.md‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,11 @@ corresponding `urn` to accept the terms for the image.
4747
| Publisher | Offer | Sku | GPU | IB/RDMA | Native Only | Native Convert |
4848
|-----------------------|-----------------------|-----|:---:|:-------:|:-----------:|:--------------:|
4949
| microsoft-azure-batch | centos-container | 7-4 | X | | X | |
50-
| microsoft-azure-batch | centos-container | 7-7 | X | | X | |
50+
| microsoft-azure-batch | centos-container | 7-8 | X | | X | |
5151
| microsoft-azure-batch | centos-container-rdma | 7-4 | X | X (4) | X | |
52-
| microsoft-azure-batch | centos-container-rdma | 7-7 | X | X (5) | X | |
52+
| microsoft-azure-batch | centos-container-rdma | 7-8 | X | X (5) | X | |
5353
| OpenLogic | CentOS | 7.4 | X | | | X |
54-
| OpenLogic | CentOS | 7.7 | X | | | X |
55-
| OpenLogic | CentOS | 8.0 | X | | | |
54+
| OpenLogic | CentOS | 7_8 | X | | | X |
5655
| OpenLogic | CentOS-HPC | 7.4 | X | X (4) | | X |
5756
| OpenLogic | CentOS-HPC | 7.7 | X | X (5) | | X |
5857

@@ -70,22 +69,22 @@ SLES is not supported at this time.
7069

7170
| Publisher | Offer | Sku | GPU | IB/RDMA | Native Only | Native Convert |
7271
|-----------------------|------------------------------|-------------|:---:|:---------:|:-----------:|:--------------:|
73-
| Canonical | UbuntuServer | 16.04-LTS | X | X (1) | | X (2,4) |
7472
| Canonical | UbuntuServer | 18.04-LTS | X | X (1) | | |
75-
| microsoft-azure-batch | ubuntu-server-container | 16-04-lts | X | | X | |
76-
| microsoft-azure-batch | ubuntu-server-container-rdma | 16-04-lts | X | X (3,4) | X | |
73+
| Canonical | 0001-com-ubuntu-server-focal | 20_04-lts | X | X (1) | | X (2,5) |
74+
| microsoft-azure-batch | ubuntu-server-container | 20-04-lts | X | | X | |
75+
| microsoft-azure-batch | ubuntu-server-container-rdma | 20-04-lts | X | X (5) | X | |
7776

7877
### Windows
7978

79+
Note that `WindowsServer` Skus ending with suffixes such as `-gs`,
80+
`-smalldisk`, `-smalldisk-gs`, and `-smalldisk-g2` are supported.
81+
8082
| Publisher | Offer | Sku | GPU | IB/RDMA | Native Only | Native Convert |
8183
|------------------------|-------------------------|------------------------------------------------|:---:|:-------:|:-----------:|:--------------:|
8284
| MicrosoftWindowsServer | WindowsServer | 2016-Datacenter-with-Containers | | | X | |
8385
| MicrosoftWindowsServer | WindowsServer | 2019-Datacenter-with-Containers | | | X | |
84-
| MicrosoftWindowsServer | WindowsServer | 2019-Datacenter-with-Containers-smalldisk | | | X | |
8586
| MicrosoftWindowsServer | WindowsServer | 2019-Datacenter-Core-with-Containers | | | X | |
86-
| MicrosoftWindowsServer | WindowsServer | 2019-Datacenter-Core-with-Containers-smalldisk | | | X | |
87-
| MicrosoftWindowsServer | WindowsServer | Datacenter-Core-1903-with-Containers-smalldisk | | | X | |
88-
| MicrosoftWindowsServer | WindowsServerSemiAnnual | Datacenter-Core-1809-with-Containers-smalldisk | | | X | |
87+
| MicrosoftWindowsServer | WindowsServer | Datacenter-Core-20H2-with-Containers | | | X | |
8988

9089
## Notes
9190
1. IB/RDMA is supported for this host OS with a custom image unless

0 commit comments

Comments
 (0)