Skip to content

Commit f4529a1

Browse files
QuanMPhmQuanMPhm
committed
Created new allocation attribute for IBM storage quota
The addition of IBM Scale storage to OpenShift resources has several implications: - A pre-existing Openshift attribute, `QUOTA_REQUESTS_STORAGE`, was renamed to include "NESE" in its name to avoid confusion with the new IBM storage attribute. - OpenShift resources now have a new attribute `IBM Spectrum Scale Storage Available` ("true" or "false") to indicate IBM Scale storage availability. If "true", new and pre-existing allocations will have a new attribute to indicate IBM storage quota, which defaults to 0 Gb. - Openshift quota key for NESE storage is now `ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage` For now, IBM storage quota is free and not included in the storage invoice.
1 parent c35be23 commit f4529a1

File tree

10 files changed

+181
-19
lines changed

10 files changed

+181
-19
lines changed

‎src/coldfront_plugin_cloud/attributes.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ class CloudAllocationAttribute:
2424
RESOURCE_API_URL = "OpenShift API Endpoint URL"
2525
RESOURCE_IDENTITY_NAME = "OpenShift Identity Provider Name"
2626
RESOURCE_ROLE = "Role for User in Project"
27+
RESOURCE_IBM_AVAILABLE = "IBM Spectrum Scale Storage Available"
2728

2829
RESOURCE_FEDERATION_PROTOCOL = "OpenStack Federation Protocol"
2930
RESOURCE_IDP = "OpenStack Identity Provider"
@@ -42,6 +43,7 @@ class CloudAllocationAttribute:
4243
CloudResourceAttribute(name=RESOURCE_IDP),
4344
CloudResourceAttribute(name=RESOURCE_PROJECT_DOMAIN),
4445
CloudResourceAttribute(name=RESOURCE_ROLE),
46+
CloudResourceAttribute(name=RESOURCE_IBM_AVAILABLE),
4547
CloudResourceAttribute(name=RESOURCE_USER_DOMAIN),
4648
CloudResourceAttribute(name=RESOURCE_EULA_URL),
4749
CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK),
@@ -86,7 +88,10 @@ class CloudAllocationAttribute:
8688
QUOTA_LIMITS_CPU = "OpenShift Limit on CPU Quota"
8789
QUOTA_LIMITS_MEMORY = "OpenShift Limit on RAM Quota (MiB)"
8890
QUOTA_LIMITS_EPHEMERAL_STORAGE_GB = "OpenShift Limit on Ephemeral Storage Quota (GiB)"
89-
QUOTA_REQUESTS_STORAGE = "OpenShift Request on Storage Quota (GiB)"
91+
QUOTA_REQUESTS_NESE_STORAGE = "OpenShift Request on NESE Storage Quota (GiB)"
92+
QUOTA_REQUESTS_IBM_STORAGE = (
93+
"OpenShift Request on IBM Spectrum Scale Storage Quota (GiB)"
94+
)
9095
QUOTA_REQUESTS_GPU = "OpenShift Request on GPU Quota"
9196
QUOTA_REQUESTS_VM_GPU_A100_SXM4 = "OpenShift Request on GPU A100 SXM4"
9297
QUOTA_REQUESTS_VM_GPU_V100 = "OpenShift Request on GPU V100"
@@ -107,7 +112,8 @@ class CloudAllocationAttribute:
107112
CloudAllocationAttribute(name=QUOTA_LIMITS_CPU),
108113
CloudAllocationAttribute(name=QUOTA_LIMITS_MEMORY),
109114
CloudAllocationAttribute(name=QUOTA_LIMITS_EPHEMERAL_STORAGE_GB),
110-
CloudAllocationAttribute(name=QUOTA_REQUESTS_STORAGE),
115+
CloudAllocationAttribute(name=QUOTA_REQUESTS_NESE_STORAGE),
116+
CloudAllocationAttribute(name=QUOTA_REQUESTS_IBM_STORAGE),
111117
CloudAllocationAttribute(name=QUOTA_REQUESTS_GPU),
112118
CloudAllocationAttribute(name=QUOTA_REQUESTS_VM_GPU_A100_SXM4),
113119
CloudAllocationAttribute(name=QUOTA_REQUESTS_VM_GPU_V100),

‎src/coldfront_plugin_cloud/management/commands/add_openshift_resource.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ def add_arguments(self, parser):
4444
action="store_true",
4545
help="Indicates this is an OpenShift Virtualization resource (default: False)",
4646
)
47+
parser.add_argument(
48+
"--ibm-storage-available",
49+
action="store_true",
50+
help="Indicates that Ibm Scale storage is available in this resource (default: False)",
51+
)
4752

4853
def handle(self, *args, **options):
4954
self.validate_role(options["role"])
@@ -86,3 +91,11 @@ def handle(self, *args, **options):
8691
resource=openshift,
8792
value=options["role"],
8893
)
94+
95+
ResourceAttribute.objects.get_or_create(
96+
resource_attribute_type=ResourceAttributeType.objects.get(
97+
name=attributes.RESOURCE_IBM_AVAILABLE
98+
),
99+
resource=openshift,
100+
value="true" if options["ibm_storage_available"] else "false",
101+
)

‎src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -300,9 +300,9 @@ def process_invoice_row(allocation, attrs, su_name, rate):
300300
allocation,
301301
[
302302
attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB,
303-
attributes.QUOTA_REQUESTS_STORAGE,
303+
attributes.QUOTA_REQUESTS_NESE_STORAGE,
304304
],
305-
"OpenShift Storage",
305+
"OpenShift NESE Storage",
306306
openshift_storage_rate,
307307
)
308308

‎src/coldfront_plugin_cloud/management/commands/register_cloud_attributes.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
("OpenStack Project ID", {"name": "Allocated Project ID"}),
2323
("OpenStack Project Name", {"name": "Allocated Project Name"}),
2424
("OpenShift Limit on RAM Quota", {"name": "OpenShift Limit on RAM Quota (MB)"}),
25+
(
26+
"OpenShift Request on Storage Quota (GiB)",
27+
{"name": "OpenShift Request on NESE Storage Quota (GiB)"},
28+
),
2529
("OpenStack Volume Quota", {"name": "OpenStack Number of Volumes Quota"}),
2630
("OpenStack Compute RAM Quota", {"name": "OpenStack Compute RAM Quota (MiB)"}),
2731
("OpenStack Volume GB Quota", {"name": "OpenStack Volume Quota (GiB)"}),

‎src/coldfront_plugin_cloud/openshift.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,12 @@ class OpenShiftResourceAllocator(base.ResourceAllocator):
7070
attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: lambda x: {
7171
"limits.ephemeral-storage": f"{x}Gi"
7272
},
73-
attributes.QUOTA_REQUESTS_STORAGE: lambda x: {"requests.storage": f"{x}Gi"},
73+
attributes.QUOTA_REQUESTS_NESE_STORAGE: lambda x: {
74+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": f"{x}Gi"
75+
},
76+
attributes.QUOTA_REQUESTS_IBM_STORAGE: lambda x: {
77+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": f"{x}Gi"
78+
},
7479
attributes.QUOTA_REQUESTS_GPU: lambda x: {"requests.nvidia.com/gpu": f"{x}"},
7580
attributes.QUOTA_PVC: lambda x: {"persistentvolumeclaims": f"{x}"},
7681
}

‎src/coldfront_plugin_cloud/openshift_vm.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@ class OpenShiftVMResourceAllocator(openshift.OpenShiftResourceAllocator):
88
attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: lambda x: {
99
"limits.ephemeral-storage": f"{x}Gi"
1010
},
11-
attributes.QUOTA_REQUESTS_STORAGE: lambda x: {"requests.storage": f"{x}Gi"},
11+
attributes.QUOTA_REQUESTS_NESE_STORAGE: lambda x: {
12+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": f"{x}Gi"
13+
},
14+
attributes.QUOTA_REQUESTS_IBM_STORAGE: lambda x: {
15+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": f"{x}Gi"
16+
},
1217
attributes.QUOTA_REQUESTS_VM_GPU_A100_SXM4: lambda x: {
1318
"requests.nvidia.com/A100_SXM4_40GB": f"{x}"
1419
},

‎src/coldfront_plugin_cloud/tasks.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,15 +34,17 @@
3434
attributes.QUOTA_LIMITS_CPU: 1,
3535
attributes.QUOTA_LIMITS_MEMORY: 4096,
3636
attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: 5,
37-
attributes.QUOTA_REQUESTS_STORAGE: 20,
37+
attributes.QUOTA_REQUESTS_NESE_STORAGE: 20,
38+
attributes.QUOTA_REQUESTS_IBM_STORAGE: 0,
3839
attributes.QUOTA_REQUESTS_GPU: 0,
3940
attributes.QUOTA_PVC: 2,
4041
},
4142
"openshift_vm": {
4243
attributes.QUOTA_LIMITS_CPU: 1,
4344
attributes.QUOTA_LIMITS_MEMORY: 4096,
4445
attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: 5,
45-
attributes.QUOTA_REQUESTS_STORAGE: 20,
46+
attributes.QUOTA_REQUESTS_NESE_STORAGE: 20,
47+
attributes.QUOTA_REQUESTS_IBM_STORAGE: 0,
4648
attributes.QUOTA_REQUESTS_VM_GPU_A100_SXM4: 0,
4749
attributes.QUOTA_REQUESTS_VM_GPU_V100: 0,
4850
attributes.QUOTA_REQUESTS_VM_GPU_H100: 0,
@@ -74,7 +76,21 @@
7476
def get_expected_attributes(allocator: base.ResourceAllocator):
7577
"""Based on the allocator's resource type, return the expected quotas attributes the allocation should have"""
7678
resource_name = allocator.resource_type
77-
return list(UNIT_QUOTA_MULTIPLIERS[resource_name].keys())
79+
resource_expected_quotas = UNIT_QUOTA_MULTIPLIERS[resource_name].copy()
80+
81+
# If the resource attribute is not set (i.e for OpenStack resources), get_attribute returns None
82+
is_ibm_storage_available = allocator.resource.get_attribute(
83+
attributes.RESOURCE_IBM_AVAILABLE
84+
)
85+
is_ibm_storage_available = (
86+
is_ibm_storage_available and is_ibm_storage_available.lower() == "true"
87+
)
88+
if "openshift" in resource_name and not is_ibm_storage_available:
89+
resource_expected_quotas.pop(
90+
attributes.QUOTA_REQUESTS_IBM_STORAGE, None
91+
) # The resource may or may not already have this attribute
92+
93+
return list(resource_expected_quotas.keys())
7894

7995

8096
def find_allocator(allocation) -> base.ResourceAllocator:

‎src/coldfront_plugin_cloud/tests/base.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,11 @@ def new_openstack_resource(name=None, auth_url=None) -> Resource:
8080

8181
@staticmethod
8282
def new_openshift_resource(
83-
name=None, api_url=None, idp=None, for_virtualization=False
83+
name=None,
84+
api_url=None,
85+
idp=None,
86+
for_virtualization=False,
87+
ibm_storage_available=False,
8488
) -> Resource:
8589
resource_name = name or uuid.uuid4().hex
8690

@@ -90,6 +94,7 @@ def new_openshift_resource(
9094
api_url=api_url or "https://onboarding-onboarding.cluster.local:6443",
9195
idp=idp or "developer",
9296
for_virtualization=for_virtualization,
97+
ibm_storage_available=ibm_storage_available,
9398
)
9499
return Resource.objects.get(name=resource_name)
95100

‎src/coldfront_plugin_cloud/tests/functional/openshift/test_allocation.py‎

Lines changed: 115 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ def setUp(self) -> None:
1818
self.resource = self.new_openshift_resource(
1919
name="Microshift",
2020
api_url=os.getenv("OS_API_URL"),
21+
ibm_storage_available=True,
2122
)
2223

2324
def test_new_allocation(self):
@@ -135,7 +136,7 @@ def test_new_allocation_quota(self):
135136
2 * 5,
136137
)
137138
self.assertEqual(
138-
allocation.get_attribute(attributes.QUOTA_REQUESTS_STORAGE), 2 * 20
139+
allocation.get_attribute(attributes.QUOTA_REQUESTS_NESE_STORAGE), 2 * 20
139140
)
140141
self.assertEqual(allocation.get_attribute(attributes.QUOTA_REQUESTS_GPU), 2 * 0)
141142
self.assertEqual(allocation.get_attribute(attributes.QUOTA_PVC), 2 * 2)
@@ -149,7 +150,8 @@ def test_new_allocation_quota(self):
149150
"limits.cpu": "2",
150151
"limits.memory": "8Gi",
151152
"limits.ephemeral-storage": "10Gi",
152-
"requests.storage": "40Gi",
153+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "40Gi",
154+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": "0",
153155
"requests.nvidia.com/gpu": "0",
154156
"persistentvolumeclaims": "4",
155157
},
@@ -164,7 +166,7 @@ def test_new_allocation_quota(self):
164166
allocation, attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB, 50
165167
)
166168
utils.set_attribute_on_allocation(
167-
allocation, attributes.QUOTA_REQUESTS_STORAGE, 100
169+
allocation, attributes.QUOTA_REQUESTS_NESE_STORAGE, 100
168170
)
169171
utils.set_attribute_on_allocation(allocation, attributes.QUOTA_REQUESTS_GPU, 1)
170172
utils.set_attribute_on_allocation(allocation, attributes.QUOTA_PVC, 10)
@@ -175,7 +177,7 @@ def test_new_allocation_quota(self):
175177
allocation.get_attribute(attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB), 50
176178
)
177179
self.assertEqual(
178-
allocation.get_attribute(attributes.QUOTA_REQUESTS_STORAGE), 100
180+
allocation.get_attribute(attributes.QUOTA_REQUESTS_NESE_STORAGE), 100
179181
)
180182
self.assertEqual(allocation.get_attribute(attributes.QUOTA_REQUESTS_GPU), 1)
181183
self.assertEqual(allocation.get_attribute(attributes.QUOTA_PVC), 10)
@@ -192,7 +194,8 @@ def test_new_allocation_quota(self):
192194
"limits.cpu": "6",
193195
"limits.memory": "8Gi",
194196
"limits.ephemeral-storage": "50Gi",
195-
"requests.storage": "100Gi",
197+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "100Gi",
198+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": "0",
196199
"requests.nvidia.com/gpu": "1",
197200
"persistentvolumeclaims": "10",
198201
},
@@ -221,7 +224,8 @@ def test_reactivate_allocation(self):
221224
"limits.cpu": "2",
222225
"limits.memory": "8Gi",
223226
"limits.ephemeral-storage": "10Gi",
224-
"requests.storage": "40Gi",
227+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "40Gi",
228+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": "0",
225229
"requests.nvidia.com/gpu": "0",
226230
"persistentvolumeclaims": "4",
227231
},
@@ -242,7 +246,8 @@ def test_reactivate_allocation(self):
242246
"limits.cpu": "3",
243247
"limits.memory": "8Gi",
244248
"limits.ephemeral-storage": "10Gi",
245-
"requests.storage": "40Gi",
249+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "40Gi",
250+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": "0",
246251
"requests.nvidia.com/gpu": "0",
247252
"persistentvolumeclaims": "4",
248253
},
@@ -375,3 +380,106 @@ def test_allocation_new_attribute(self):
375380
"limits.memory": "8Gi",
376381
},
377382
)
383+
384+
def test_migrate_quota_field_names(self):
385+
"""When a quota key in QUOTA_KEY_MAPPING changes to a new value, validate_allocations should update the quota."""
386+
user = self.new_user()
387+
project = self.new_project(pi=user)
388+
allocation = self.new_allocation(project, self.resource, 1)
389+
allocator = openshift.OpenShiftResourceAllocator(self.resource, allocation)
390+
391+
tasks.activate_allocation(allocation.pk)
392+
allocation.refresh_from_db()
393+
394+
project_id = allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
395+
396+
quota = allocator.get_quota(project_id)
397+
self.assertEqual(
398+
quota,
399+
{
400+
"limits.cpu": "1",
401+
"limits.memory": "4Gi",
402+
"limits.ephemeral-storage": "5Gi",
403+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "20Gi",
404+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": "0",
405+
"requests.nvidia.com/gpu": "0",
406+
"persistentvolumeclaims": "2",
407+
},
408+
)
409+
410+
# Now migrate NESE Storage quota field (ocs-external...) to fake storage quota
411+
with unittest.mock.patch.dict(
412+
openshift.OpenShiftResourceAllocator.QUOTA_KEY_MAPPING,
413+
{
414+
attributes.QUOTA_REQUESTS_NESE_STORAGE: lambda x: {
415+
"fake-storage.storageclass.storage.k8s.io/requests.storage": f"{x}Gi"
416+
}
417+
},
418+
):
419+
call_command("validate_allocations", apply=True)
420+
421+
# Check the quota after migration
422+
quota = allocator.get_quota(project_id)
423+
self.assertEqual(
424+
quota,
425+
{
426+
"limits.cpu": "1",
427+
"limits.memory": "4Gi",
428+
"limits.ephemeral-storage": "5Gi",
429+
"fake-storage.storageclass.storage.k8s.io/requests.storage": "20Gi", # Migrated key
430+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": "0",
431+
"requests.nvidia.com/gpu": "0",
432+
"persistentvolumeclaims": "2",
433+
},
434+
)
435+
436+
def test_ibm_storage_not_available(self):
437+
"""If IBM Scale storage is not available, the corresponding quotas should not be set."""
438+
user = self.new_user()
439+
project = self.new_project(pi=user)
440+
441+
# Set ibm storage as not available
442+
self.resource.resourceattribute_set.filter(
443+
resource_attribute_type__name=attributes.RESOURCE_IBM_AVAILABLE
444+
).update(value="false")
445+
allocation = self.new_allocation(project, self.resource, 1)
446+
allocator = openshift.OpenShiftResourceAllocator(self.resource, allocation)
447+
448+
tasks.activate_allocation(allocation.pk)
449+
allocation.refresh_from_db()
450+
451+
project_id = allocation.get_attribute(attributes.ALLOCATION_PROJECT_ID)
452+
453+
quota = allocator.get_quota(project_id)
454+
self.assertEqual(
455+
quota,
456+
{
457+
"limits.cpu": "1",
458+
"limits.memory": "4Gi",
459+
"limits.ephemeral-storage": "5Gi",
460+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "20Gi",
461+
"requests.nvidia.com/gpu": "0",
462+
"persistentvolumeclaims": "2",
463+
},
464+
)
465+
466+
# Now set IBM Scale storage as available
467+
self.resource.resourceattribute_set.filter(
468+
resource_attribute_type__name=attributes.RESOURCE_IBM_AVAILABLE
469+
).update(value="true")
470+
471+
call_command("validate_allocations", apply=True)
472+
473+
quota = allocator.get_quota(project_id)
474+
self.assertEqual(
475+
quota,
476+
{
477+
"limits.cpu": "1",
478+
"limits.memory": "4Gi",
479+
"limits.ephemeral-storage": "5Gi",
480+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "20Gi",
481+
"ibm-spectrum-scale-fileset.storageclass.storage.k8s.io/requests.storage": "0", # Newly added IBM key
482+
"requests.nvidia.com/gpu": "0",
483+
"persistentvolumeclaims": "2",
484+
},
485+
)

‎src/coldfront_plugin_cloud/tests/functional/openshift_vm/test_allocation.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ def test_new_allocation(self):
3636
2 * 5,
3737
)
3838
self.assertEqual(
39-
allocation.get_attribute(attributes.QUOTA_REQUESTS_STORAGE), 2 * 20
39+
allocation.get_attribute(attributes.QUOTA_REQUESTS_NESE_STORAGE), 2 * 20
4040
)
4141
self.assertEqual(
4242
allocation.get_attribute(attributes.QUOTA_REQUESTS_VM_GPU_A100_SXM4), 2 * 0
@@ -58,7 +58,7 @@ def test_new_allocation(self):
5858
"limits.cpu": "2",
5959
"limits.memory": "8Gi",
6060
"limits.ephemeral-storage": "10Gi",
61-
"requests.storage": "40Gi",
61+
"ocs-external-storagecluster-ceph-rbd.storageclass.storage.k8s.io/requests.storage": "40Gi",
6262
"requests.nvidia.com/A100_SXM4_40GB": "0",
6363
"requests.nvidia.com/GV100GL_Tesla_V100": "0",
6464
"requests.nvidia.com/H100_SXM5_80GB": "0",

0 commit comments

Comments
 (0)