Skip to content

Commit ebfaa1a

Browse files
mbuechsegarloff
andauthored
Stabilize SCS-compatible IaaS v5 (#824)
* Stabilize SCS-compatible IaaS v5 make it effective from 2024-11-16 and turn v4 into warn from 2025-01-01 * Include scs-0123-v1 and adapt test script accordingly * move manual docs check to target preview for the time being * Make it effective today, not in the past. Signed-off-by: Matthias Büchse <[email protected]> Signed-off-by: Kurt Garloff <[email protected]> Co-authored-by: Kurt Garloff <[email protected]>
1 parent 7a2662a commit ebfaa1a

File tree

4 files changed

+95
-81
lines changed

4 files changed

+95
-81
lines changed

Tests/iaas/mandatory-services/mandatory-iaas-services.py

100644100755
Lines changed: 50 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""Mandatory APIs checker
23
This script retrieves the endpoint catalog from Keystone using the OpenStack
34
SDK and checks whether all mandatory APi endpoints, are present.
@@ -26,54 +27,30 @@
2627
block_storage_service = ["volume", "volumev3", "block-storage"]
2728

2829

29-
def connect(cloud_name: str) -> openstack.connection.Connection:
30-
"""Create a connection to an OpenStack cloud
31-
:param string cloud_name:
32-
The name of the configuration to load from clouds.yaml.
33-
:returns: openstack.connnection.Connection
34-
"""
35-
return openstack.connect(
36-
cloud=cloud_name,
37-
)
38-
39-
40-
def check_presence_of_mandatory_services(cloud_name: str, s3_credentials=None):
41-
try:
42-
connection = connect(cloud_name)
43-
services = connection.service_catalog
44-
except Exception as e:
45-
print(str(e))
46-
raise Exception(
47-
f"Connection to cloud '{cloud_name}' was not successfully. "
48-
f"The Catalog endpoint could not be accessed. "
49-
f"Please check your cloud connection and authorization."
50-
)
30+
def check_presence_of_mandatory_services(conn: openstack.connection.Connection, s3_credentials=None):
31+
services = conn.service_catalog
5132

5233
if s3_credentials:
5334
mandatory_services.remove("object-store")
5435
for svc in services:
5536
svc_type = svc['type']
5637
if svc_type in mandatory_services:
5738
mandatory_services.remove(svc_type)
58-
continue
59-
if svc_type in block_storage_service:
39+
elif svc_type in block_storage_service:
6040
block_storage_service.remove(svc_type)
6141

6242
bs_service_not_present = 0
6343
if len(block_storage_service) == 3:
6444
# neither block-storage nor volume nor volumev3 is present
6545
# we must assume, that there is no volume service
66-
logger.error("FAIL: No block-storage (volume) endpoint found.")
46+
logger.error("No block-storage (volume) endpoint found.")
6747
mandatory_services.append(block_storage_service[0])
6848
bs_service_not_present = 1
69-
if not mandatory_services:
70-
# every mandatory service API had an endpoint
71-
return 0 + bs_service_not_present
72-
else:
73-
# there were multiple mandatory APIs not found
74-
logger.error(f"FAIL: The following endpoints are missing: "
75-
f"{mandatory_services}")
76-
return len(mandatory_services) + bs_service_not_present
49+
if mandatory_services:
50+
# some mandatory APIs were not found
51+
logger.error(f"The following endpoints are missing: "
52+
f"{', '.join(mandatory_services)}.")
53+
return len(mandatory_services) + bs_service_not_present
7754

7855

7956
def list_containers(conn):
@@ -167,8 +144,8 @@ def s3_from_ostack(creds, conn, endpoint):
167144
# pass
168145

169146

170-
def check_for_s3_and_swift(cloud_name: str, s3_credentials=None):
171-
# If we get credentials we assume, that there is no Swift and only test s3
147+
def check_for_s3_and_swift(conn: openstack.connection.Connection, s3_credentials=None):
148+
# If we get credentials, we assume that there is no Swift and only test s3
172149
if s3_credentials:
173150
try:
174151
s3 = s3_conn(s3_credentials)
@@ -183,58 +160,46 @@ def check_for_s3_and_swift(cloud_name: str, s3_credentials=None):
183160
if s3_buckets == [TESTCONTNAME]:
184161
del_bucket(s3, TESTCONTNAME)
185162
# everything worked, and we don't need to test for Swift:
186-
print("SUCCESS: S3 exists")
163+
logger.info("SUCCESS: S3 exists")
187164
return 0
188165
# there were no credentials given, so we assume s3 is accessable via
189166
# the service catalog and Swift might exist too
190-
try:
191-
connection = connect(cloud_name)
192-
connection.authorize()
193-
except Exception as e:
194-
print(str(e))
195-
raise Exception(
196-
f"Connection to cloud '{cloud_name}' was not successfully. "
197-
f"The Catalog endpoint could not be accessed. "
198-
f"Please check your cloud connection and authorization."
199-
)
200167
s3_creds = {}
201168
try:
202-
endpoint = connection.object_store.get_endpoint()
203-
except Exception as e:
204-
logger.error(
205-
f"FAIL: No object store endpoint found in cloud "
206-
f"'{cloud_name}'. No testing for the s3 service possible. "
207-
f"Details: %s", e
169+
endpoint = conn.object_store.get_endpoint()
170+
except Exception:
171+
logger.exception(
172+
"No object store endpoint found. No testing for the s3 service possible."
208173
)
209174
return 1
210175
# Get S3 endpoint (swift) and ec2 creds from OpenStack (keystone)
211-
s3_from_ostack(s3_creds, connection, endpoint)
176+
s3_from_ostack(s3_creds, conn, endpoint)
212177
# Overrides (var names are from libs3, in case you wonder)
213178
s3_from_env(s3_creds, "HOST", "S3_HOSTNAME", "https://")
214179
s3_from_env(s3_creds, "AK", "S3_ACCESS_KEY_ID")
215180
s3_from_env(s3_creds, "SK", "S3_SECRET_ACCESS_KEY")
216181

217-
s3 = s3_conn(s3_creds, connection)
182+
s3 = s3_conn(s3_creds, conn)
218183
s3_buckets = list_s3_buckets(s3)
219184
if not s3_buckets:
220185
s3_buckets = create_bucket(s3, TESTCONTNAME)
221186
assert s3_buckets
222187

223188
# If we got till here, s3 is working, now swift
224-
swift_containers = list_containers(connection)
189+
swift_containers = list_containers(conn)
225190
# if not swift_containers:
226-
# swift_containers = create_container(connection, TESTCONTNAME)
191+
# swift_containers = create_container(conn, TESTCONTNAME)
227192
result = 0
228193
if Counter(s3_buckets) != Counter(swift_containers):
229-
print("WARNING: S3 buckets and Swift Containers differ:\n"
230-
f"S3: {sorted(s3_buckets)}\nSW: {sorted(swift_containers)}")
194+
logger.warning("S3 buckets and Swift Containers differ:\n"
195+
f"S3: {sorted(s3_buckets)}\nSW: {sorted(swift_containers)}")
231196
result = 1
232197
else:
233-
print("SUCCESS: S3 and Swift exist and agree")
198+
logger.info("SUCCESS: S3 and Swift exist and agree")
234199
# Clean up
235200
# FIXME: Cleanup created EC2 credential
236201
# if swift_containers == [TESTCONTNAME]:
237-
# del_container(connection, TESTCONTNAME)
202+
# del_container(conn, TESTCONTNAME)
238203
# Cleanup created S3 bucket
239204
if s3_buckets == [TESTCONTNAME]:
240205
del_bucket(s3, TESTCONTNAME)
@@ -266,34 +231,47 @@ def main():
266231
help="Enable OpenStack SDK debug logging"
267232
)
268233
args = parser.parse_args()
234+
logging.basicConfig(
235+
format="%(levelname)s: %(message)s",
236+
level=logging.DEBUG if args.debug else logging.INFO,
237+
)
269238
openstack.enable_logging(debug=args.debug)
270239

271240
# parse cloud name for lookup in clouds.yaml
272-
cloud = os.environ.get("OS_CLOUD", None)
273-
if args.os_cloud:
274-
cloud = args.os_cloud
275-
assert cloud, (
276-
"You need to have the OS_CLOUD environment variable set to your cloud "
277-
"name or pass it via --os-cloud"
278-
)
241+
cloud = args.os_cloud or os.environ.get("OS_CLOUD", None)
242+
if not cloud:
243+
raise RuntimeError(
244+
"You need to have the OS_CLOUD environment variable set to your "
245+
"cloud name or pass it via --os-cloud"
246+
)
279247

280248
s3_credentials = None
281249
if args.s3_endpoint:
282250
if (not args.s3_access) or (not args.s3_access_secret):
283-
print("WARNING: test for external s3 needs access key and access secret.")
251+
logger.warning("test for external s3 needs access key and access secret.")
284252
s3_credentials = {
285253
"AK": args.s3_access,
286254
"SK": args.s3_access_secret,
287255
"HOST": args.s3_endpoint
288256
}
289257
elif args.s3_access or args.s3_access_secret:
290-
print("WARNING: access to s3 was given, but no endpoint provided.")
258+
logger.warning("access to s3 was given, but no endpoint provided.")
291259

292-
result = check_presence_of_mandatory_services(cloud, s3_credentials)
293-
result = result + check_for_s3_and_swift(cloud, s3_credentials)
260+
with openstack.connect(cloud) as conn:
261+
result = check_presence_of_mandatory_services(conn, s3_credentials)
262+
result += check_for_s3_and_swift(conn, s3_credentials)
263+
264+
print('service-apis-check: ' + ('PASS', 'FAIL')[min(1, result)])
294265

295266
return result
296267

297268

298269
if __name__ == "__main__":
299-
main()
270+
try:
271+
sys.exit(main())
272+
except SystemExit:
273+
raise
274+
except BaseException as exc:
275+
logging.debug("traceback", exc_info=True)
276+
logging.critical(str(exc))
277+
sys.exit(1)

Tests/requirements.in

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
aiohttp
2+
boto3 # TODO: move into iaas/requirements.in
23
click
3-
kubernetes_asyncio
4+
kubernetes_asyncio # TODO: move into kaas/requirements.in
45
python-dateutil
56
PyYAML
6-
openstacksdk
7+
openstacksdk # TODO: move into iaas/requirements.in
78
requests
89
tomli

Tests/requirements.txt

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# This file is autogenerated by pip-compile with Python 3.10
2+
# This file is autogenerated by pip-compile with Python 3.12
33
# by the following command:
44
#
55
# pip-compile requirements.in
@@ -12,10 +12,14 @@ aiohttp==3.10.11
1212
# kubernetes-asyncio
1313
aiosignal==1.3.1
1414
# via aiohttp
15-
async-timeout==4.0.3
16-
# via aiohttp
1715
attrs==24.2.0
1816
# via aiohttp
17+
boto3==1.35.65
18+
# via -r requirements.in
19+
botocore==1.35.65
20+
# via
21+
# boto3
22+
# s3transfer
1923
certifi==2024.7.4
2024
# via
2125
# kubernetes-asyncio
@@ -47,7 +51,10 @@ iso8601==2.1.0
4751
# keystoneauth1
4852
# openstacksdk
4953
jmespath==1.0.1
50-
# via openstacksdk
54+
# via
55+
# boto3
56+
# botocore
57+
# openstacksdk
5158
jsonpatch==1.33
5259
# via openstacksdk
5360
jsonpointer==3.0.0
@@ -83,6 +90,7 @@ pycparser==2.22
8390
python-dateutil==2.9.0.post0
8491
# via
8592
# -r requirements.in
93+
# botocore
8694
# kubernetes-asyncio
8795
pyyaml==6.0.2
8896
# via
@@ -95,6 +103,8 @@ requests==2.32.3
95103
# keystoneauth1
96104
requestsexceptions==1.4.0
97105
# via openstacksdk
106+
s3transfer==0.10.3
107+
# via boto3
98108
six==1.16.0
99109
# via
100110
# kubernetes-asyncio
@@ -105,10 +115,9 @@ stevedore==5.2.0
105115
# keystoneauth1
106116
tomli==2.0.1
107117
# via -r requirements.in
108-
typing-extensions==4.12.2
109-
# via dogpile-cache
110118
urllib3==2.2.2
111119
# via
120+
# botocore
112121
# kubernetes-asyncio
113122
# requests
114123
yarl==1.17.2

Tests/scs-compatible-iaas.yaml

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,21 @@ modules:
206206
tags: [availability-zones]
207207
description: >
208208
Note: manual check! Must fulfill all requirements of <https://docs.scs.community/standards/scs-0121-v1-Availability-Zones-Standard>
209+
- id: scs-0123-v1
210+
name: Mandatory and Supported IaaS Services
211+
url: https://docs.scs.community/standards/scs-0123-v1-mandatory-and-supported-IaaS-services
212+
run:
213+
- executable: ./iaas/mandatory-services/mandatory-iaas-services.py
214+
args: --os-cloud {os_cloud} --debug
215+
testcases:
216+
- id: service-apis-check
217+
tags: [mandatory]
218+
description: >
219+
Must fulfill all requirements of <https://docs.scs.community/standards/scs-0123-v1-mandatory-and-supported-IaaS-services> (except for documentation requirements, which are tested manually with service-apis-docs-check).
220+
- id: service-apis-docs-check
221+
tags: [service-apis-docs]
222+
description: >
223+
Note: manual check! Must fulfill documentation requirements of <https://docs.scs.community/standards/scs-0123-v1-mandatory-and-supported-IaaS-services>.
209224
- id: scs-0302-v1
210225
name: Domain Manager Role
211226
url: https://docs.scs.community/standards/scs-0302-v1-domain-manager-role
@@ -218,6 +233,16 @@ modules:
218233
description: >
219234
Note: manual check! Must fulfill all requirements of <https://docs.scs.community/standards/scs-0302-v1-domain-manager-role>
220235
timeline:
236+
- date: 2025-01-01
237+
versions:
238+
v5: effective
239+
v4: warn
240+
v3: deprecated
241+
- date: 2024-11-21
242+
versions:
243+
v5: effective
244+
v4: effective
245+
v3: deprecated
221246
- date: 2024-11-08
222247
versions:
223248
v5: draft
@@ -261,6 +286,7 @@ timeline:
261286
v1: effective
262287
versions:
263288
- version: v5
289+
stabilized_at: 2024-11-14
264290
include:
265291
- opc-v2022.11
266292
- scs-0100-v3.1
@@ -278,7 +304,7 @@ versions:
278304
- scs-0302-v1
279305
targets:
280306
main: mandatory
281-
preview: domain-manager/availability-zones
307+
preview: domain-manager/availability-zones/service-apis-docs
282308
- version: v4
283309
stabilized_at: 2024-02-28
284310
include:

0 commit comments

Comments
 (0)