Provide details page for subject group (multiple regions or k8s versions)

Signed-off-by: Matthias Büchse <[email protected]>

Author: mbuechse

compliance-monitor/bootstrap.yaml

@@ -31,15 +31,19 @@ accounts:
         public_key_type: "ssh-ed25519"
         public_key_name: "primary"
   - subject: pco-prod1
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: pco-prod2
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: pco-prod3
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: pco-prod4
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: poc-wgcloud
@@ -56,9 +60,11 @@ accounts:
     delegates:
       - zuul_ci
   - subject: syseleven-dus2
+    group: syseleven
     delegates:
       - zuul_ci
   - subject: syseleven-ham1
+    group: syseleven
     delegates:
       - zuul_ci
   - subject: wavestack

compliance-monitor/monitor.py

@@ -41,7 +41,7 @@
     db_find_account, db_update_account, db_update_publickey, db_filter_publickeys, db_get_reports,
     db_get_keys, db_insert_report, db_get_recent_results2, db_patch_approval2, db_get_report,
     db_ensure_schema, db_get_apikeys, db_update_apikey, db_filter_apikeys, db_clear_delegates,
-    db_find_subjects, db_insert_result2, db_get_relevant_results2, db_add_delegate,
+    db_find_subjects, db_insert_result2, db_get_relevant_results2, db_add_delegate, db_get_group,
 )
 
 
@@ -79,6 +79,7 @@ def __init__(self):
         self.yaml_path = os.path.abspath("../Tests")
 
 
+GROUP_PREFIX = 'group-'
 ROLES = {'read_any': 1, 'append_any': 2, 'admin': 4, 'approve': 8}
 # number of days that expired results will be considered in lieu of more recent, but unapproved ones
 GRACE_PERIOD_DAYS = 7
@@ -226,7 +227,8 @@ def import_bootstrap(bootstrap_path, conn):
     with conn.cursor() as cur:
         for account in accounts:
             roles = sum(ROLES[r] for r in account.get('roles', ()))
-            accountid = db_update_account(cur, {'subject': account['subject'], 'roles': roles})
+            acc_record = {'subject': account['subject'], 'roles': roles, 'group': account.get('group')}
+            accountid = db_update_account(cur, acc_record)
             db_clear_delegates(cur, accountid)
             for delegate in account.get('delegates', ()):
                 db_add_delegate(cur, accountid, delegate)
@@ -637,6 +639,13 @@ async def get_report_view_full(
     )
 
 
+def _resolve_group(cur, subject, prefix=GROUP_PREFIX):
+    group = subject.removeprefix(prefix)
+    if subject != group:
+        return group, db_get_group(cur, group)
+    return None, [subject]
+
+
 @app.get("/{view_type}/detail/{subject}/{scopeuuid}")
 async def get_detail(
     request: Request,
@@ -646,14 +655,18 @@ async def get_detail(
     scopeuuid: str,
 ):
     with conn.cursor() as cur:
-        rows2 = db_get_relevant_results2(cur, subject, scopeuuid, approved_only=True)
+        group, subjects = _resolve_group(cur, subject)
+        rows2 = []
+        for subj in subjects:
+            rows2.extend(db_get_relevant_results2(cur, subj, scopeuuid, approved_only=True))
     results2 = convert_result_rows_to_dict2(
         rows2, get_scopes(), include_report=True, grace_period_days=GRACE_PERIOD_DAYS,
         subjects=(subject, ), scopes=(scopeuuid, ),
     )
+    title = f'Details for group {group}' if group else f'Details for subject {subject}'
     return render_view(
         VIEW_DETAIL, view_type, results=results2, base_url=settings.base_url,
-        title=f'{subject} compliance',
+        title=title,
     )
 
 
@@ -666,13 +679,17 @@ async def get_detail_full(
     scopeuuid: str,
 ):
     with conn.cursor() as cur:
-        rows2 = db_get_relevant_results2(cur, subject, scopeuuid, approved_only=False)
+        group, subjects = _resolve_group(cur, subject)
+        rows2 = []
+        for subject in subjects:
+            rows2.extend(db_get_relevant_results2(cur, subject, scopeuuid, approved_only=False))
     results2 = convert_result_rows_to_dict2(
         rows2, get_scopes(), include_report=True, subjects=(subject, ), scopes=(scopeuuid, ),
     )
+    title = f'Details for group {group}' if group else f'Details for subject {subject}'
     return render_view(
         VIEW_DETAIL, view_type, results=results2, base_url=settings.base_url,
-        title=f'{subject} compliance (incl. unverified results)',
+        title=f'{title} (incl. unverified results)',
     )
 
 

compliance-monitor/sql.py

@@ -3,9 +3,9 @@
 
 # list schema versions in ascending order
 SCHEMA_VERSION_KEY = 'version'
-SCHEMA_VERSIONS = ['v1', 'v2', 'v3']
+SCHEMA_VERSIONS = ['v1', 'v2', 'v3', 'v4']
 # use ... (Ellipsis) here to indicate that no default value exists (will lead to error if no value is given)
-ACCOUNT_DEFAULTS = {'subject': ..., 'api_key': ..., 'roles': ...}
+ACCOUNT_DEFAULTS = {'subject': ..., 'api_key': ..., 'roles': ..., 'group': None}
 PUBLIC_KEY_DEFAULTS = {'public_key': ..., 'public_key_type': ..., 'public_key_name': ...}
 
 
@@ -135,6 +135,14 @@ def db_ensure_schema_v3(cur: cursor):
     ''')
 
 
+def db_ensure_schema_v4(cur: cursor):
+    # start from v3, do small alteration
+    db_ensure_schema_v2(cur)
+    cur.execute('''
+    ALTER TABLE account ADD COLUMN IF NOT EXISTS "group" text;
+    ''')
+
+
 def db_upgrade_data_v1_v2(cur):
     # we are going to drop table result, but use delete anyway to have the transaction safety
     cur.execute('''
@@ -207,6 +215,10 @@ def db_upgrade_schema(conn: connection, cur: cursor):
             db_ensure_schema_v3(cur)
             db_set_schema_version(cur, 'v3')
             conn.commit()
+        elif current == 'v3':
+            db_ensure_schema_v4(cur)
+            db_set_schema_version(cur, 'v4')
+            conn.commit()
 
 
 def db_ensure_schema(conn: connection):
@@ -229,11 +241,12 @@ def db_ensure_schema(conn: connection):
 def db_update_account(cur: cursor, record: dict):
     sanitized = sanitize_record(record, ACCOUNT_DEFAULTS)
     cur.execute('''
-    INSERT INTO account (subject, roles)
-    VALUES (%(subject)s, %(roles)s)
+    INSERT INTO account (subject, roles, "group")
+    VALUES (%(subject)s, %(roles)s, %(group)s)
     ON CONFLICT (subject)
     DO UPDATE
-    SET roles = EXCLUDED.roles
+    SET roles = EXCLUDED.roles,
+    "group" = EXCLUDED."group"
     RETURNING accountid;''', sanitized)
     accountid, = cur.fetchone()
     return accountid
@@ -262,6 +275,11 @@ def db_find_subjects(cur: cursor, delegate):
     return [row[0] for row in cur.fetchall()]
 
 
+def db_get_group(cur: cursor, group):
+    cur.execute('''SELECT subject FROM account WHERE "group" = %s;''', (group, ))
+    return [row[0] for row in cur.fetchall()]
+
+
 def db_update_apikey(cur: cursor, accountid, apikey_hash):
     sanitized = dict(accountid=accountid, apikey_hash=apikey_hash)
     cur.execute('''

compliance-monitor/templates/details.md.j2

@@ -2,7 +2,7 @@
 {# omit h1 title here because we can only have one of those,
    and the html wrapper template will add one anyway -#}
 {% for scopeuuid, scope_result in subject_result.items() -%}
-## {{ scope_result.name }}
+## {{ subject }}: {{ scope_result.name }}
 
 - [spec overview]({{ scope_url(scopeuuid) }})
 

compliance-monitor/templates/overview.md.j2

@@ -23,7 +23,7 @@ Version numbers are suffixed by a symbol depending on state: * for _draft_, †
 {#- #} [{{ results | pick(iaas, 'artcodix') | summary }}]({{ detail_url('artcodix', iaas) }}) {# -#}
 | [HM](https://ohm.muc.cloud.cnds.io/) |
 | [pluscloud open](https://www.plusserver.com/en/products/pluscloud-open) | Public cloud for customers (4 regions)  | plusserver GmbH | {# #}
-{#- #}[{{ results | pick(iaas, 'pco-prod1', 'pco-prod2', 'pco-prod3', 'pco-prod4') | summary }}]({{ detail_url('pco-prod1', iaas) }}) {# -#}
+{#- #}[{{ results | pick(iaas, 'pco-prod1', 'pco-prod2', 'pco-prod3', 'pco-prod4') | summary }}]({{ detail_url('group-pco-prod', iaas) }}) {# -#}
 | [HM1](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-pco) [HM2](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod2) [HM3](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod3) [HM4](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod4) |
 | PoC WG-Cloud OSBA | Cloud PoC for FITKO | Cloud&Heat Technologies GmbH |
 {#- #} [{{ results | pick(iaas, 'poc-wgcloud') | summary }}]({{ detail_url('poc-wgcloud', iaas) }}) {# -#}
@@ -35,7 +35,7 @@ Version numbers are suffixed by a symbol depending on state: * for _draft_, †
 {#- #} [{{ results | pick(iaas, 'scaleup-occ2') | summary }}]({{ detail_url('scaleup-occ2', iaas) }}) {# -#}
 | [HM](https://health.occ2.scaleup.sovereignit.cloud) |
 | [syseleven](https://www.syseleven.de/en/products-services/openstack-cloud/) | Public OpenStack Cloud (2 SCS regions) | SysEleven GmbH | {# #}
-{#- #} [{{ results | pick(iaas, 'syseleven-dus2', 'syseleven-ham1') | summary }}]({{ detail_url('syseleven-dus2', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'syseleven-dus2', 'syseleven-ham1') | summary }}]({{ detail_url('group-syseleven', iaas) }}) {# -#}
 | (soon) |
 | [Wavestack](https://www.noris.de/wavestack-cloud/) | Public cloud for customers | noris network AG/Wavecon GmbH |
 {#- #} [{{ results | pick(iaas, 'wavestack') | summary }}]({{ detail_url('wavestack', iaas) }}) {# -#}