Merge pull request #433 from cloud-pi-native/develop

Develop

Author: this-is-tobi

roles/gitlab/tasks/gitaly-priority-class.yaml

@@ -0,0 +1,12 @@
+- name: Declare PriorityClass for Gitaly
+  kubernetes.core.k8s:
+    state: present
+    namespace: "{{ dsc.gitlab.namespace }}"
+    definition:
+      apiVersion: scheduling.k8s.io/v1
+      kind: PriorityClass
+      metadata:
+        name: gitlab-gitaly
+      value: 1000000
+      globalDefault: false
+      description: "GitLab Gitaly priority class"

roles/gitlab/tasks/main.yaml

@@ -180,6 +180,10 @@
       data:
         tls.crt: "{{ exposed_ca_pem | b64encode }}"
 
+- name: Declare a PriorityClass for gitaly
+  ansible.builtin.include_tasks:
+    file: gitaly-priority-class.yaml
+
 - name: Set path fact
   ansible.builtin.set_fact:
     path: "{{ role_path + '/templates/values' }}"
@@ -382,6 +386,7 @@
   ansible.builtin.set_fact:
     extra_kaniko_args: "{{ extra_kaniko_args | default('') }} --registry-mirror {{ item.registry.endpointUrl | regex_replace('^https?://', '') }}/{{ item.name }}"
   loop: "{{ dsc.harbor.proxyCache }}"
+  when: dsc.harbor.proxyCache is defined
 
 - name: Set or update insecure args variables
   community.general.gitlab_group_variable:
@@ -393,7 +398,7 @@
     state: "{{ dsc.gitlab.insecureCI | ternary('present', 'absent') }}"
     variables:
       - name: EXTRA_KANIKO_ARGS
-        value: --skip-tls-verify {{ extra_kaniko_args }}
+        value: --skip-tls-verify {{ extra_kaniko_args | default('') }}
       - name: EXTRA_GIT_ARGS
         value: -c http.sslVerify=false
       - name: EXTRA_VAULT_ARGS

roles/gitlab/templates/pg-cluster-gitlab.yaml.j2

@@ -53,6 +53,8 @@ spec:
   - name: pg-cluster-gitlab
 {% if dsc.gitlab.cnpg.mode == "restore" %}
     barmanObjectStore:
+      wal:
+        maxParallel: 8
       destinationPath: "s3://{{ dsc.global.backup.s3.bucketName }}/{{ dsc.global.backup.cnpg.pathPrefix }}"
       endpointURL: "{{ dsc.global.backup.s3.endpointURL }}"
 {% if dsc.global.backup.s3.endpointCA.key is defined %}

roles/gitlab/templates/values/00-main.j2

@@ -28,6 +28,16 @@ gitlab:
     registry:
       enabled: false
   gitaly:
+    priorityClassName: gitlab-gitaly
+    resources:
+      requests:
+        cpu: 1000m
+        memory: 2Gi
+      limits:
+        cpu: 1000m
+        memory: 2Gi
+    securityContext:
+      fsGroupChangePolicy: OnRootMismatch
 {% if dsc.global.metrics.enabled %}
     metrics:
       serviceMonitor:

roles/harbor/templates/pg-cluster-harbor.yaml.j2

@@ -53,6 +53,8 @@ spec:
   - name: pg-cluster-harbor
 {% if dsc.harbor.cnpg.mode == "restore" %}
     barmanObjectStore:
+      wal:
+        maxParallel: 8
       destinationPath: "s3://{{ dsc.global.backup.s3.bucketName }}/{{ dsc.global.backup.cnpg.pathPrefix }}"
       endpointURL: "{{ dsc.global.backup.s3.endpointURL }}"
 {% if dsc.global.backup.s3.endpointCA.key is defined %}

roles/keycloak/tasks/main.yml

@@ -6,6 +6,8 @@
     kind: Namespace
     state: present
 
+# Setup CNPG s3 secret
+
 - name: CNPG s3 CA (secret)
   when: >
     dsc.global.backup.cnpg.enabled and
@@ -34,7 +36,10 @@
           data:
             ca.pem: "{{ cnpg_s3_ca_pem }}"
 
+# Setup CNPG backup
+
 - name: Set cnpg backup secret
+  when: dsc.global.backup.cnpg.enabled
   kubernetes.core.k8s:
     name: "{{ dsc.global.backup.s3.credentials.name }}"
     namespace: "{{ dsc.keycloak.namespace }}"
@@ -44,7 +49,6 @@
       data:
         accessKeyId: "{{ dsc.global.backup.s3.credentials.accessKeyId.value | b64encode }}"
         secretAccessKey: "{{ dsc.global.backup.s3.credentials.secretAccessKey.value | b64encode }}"
-  when: dsc.global.backup.cnpg.enabled
 
 - name: Remove cnpg scheduled backup
   kubernetes.core.k8s:
@@ -55,7 +59,9 @@
     state: absent
   when: not dsc.global.backup.cnpg.enabled
 
-- name: Create PostgreSQL cluster and keycloak database
+# Create CNPG cluster and Keycloak database
+
+- name: Create PostgreSQL cluster and Keycloak database
   kubernetes.core.k8s:
     template: "{{ item }}"
   with_items:
@@ -95,6 +101,8 @@
   retries: 30
   delay: 5
 
+# Set Keycloak admin password
+
 - name: Get Keycloak admin password secret
   kubernetes.core.k8s_info:
     namespace: "{{ dsc.keycloak.namespace }}"
@@ -116,67 +124,7 @@
         namespace: "{{ dsc.keycloak.namespace }}"
       type: Opaque
 
-- name: Check Keycloak helm release
-  kubernetes.core.helm_info:
-    name: keycloak
-    namespace: "{{ dsc.keycloak.namespace }}"
-  register: kc_helm_release
-
-- name: Reset Keycloak admin password
-  when: >
-    kc_helm_release.status is defined and
-    kc_adm_pass_secret.resources | length == 0
-  block:
-    - name: Get Keycloak primary BDD pod
-      kubernetes.core.k8s_info:
-        kind: Pod
-        label_selectors:
-          - "cnpg.io/cluster=pg-cluster-keycloak"
-          - "cnpg.io/instanceRole=primary"
-      register: kc_bdd_pod
-
-    - name: Get Keycloak admin ID from database
-      kubernetes.core.k8s_exec:
-        pod: "{{ kc_bdd_pod.resources[0].metadata.name }}"
-        namespace: "{{ dsc.keycloak.namespace }}"
-        command: >
-          psql -U postgres -d keycloak --csv -c "\x" -c "select id from user_entity where username = 'admin';"
-      register: kc_admin_id
-
-    - name: Set kc_admin_id fact
-      ansible.builtin.set_fact:
-        kc_admin_id: "{{ kc_admin_id.stdout | regex_search('^id.*', multiline=True) | regex_search('id,(.+)', '\\1') | first }}"
-
-    - name: Delete Keycloak admin in database
-      kubernetes.core.k8s_exec:
-        pod: "{{ kc_bdd_pod.resources[0].metadata.name }}"
-        namespace: "{{ dsc.keycloak.namespace }}"
-        command: >
-          psql -U postgres -d keycloak -c "delete from credential where user_id = '"{{ kc_admin_id }}"';"
-          -c "delete from user_role_mapping where user_id = '"{{ kc_admin_id }}"';"
-          -c "delete from user_entity where id = '"{{ kc_admin_id }}"';"
-          -c "delete from user_required_action where user_id = '"{{ kc_admin_id }}"';"
-
-    - name: Restart Keycloak pods to reset admin password
-      kubernetes.core.k8s:
-        kind: Pod
-        namespace: "{{ dsc.keycloak.namespace }}"
-        label_selectors:
-          - "app.kubernetes.io/component=keycloak"
-          - "app.kubernetes.io/instance=keycloak"
-        state: absent
-
-    - name: Wait Keycloak URL
-      ansible.builtin.uri:
-        url: https://{{ keycloak_domain }}
-        validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
-        method: GET
-        status_code: [200, 202]
-        return_content: false
-      register: kc_response
-      until: kc_response is not failed
-      retries: 30
-      delay: 5
+# Deploy Keycloak
 
 - name: Add bitnami helm repo
   kubernetes.core.helm_repository:
@@ -216,18 +164,126 @@
   retries: 30
   delay: 5
 
+# Set admin facts and check access to Keycloak API
+
 - name: Get Keycloak admin password
   kubernetes.core.k8s_info:
     namespace: "{{ dsc.keycloak.namespace }}"
     kind: Secret
     name: keycloak
   register: kc_adm_pass
 
-- name: Set Keycloak admin name fact
+- name: Set Keycloak admin facts
   ansible.builtin.set_fact:
     keycloak_admin_password: "{{ kc_adm_pass.resources[0].data['admin-password'] | b64decode }}"
     keycloak_admin: admin
 
+- name: Get Keycloak API token
+  ansible.builtin.uri:
+    url: https://{{ keycloak_domain }}/realms/master/protocol/openid-connect/token
+    method: POST
+    status_code: [200, 202]
+    validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
+    return_content: true
+    body: username={{ keycloak_admin }}&password={{ keycloak_admin_password }}&grant_type=password&client_id=admin-cli
+  register: kc_token
+  ignore_errors: true
+
+- name: Reset Keycloak admin fact and API token
+  when: kc_token is failed
+  block:
+    - name: Reset Keycloak admin fact
+      ansible.builtin.set_fact:
+        keycloak_admin: dsoadmin
+
+    - name: Get Keycloak API token
+      ansible.builtin.uri:
+        url: https://{{ keycloak_domain }}/realms/master/protocol/openid-connect/token
+        method: POST
+        status_code: [200, 202]
+        validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
+        return_content: true
+        body: username={{ keycloak_admin }}&password={{ keycloak_admin_password }}&grant_type=password&client_id=admin-cli
+      register: kc_token
+
+- name: Set kc_access_token fact
+  ansible.builtin.set_fact:
+    kc_access_token: "{{ kc_token.json.access_token }}"
+
+# Create permanent Keycloak admin and update DSO Console inventory
+
+- name: Get keycloak master realm users from API
+  ansible.builtin.uri:
+    url: https://{{ keycloak_domain }}/admin/realms/master/users
+    method: GET
+    status_code: [200, 202]
+    return_content: true
+    validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
+    body_format: json
+    headers:
+      Authorization: bearer {{ kc_access_token }}
+  register: kc_master_users
+
+- name: Set permanent_admin_present fact
+  ansible.builtin.set_fact:
+    permanent_admin_present: false
+
+- name: Update admin_present fact
+  when: kc_master_users.json | selectattr('username', 'equalto', 'dsoadmin')
+  ansible.builtin.set_fact:
+    permanent_admin_present: true
+
+- name: Create permanent admin group and user into master realm
+  when: not permanent_admin_present
+  block:
+    - name: Create admin group
+      community.general.keycloak_group:
+        auth_client_id: admin-cli
+        auth_keycloak_url: https://{{ keycloak_domain }}
+        auth_realm: master
+        auth_username: "{{ keycloak_admin }}"
+        auth_password: "{{ keycloak_admin_password }}"
+        name: admin
+        realm: master
+        state: present
+
+    - name: Map admin realm role from admin group
+      community.general.keycloak_realm_rolemapping:
+        realm: master
+        auth_client_id: admin-cli
+        auth_keycloak_url: https://{{ keycloak_domain }}
+        auth_realm: master
+        auth_username: "{{ keycloak_admin }}"
+        auth_password: "{{ keycloak_admin_password }}"
+        state: present
+        group_name: admin
+        roles:
+          - name: admin
+
+    - name: Create master realm permanent admin user
+      community.general.keycloak_user:
+        validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
+        auth_client_id: admin-cli
+        auth_keycloak_url: https://{{ keycloak_domain }}
+        auth_realm: master
+        auth_username: "{{ keycloak_admin }}"
+        auth_password: "{{ keycloak_admin_password }}"
+        state: present
+        realm: master
+        credentials:
+          - temporary: false
+            type: password
+            value: "{{ keycloak_admin_password }}"
+        username: dsoadmin
+        first_name: Admin
+        last_name: Admin
+        email: admin@example.com
+        enabled: true
+        email_verified: true
+        groups:
+          - name: admin
+            state: present
+
 - name: Update console inventory
   kubernetes.core.k8s:
     kind: Secret
@@ -237,7 +293,49 @@
     definition:
       data:
         KEYCLOAK_ADMIN_PASSWORD: "{{ keycloak_admin_password | b64encode }}"
-        KEYCLOAK_ADMIN: "{{ keycloak_admin | b64encode }}"
+        KEYCLOAK_ADMIN: "{{ 'dsoadmin' | b64encode }}"
+
+# Remove Keycloak temporary admin
+
+- name: Set temporary_admin_present fact
+  ansible.builtin.set_fact:
+    temporary_admin_present: false
+
+- name: Update temporary_admin_present fact
+  when: kc_master_users.json | selectattr('username', 'equalto', 'admin')
+  ansible.builtin.set_fact:
+    temporary_admin_present: true
+
+- name: Remove temporary admin from master realm
+  when: temporary_admin_present
+  community.general.keycloak_user:
+    validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
+    auth_client_id: admin-cli
+    auth_keycloak_url: https://{{ keycloak_domain }}
+    auth_realm: master
+    auth_username: "{{ keycloak_admin }}"
+    auth_password: "{{ keycloak_admin_password }}"
+    state: absent
+    realm: master
+    username: admin
+
+# Ensure we will use permanent admin for subsequent tasks
+
+- name: Get Keycloak API token
+  ansible.builtin.uri:
+    url: https://{{ keycloak_domain }}/realms/master/protocol/openid-connect/token
+    method: POST
+    status_code: [200, 202]
+    validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
+    return_content: true
+    body: username={{ keycloak_admin }}&password={{ keycloak_admin_password }}&grant_type=password&client_id=admin-cli
+  register: kc_token
+  ignore_errors: true
+
+- name: Reset Keycloak admin fact
+  when: kc_token is failed
+  ansible.builtin.set_fact:
+    keycloak_admin: dsoadmin
 
 - name: Get Keycloak API token
   ansible.builtin.uri:
@@ -253,6 +351,8 @@
   ansible.builtin.set_fact:
     kc_access_token: "{{ kc_token.json.access_token }}"
 
+# Create and setup dso realm
+
 - name: Create dso realm
   community.general.keycloak_realm:
     validate_certs: "{{ dsc.exposedCA.type == 'none' }}"
@@ -433,6 +533,8 @@
     realm: dso
     otp_policy_algorithm: SHA256
 
+# Patch some metrics resources
+
 - name: Patch serviceMonitors
   when: >
     dsc.global.metrics.enabled and

roles/keycloak/templates/pg-cluster-keycloak.yaml.j2

@@ -53,6 +53,8 @@ spec:
   - name: pg-cluster-keycloak
 {% if dsc.keycloak.cnpg.mode == "restore" %}
     barmanObjectStore:
+      wal:
+        maxParallel: 8
       destinationPath: "s3://{{ dsc.global.backup.s3.bucketName }}/{{ dsc.global.backup.cnpg.pathPrefix }}"
       endpointURL: "{{ dsc.global.backup.s3.endpointURL }}"
 {% if dsc.global.backup.s3.endpointCA.key is defined %}