Merge pull request ceph#62561 from rkachach/fix_issue_70359_v2

mgr/cephadm: harmonize mgmt-gateway and oauth2-proxy spec fields

Reviewed-by: Adam King <[email protected]>
Reviewed-by: Pedro Gonzalez Gomez <[email protected]>

Author: adk3798

doc/cephadm/services/mgmt-gateway.rst

@@ -128,6 +128,7 @@ A ``mgmt-gateway`` service can be applied using a specification. An example in Y
         - ceph0
     spec:
      port: 5000
+     ssl: True
      ssl_protocols:
        - TLSv1.2
        - TLSv1.3
@@ -136,13 +137,13 @@ A ``mgmt-gateway`` service can be applied using a specification. An example in Y
        - AES128-SHA
        - AES256-SHA
        - ...
-     ssl_certificate: |
+     ssl_cert: |
        -----BEGIN CERTIFICATE-----
        MIIDtTCCAp2gAwIBAgIYMC4xNzc1NDQxNjEzMzc2MjMyXzxvQ7EcMA0GCSqGSIb3
        DQEBCwUAMG0xCzAJBgNVBAYTAlVTMQ0wCwYDVQQIDARVdGFoMRcwFQYDVQQHDA5T
        [...]
        -----END CERTIFICATE-----
-    ssl_certificate_key: |
+     ssl_key: |
        -----BEGIN PRIVATE KEY-----
        MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC5jdYbjtNTAKW4
        /CwQr/7wOiLGzVxChn3mmCIF3DwbL/qvTFTX2d8bDf6LjGwLYloXHscRfxszX/4h

qa/suites/orch/cephadm/workunits/task/test_mgmt_gateway.yaml

@@ -48,9 +48,9 @@ tasks:
           enable_health_check_endpoint: True
         EOT
         # Add generated certificates to spec file
-        echo "  ssl_certificate: |" >> /tmp/mgmt.spec 
+        echo "  ssl_cert: |" >> /tmp/mgmt.spec
         while read LINE; do echo $LINE | sed -e "s/^/    /"; done < /tmp/cert.pem >> /tmp/mgmt.spec
-        echo "  ssl_certificate_key: |" >> /tmp/mgmt.spec
+        echo "  ssl_key: |" >> /tmp/mgmt.spec
         while read LINE; do echo $LINE | sed -e "s/^/    /"; done < /tmp/key.pem >> /tmp/mgmt.spec
         # Apply spec
         ceph orch apply -i /tmp/mgmt.spec
@@ -60,18 +60,42 @@ tasks:
     host.a:
       - |
         set -ex
+
+        # Function to wait for a service to be healthy and log response on error
+        wait_for_service() {
+          local name="$1"
+          local url="$2"
+          local jq_filter="$3"
+
+          echo "Waiting for service $name to be healthy at $url..."
+          for i in {1..30}; do
+            local response
+            response=$(curl -k -s -u admin:admin "$url")
+            if echo "$response" | jq -e "$jq_filter" > /dev/null; then
+              echo "Service $name is healthy."
+              return 0
+            fi
+            echo "Attempt $i: service $name not ready yet"
+            sleep 10
+          done
+
+          echo "Timeout waiting for $name at $url"
+          echo "Last HTTP response:"
+          echo "$response"
+          echo "jq output:"
+          echo "$response" | jq "$jq_filter" || echo "(jq parse error or no match)"
+          return 1
+        }
+
         # retrieve mgmt hostname and ip
         MGMT_GTW_HOST=$(ceph orch ps --daemon-type mgmt-gateway -f json | jq -e '.[]' | jq -r '.hostname')
         MGMT_GTW_IP=$(ceph orch host ls -f json | jq -r --arg MGMT_GTW_HOST "$MGMT_GTW_HOST" '.[] | select(.hostname==$MGMT_GTW_HOST) | .addr')
+
         # check mgmt-gateway health
         curl -k -s https://${MGMT_GTW_IP}/health
         curl -k -s https://${MGMT_GTW_IP}:29443/health
-        # wait for background services to be reconfigured following mgmt-gateway installation
-        sleep 180
-        # check grafana endpoints are responsive and database health is okay
-        curl -k -s https://${MGMT_GTW_IP}/grafana/api/health | jq -e '.database == "ok"'
-        # check prometheus endpoints are responsive
-        curl -k -s -u admin:admin https://${MGMT_GTW_IP}/prometheus/api/v1/status/config | jq -e '.status == "success"'
-        # check alertmanager endpoints are responsive
-        curl -k -s -u admin:admin https://${MGMT_GTW_IP}/alertmanager/api/v2/status
 
+        # wait for monitoring services
+        wait_for_service "Grafana" "https://${MGMT_GTW_IP}/grafana/api/health" '.database == "ok"' || exit 1
+        wait_for_service "Prometheus" "https://${MGMT_GTW_IP}/prometheus/api/v1/status/config" '.status == "success"' || exit 1
+        wait_for_service "Alertmanager" "https://${MGMT_GTW_IP}/alertmanager/api/v2/status" '.cluster.status == "ready"' || exit 1

src/pybind/mgr/cephadm/module.py

@@ -751,7 +751,7 @@ def _get_mgmt_gw_endpoint(self, is_internal: bool) -> Optional[str]:
             endpoint_suffix = '/internal'
         else:
             mgmt_gw_port = dd.ports[0] if dd.ports else None
-            protocol = 'http' if mgmt_gw_spec.disable_https else 'https'
+            protocol = 'https' if mgmt_gw_spec.ssl else 'http'
             endpoint_suffix = ''
 
         mgmt_gw_endpoint = build_url(scheme=protocol, host=mgmt_gw_addr, port=mgmt_gw_port)

src/pybind/mgr/cephadm/services/mgmt_gateway.py

@@ -57,10 +57,10 @@ def get_external_certificates(self, svc_spec: MgmtGatewaySpec, daemon_spec: Ceph
         user_made = False
         if not (cert and key):
             # not available on store, check if provided on the spec
-            if svc_spec.ssl_certificate and svc_spec.ssl_certificate_key:
+            if svc_spec.ssl_cert and svc_spec.ssl_key:
                 user_made = True
-                cert = svc_spec.ssl_certificate
-                key = svc_spec.ssl_certificate_key
+                cert = svc_spec.ssl_cert
+                key = svc_spec.ssl_key
             else:
                 # not provided on the spec, let's generate self-sigend certificates
                 ip = self.get_mgmt_gw_ip(svc_spec, daemon_spec)
@@ -147,7 +147,6 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
             'enable_oauth2_proxy': bool(oauth2_proxy_endpoints),
         }
 
-        cert, key = self.get_external_certificates(svc_spec, daemon_spec)
         internal_cert, internal_pkey = self.get_internal_certificates(svc_spec, daemon_spec)
         daemon_config = {
             "files": {
@@ -159,7 +158,8 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st
                 "ca.crt": self.mgr.cert_mgr.get_root_ca()
             }
         }
-        if not svc_spec.disable_https:
+        if svc_spec.ssl:
+            cert, key = self.get_external_certificates(svc_spec, daemon_spec)
             daemon_config["files"]["nginx.crt"] = cert
             daemon_config["files"]["nginx.key"] = key
 

src/pybind/mgr/cephadm/services/oauth2_proxy.py

@@ -64,10 +64,10 @@ def get_certificates(self, svc_spec: OAuth2ProxySpec, daemon_spec: CephadmDaemon
         user_made = False
         if not (cert and key):
             # not available on store, check if provided on the spec
-            if svc_spec.ssl_certificate and svc_spec.ssl_certificate_key:
+            if svc_spec.ssl_cert and svc_spec.ssl_key:
                 user_made = True
-                cert = svc_spec.ssl_certificate
-                key = svc_spec.ssl_certificate_key
+                cert = svc_spec.ssl_cert
+                key = svc_spec.ssl_key
             else:
                 # not provided on the spec, let's generate self-sigend certificates
                 addr = self.mgr.inventory.get_addr(daemon_spec.host)

src/pybind/mgr/cephadm/templates/services/mgmt-gateway/external_server.conf.j2

@@ -1,6 +1,6 @@
 
 server {
-{% if spec.disable_https %}
+{% if not spec.ssl %}
     listen {{ spec.port or 80 }};
 {% else %}
     listen                    {{ spec.port or 443 }} ssl;

src/pybind/mgr/cephadm/tests/test_services.py

@@ -1596,8 +1596,8 @@ def inline_certificate(multi_line_cert):
                                       client_secret='my_client_secret',
                                       oidc_issuer_url='http://192.168.10.10:8888/dex',
                                       cookie_secret='kbAEM9opAmuHskQvt0AW8oeJRaOM2BYy5Loba0kZ0SQ=',
-                                      ssl_certificate=ceph_generated_cert,
-                                      ssl_certificate_key=ceph_generated_key)
+                                      ssl_cert=ceph_generated_cert,
+                                      ssl_key=ceph_generated_key)
 
         with with_host(cephadm_module, "test"):
             cephadm_module.cert_mgr.save_cert('grafana_cert', ceph_generated_cert, host='test')
@@ -4013,8 +4013,8 @@ def get_services_endpoints(name):
 
         server_port = 5555
         spec = MgmtGatewaySpec(port=server_port,
-                               ssl_certificate=ceph_generated_cert,
-                               ssl_certificate_key=ceph_generated_key)
+                               ssl_cert=ceph_generated_cert,
+                               ssl_key=ceph_generated_key)
 
         expected = {
             "fsid": "fsid",
@@ -4261,8 +4261,8 @@ def get_services_endpoints(name):
 
         server_port = 5555
         spec = MgmtGatewaySpec(port=server_port,
-                               ssl_certificate=ceph_generated_cert,
-                               ssl_certificate_key=ceph_generated_key,
+                               ssl_cert=ceph_generated_cert,
+                               ssl_key=ceph_generated_key,
                                enable_auth=True)
 
         expected = {
@@ -4611,8 +4611,8 @@ def get_services_endpoints(name):
 
         server_port = 5555
         mgmt_gw_spec = MgmtGatewaySpec(port=server_port,
-                                       ssl_certificate=ceph_generated_cert,
-                                       ssl_certificate_key=ceph_generated_key,
+                                       ssl_cert=ceph_generated_cert,
+                                       ssl_key=ceph_generated_key,
                                        enable_auth=True,
                                        virtual_ip=virtual_ip)
 
@@ -4622,8 +4622,8 @@ def get_services_endpoints(name):
                                       client_secret='my_client_secret',
                                       oidc_issuer_url='http://192.168.10.10:8888/dex',
                                       cookie_secret='kbAEM9opAmuHskQvt0AW8oeJRaOM2BYy5Loba0kZ0SQ=',
-                                      ssl_certificate=ceph_generated_cert,
-                                      ssl_certificate_key=ceph_generated_key,
+                                      ssl_cert=ceph_generated_cert,
+                                      ssl_key=ceph_generated_key,
                                       allowlist_domains=[allowed_domain])
 
         whitelist_domains = f"{allowed_domain},1::4,ceph-node" if virtual_ip is None else f"{allowed_domain},{virtual_ip},1::4,ceph-node"

src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts

@@ -752,10 +752,10 @@ export class ServiceFormComponent extends CdForm implements OnInit {
                   .setValue(response[0].spec?.ssl_ciphers.join(':'));
               }
               if (response[0].spec?.ssl_cert) {
-                this.serviceForm.get('ssl_cert').setValue(response[0].spec.ssl_certificate);
+                this.serviceForm.get('ssl_cert').setValue(response[0].spec.ssl_cert);
               }
               if (response[0].spec?.ssl_key) {
-                this.serviceForm.get('ssl_key').setValue(response[0].spec.ssl_certificate_key);
+                this.serviceForm.get('ssl_key').setValue(response[0].spec.ssl_key);
               }
               if (response[0].spec?.enable_auth) {
                 this.serviceForm.get('enable_auth').setValue(response[0].spec.enable_auth);
@@ -1284,8 +1284,8 @@ export class ServiceFormComponent extends CdForm implements OnInit {
           serviceSpec['virtual_interface_networks'] = values['virtual_interface_networks'];
           break;
         case 'mgmt-gateway':
-          serviceSpec['ssl_certificate'] = values['ssl_cert']?.trim();
-          serviceSpec['ssl_certificate_key'] = values['ssl_key']?.trim();
+          serviceSpec['ssl_cert'] = values['ssl_cert']?.trim();
+          serviceSpec['ssl_key'] = values['ssl_key']?.trim();
           serviceSpec['enable_auth'] = values['enable_auth'];
           serviceSpec['port'] = values['port'];
           if (serviceSpec['port'] === (443 || 80)) {

src/pybind/mgr/orchestrator/module.py

@@ -2048,7 +2048,7 @@ def _apply_iscsi(self,
     @_cli_write_command('orch apply mgmt-gateway')
     def _apply_mgmt_gateway(self,
                             port: Optional[int] = None,
-                            disable_https: Optional[bool] = False,
+                            ssl: Optional[bool] = True,
                             enable_auth: Optional[bool] = False,
                             virtual_ip: Optional[str] = None,
                             placement: Optional[str] = None,
@@ -2066,7 +2066,7 @@ def _apply_mgmt_gateway(self,
             unmanaged=unmanaged,
             port=port,
             virtual_ip=virtual_ip,
-            disable_https=disable_https,
+            ssl=ssl,
             enable_auth=enable_auth,
             preview_only=dry_run
         )

src/python-common/ceph/deployment/service_spec.py

@@ -1935,11 +1935,11 @@ def __init__(self,
                  config: Optional[Dict[str, str]] = None,
                  networks: Optional[List[str]] = None,
                  placement: Optional[PlacementSpec] = None,
-                 disable_https: Optional[bool] = False,
+                 ssl: Optional[bool] = True,
                  enable_auth: Optional[bool] = False,
                  port: Optional[int] = None,
-                 ssl_certificate: Optional[str] = None,
-                 ssl_certificate_key: Optional[str] = None,
+                 ssl_cert: Optional[str] = None,
+                 ssl_key: Optional[str] = None,
                  ssl_prefer_server_ciphers: Optional[str] = None,
                  ssl_session_tickets: Optional[str] = None,
                  ssl_session_timeout: Optional[str] = None,
@@ -1968,16 +1968,16 @@ def __init__(self,
             extra_entrypoint_args=extra_entrypoint_args,
             custom_configs=custom_configs
         )
-        #: Is a flag to disable HTTPS. If True, the server will use unsecure HTTP
-        self.disable_https = disable_https
+        #: Is a flag to enable/disable HTTPS. By default set to True.
+        self.ssl = ssl
         #: Is a flag to enable SSO auth. Requires oauth2-proxy to be active for SSO authentication.
         self.enable_auth = enable_auth
         #: The port number on which the server will listen
         self.port = port
         #: A multi-line string that contains the SSL certificate
-        self.ssl_certificate = ssl_certificate
+        self.ssl_cert = ssl_cert
         #: A multi-line string that contains the SSL key
-        self.ssl_certificate_key = ssl_certificate_key
+        self.ssl_key = ssl_key
         #: Prefer server ciphers over client ciphers: on | off
         self.ssl_prefer_server_ciphers = ssl_prefer_server_ciphers
         #: A multioption flag to control session tickets: on | off
@@ -2009,8 +2009,8 @@ def get_port_start(self) -> List[int]:
     def validate(self) -> None:
         super(MgmtGatewaySpec, self).validate()
         self._validate_port(self.port)
-        self._validate_certificate(self.ssl_certificate, "ssl_certificate")
-        self._validate_private_key(self.ssl_certificate_key, "ssl_certificate_key")
+        self._validate_certificate(self.ssl_cert, "ssl_cert")
+        self._validate_private_key(self.ssl_key, "ssl_key")
         self._validate_boolean_switch(self.ssl_prefer_server_ciphers, "ssl_prefer_server_ciphers")
         self._validate_boolean_switch(self.ssl_session_tickets, "ssl_session_tickets")
         self._validate_session_timeout(self.ssl_session_timeout)
@@ -2079,8 +2079,8 @@ def __init__(self,
                  oidc_issuer_url: Optional[str] = None,
                  redirect_url: Optional[str] = None,
                  cookie_secret: Optional[str] = None,
-                 ssl_certificate: Optional[str] = None,
-                 ssl_certificate_key: Optional[str] = None,
+                 ssl_cert: Optional[str] = None,
+                 ssl_key: Optional[str] = None,
                  allowlist_domains: Optional[List[str]] = None,
                  unmanaged: bool = False,
                  extra_container_args: Optional[GeneralArgList] = None,
@@ -2114,9 +2114,9 @@ def __init__(self,
         # 24, or 32 bytes to create an AES cipher.
         self.cookie_secret = cookie_secret or self.generate_random_secret()
         #: The multi-line SSL certificate for encrypting communications.
-        self.ssl_certificate = ssl_certificate
+        self.ssl_cert = ssl_cert
         #: The multi-line SSL certificate private key for decrypting communications.
-        self.ssl_certificate_key = ssl_certificate_key
+        self.ssl_key = ssl_key
         #: List of allowed domains for safe redirection after login or logout,
         # preventing unauthorized redirects.
         self.allowlist_domains = allowlist_domains