Merge pull request #3854 from reubenmiller/test-support-custom-mqtt-endpoints

tests: allow using a custom Cumulocity mqtt endpoint in tests

Author: reubenmiller

tests/RobotFramework/devdata/env.template

@@ -19,6 +19,8 @@ PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
 C8Y_BASEURL=
 C8Y_USER=
 C8Y_PASSWORD=
+# Custom MQTT endpoint, e.g. example.cumulocity:9883
+#C8Y_MQTT=
 
 #
 # AWS IoT

tests/RobotFramework/libraries/ThinEdgeIO/ThinEdgeIO.py

@@ -22,6 +22,7 @@
 
 from paho.mqtt import matcher
 from robot.api.deco import keyword, library
+from robot.libraries.BuiltIn import BuiltIn
 from DeviceLibrary import DeviceLibrary, DeviceAdapter
 from DeviceLibrary.DeviceLibrary import timestamp
 from Cumulocity import Cumulocity, retry
@@ -81,6 +82,14 @@ def date(self) -> datetime:
     # {"tst":"2022-12-27T16:55:44.923776Z+0000","topic":"c8y/s/us","qos":0,"retain":0,"payloadlen":99,"payload":"119,c8y-bridge.conf,c8y-configuration-plugin,example,mosquitto.conf,tedge-mosquitto.conf,tedge.toml"}
 
 
+def strip_scheme(url: Optional[str]) -> Optional[str]:
+    """Strip the scheme from a URL"""
+    if isinstance(url, str):
+        # strip any scheme if present
+        return url.strip().split("://", 1)[-1]
+    return url
+
+
 @library(scope="SUITE", auto_keywords=False)
 class ThinEdgeIO(DeviceLibrary):
     """ThinEdgeIO Library"""
@@ -102,6 +111,26 @@ def __init__(
         # Configure retries
         retry.configure_retry_on_members(self, "^_assert_")
 
+        # Cumulocity configuration cache
+        self._c8y_config = {}
+
+    @property
+    def c8y_config(self) -> Dict[str, Any]:
+        """Get the Cumulocity configuration"""
+        if not self._c8y_config:
+            self._c8y_config = BuiltIn().get_variable_value(r"&{C8Y_CONFIG}", {}) or {}
+        return self._c8y_config
+
+    @property
+    def c8y_host(self) -> str:
+        """Get the Cumulocity domain/host"""
+        return c8y_lib.get_domain()
+
+    @property
+    def c8y_mqtt(self) -> Optional[str]:
+        """Get the Cumulocity MQTT broker URL"""
+        return strip_scheme(self.c8y_config.get("mqtt"))
+
     def end_suite(self, _data: Any, result: Any):
         """End suite hook which is called by Robot Framework
         when the test suite has finished
@@ -201,6 +230,8 @@ def register_device_with_cumulocity_ca(
         device_type: str = "thin-edge.io",
         device_name: Optional[str] = None,
         csr_path: Optional[str] = None,
+        http: Optional[str] = None,
+        mqtt: Optional[str] = None,
         **kwargs,
     ):
         """Register a device with Cumulocity using the Cumulocity Certificate Authority feature
@@ -217,7 +248,9 @@ def register_device_with_cumulocity_ca(
             device_type=device_type,
         )
 
-        cmd = f"""tedge config set c8y.url '{credentials.url}' && tedge cert download c8y --device-id '{external_id}' --one-time-password '{credentials.one_time_password}' --retry-every 5s --max-timeout 60s"""
+        self.set_cumulocity_urls(http=http, mqtt=mqtt, device_name=device_name)
+
+        cmd = f"tedge cert download c8y --device-id '{external_id}' --one-time-password '{credentials.one_time_password}' --retry-every 5s --max-timeout 60s"
         if csr_path:
             cmd += f" --csr-path '{csr_path}'"
 
@@ -226,12 +259,57 @@ def register_device_with_cumulocity_ca(
             device_name=device_name,
         )
 
+    @keyword("Set Cumulocity URLs")
+    def set_cumulocity_urls(
+        self,
+        http: Optional[str] = None,
+        mqtt: Optional[str] = None,
+        profile: Optional[str] = None,
+        device_name: Optional[str] = None,
+    ):
+        """Set the Cumulocity URLs. If an mqtt url is defined then the individual urls will be set
+        otherwise only the single c8y.url will be set.
+
+        Args:
+            http (Optional[str]): The HTTP URL to set. If set to None, then the default host will be used.
+            mqtt (Optional[str]): The MQTT URL to set. If set to None, then the default mqtt broker will be used (based on the host).
+            device_name (Optional[str]): The device name to set.
+            profile (Optional[str]): Cloud profile. Defaults to None.
+        """
+
+        if mqtt is None:
+            mqtt = self.c8y_mqtt
+
+        if http is None:
+            http = self.c8y_host
+
+        def _append_command(cmd):
+            if profile:
+                commands.append(f"{cmd} --profile {profile}")
+            else:
+                commands.append(cmd)
+
+        commands = []
+        if not mqtt:
+            _append_command(f"tedge config set c8y.url '{http}'")
+        else:
+            _append_command(f"tedge config set c8y.mqtt '{mqtt}'")
+            _append_command(f"tedge config set c8y.http '{http}'")
+
+        cmd = " && ".join(commands)
+        self.execute_command(
+            cmd,
+            device_name=device_name,
+        )
+
     @keyword("Register Device With Self-Signed Certificate")
     def register_device_with_self_signed_certificate(
         self,
         external_id: str,
         device_name: Optional[str] = None,
         auto_registration_enabled: bool = True,
+        http: Optional[str] = None,
+        mqtt: Optional[str] = None,
         **kwargs,
     ):
         """Register a device with Cumulocity using a self-signed certificate. The self-signed
@@ -241,10 +319,8 @@ def register_device_with_self_signed_certificate(
 
         | Register Device With Self-Signed Certificate | external_id=tedge001 |
         """
-        domain = c8y_lib.get_domain()
-        self.execute_command(
-            f"tedge config set c8y.url '{domain}' && tedge cert create --device-id '{external_id}'"
-        )
+        self.set_cumulocity_urls(http=http, mqtt=mqtt, device_name=device_name)
+        self.execute_command(f"tedge cert create --device-id '{external_id}'")
         pem_contents = self.execute_command(
             'cat "$(tedge config get device.cert_path)"'
         )

tests/RobotFramework/resources/common.resource

@@ -20,6 +20,7 @@ ${DEVICE_ADAPTER}       %{DEVICE_ADAPTER=docker}
 # Cumulocity settings
 &{C8Y_CONFIG}
 ...                     host=%{C8Y_BASEURL= }
+...                     mqtt=%{C8Y_MQTT= }
 ...                     tenant=%{C8Y_TENANT= }
 ...                     username=%{C8Y_USER= }
 ...                     password=%{C8Y_PASSWORD= }

tests/RobotFramework/tests/cumulocity/configuration/configuration_operation_multi_cloud.robot

@@ -81,7 +81,8 @@ Setup Second Device
     ...    tedge config set c8y.device.key_path --profile second /etc/tedge/device-certs/[email protected]
     Execute Command    tedge config set c8y.proxy.bind.port --profile second 8002
     Execute Command    tedge config set c8y.bridge.topic_prefix --profile second c8y-second
-    Execute Command    tedge config set c8y.url --profile second "$(tedge config get c8y.url)"
+
+    Set Cumulocity URLs    profile=second
 
     Execute Command    tedge cert create --device-id ${second_device_sn} c8y --profile second
     Register Certificate For Cleanup    cloud_profile=second    common_name=${second_device_sn}

tests/RobotFramework/tests/cumulocity/http_proxy/http_proxy_support.robot

@@ -62,8 +62,7 @@ Setup Device Without thin-edge.io
     Start Service    gost-http-proxy
 
 Configure and Connect to Cumulocity
-    ${domain}=    Cumulocity.Get Domain
-    Execute Command    tedge config set c8y.url "${domain}"
+    Set Cumulocity URLs
     Execute Command    tedge cert create --device-id "${DEVICE_SN}"
     Execute Command
     ...    cmd=sudo env C8Y_USER='${C8Y_CONFIG.username}' C8Y_PASSWORD='${C8Y_CONFIG.password}' tedge cert upload c8y

tests/RobotFramework/tests/cumulocity/registration/cumulocity_ca.robot

@@ -11,8 +11,7 @@ Test Tags       theme:c8y
 Register Device Using Cumulocity CA
     [Setup]    Custom Setup
     ${credentials}=    Bulk Register Device With Cumulocity CA    ${DEVICE_SN}
-    ${DOMAIN}=    Cumulocity.Get Domain
-    Execute Command    tedge config set c8y.url "${DOMAIN}"
+    Set Cumulocity URLs
     Execute Command
     ...    tedge cert download c8y --device-id "${DEVICE_SN}" --one-time-password '${credentials.one_time_password}' --retry-every 5s --max-timeout 30s
     Execute Command    tedge connect c8y
@@ -23,7 +22,7 @@ Register Device Using Cumulocity CA with url flag
     ${DOMAIN}=    Cumulocity.Get Domain
     Execute Command
     ...    tedge cert download c8y --device-id "${DEVICE_SN}" --one-time-password '${credentials.one_time_password}' --url ${DOMAIN} --retry-every 5s --max-timeout 30s
-    Execute Command    tedge config set c8y.url "${DOMAIN}"
+    Set Cumulocity URLs
     Execute Command    tedge connect c8y
 
 Certificate Renewal Service Using Cumulocity Certificate Authority
@@ -52,9 +51,8 @@ Certificate Renewal with Cloud Profiles
     ${DEVICE_SN_2}=    Set Variable    ${DEVICE_SN}_2
     ThinEdgeIO.Register Device With Cumulocity CA    external_id
     ${credentials}=    Cumulocity.Bulk Register Device With Cumulocity CA    external_id=${DEVICE_SN_2}
-    ${DOMAIN}=    Cumulocity.Get Domain
 
-    Execute Command    tedge config set c8y.url "${DOMAIN}" --profile customer
+    Set Cumulocity URLs    profile=customer
     Execute Command
     ...    tedge config set c8y.device.cert_path /etc/tedge/device-certs/tedge-certificate-customer.pem --profile customer
     Execute Command    tedge config set c8y.bridge.topic_prefix c8y-customer --profile customer

tests/RobotFramework/tests/cumulocity/self-update/tedge_self_update.robot

@@ -160,7 +160,9 @@ Update tedge Using a Custom Software Update Workflow
 
     # Register device (using already installed version)
     # Use self-signed certificate as the older thin-edge.io version does not support Cumulocity CA
-    Register Device With Self-Signed Certificate    external_id=${DEVICE_SN}
+    # and don't explicitly set the mqtt url so that the default value is used (e.g. Core MQTT)
+    Register Device With Self-Signed Certificate    external_id=${DEVICE_SN}    mqtt=${EMPTY}
+
     Execute Command    cmd=tedge connect c8y
     Device Should Exist    ${DEVICE_SN}
 
@@ -231,8 +233,10 @@ Unpin thin-edge.io APT Packages
 
 Register Legacy thin-edge.io
     [Arguments]    ${external_id}
+    # Don't explicitly set the mqtt url so that the default value is used (e.g. Core MQTT)
+    # as older versions of thin-edge.io don't support the mqtt service
     ${domain}=    Cumulocity.Get Domain
-    Execute Command    cmd=tedge config set c8y.url ${domain}
+    ThinEdgeIO.Set Cumulocity URLs    http=${domain}    mqtt=${EMPTY}
     Execute Command    cmd=tedge cert create --device-id '${external_id}'
     ${pem_contents}=    Execute Command    cmd=cat "$(tedge config get device.cert.path)"    strip=${True}
     Cumulocity.Upload Certificate

tests/RobotFramework/tests/cumulocity/smartrest_one/smartrest_one.robot

@@ -89,7 +89,7 @@ SmartREST1 Template Should Exist
 Custom Setup
     [Arguments]    ${use_builtin_bridge}
     ${DEVICE_SN}=    Setup    register=${False}
-    Execute Command    cmd=tedge config set c8y.url "${C8Y_CONFIG.host}"
+    Set Cumulocity URLs
     Execute Command    tedge config set mqtt.bridge.built_in ${use_builtin_bridge}
     Execute Command    tedge config set c8y.auth_method basic
 

tests/RobotFramework/tests/pkcs11/compatibility.robot

@@ -182,8 +182,7 @@ Custom Setup
     Execute Command    sudo -u tedge /usr/bin/init_softhsm.sh --device-id "${DEVICE_SN}" --pin 123456
 
     # configure tedge
-    ${domain}=    Cumulocity.Get Domain
-    Execute Command    tedge config set c8y.url "${domain}"
+    Set Cumulocity URLs
     Execute Command    tedge config set mqtt.bridge.built_in true
     Execute Command    tedge config set device.cryptoki.mode socket
 

tests/RobotFramework/tests/pkcs11/compatibility_16x.robot

@@ -49,8 +49,7 @@ Custom Setup
     Execute Command    sudo -u tedge /usr/bin/init_softhsm.sh --device-id "${DEVICE_SN}" --pin 123456
 
     # configure tedge
-    ${domain}=    Cumulocity.Get Domain
-    Execute Command    tedge config set c8y.url "${domain}"
+    Set Cumulocity URLs
     Execute Command    tedge config set mqtt.bridge.built_in true
     Execute Command    tedge config set device.cryptoki.mode socket