Merge remote-tracking branch 'origin/main' into prometheus-alerts

Author: maxstack

requirements.txt

@@ -1,6 +1,8 @@
 grafana-client==4.1.*
 opensearch-py==2.5.*
+packaging==24.*
 prometheus-api-client==0.5.*
 pylint==3.3.*
+pytest-subtests==0.12.*
 pytest-testinfra==10.1.*
 requests==2.31.*

stackhpc_cloud_tests/host/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 StackHPC Ltd.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.

stackhpc_cloud_tests/host/test_docker.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2024 StackHPC Ltd.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import os
+from packaging.version import parse
+import pytest
+
+
+@pytest.fixture
+def docker_info(host, scope="session"):
+    """Pytest fixture that provides the output of 'docker info'."""
+    with host.sudo("stack"):
+        docker_info = host.check_output("docker info --format json")
+    return json.loads(docker_info)
+
+
+def test_docker_version(host):
+    """Check that Docker is accessible and optionally check version."""
+    # An optional inclusive minimum version.
+    min_version = os.environ.get("DOCKER_VERSION_MIN")
+    # An optional exclusive maximum version.
+    max_version = os.environ.get("DOCKER_VERSION_MAX")
+    with host.sudo("stack"):
+        client_version = parse(host.docker.client_version())
+        server_version = parse(host.docker.server_version())
+    if min_version:
+        min_version = parse(min_version)
+        assert client_version >= min_version
+        assert server_version >= min_version
+    if max_version:
+        max_version = parse(max_version)
+        assert client_version < max_version
+        assert server_version < max_version
+
+
+def test_docker_containers(subtests, host):
+    """Check that Docker containers are healthy."""
+    with host.sudo("stack"):
+        docker_containers = host.docker.get_containers()
+    for container in docker_containers:
+        # Use the subtests fixture to create a dynamically parametrised test
+        # based on the containers on the system.
+        with subtests.test(msg="container=" + container.name):
+            state = container.inspect()["State"]
+            assert state["Running"]
+            assert not state["Restarting"]
+            assert not state["Dead"]
+            assert not state["OOMKilled"]
+            if "Health" in state:
+                assert state["Health"]["Status"] == "healthy"
+            if "HostConfig" in state:
+                assert state["HostConfig"]["LogConfig"]["Type"] == "json-file"
+                assert "max-file" in state["HostConfig"]["LogConfig"]["Config"]
+                assert "max-size" in state["HostConfig"]["LogConfig"]["Config"]
+
+
+def test_docker_driver(docker_info):
+    """Check that Docker is using the overlay2 storage driver."""
+    assert docker_info["Driver"] == "overlay2"
+
+
+def test_no_bridge_network_exists(host):
+    """Check that no bridge network exists."""
+    with host.sudo("stack"):
+        docker_networks = host.check_output("docker network ls --format json")
+    for network in docker_networks.splitlines():
+        network = json.loads(network)
+        assert network["Name"] != "bridge"
+        assert network["Driver"] != "bridge"
+
+
+def test_ip_forwarding_disabled(docker_info):
+    """Check that IP forwarding is disabled."""
+    assert not docker_info["IPv4Forwarding"]
+
+
+def test_iptables_disabled(host):
+    """Check that IPTables manipulation is disabled."""
+    # (MaxN) "docker system info" for version 27.4.1 will report "true" for "BridgeNfIptables" and "BridgeNfIp6tables"
+    # regardless of the setting of "iptables" in /etc/docker/daemon.json,
+    # however correct creation of iptables rules will follow this setting - so test on the iptables rules instead.
+    iptables_chains = host.check_output("iptables -L")
+    assert "FORWARD" in iptables_chains
+    assert "DOCKER" not in iptables_chains
+
+def test_live_restore_enabled(docker_info):
+    """Check that live restore is enabled."""
+    assert docker_info["LiveRestoreEnabled"]

stackhpc_cloud_tests/host/test_selinux.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2024 StackHPC Ltd.
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import pytest
+
+
+def test_selinux(host):
+    """Check that SELinux is enabled and permissive on supported systems."""
+    # Adapted from Kayobe host configure tests:
+    # https://opendev.org/openstack/kayobe/src/commit/5333596afd6b93151e8e5a58257944b892d90060/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py#L350
+    if host.system_info.distribution in {"debian", "ubuntu"}:
+        pytest.skip(reason="SELinux is not supported on Debian or Ubuntu")
+    # Desired state: enforcing, permissive or disabled
+    expected_mode = os.environ["SELINUX_STATE"]
+    assert expected_mode in {"enforcing", "permissive", "disabled"}
+    expected_status = "disabled" if expected_mode == "disabled" else "enabled"
+    selinux = host.check_output("sestatus")
+    selinux = selinux.splitlines()
+    # Remove duplicate whitespace characters in output
+    selinux = [" ".join(x.split()) for x in selinux]
+
+    assert f"SELinux status: {expected_status}" in selinux
+    if expected_status == "enabled":
+        assert f"Current mode: {expected_mode}" in selinux
+        assert f"Mode from config file: {expected_mode}" in selinux

stackhpc_cloud_tests/monitoring/test_opensearch.py

@@ -68,7 +68,11 @@ def test_opensearch_dashboards_status():
     dashboard_username = os.environ["OPENSEARCH_DASHBOARDS_USERNAME"]
     dashboard_password = os.environ["OPENSEARCH_DASHBOARDS_PASSWORD"]
     dashboard_url += "/api/status"
-    result = requests.get(dashboard_url, auth=(dashboard_username, dashboard_password))
+    dashboard_cacert = os.environ.get("OPENSEARCH_DASHBOARDS_CACERT")
+    kwargs = {}
+    if dashboard_cacert:
+        kwargs["verify"] = dashboard_cacert
+    result = requests.get(dashboard_url, auth=(dashboard_username, dashboard_password), **kwargs)
     assert result.ok
     result = result.json()
     assert result["status"]["overall"]["state"] == "green"