Tests: Improve test quality, specifically for explore dashboards

amotl · amotl · commit fe736e5a7feb · 2022-03-25T01:11:37.000+01:00
Grafana 6 vs. Grafana &gt;= 7 have a different behaviour here because
Grafana 6 did not know about data source UIDs yet.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -13,6 +13,8 @@ in progress
 - Tests: Run Grafana on non-standard port 33333
 - Tests: Add flag ``CLEANUP_RESOURCES`` to determine whether to clean up
   all resources provisioned to Grafana.
+- Tests: Improve test quality, specifically for ``explore dashboards`` on
+  Grafana 6 vs. Grafana >= 7
 
 
 2022-02-03 0.13.1
diff --git a/doc/backlog.rst b/doc/backlog.rst
@@ -7,6 +7,8 @@ grafana-wtf backlog
 Prio 1
 ******
 - [o] With Grafana >8.3, resolve datasource name and add to ``{'type': 'influxdb', 'uid': 'PDF2762CDFF14A314'}``
+- [o] Add "folder name/uid" to "explore dashboards" response.
+- [o] Check if "datasources" is always present in responses to "explore dashboards".
 
 
 *********
@@ -27,6 +29,7 @@ Prio 1.5
 - [o] Show dependencies
 - [o] Optionally apply "replace" to data sources also
 - [o] Add software tests for authenticated access to Grafana (--grafana-token)
+- [o] Add output format RSS
 
 
 ******
diff --git a/grafana_wtf/commands.py b/grafana_wtf/commands.py
@@ -148,7 +148,7 @@ def run():
     setup_logging(log_level)
 
     # Debugging
-    log.debug("Options: {}".format(json.dumps(options, indent=4)))
+    # log.debug("Options: {}".format(json.dumps(options, indent=4)))
 
     configure_http_logging(options)
 
diff --git a/grafana_wtf/core.py b/grafana_wtf/core.py
@@ -295,6 +295,17 @@ def info(self):
 
         return response
 
+    def version(self):
+
+        try:
+            health = self.grafana.client.GET("/health")
+        except Exception as ex:
+            log.error(f"Request to /health endpoint failed: {ex}")
+            health = {}
+
+        version = health.get("version")
+        return version
+
     def dashboard_details(self):
         for dashboard in self.data.dashboards:
             yield DashboardDetails(dashboard=dashboard)
diff --git a/setup.py b/setup.py
@@ -28,7 +28,7 @@
 
 extras = {
     "test": [
-        "pytest>=5,<7",
+        "pytest>=5,<8",
         "lovely-pytest-docker>=0.2.1,<3",
         "grafanalib>=0.6,<0.7",
     ]
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -3,6 +3,7 @@
 import re
 from io import StringIO
 from pathlib import Path
+from typing import List, Union
 
 import grafanalib.core
 import pytest
@@ -12,6 +13,9 @@
 from grafana_wtf.core import GrafanaWtf
 
 # Whether to clean up all resources provisioned to Grafana.
+# Note that the test suite will not complete successfully when toggling this
+# setting. It can be used when running individual test cases in order to
+# investigate the resources provisioned to Grafana.
 CLEANUP_RESOURCES = True
 
 
@@ -56,6 +60,38 @@ def create_datasource(docker_grafana):
 
     - https://grafana.com/docs/grafana/latest/http_api/data_source/
     - https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+
+    The JSON response to `create_datasource` looks like this::
+
+        {
+          "datasource": {
+            "id": 3,
+            "uid": "PDF2762CDFF14A314",
+            "orgId": 1,
+            "name": "ldi_v2",
+            "type": "influxdb",
+            "typeLogoUrl": "",
+            "access": "proxy",
+            "url": "http://localhost:8086/",
+            "password": "root",
+            "user": "root",
+            "database": "ldi_v2",
+            "basicAuth": false,
+            "basicAuthUser": "",
+            "basicAuthPassword": "",
+            "withCredentials": false,
+            "isDefault": false,
+            "jsonData": {},
+            "secureJsonFields": {
+              "password": true
+            },
+            "version": 1,
+            "readOnly": false
+          },
+          "id": 3,
+          "message": "Datasource added",
+          "name": "ldi_v2"
+        }
     """
 
     # Reference to `grafana-client`.
@@ -64,7 +100,19 @@ def create_datasource(docker_grafana):
     # Keep track of the datasource ids in order to delete them afterwards.
     datasource_ids = []
 
-    def _create_datasource(name: str, type: str, access: str, **kwargs):
+    def _create_datasource(name: str, type: str = "testdata", access: str = "proxy", **kwargs):
+
+        # Reuse existing datasource.
+        try:
+            response = grafana.datasource.get_datasource_by_name(name)
+            datasource_id = response["id"]
+            datasource_ids.append(datasource_id)
+            return
+        except GrafanaClientError as ex:
+            if ex.status_code != 404:
+                raise
+
+        # Create new datasource.
         datasource = dict(name=name, type=type, access=access)
         datasource.update(kwargs)
         try:
@@ -91,60 +139,74 @@ def _create_datasource(name: str, type: str, access: str, **kwargs):
 def create_folder(docker_grafana):
     """
     Create a Grafana folder from a test case.
-    After the test case finished, it will remove the dashboard again.
+    After the test case finished, it will remove the folder again.
 
     - https://grafana.com/docs/grafana/latest/http_api/folder/
     - https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+
+    The JSON response to `create_folder` looks like this::
+
+        {
+          "id": 44,
+          "uid": "iga1UrEnz",
+          "title": "Testdrive",
+          "url": "/dashboards/f/iga1UrEnz/testdrive",
+          "hasAcl": false,
+          "canSave": true,
+          "canEdit": true,
+          "canAdmin": true,
+          "createdBy": "admin",
+          "created": "2022-03-22T23:44:38Z",
+          "updatedBy": "admin",
+          "updated": "2022-03-22T23:44:38Z",
+          "version": 1
+        }
     """
 
     # Reference to `grafana-client`.
     grafana = GrafanaWtf.grafana_client_factory(docker_grafana)
 
-    # Keep track of the dashboard uids in order to delete them afterwards.
+    # Keep track of the folder uids in order to delete them afterwards.
     folder_uids = []
 
-    # https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
     def _create_folder(title: str, uid: str = None):
 
-        # Create dashboard in Grafana.
+        # Reuse folder when it already exists.
         try:
-            response = grafana.folder.create_folder(title=title, uid=uid)
+            response = grafana.folder.get_folder(uid=uid)
             folder_id = response["id"]
             folder_uid = response["uid"]
+            folder_uids.append(folder_uid)
+            return folder_id
+        except GrafanaClientError as ex:
+            if ex.status_code != 404:
+                raise
 
-            # Response is like:
-            """
-            {
-              "id": 44,
-              "uid": "iga1UrEnz",
-              "title": "Testdrive",
-              "url": "/dashboards/f/iga1UrEnz/testdrive",
-              "hasAcl": false,
-              "canSave": true,
-              "canEdit": true,
-              "canAdmin": true,
-              "createdBy": "admin",
-              "created": "2022-03-22T23:44:38Z",
-              "updatedBy": "admin",
-              "updated": "2022-03-22T23:44:38Z",
-              "version": 1
-            }
-            """
-
+        # Create folder.
+        try:
+            response = grafana.folder.create_folder(title=title, uid=uid)
+            folder_id = response["id"]
+            folder_uid = response["uid"]
             folder_uids.append(folder_uid)
             return folder_id
         except GrafanaClientError as ex:
             # TODO: Mimic the original response in order to make the removal work.
-            if not re.match(
+            error_exists = re.match(
                 "Client Error 409: a folder or dashboard in the general folder with the same name already exists",
                 str(ex),
                 re.IGNORECASE,
-            ):
+            )
+            error_modified = re.match(
+                "Client Error 412: The folder has been changed by someone else",
+                str(ex),
+                re.IGNORECASE,
+            )
+            if not (error_exists or error_modified):
                 raise
 
     yield _create_folder
 
-    # Delete dashboard again.
+    # Delete folder again.
     if CLEANUP_RESOURCES:
         if folder_uids:
             for folder_uid in folder_uids:
@@ -159,6 +221,17 @@ def create_dashboard(docker_grafana):
 
     - https://grafana.com/docs/grafana/latest/http_api/dashboard/
     - https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+
+    The JSON response to `update_dashboard` looks like this::
+
+        {
+          "id": 2,
+          "slug": "luftdaten-info-generic-trend-v33",
+          "status": "success",
+          "uid": "jpVsQxRja",
+          "url": "/d/jpVsQxRja/luftdaten-info-generic-trend-v33",
+          "version": 1
+        }
     """
 
     # Reference to `grafana-client`.
@@ -177,10 +250,6 @@ def _create_dashboard(dashboard: dict = None, folder_id: str = None, folder_uid:
         if folder_uid:
             payload["folderUid"] = folder_uid
         response = grafana.dashboard.update_dashboard(dashboard=payload)
-
-        # Response is like:
-        # {'id': 3, 'slug': 'foo', 'status': 'success', 'uid': 'iO0xgE2nk', 'url': '/d/iO0xgE2nk/foo', 'version': 1}
-
         dashboard_uid = response["uid"]
         dashboard_uids.append(dashboard_uid)
 
@@ -202,26 +271,46 @@ def ldi_resources(create_datasource, create_folder, create_dashboard):
     https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
     """
 
-    # Create LDI datasource.
-    create_datasource(
-        name="ldi_v2",
-        type="influxdb",
-        access="proxy",
-        url="http://localhost:8086/",
-        user="root",
-        password="root",
-        database="ldi_v2",
-        secureJsonData={"password": "root"},
-    )
-
-    # Create folder.
-    folder_id = create_folder(title="Testdrive", uid="testdrive")
-
-    # Create LDI dashboards.
-    for file in Path("tests/grafana/dashboards").glob("*.json"):
-        with open(file, "r") as f:
-            dashboard = json.load(f)
-            create_dashboard(dashboard=dashboard, folder_id=folder_id)
+    def _ldi_resources(dashboards: List[Union[Path, str]] = None):
+        # Create LDI datasource.
+        create_datasource(
+            name="ldi_v2",
+            type="influxdb",
+            access="proxy",
+            uid="PDF2762CDFF14A314",
+            url="http://localhost:8086/",
+            user="root",
+            password="root",
+            database="ldi_v2",
+            secureJsonData={"password": "root"},
+        )
+
+        # Create folder.
+        folder_id = create_folder(title="Testdrive", uid="testdrive")
+
+        # Create LDI dashboards.
+        if dashboards:
+            dashboard_files = dashboards
+        else:
+            dashboard_files = Path("tests/grafana/dashboards").glob("*.json")
+
+        for file in dashboard_files:
+            with open(file, "r") as f:
+                dashboard = json.load(f)
+                create_dashboard(dashboard=dashboard, folder_id=folder_id)
+
+    return _ldi_resources
+
+
+@pytest.fixture
+def grafana_version(docker_grafana):
+    """
+    Return Grafana version number.
+    """
+    engine = GrafanaWtf(grafana_url=docker_grafana, grafana_token=None)
+    engine.setup()
+    grafana_version = engine.version()
+    return grafana_version
 
 
 def mkdashboard(title: str, datasource: str):
diff --git a/tests/test_commands.py b/tests/test_commands.py

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@`
`28`	`28`
`29`	`29`	`extras = {`
`30`	`30`	`"test": [`
`31`		`- "pytest>=5,<7",`
	`31`	`+ "pytest>=5,<8",`
`32`	`32`	`"lovely-pytest-docker>=0.2.1,<3",`
`33`	`33`	`"grafanalib>=0.6,<0.7",`
`34`	`34`	`]`