Tests: Make test suite clean up its provisioned assets from Grafana

Author: amotl

CHANGES.rst

@@ -9,6 +9,7 @@ in progress
 - Tests: Improve fixture ``create_datasource`` to clean up afterwards
 - Tests: Add fixture ``create_dashboard`` to create dashboards at runtime
 - Tests: Disable caching in test mode
+- Tests: Make test suite clean up its provisioned assets from Grafana
 
 
 2022-02-03 0.13.1

doc/backlog.rst

@@ -7,22 +7,20 @@ grafana-wtf backlog
 Prio 1
 ******
 - [o] With Grafana >8.3, resolve datasource name and add to ``{'type': 'influxdb', 'uid': 'PDF2762CDFF14A314'}``
+- [o] Move test harness Grafana to a different port than 3000
 
 
 *********
 Prio 1.25
 *********
 - [o] Statistics reports for data sources and panels: https://github.com/panodata/grafana-wtf/issues/18
 - [o] Finding invalid data sources: https://github.com/panodata/grafana-wtf/issues/19
-- [o] Add test fixture for adding dashboards at runtime from branch ``amo/test-dashboard-runtime``
 
 
 ********
 Prio 1.5
 ********
 - [o] Search through more Grafana entities (users, organizations, teams)
-- [o] Add test fixture which completely resets everything in Grafana before running the test harness.
-      Move to a different port than 3000 then!
 - [o] Improve output format handling and error cases
 - [o] Introduce paging to reach beyond the 5000 results limit,
   see https://grafana.com/docs/http_api/folder_dashboard_search/
@@ -63,3 +61,6 @@ Done
     - Display number of dashboards, folders, users, and playlists
 - [x] Blackify
 - [x] Dockerize
+- [x] Add test fixture for adding dashboards at runtime from branch ``amo/test-dashboard-runtime``
+- [x] Improve test suite wrt. test case isolation vs. Grafana resources
+- [x] Add test fixture which completely resets everything in Grafana before running the test harness.

tests/conftest.py

@@ -51,7 +51,8 @@ def create_datasource(docker_grafana):
     Create a Grafana data source from a test case.
     After the test case finished, it will remove the data source again.
 
-    https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+    - https://grafana.com/docs/grafana/latest/http_api/data_source/
+    - https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
     """
 
     # Reference to `grafana-client`.
@@ -60,14 +61,16 @@ def create_datasource(docker_grafana):
     # Keep track of the datasource ids in order to delete them afterwards.
     datasource_ids = []
 
-    def _create_datasource(name: str, type: str, access: str):
+    def _create_datasource(name: str, type: str, access: str, **kwargs):
+        datasource = dict(name=name, type=type, access=access)
+        datasource.update(kwargs)
         try:
-            response = grafana.datasource.create_datasource(dict(name=name, type=type, access=access))
+            response = grafana.datasource.create_datasource(datasource)
             datasource_id = response["datasource"]["id"]
             datasource_ids.append(datasource_id)
         except GrafanaClientError as ex:
-            # TODO: Mimic the original response `{'datasource': {'id': 5, 'uid': 'u9wNRyEnk', 'orgId': 1, ...`.
-            #       in order to make the removal work.
+            # TODO: Mimic the original response in order to make the removal work.
+            # `{'datasource': {'id': 5, 'uid': 'u9wNRyEnk', 'orgId': 1, ...`.
             if not re.match(
                 "Client Error 409: Data source with (the )?same name already exists", str(ex), re.IGNORECASE
             ):
@@ -80,13 +83,77 @@ def _create_datasource(name: str, type: str, access: str):
             grafana.datasource.delete_datasource_by_id(datasource_id)
 
 
+@pytest.fixture
+def create_folder(docker_grafana):
+    """
+    Create a Grafana folder from a test case.
+    After the test case finished, it will remove the dashboard again.
+
+    - https://grafana.com/docs/grafana/latest/http_api/folder/
+    - https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+    """
+
+    # Reference to `grafana-client`.
+    grafana = GrafanaWtf.grafana_client_factory(docker_grafana)
+
+    # Keep track of the dashboard uids in order to delete them afterwards.
+    folder_uids = []
+
+    # https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+    def _create_folder(title: str, uid: str = None):
+
+        # Create dashboard in Grafana.
+        try:
+            response = grafana.folder.create_folder(title=title, uid=uid)
+            folder_id = response["id"]
+            folder_uid = response["uid"]
+
+            # Response is like:
+            """
+            {
+              "id": 44,
+              "uid": "iga1UrEnz",
+              "title": "Testdrive",
+              "url": "/dashboards/f/iga1UrEnz/testdrive",
+              "hasAcl": false,
+              "canSave": true,
+              "canEdit": true,
+              "canAdmin": true,
+              "createdBy": "admin",
+              "created": "2022-03-22T23:44:38Z",
+              "updatedBy": "admin",
+              "updated": "2022-03-22T23:44:38Z",
+              "version": 1
+            }
+            """
+
+            folder_uids.append(folder_uid)
+            return folder_id
+        except GrafanaClientError as ex:
+            # TODO: Mimic the original response in order to make the removal work.
+            if not re.match(
+                "Client Error 409: a folder or dashboard in the general folder with the same name already exists",
+                str(ex),
+                re.IGNORECASE,
+            ):
+                raise
+
+    yield _create_folder
+
+    # Delete dashboard again.
+    if folder_uids:
+        for folder_uid in folder_uids:
+            grafana.folder.delete_folder(uid=folder_uid)
+
+
 @pytest.fixture
 def create_dashboard(docker_grafana):
     """
     Create a Grafana dashboard from a test case.
     After the test case finished, it will remove the dashboard again.
 
-    https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+    - https://grafana.com/docs/grafana/latest/http_api/dashboard/
+    - https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
     """
 
     # Reference to `grafana-client`.
@@ -96,11 +163,15 @@ def create_dashboard(docker_grafana):
     dashboard_uids = []
 
     # https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
-    def _create_dashboard(title: str, datasource: str):
+    def _create_dashboard(dashboard: dict = None, folder_id: str = None, folder_uid: str = None):
 
         # Create dashboard in Grafana.
-        dashboard = mkdashboard(title=title, datasource=datasource)
-        response = grafana.dashboard.update_dashboard(dashboard={"dashboard": dashboard, "overwrite": True})
+        payload = {"dashboard": dashboard, "overwrite": True}
+        if folder_id:
+            payload["folderId"] = folder_id
+        if folder_uid:
+            payload["folderUid"] = folder_uid
+        response = grafana.dashboard.update_dashboard(dashboard=payload)
 
         # Response is like:
         # {'id': 3, 'slug': 'foo', 'status': 'success', 'uid': 'iO0xgE2nk', 'url': '/d/iO0xgE2nk/foo', 'version': 1}
@@ -116,6 +187,37 @@ def _create_dashboard(title: str, datasource: str):
             grafana.dashboard.delete_dashboard(dashboard_uid=dashboard_uid)
 
 
+@pytest.fixture
+def ldi_resources(create_datasource, create_folder, create_dashboard):
+    """
+    Create a Grafana dashboard from a test case.
+    After the test case finished, it will remove the dashboard again.
+
+    https://docs.pytest.org/en/4.6.x/fixture.html#factories-as-fixtures
+    """
+
+    # Create LDI datasource.
+    create_datasource(
+        name="ldi_v2",
+        type="influxdb",
+        access="proxy",
+        url="http://localhost:8086/",
+        user="root",
+        password="root",
+        database="ldi_v2",
+        secureJsonData={"password": "root"},
+    )
+
+    # Create folder.
+    folder_id = create_folder(title="Testdrive", uid="testdrive")
+
+    # Create LDI dashboards.
+    for file in Path("tests/grafana/dashboards").glob("*.json"):
+        with open(file, "r") as f:
+            dashboard = json.load(f)
+            create_dashboard(dashboard=dashboard, folder_id=folder_id)
+
+
 def mkdashboard(title: str, datasource: str):
     """
     Build dashboard with single panel.

tests/docker-compose.yml

@@ -4,12 +4,8 @@ services:
   grafana:
     container_name: grafana-wtf-test
     image: grafana/grafana:${GRAFANA_VERSION}
-    volumes:
-      - ./grafana/datasources:/etc/grafana/provisioning/datasources/
-      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards/
     ports:
       - "3000:3000"
     environment:
       GF_LOG_LEVEL: info
       #GF_LOG_LEVEL: debug
-      #GF_PATHS_PROVISIONING: /etc/grafana/provisioning

tests/grafana/dashboards/all.yml

@@ -16,6 +16,7 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
+  "id": null,
   "iteration": 1632507348685,
   "links": [
     {

tests/grafana/dashboards/ldi-v27.json

@@ -22,7 +22,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 3,
+  "id": null,
   "iteration": 1642798659207,
   "links": [
     {