support for multiple prefixes and related fixes (#116)

istrator2 · web-flow · commit eab4be0ec685 · 2023-11-27T21:32:48.000Z
* make code multiprefix-able
* fix some autoformatting
* change quoting
* add unittests
diff --git a/.gitignore b/.gitignore
@@ -132,7 +132,6 @@ dmypy.json
 bazel-*
 
 # docker-compose
-.env
 docker-compose.override.yaml
 # docker-compose volumes
 /volumes
@@ -141,3 +140,6 @@ docker-compose.override.yaml
 
 # config file
 wgkex.yaml
+
+# pycharm project metadata
+.idea/
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -9,19 +9,19 @@ services:
       - ./volumes/mosquitto/data:/mosquitto/data
       - ./volumes/mosquitto/log:/mosquitto/log
     ports:
-     - "9001:9001"
+      - "9001:9001"
 
   broker:
     image: ghcr.io/freifunkmuc/wgkex:latest
     command: broker
     restart: unless-stopped
     ports:
       - "5000:5000"
-    #volumes:
+      #volumes:
       #- ./config/broker/wgkex.yaml:/etc/wgkex.yaml
     environment:
-      WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt}
-      WGKEX_DOMAIN_PREFIX: ${WGKEX_DOMAIN_PREFIX-ffmuc_}
+      WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city}
+      WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_, ffdon_, ffwert_}
       WGKEX_DEBUG: ${WGKEX_DEBUG-DEBUG}
       MQTT_BROKER_URL: ${MQTT_BROKER_URL-mqtt}
       MQTT_BROKER_PORT: ${MQTT_BROKER_PORT-1883}
@@ -35,10 +35,10 @@ services:
     command: worker
     restart: unless-stopped
     #volumes:
-      #- ./config/worker/wgkex.yaml:/etc/wgkex.yaml
+    #- ./config/worker/wgkex.yaml:/etc/wgkex.yaml
     environment:
-      WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt}
-      WGKEX_DOMAIN_PREFIX: ${WGKEX_DOMAIN_PREFIX-ffmuc_}
+      WGKEX_DOMAINS: ${WGKEX_DOMAINS-ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city}
+      WGKEX_DOMAIN_PREFIXES: ${WGKEX_DOMAIN_PREFIXES-ffmuc_, ffdon_, ffwert_}
       WGKEX_DEBUG: ${WGKEX_DEBUG-DEBUG}
       MQTT_BROKER_URL: ${MQTT_BROKER_URL-mqtt}
       MQTT_BROKER_PORT: ${MQTT_BROKER_PORT-1883}
diff --git a/entrypoint b/entrypoint
@@ -1,15 +1,15 @@
 #!/bin/bash
 set -e
 
-: ${WGKEX_DOMAINS:="ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt"}
-: ${WGKEX_DOMAIN_PREFIX:="ffmuc_"}
-: ${WGKEX_DEBUG:="DEBUG"}
-: ${MQTT_BROKER_URL:="mqtt"}
-: ${MQTT_BROKER_PORT:="1883"}
-: ${MQTT_USERNAME:=""}
-: ${MQTT_PASSWORD:=""}
-: ${MQTT_KEEPALIVE:="5"}
-: ${MQTT_TLS:="False"}
+: "${WGKEX_DOMAINS:=ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city}"
+: "${WGKEX_DOMAIN_PREFIXES:=ffmuc_, ffdon_, ffwert_}"
+: "${WGKEX_DEBUG:=DEBUG}"
+: "${MQTT_BROKER_URL:=mqtt}"
+: "${MQTT_BROKER_PORT:=1883}"
+: "${MQTT_USERNAME:=}"
+: "${MQTT_PASSWORD:=}"
+: "${MQTT_KEEPALIVE:=5}"
+: "${MQTT_TLS:=False}"
 
 mk_config() {
 if [ ! -e /etc/wgkex.yaml ] ; then
@@ -19,9 +19,12 @@ IFS=", "
 for i in $WGKEX_DOMAINS; do
   echo "  - $i"
 done
+echo "domain_prefixes:"
+for i in $WGKEX_DOMAIN_PREFIXES; do
+  echo "  - $i"
+done
 cat <<EOF
 log_level: $WGKEX_DEBUG
-domain_prefix: $WGKEX_DOMAIN_PREFIX
 mqtt:
   broker_url: $MQTT_BROKER_URL
   broker_port: $MQTT_BROKER_PORT
diff --git a/env.example b/env.example
@@ -1,7 +1,7 @@
 # Copy or rename this file to .env and modify if for your needs
 
-#WGKEX_DOMAINS="ffmuc_freising, ffmuc_gauting, ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_uml_nord, ffmuc_uml_ost, ffmuc_uml_sued, ffmuc_uml_west, ffmuc_welt"
-#WGKEX_DOMAIN_PREFIX="ffmuc_"
+#WGKEX_DOMAINS="ffmuc_muc_cty, ffmuc_muc_nord, ffmuc_muc_ost, ffmuc_muc_sued, ffmuc_muc_west, ffmuc_welt, ffwert_city"
+#WGKEX_DOMAIN_PREFIXES="ffmuc_, ffdon_, ffwert_"
 #WGKEX_DEBUG="DEBUG"
 
 #MQTT_BROKER_URL="mqtt"
@@ -10,5 +10,3 @@
 #MQTT_PASSWORD=""
 #MQTT_KEEPALIVE="5"
 #MQTT_TLS="False"
-
-
diff --git a/requirements.txt b/requirements.txt
@@ -1,11 +1,12 @@
-NetLink
+NetLink~=0.1
 flask-mqtt
-pyroute2
-PyYAML
-Flask
-waitress
+pyroute2~=0.7.9
+PyYAML~=6.0.1
+Flask~=3.0.0
+waitress~=2.1.2
 
 # Common
-ipaddress
-mock
-coverage
+ipaddress~=1.0.23
+mock~=5.1.0
+coverage
+paho-mqtt~=1.6.1
diff --git a/wgkex.yaml.example b/wgkex.yaml.example
@@ -1,16 +1,11 @@
 domains:
-  - ffmuc_freising
-  - ffmuc_gauting
   - ffmuc_muc_cty
   - ffmuc_muc_nord
   - ffmuc_muc_ost
   - ffmuc_muc_sued
   - ffmuc_muc_west
-  - ffmuc_uml_nord
-  - ffmuc_uml_ost
-  - ffmuc_uml_sued
-  - ffmuc_uml_west
   - ffmuc_welt
+  - ffwert_city
 mqtt:
   broker_url: broker.hivemq.com
   broker_port: 1883
@@ -21,17 +16,20 @@ mqtt:
 broker_listen:
   host: 0.0.0.0
   port: 5000
-domain_prefix: myprefix-
+domain_prefixes:
+  - ffmuc_
+  - ffdon_
+  - ffwert_
 logging_config:
-    formatters:
-      standard:
-        format: '%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
+  formatters:
+    standard:
+      format: '%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
+  handlers:
+    console:
+      class: logging.StreamHandler
+      formatter: standard
+  root:
     handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: standard
-    root:
-      handlers:
       - console
-      level: DEBUG
-    version: 1
+    level: DEBUG
+  version: 1
diff --git a/wgkex/broker/app.py b/wgkex/broker/app.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+"""wgkex broker"""
 import re
 import dataclasses
 import logging
@@ -17,7 +18,6 @@
 from wgkex.config import config
 from wgkex.common import logger
 
-
 WG_PUBKEY_PATTERN = re.compile(r"^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw480]=$")
 
 
diff --git a/wgkex/broker/templates/index.html b/wgkex/broker/templates/index.html
@@ -1,6 +1,6 @@
 <html>
 <head>
-<title>wgkex</title>
+    <title>wgkex</title>
 </head>
 <body>
 <h1>WGKEX</h1>
diff --git a/wgkex/config/config.py b/wgkex/config/config.py
@@ -1,4 +1,5 @@
 """Configuration handling class."""
+import logging
 import os
 import sys
 import yaml
@@ -41,6 +42,14 @@ class MQTT:
 
     @classmethod
     def from_dict(cls, mqtt_cfg: Dict[str, str]) -> "MQTT":
+        """seems to generate a mqtt config object from dictionary
+
+        Args:
+            mqtt_cfg ():
+
+        Returns:
+            mqtt config object
+        """
         return cls(
             broker_url=mqtt_cfg["broker_url"],
             username=mqtt_cfg["username"],
@@ -60,12 +69,11 @@ class Config:
     Attributes:
         domains: The list of domains to listen for.
         mqtt: The MQTT configuration.
-        domain_prefix: The prefix to pre-pend to a given domain.
-    """
+        domain_prefixes: The list of prefixes to pre-pend to a given domain."""
 
     domains: List[str]
     mqtt: MQTT
-    domain_prefix: str
+    domain_prefixes: List[str]
 
     @classmethod
     def from_dict(cls, cfg: Dict[str, str]) -> "Config":
@@ -79,7 +87,7 @@ def from_dict(cls, cfg: Dict[str, str]) -> "Config":
         return cls(
             domains=cfg["domains"],
             mqtt=mqtt_cfg,
-            domain_prefix=cfg["domain_prefix"],
+            domain_prefixes=cfg["domain_prefixes"],
         )
 
 
@@ -124,6 +132,7 @@ def fetch_config_from_disk() -> str:
         The file contents as string.
     """
     config_file = os.environ.get(WG_CONFIG_OS_ENV, WG_CONFIG_DEFAULT_LOCATION)
+    logging.debug("getting config_file: %s", repr(config_file))
     try:
         with open(config_file, "r") as stream:
             return stream.read()
diff --git a/wgkex/config/config_test.py b/wgkex/config/config_test.py
@@ -1,10 +1,17 @@
+"""Tests for configuration handling class."""
 import unittest
 import mock
 import config
 import yaml
 
-_VALID_CFG = "domain_prefix: ffmuc_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n  broker_port: 1883\n  broker_url: mqtt://broker\n  keepalive: 5\n  password: pass\n  tls: true\n  username: user\n"
-_INVALID_LINT = "domain_prefix: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n  broker_port: 1883\n  broker_url: mqtt://broker\n  keepalive: 5\n  password: pass\n  tls: true\n  username: user\n"
+_VALID_CFG = (
+    "domain_prefixes:\n- ffmuc_\n- ffdon_\n- ffwert_\nlog_level: DEBUG\ndomains:\n- a\n- b\nmqtt:\n  broker_port: 1883"
+    "\n  broker_url: mqtt://broker\n  keepalive: 5\n  password: pass\n  tls: true\n  username: user\n"
+)
+_INVALID_LINT = (
+    "domain_prefixes: ffmuc_\nBAD_KEY_FOR_DOMAIN:\n- a\n- b\nmqtt:\n  broker_port: 1883\n  broker_url: "
+    "mqtt://broker\n  keepalive: 5\n  password: pass\n  tls: true\n  username: user\n"
+)
 _INVALID_CFG = "asdasdasdasd"
 
 
@@ -52,7 +59,7 @@ def test_fetch_from_config_success(self):
             self.assertListEqual(["a", "b"], config.fetch_from_config("domains"))
 
     def test_fetch_from_config_no_key_in_config(self):
-        """Test fetch non existent key from configuration."""
+        """Test fetch non-existent key from configuration."""
         mock_open = mock.mock_open(read_data=_VALID_CFG)
         with mock.patch("builtins.open", mock_open):
             self.assertIsNone(config.fetch_from_config("key_does_not_exist"))
diff --git a/wgkex/worker/app.py b/wgkex/worker/app.py
@@ -20,6 +20,14 @@ class DomainsNotInConfig(Error):
     """If no domains exist in configuration file."""
 
 
+class PrefixesNotInConfig(Error):
+    """If no prefixes exist in configuration file."""
+
+
+class DomainsAreNotUnique(Error):
+    """If non-unique domains exist in configuration file."""
+
+
 def flush_workers(domain: Text) -> None:
     """Calls peer flush every _CLEANUP_TIME interval."""
     while True:
@@ -35,31 +43,73 @@ def clean_up_worker(domains: List[Text]) -> None:
         domains: list of domains.
     """
     logger.debug("Cleaning up the following domains: %s", domains)
-    prefix = config.load_config().get("domain_prefix")
+    prefixes = config.load_config().get("domain_prefixes")
+    cleanup_counter = 0
+    # ToDo: do we need a check if every domain got gleaned?
+    for prefix in prefixes:
+        for domain in domains:
+            if prefix in domain:
+                logger.info("Scheduling cleanup task for %s, ", domain)
+                try:
+                    cleaned_domain = domain.split(prefix)[1]
+                    cleanup_counter += 1
+                except IndexError:
+                    logger.error(
+                        "Cannot strip domain with prefix %s from passed value %s. Skipping cleanup operation",
+                        prefix,
+                        domain,
+                    )
+                    continue
+                thread = threading.Thread(target=flush_workers, args=(cleaned_domain,))
+                thread.start()
+    if cleanup_counter < len(domains):
+        logger.error(
+            "Not every domain got cleaned. Check domains for missing prefixes",
+            repr(domains),
+            repr(prefixes),
+        )
+
+
+def check_all_domains_unique(domains, prefixes):
+    """strips off prefixes and checks if domains are unique
+
+    Args:
+        domains: [str]
+    Returns:
+        boolean
+    """
+    if not prefixes:
+        raise PrefixesNotInConfig("Could not locate prefixes in configuration.")
+    if not isinstance(prefixes, list):
+        raise TypeError("prefixes is not a list")
+    unique_domains = []
     for domain in domains:
-        logger.info("Scheduling cleanup task for %s, ", domain)
-        try:
-            cleaned_domain = domain.split(prefix)[1]
-        except IndexError:
-            logger.error(
-                "Cannot strip domain with prefix %s from passed value %s. Skipping cleanup operation",
-                prefix,
-                domain,
-            )
-            continue
-        thread = threading.Thread(target=flush_workers, args=(cleaned_domain,))
-        thread.start()
+        for prefix in prefixes:
+            if prefix in domain:
+                stripped_domain = domain.split(prefix)[1]
+                if stripped_domain in unique_domains:
+                    logger.error(
+                        "We have a non-unique domain here",
+                        domain,
+                    )
+                    return False
+                unique_domains.append(stripped_domain)
+    return True
 
 
 def main():
     """Starts MQTT listener.
 
     Raises:
         DomainsNotInConfig: If no domains were found in configuration file.
+        DomainsAreNotUnique: If there were non-unique domains after stripping prefix
     """
     domains = config.load_config().get("domains")
+    prefixes = config.load_config().get("domain_prefixes")
     if not domains:
         raise DomainsNotInConfig("Could not locate domains in configuration.")
+    if not check_all_domains_unique(domains, prefixes):
+        raise DomainsAreNotUnique("There are non-unique domains! Check config.")
     clean_up_worker(domains)
     watch_queue()
     mqtt.connect()
diff --git a/wgkex/worker/app_test.py b/wgkex/worker/app_test.py
diff --git a/wgkex/worker/mqtt.py b/wgkex/worker/mqtt.py
