ENH: Rewrite the configuration file handling

In IntelMQ 3 the defaults.conf and the pipeline.conf were dropped,
therefore access to those files via the API is not useful anymore.
At the same time, the runtime.conf now includes pipeline configuration
and is stored as a YAML file. Given those changes, the whole
configuration file handling had to be rewritten: There are now 3 API
endpoints:
* one to read (get) and write (post) the runtime configuration
* one to read (get) and write (post) the positions configuration
* one to read (get) the harmonization information

Everything is still done using JSON objects, the conversion to YAML is
done internally.
The removal of the save_file method also includes the removal of some
sanity checks that ran before the file was saved. Some of those should
probably be part of intelmq itself (i.e. checking for allowed characters
in bot names).

Author: Birger Schacht

intelmq_api/api.py

@@ -13,6 +13,7 @@
 
 import sys
 import os
+import pathlib
 import string
 import typing
 
@@ -23,6 +24,7 @@
 import intelmq_api.config
 import intelmq_api.session as session
 
+from intelmq.lib import utils  # type: ignore
 
 Levels = hug.types.OneOf(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL",
                           "ALL"])
@@ -73,7 +75,6 @@ def initialize_api(config: intelmq_api.config.Config) -> None:
 
     runner = runctl.RunIntelMQCtl(api_config.intelmq_ctl_cmd)
     file_access = files.FileAccess(api_config)
-    file_access.update_from_runctl(runner.get_paths())
 
     session_file = api_config.session_store
     if session_file is not None:
@@ -181,18 +182,6 @@ def config(response, file: str, fetch: bool=False):
     return contents
 
 
-@hug.post("/api/save", parse_body=True,
-          inputs={"application/x-www-form-urlencoded": hug.input_format.text},
-          requires=token_authentication, versions=1)
-def save(body, file: str):
-    global file_access
-    try:
-        file_access.save_file(file, body)
-        return "success"
-    except files.SaveFileException as e:
-        return str(e)
-
-
 @hug.post("/api/login", versions=1)
 def login(username: str, password: str):
     if session_store is not None:
@@ -203,3 +192,41 @@ def login(username: str, password: str):
                     "username": username,
                     }
     return "Invalid username and/or password"
+
+
+@hug.get("/api/harmonization", requires=token_authentication, versions=1)
+def get_harmonization():
+    positions = pathlib.Path('/opt/intelmq/etc/harmonization.conf')
+    paths = runner.get_paths()
+    if 'CONFIG_DIR' in paths:
+        positions = pathlib.Path(paths['CONFIG_DIR']) / 'harmonization.conf'
+    return positions.read_text()
+
+
+@hug.get("/api/runtime", requires=token_authentication, versions=1)
+def get_runtime():
+    return utils.get_runtime()
+
+
+@hug.post("/api/runtime", requires=token_authentication, version=1)
+def post_runtime(body):
+    return utils.set_runtime(body)
+
+
+@hug.get("/api/positions", requires=token_authentication, versions=1)
+def get_positions():
+    positions = pathlib.Path('/opt/intelmq/etc/manager/positions.conf')
+    paths = runner.get_paths()
+    if 'CONFIG_DIR' in paths:
+        positions = pathlib.Path(paths['CONFIG_DIR']) / 'manager/positions.conf'
+    return positions.read_text()
+
+
+@hug.post("/api/positions", requires=token_authentication, version=1)
+def post_positions(body):
+    positions = pathlib.Path('/opt/intelmq/etc/manager/positions.conf')
+    paths = runner.get_paths()
+    if 'CONFIG_DIR' in paths:
+        positions = pathlib.Path(paths['CONFIG_DIR']) / 'manager/positions.conf'
+    positions.write_text(body)
+    return positions.read_text()

intelmq_api/files.py

@@ -21,15 +21,6 @@
 from intelmq_api.config import Config
 
 
-BOT_CONFIG_CHARS = set(string.ascii_letters + string.digits + string.punctuation
-                       + " \n\r\t")
-
-
-class SaveFileException(Exception):
-
-    """Exception thrown for errors/invalid input in save_file"""
-
-
 def path_starts_with(path: PurePath, prefix: PurePath) -> bool:
     """Return whether the path starts with prefix.
 
@@ -51,33 +42,8 @@ class FileAccess:
     def __init__(self, config: Config):
         self.allowed_path = config.allowed_path
 
-        self.writable_files = {
-            'defaults': Path('/opt/intelmq/etc/defaults.conf'),
-            'pipeline': Path('/opt/intelmq/etc/pipeline.conf'),
-            'runtime': Path('/opt/intelmq/etc/runtime.conf'),
-            'positions': Path('/opt/intelmq/etc/manager/positions.conf'),
-            } # type: Dict[str, Path]
-
-        self.readonly_files = {
-            'harmonization': Path('/opt/intelmq/etc/harmonization.conf'),
-            } # type: Dict[str, Path]
-
-    def update_from_runctl(self, paths: Dict[str, str]):
-        self.writable_files["defaults"] = Path(paths["DEFAULTS_CONF_FILE"])
-        self.writable_files["pipeline"] = Path(paths["PIPELINE_CONF_FILE"])
-        self.writable_files["runtime"] = Path(paths["RUNTIME_CONF_FILE"])
-        self.writable_files["positions"] = Path(paths["CONFIG_DIR"]
-                                                + "/manager/positions.conf")
-        self.readonly_files["harmonization"] = Path(paths["HARMONIZATION_CONF_FILE"])
-
     def file_name_allowed(self, filename: str) -> Optional[Tuple[bool, Path]]:
         """Determine wether the API should allow access to a file."""
-        predefined = self.writable_files.get(filename)
-        if predefined is None:
-            predefined = self.readonly_files.get(filename)
-        if predefined is not None:
-            return (True, predefined)
-
         resolved = Path(filename).resolve()
         if not path_starts_with(resolved, self.allowed_path):
             return None
@@ -114,34 +80,3 @@ def load_file_or_directory(self, unvalidated_filename: str, fetch: bool) \
                 obj = {"size": stat.st_size, "path": str(path.resolve())}
             result["files"][path.name] = obj
         return (content_type, result)
-
-    def save_file(self, unvalidated_filename: str, contents: str) -> None:
-        target_path = self.writable_files.get(unvalidated_filename)
-        if target_path is None:
-            raise SaveFileException("Invalid filename: {!r}"
-                                    .format(unvalidated_filename))
-
-        try:
-            parsed = json.loads(contents)
-        except json.JSONDecodeError as e:
-            raise SaveFileException("File contents for {!r} are not JSON: {}"
-                                    .format(unvalidated_filename, str(e)))
-        if not isinstance(parsed, dict):
-            raise SaveFileException("File must contain a JSON object.")
-
-        if unvalidated_filename not in ("defaults", "positions"):
-            for key in parsed.keys():
-                if key != "__default__" and re.search("[^A-Za-z0-9.-]", key):
-                    raise SaveFileException("Invalid bot ID: {!r}".format(key))
-
-        if not set(contents) < BOT_CONFIG_CHARS:
-            raise SaveFileException("Config has invalid characters");
-
-
-        old_contents = target_path.read_text()
-        if contents != old_contents:
-            try:
-                target_path.write_text(contents, encoding="utf-8")
-            except IOError:
-                # TODO: log details of this error
-                raise SaveFileException("Could not write file.")