Merge remote-tracking branch 'upstream/pr/1787' into maintenance

Author: Sebastian Wagner

CHANGELOG.md

@@ -36,6 +36,10 @@ CHANGELOG
 - Add missing newlines at end of various test input files (PR#1785 by Sebastian Wagner, fixes #1777).
 
 ### Tools
+- `intelmqsetup`:
+  - Also cover required directory layout and file permissions for `intelmq-api` (PR#1787 by Sebastian Wagner, fixes #1783).
+- `intelmqctl`:
+  - Do not log an error message if logging to file is explicitly disabled, e.g. in calls from `intelmsetup`. The error message would not be useful for the user and is not necessary.
 
 ### Contrib
 

intelmq/bin/intelmqctl.py

@@ -710,7 +710,7 @@ def __init__(self, interactive: bool = False, return_type: str = "python", quiet
 
         try:
             if no_file_logging:
-                raise FileNotFoundError
+                raise FileNotFoundError('Logging to file disabled.')
             logger = utils.log('intelmqctl', log_level=self.logging_level,
                                log_format_stream=utils.LOG_FORMAT_SIMPLE,
                                logging_level_stream=logging_level_stream,
@@ -720,7 +720,8 @@ def __init__(self, interactive: bool = False, return_type: str = "python", quiet
             logger = utils.log('intelmqctl', log_level=self.logging_level, log_path=False,
                                log_format_stream=utils.LOG_FORMAT_SIMPLE,
                                logging_level_stream=logging_level_stream)
-            logger.error('Not logging to file: %s', exc)
+            if not isinstance(exc, FileNotFoundError) and exc.args[0] != 'Logging to file disabled.':
+                logger.error('Not logging to file: %s', exc)
         self.logger = logger
         if defaults_loading_exc:
             self.logger.exception('Loading the defaults configuration failed!',

intelmq/bin/intelmqsetup.py

@@ -1,75 +1,162 @@
 # -*- coding: utf-8 -*-
 """
-© 2019 Sebastian Wagner <wagner@cert.at>
+© 2019-2021 nic.at GmbH <intelmq-team@cert.at>
 
-SPDX-License-Identifier: AGPL-3.0
+SPDX-License-Identifier: AGPL-3.0-only
 
 Sets up an intelmq environment after installation or upgrade by
  * creating needed directories
  * set intelmq as owner for those
  * providing example configuration files if not already existing
 
+If intelmq-api is installed, the similar steps are performed:
+ * creates needed directories
+ * sets the webserver as group for them
+ * sets group write permissions
+
 Reasoning:
 Pip does not (and cannot) create `/opt/intelmq`/user-given ROOT_DIR, as described in
 https://github.com/certtools/intelmq/issues/819
 """
 import argparse
-import glob
 import os
 import shutil
+import stat
 import sys
 import pkg_resources
 
-from pwd import getpwuid
+from grp import getgrnam
+from pathlib import Path
+from pwd import getpwnam
+from typing import Optional
+
+try:
+    import intelmq_api
+except ImportError:
+    intelmq_api = None
 
+from termstyle import red
 from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
                      VAR_STATE_PATH, BOTS_FILE, STATE_FILE_PATH)
 from intelmq.bin.intelmqctl import IntelMQController
 
 
-def intelmqsetup(ownership=True, state_file=STATE_FILE_PATH):
-    if os.geteuid() != 0 and ownership:
-        sys.exit('You need to run this program as root (for setting file ownership)')
+MANAGER_CONFIG_DIR = Path(CONFIG_DIR) / 'manager/'
+FILE_OUTPUT_PATH = Path(VAR_STATE_PATH) / 'file-output/'
 
+
+def basic_checks(skip_ownership):
+    if os.geteuid() != 0 and not skip_ownership:
+        sys.exit(red('You need to run this program as root for setting file ownership!'))
     if not ROOT_DIR:
-        sys.exit('Not a pip-installation of IntelMQ, nothing to initialize.')
-
-    create_dirs = ('%s/file-output' % VAR_STATE_PATH,
-                   VAR_RUN_PATH,
-                   DEFAULT_LOGGING_PATH,
-                   CONFIG_DIR)
-    for create_dir in create_dirs:
-        if not os.path.isdir(create_dir):
-            os.makedirs(create_dir, mode=0o755,
-                        exist_ok=True)
-            print('Created directory %r.' % create_dir)
-
-    example_confs = glob.glob(pkg_resources.resource_filename('intelmq', 'etc/*.conf'))
+        sys.exit(red('Not a pip-installation of IntelMQ, nothing to initialize.'))
+
+    if skip_ownership:
+        return
+    try:
+        getpwnam('intelmq')
+    except KeyError:
+        sys.exit(red("User 'intelmq' does not exist. Please create it and then re-run this program."))
+    try:
+        getgrnam('intelmq')
+    except KeyError:
+        sys.exit(red("Group 'intelmq' does not exist. Please create it and then re-run this program."))
+
+
+def create_directory(directory: str, octal_mode: int):
+    directory = Path(directory)
+    readable_mode = stat.filemode(octal_mode)
+    if not directory.is_dir():
+        directory.mkdir(mode=octal_mode, exist_ok=True, parents=True)
+        print(f'Created directory {directory!s} with permissions {readable_mode}.')
+    else:
+        current_mode = directory.stat().st_mode
+        if current_mode != octal_mode:
+            current_mode_readable = stat.filemode(current_mode)
+            print(f'Fixed wrong permissions of {directory!s}: {current_mode_readable!r} -> {readable_mode!r}.')
+            directory.chmod(octal_mode)
+
+
+def change_owner(file: str, owner=None, group=None, log: bool = True):
+    if owner and Path(file).owner() != owner:
+        if log:
+            print(f'Fixing owner of {file!s}.')
+        shutil.chown(file, user=owner)
+    if group and Path(file).group() != group:
+        if log:
+            print(f'Fixing group of {file!s}.')
+        shutil.chown(file, group=group)
+
+
+def find_webserver_user():
+    candidates = ('www-data', 'wwwrun', 'httpd', 'apache')
+    for candidate in candidates:
+        try:
+            getpwnam(candidate)
+        except KeyError:
+            pass
+        else:
+            print(f'Detected webserver username {candidate!r}.')
+            return candidate
+
+
+def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
+    create_directory(FILE_OUTPUT_PATH, 0o40755)
+    create_directory(VAR_RUN_PATH, 0o40755)
+    create_directory(DEFAULT_LOGGING_PATH, 0o40755)
+    create_directory(CONFIG_DIR, 0o40775)
+
+    example_confs = Path(pkg_resources.resource_filename('intelmq', 'etc')).glob('*.conf')
     for example_conf in example_confs:
-        fname = os.path.split(example_conf)[-1]
-        if os.path.exists(os.path.join(CONFIG_DIR, fname)):
-            print('Not overwriting existing %r with example.' % fname)
+        fname = Path(example_conf).name
+        destination_file = Path(CONFIG_DIR) / fname
+        if destination_file.exists():
+            print(f'Not overwriting existing {fname!r} with example.')
+            log_ownership_change = True
         else:
             shutil.copy(example_conf, CONFIG_DIR)
-            print('Use example %r.' % fname)
-
-    print('Writing BOTS file.')
-    shutil.copy(pkg_resources.resource_filename('intelmq', 'bots/BOTS'),
-                BOTS_FILE)
+            print(f'Installing example {fname!r} to {CONFIG_DIR}.')
+            log_ownership_change = False  # For installing the new files, we don't need to inform the admin that the permissions have been "fixed"
+        if ownership:
+            change_owner(destination_file, owner='intelmq', group='intelmq', log=log_ownership_change)
+
+    if Path(BOTS_FILE).is_symlink():
+        print('Skip writing BOTS file as it is a link.')
+    else:
+        print('Writing BOTS file.')
+        shutil.copy(pkg_resources.resource_filename('intelmq', 'bots/BOTS'),
+                    BOTS_FILE)
 
     if ownership:
         print('Setting intelmq as owner for it\'s directories.')
         for obj in (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
-                    VAR_STATE_PATH, VAR_STATE_PATH + 'file-output'):
-            if getpwuid(os.stat(obj).st_uid).pw_name != 'intelmq':
-                shutil.chown(obj, user='intelmq')
+                    VAR_STATE_PATH, FILE_OUTPUT_PATH):
+            change_owner(obj, owner='intelmq')
 
-    print('Calling `intelmqctl upgrade-config to update/create state file')
+    print('Calling `intelmqctl upgrade-config` to update/create state file.')
     controller = IntelMQController(interactive=False, no_file_logging=True,
                                    drop_privileges=False)
     controller.upgrade_conf(state_file=state_file, no_backup=True)
 
 
+def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = None):
+    if ownership:
+        change_owner(CONFIG_DIR, group='intelmq')
+
+    # Manager configuration directory
+    create_directory(MANAGER_CONFIG_DIR, 0o40775)
+    if ownership:
+        change_owner(MANAGER_CONFIG_DIR, group='intelmq')
+
+    intelmq_group = getgrnam('intelmq')
+    webserver_user = webserver_user or find_webserver_user()
+    if webserver_user not in intelmq_group.gr_mem:
+        sys.exit(red(f"Webserver user {webserver_user} is not a member of the 'intelmq' group. "
+                     f"Please add it with: 'usermod -aG intelmq {webserver_user}'."))
+
+    print('Setup of intelmq-api successful.')
+
+
 def main():
     parser = argparse.ArgumentParser("Set's up directories and example "
                                      "configurations for IntelMQ.")
@@ -78,9 +165,17 @@ def main():
     parser.add_argument('--state-file',
                         help='The state file location to use.',
                         default=STATE_FILE_PATH)
+    parser.add_argument('--webserver-user',
+                        help='The webserver to use instead of auto-detection.')
     args = parser.parse_args()
-    intelmqsetup(ownership=not args.skip_ownership,
-                 state_file=args.state_file)
+
+    basic_checks(skip_ownership=args.skip_ownership)
+    intelmqsetup_core(ownership=not args.skip_ownership,
+                      state_file=args.state_file)
+    if intelmq_api:
+        print('Running setup for intelmq-api.')
+        intelmqsetup_api(ownership=not args.skip_ownership,
+                         webserver_user=args.webserver_user)
 
 
 if __name__ == '__main__':