Merge branch 'maintenance' into develop

Author: Sebastian Wagner

CHANGELOG.md

@@ -104,6 +104,7 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). Old n
 ### Tools
 - `intelmqsetup`:
   - Also cover required directory layout and file permissions for `intelmq-api` (PR#1787 by Sebastian Wagner, fixes #1783).
+  - Also cover webserver and sudoers configuration for `intelmq-api` and `intelmq-manger` (PR#1805 by Sebastian Wagner, fixes #1803).
 - `intelmqctl`:
   - Do not log an error message if logging to file is explicitly disabled, e.g. in calls from `intelmsetup`. The error message would not be useful for the user and is not necessary.
 

debian/rules

@@ -61,7 +61,7 @@ override_dh_auto_install: $(BOTDOCS)
 	# remove program not needed for packages
 	rm debian/intelmq/usr/bin/intelmqsetup
 	# create directory layout and empty state file
-	ROOT_DIR=debian/intelmq/ PYTHONPATH=. python3 intelmq/bin/intelmqsetup.py --skip-ownership --state-file debian/intelmq/var/lib/intelmq/state.json
+	ROOT_DIR=debian/intelmq/ PYTHONPATH=. python3 intelmq/bin/intelmqsetup.py --skip-ownership --state-file debian/intelmq/var/lib/intelmq/state.json --skip-api
 
 override_dh_install:
 	dh_install

intelmq/bin/intelmqsetup.py

@@ -28,21 +28,31 @@
 from grp import getgrnam
 from pathlib import Path
 from pwd import getpwnam
+from subprocess import run, CalledProcessError
+from tempfile import NamedTemporaryFile
 from typing import Optional
 
 try:
     import intelmq_api
 except ImportError:
     intelmq_api = None
 
+try:
+    import intelmq_manager
+except ImportError:
+    intelmq_manager = None
+
 from termstyle import red
 from intelmq import (CONFIG_DIR, DEFAULT_LOGGING_PATH, ROOT_DIR, VAR_RUN_PATH,
                      VAR_STATE_PATH, BOTS_FILE, STATE_FILE_PATH)
 from intelmq.bin.intelmqctl import IntelMQController
 
 
-MANAGER_CONFIG_DIR = Path(CONFIG_DIR) / 'manager/'
 FILE_OUTPUT_PATH = Path(VAR_STATE_PATH) / 'file-output/'
+ETC_INTELMQ = Path('/etc/intelmq/')
+ETC_INTELMQ_MANAGER = ETC_INTELMQ / 'manager/'
+WEBSERVER_CONFIG_DIR = None  # "cache" for the webserver configuration directory
+NOTE_WEBSERVER_RELOAD = False  # if the webserver needs to be reloaded
 
 
 def basic_checks(skip_ownership):
@@ -77,7 +87,7 @@ def create_directory(directory: str, octal_mode: int):
             directory.chmod(octal_mode)
 
 
-def change_owner(file: str, owner=None, group=None, log: bool = True):
+def change_owner(file: str, owner: Optional[str] = None, group: Optional[str] = None, log: bool = True):
     if owner and Path(file).owner() != owner:
         if log:
             print(f'Fixing owner of {file!s}.')
@@ -96,8 +106,39 @@ def find_webserver_user():
         except KeyError:
             pass
         else:
-            print(f'Detected webserver username {candidate!r}.')
+            print(f'Detected Apache username {candidate!r}.')
             return candidate
+    else:
+        sys.exit(red("Unable to detect Apache user name. "
+                     "Please re-run this program and give the Apache user name with '--webserver-user'."))
+
+
+def find_webserver_configuration_directory():
+    global WEBSERVER_CONFIG_DIR
+    if WEBSERVER_CONFIG_DIR:
+        return WEBSERVER_CONFIG_DIR
+    webserver_configuration_dir_candidates = (Path('/etc/apache2/conf-available/'),
+                                              Path('/etc/apache2/conf.d/'),
+                                              Path('/etc/httpd/conf.d/'))
+    for webserver_configuration_dir_candidate in webserver_configuration_dir_candidates:
+        if webserver_configuration_dir_candidate.exists():
+            print(f'Detected Apache configuration directory {webserver_configuration_dir_candidate!s}.')
+            WEBSERVER_CONFIG_DIR = webserver_configuration_dir_candidate
+            webserver_configuration_dir_candidate.as_posix
+            return webserver_configuration_dir_candidate
+    else:
+        sys.exit(red("Unable to detect Apache configuration directory. "
+                     "Please re-run this program and give the Apache configuration directory with '--webserver-configuration-directory'."))
+
+
+def debian_activate_apache_config(config_name: str):
+    if 'available' not in WEBSERVER_CONFIG_DIR.as_posix():
+        return  # not a Debian system
+    available = WEBSERVER_CONFIG_DIR / config_name
+    enabled = Path(WEBSERVER_CONFIG_DIR.as_posix().replace('available', 'enabled')) / config_name
+    if not enabled.exists():
+        enabled.symlink_to(available)
+        print('Created symbolic link {enabled!s} pointing to {available!s}.')
 
 
 def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
@@ -140,23 +181,109 @@ def intelmqsetup_core(ownership=True, state_file=STATE_FILE_PATH):
 
 
 def intelmqsetup_api(ownership: bool = True, webserver_user: Optional[str] = None):
+    intelmq_group = getgrnam('intelmq')
+    webserver_user = webserver_user or find_webserver_user()
+
+    create_directory(ETC_INTELMQ, 0o40775)
     if ownership:
         change_owner(CONFIG_DIR, group='intelmq')
+        change_owner(ETC_INTELMQ, owner='intelmq', group='intelmq')
 
     # Manager configuration directory
-    create_directory(MANAGER_CONFIG_DIR, 0o40775)
+    create_directory(ETC_INTELMQ_MANAGER, 0o40775)
     if ownership:
-        change_owner(MANAGER_CONFIG_DIR, group='intelmq')
+        change_owner(ETC_INTELMQ_MANAGER, group='intelmq')
+
+    base = Path(pkg_resources.resource_filename('intelmq_api', '')).parent
+    api_config = base / 'etc/intelmq/api-config.json'
+    etc_intelmq_config = ETC_INTELMQ / 'api-config.json'
+    api_sudoers = base / 'etc/intelmq/api-sudoers.conf'
+    etc_sudoers_api = Path('/etc/sudoers.d/01_intelmq-api')  # same path as used in the packages
+    api_manager_positions = base / 'etc/intelmq/manager/positions.conf'
+    etc_intelmq_manager_positions = ETC_INTELMQ_MANAGER / 'positions.conf'
+
+    if not base.as_posix().startswith('/usr/'):
+        # Paths differ in editable installations
+        print(red("Detected an editable (egg-link) pip-installation of 'intelmq-api'. Some feature of this program may not work."))
+
+    if api_config.exists() and not etc_intelmq_config.exists():
+        shutil.copy(api_config, etc_intelmq_config)
+        print(f'Copied {api_config!s} to {ETC_INTELMQ!s}.')
+    elif not api_config.exists() and not etc_intelmq_config.exists():
+        print(red(f'Unable to install api-config.json: Neither {api_config!s} nor {etc_intelmq_config!s} exists.'))
+    if api_sudoers.exists() and not etc_sudoers_api.exists():
+        with open(api_sudoers) as sudoers:
+            original_sudoers = sudoers.read()
+        sudoers = original_sudoers.replace('www-data', webserver_user)
+        with NamedTemporaryFile(mode='w') as tmp_file:
+            tmp_file.write(sudoers)
+            tmp_file.flush()
+            try:
+                run(('visudo', '-c', tmp_file.name))
+            except CalledProcessError:
+                sys.exit(red('Fatal error: Validation of adapted sudoers-file failed. Please report this bug.'))
+            change_owner(tmp_file.name, owner='root', group='root', log=False)
+            Path(tmp_file.name).chmod(0o440)
+            shutil.copy(tmp_file.name, etc_sudoers_api)
+        print(f'Copied {api_sudoers!s} to {etc_sudoers_api!s}.')
+    elif not api_sudoers.exists() and not etc_sudoers_api.exists():
+        print(red(f'Unable to install api-sudoers.conf: Neither {api_sudoers!s} nor {etc_sudoers_api!s} exists.'))
+    if api_manager_positions.exists() and not etc_intelmq_manager_positions.exists():
+        shutil.copy(api_manager_positions, etc_intelmq_manager_positions)
+        print(f'Copied {api_manager_positions!s} to {etc_intelmq_manager_positions!s}.')
+        etc_intelmq_manager_positions.chmod(0o664)
+        change_owner(etc_intelmq_manager_positions, owner='intelmq', group='intelmq', log=False)
+    elif not api_manager_positions.exists() and not etc_intelmq_manager_positions.exists():
+        print(red(f'Unable to install positions.conf: Neither {api_manager_positions!s} nor {etc_intelmq_manager_positions!s} exists.'))
 
-    intelmq_group = getgrnam('intelmq')
-    webserver_user = webserver_user or find_webserver_user()
     if webserver_user not in intelmq_group.gr_mem:
         sys.exit(red(f"Webserver user {webserver_user} is not a member of the 'intelmq' group. "
                      f"Please add it with: 'usermod -aG intelmq {webserver_user}'."))
 
+
+def intelmqsetup_api_webserver_configuration(webserver_configuration_directory: Optional[str] = None):
+    webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory()
+    api_config = Path(pkg_resources.resource_filename('intelmq_api', '')).parent / 'etc/intelmq/api-apache.conf'
+    apache_api_config = webserver_configuration_dir / 'api-apache.conf'
+    if api_config.exists() and not apache_api_config.exists():
+        shutil.copy(api_config, apache_api_config)
+        print(f'Copied {api_config!s} to {ETC_INTELMQ!s}.')
+        debian_activate_apache_config('api-apache.conf')
+
+        global NOTE_WEBSERVER_RELOAD
+        NOTE_WEBSERVER_RELOAD = True
+    elif not api_config.exists() and not apache_api_config.exists():
+        print(red(f'Unable to install webserver configuration api-config.conf: Neither {api_config!s} nor {apache_api_config!s} exists.'))
+
     print('Setup of intelmq-api successful.')
 
 
+def intelmqsetup_manager_webserver_configuration(webserver_configuration_directory: Optional[str] = None):
+    html_dir = Path(pkg_resources.resource_filename('intelmq_manager', '')).parent / '/usr/share/intelmq-manager/html'
+    html_dir_destination = Path('/usr/share/intelmq-manager/html')
+
+    if not html_dir_destination.as_posix().startswith('/usr/'):
+        # Paths differ in editable installations
+        print(red("Detected an editable (egg-link) pip-installation of intelmq-manager. Some feature of this program may not work."))
+
+    webserver_configuration_dir = webserver_configuration_directory or find_webserver_configuration_directory()
+    manager_config = Path(pkg_resources.resource_filename('intelmq_manager', '')).parent / 'etc/intelmq/manager-apache.conf'
+    apache_manager_config = webserver_configuration_dir / 'manager-apache.conf'
+    if manager_config.exists() and not apache_manager_config.exists():
+        shutil.copy(manager_config, apache_manager_config)
+        print(f'Copied {manager_config!s} to {apache_manager_config!s}.')
+        debian_activate_apache_config('manager-apache.conf')
+
+        global NOTE_WEBSERVER_RELOAD
+        NOTE_WEBSERVER_RELOAD = True
+    elif not manager_config.exists() and not apache_manager_config.exists():
+        print(red(f'Unable to install webserver configuration manager-config.conf: Neither {manager_config!s} nor {apache_manager_config!s} exists.'))
+
+    if html_dir.exists():
+        shutil.copy(html_dir, '/')
+        print(f'Copied {html_dir!s} to {html_dir_destination!s}.')
+
+
 def main():
     parser = argparse.ArgumentParser("Set's up directories and example "
                                      "configurations for IntelMQ.")
@@ -167,15 +294,34 @@ def main():
                         default=STATE_FILE_PATH)
     parser.add_argument('--webserver-user',
                         help='The webserver to use instead of auto-detection.')
+    parser.add_argument('--webserver-configuration-directory',
+                        help='The webserver configuration directory to use instead of auto-detection.')
+    parser.add_argument('--skip-api',
+                        help='Skip set-up of intelmq-api.',
+                        action='store_true')
     args = parser.parse_args()
 
     basic_checks(skip_ownership=args.skip_ownership)
     intelmqsetup_core(ownership=not args.skip_ownership,
                       state_file=args.state_file)
-    if intelmq_api:
+    if intelmq_api and not args.skip_api:
         print('Running setup for intelmq-api.')
         intelmqsetup_api(ownership=not args.skip_ownership,
                          webserver_user=args.webserver_user)
+        print('Running webserver setup for intelmq-api.')
+        intelmqsetup_api_webserver_configuration(webserver_configuration_directory=args.webserver_configuration_directory)
+    else:
+        print('Skipping set-up of intelmq-api.')
+    if intelmq_manager and not args.skip_api:
+        print('Running webserver setup for intelmq-manager.')
+        intelmqsetup_manager_webserver_configuration(webserver_configuration_directory=args.webserver_configuration_directory)
+    else:
+        print('Skipping set-up of intelmq-manager.')
+
+    if NOTE_WEBSERVER_RELOAD:
+        print('Reload the webserver to make the changes effective.')
+
+    print("'intelmqsetup' completed.")
 
 
 if __name__ == '__main__':