Make server more flexible (#576)

dsblank · web-flow · commit 7615c84ec82f · 2024-11-26T20:52:54.000+01:00
* Added pythonish query language

* Moved to object-ql

* New version of object_ql

* Poke it to make it run tests again

* Update pyproject.toml, oql 0.1.2

* Added tests for object-ql

* Add more options to server

* Update requirements

* Update requirements

* Use app.logger

* Fix log message level on adding a user

* Revise flags; messages

* Update translogger; linting

* Allow gramps_webapi to be called without exiting

* Allow gramps_webapi to be called without exiting

* Fixed typo

* Change message to gramps-web-api server

* Added copyright
diff --git a/gramps_webapi/__main__.py b/gramps_webapi/__main__.py
@@ -2,6 +2,7 @@
 # Gramps Web API - A RESTful API for the Gramps genealogy program
 #
 # Copyright (C) 2020      David Straub
+# Copyright (C) 2024      Doug Blank
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -27,18 +28,19 @@
 import sys
 import time
 import warnings
+from threading import Thread
 
 import click
+import waitress
+import webbrowser
 
 from .api.search import get_search_indexer
 from .api.util import get_db_manager, list_trees, close_db
 from .app import create_app
 from .auth import add_user, delete_user, fill_tree, user_db
 from .const import ENV_CONFIG_FILE, TREE_MULTI
 from .dbmanager import WebDbManager
-
-logging.basicConfig()
-LOG = logging.getLogger("gramps_webapi")
+from .translogger import TransLogger
 
 
 @click.group("cli")
@@ -58,12 +60,80 @@ def cli(ctx, config):
 
 @cli.command("run")
 @click.option("-p", "--port", help="Port to use (default: 5000)", default=5000)
-@click.option("--tree", help="Tree ID", default=None)
+@click.option("-t", "--tree", help="Tree ID: '*' for multi-trees", default=None)
+@click.option(
+    "-o",
+    "--open-browser",
+    help="Open gramps-web in browser: 'tab', 'window', or 'no'",
+    default="no",
+    type=click.Choice(["tab", "window", "no"], case_sensitive=False),
+)
+@click.option(
+    "-d",
+    "--debug-level",
+    help="Debug level: 'info', 'debug', 'warning', 'critical'",
+    default="info",
+    type=click.Choice(["info", "debug", "warning", "critical"], case_sensitive=False),
+)
+@click.option("-l", "--log-file", help="Set logging file to this path", default=None)
+@click.option(
+    "--host", help="Set the host address for server to listen on", default="127.0.0.1"
+)
+@click.option(
+    "--max-workers",
+    help="Maximum number of workers for frontend; requires --use-wsgi",
+    default=None,
+)
+@click.option("--use-wsgi", is_flag=True, help="Add a wsgi wrapper around server")
 @click.pass_context
-def run(ctx, port, tree):
+def run(
+    ctx, port, tree, host, open_browser, debug_level, log_file, max_workers, use_wsgi
+):
     """Run the app."""
     app = ctx.obj["app"]
-    app.run(port=port, threaded=True)
+    debug_level = debug_level.upper()
+    open_browser = open_browser.lower()
+
+    if max_workers is None:
+        max_workers = min(32, os.cpu_count() + 4)
+
+    def open_webbrowser_after_start():
+        # Wait a bit for for server to start:
+        time.sleep(1.0)
+        new = {"tab": 2, "window": 1}[open_browser]
+        webbrowser.open("http://%s:%s" % (host, port), new=0, autoraise=True)
+
+    if open_browser != "no":
+        thread = Thread(target=open_webbrowser_after_start)
+        thread.start()
+
+    if log_file:
+        file_handler = logging.FileHandler(log_file, "w+")
+        app.logger.addHandler(file_handler)
+        app.logger.setLevel(debug_level)
+
+    print("Running gramps-web-api server...")
+    if open_browser != "no":
+        print(
+            f"    Opening gramps-web in browser {open_browser} on http://{host}:{port}..."
+        )
+
+    print("    Control+C to quit")
+    if use_wsgi:
+        waitress.serve(
+            TransLogger(
+                app,
+                setup_console_handler=False,
+                set_logger_level=debug_level,
+            ),
+            host=host,
+            port=port,
+            threads=max_workers,
+        )
+    else:
+        app.run(port=port, threaded=True)
+    print()
+    print("Stopping gramps-web-api server...")
 
 
 @cli.group("user", help="Manage users.")
@@ -82,8 +152,8 @@ def user(ctx):
 @click.pass_context
 def user_add(ctx, name, password, fullname, email, role, tree):
     """Add a user."""
-    LOG.error(f"Adding user {name} ...")
     app = ctx.obj["app"]
+    app.logger.info(f"Adding user {name} ...")
     with app.app_context():
         user_db.create_all()
         add_user(name, password, fullname, email, role, tree)
@@ -94,8 +164,8 @@ def user_add(ctx, name, password, fullname, email, role, tree):
 @click.pass_context
 def user_del(ctx, name):
     """Delete a user."""
-    LOG.info(f"Deleting user {name} ...")
     app = ctx.obj["app"]
+    app.logger.info(f"Deleting user {name} ...")
     with app.app_context():
         delete_user(name)
 
@@ -146,51 +216,55 @@ def search(ctx, tree, semantic):
         ctx.obj["search_indexer"] = get_search_indexer(tree=tree, semantic=semantic)
 
 
-def progress_callback_count(current: int, total: int, prev: int | None = None) -> None:
+def progress_callback_count(
+    app, current: int, total: int, prev: int | None = None
+) -> None:
     if total == 0:
         return
     pct = int(100 * current / total)
     if prev is None:
         prev = current - 1
     pct_prev = int(100 * prev / total)
     if current == 0 or pct != pct_prev:
-        LOG.info(f"Progress: {pct}%")
+        app.logger.info(f"Progress: {pct}%")
 
 
 @search.command("index-full")
 @click.pass_context
 def index_full(ctx):
     """Perform a full reindex."""
-    LOG.info("Rebuilding search index ...")
+    app = ctx.obj["app"]
+    app.logger.info("Rebuilding search index ...")
     db_manager = ctx.obj["db_manager"]
     indexer = ctx.obj["search_indexer"]
     db = db_manager.get_db().db
 
     t0 = time.time()
     try:
-        indexer.reindex_full(db, progress_cb=progress_callback_count)
+        indexer.reindex_full(app, db, progress_cb=progress_callback_count)
     except:
-        LOG.exception("Error during indexing")
+        app.logger.exception("Error during indexing")
     finally:
         close_db(db)
-    LOG.info(f"Done building search index in {time.time() - t0:.0f} seconds.")
+    app.logger.info(f"Done building search index in {time.time() - t0:.0f} seconds.")
 
 
 @search.command("index-incremental")
 @click.pass_context
 def index_incremental(ctx):
     """Perform an incremental reindex."""
+    app = ctx.obj["app"]
     db_manager = ctx.obj["db_manager"]
     indexer = ctx.obj["search_indexer"]
     db = db_manager.get_db().db
 
     try:
         indexer.reindex_incremental(db, progress_cb=progress_callback_count)
     except Exception:
-        LOG.exception("Error during indexing")
+        app.logger.exception("Error during indexing")
     finally:
         close_db(db)
-    LOG.info("Done updating search index.")
+    app.logger.info("Done updating search index.")
 
 
 @cli.group("tree", help="Manage trees.")
@@ -212,7 +286,7 @@ def tree_list(ctx):
         print(f"{dirname:>36}  {name:<}")
 
 
-@cli.group("grampsdb", help="Manage a Gramps daabase.")
+@cli.group("grampsdb", help="Manage a Gramps database.")
 @click.option("--tree", help="Tree ID", default=None)
 @click.pass_context
 def grampsdb(ctx, tree):
@@ -236,8 +310,10 @@ def migrate_gramps_db(ctx):
 
 
 if __name__ == "__main__":
-    LOG.setLevel(logging.INFO)
-
-    cli(
-        prog_name="python3 -m gramps_webapi"
-    )  # pylint:disable=no-value-for-parameter,unexpected-keyword-arg
+    try:
+        cli(
+            prog_name="python3 -m gramps_webapi"
+        )  # pylint:disable=no-value-for-parameter,unexpected-keyword-arg
+    except SystemExit as e:
+        if e.code != 0:
+            raise
diff --git a/gramps_webapi/auth/__init__.py b/gramps_webapi/auth/__init__.py
@@ -35,7 +35,6 @@
 
 user_db = SQLAlchemy()
 
-
 def add_user(
     name: str,
     password: str,
diff --git a/gramps_webapi/translogger.py b/gramps_webapi/translogger.py
@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 Doug Blank <doug.blank@gmail.com>
+# Based on:
+# Copyright (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
+# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
+"""
+Middleware for logging requests, using Apache combined log format
+"""
+
+import logging
+import time
+import urllib.parse
+
+
+class TransLogger:
+    """
+    This logging middleware will log all requests as they go through.
+    They are, by default, sent to a logger named ``'wsgi'`` at the
+    INFO level.
+
+    If ``setup_console_handler`` is true, then messages for the named
+    logger will be sent to the console.
+    """
+
+    format = (
+        "%(REMOTE_ADDR)s - %(REMOTE_USER)s [%(time)s] "
+        '"%(REQUEST_METHOD)s %(REQUEST_URI)s %(HTTP_VERSION)s" '
+        '%(status)s %(bytes)s "%(HTTP_REFERER)s" "%(HTTP_USER_AGENT)s"'
+    )
+
+    def __init__(
+        self,
+        application,
+        logger=None,
+        format=None,
+        logging_level=logging.INFO,
+        logger_name="wsgi",
+        setup_console_handler=True,
+        set_logger_level=logging.DEBUG,
+    ):
+        if format is not None:
+            self.format = format
+        self.application = application
+        self.logging_level = logging_level
+        self.logger_name = logger_name
+        if logger is None:
+            self.logger = logging.getLogger(self.logger_name)
+            if setup_console_handler:
+                console = logging.StreamHandler()
+                console.setLevel(logging.DEBUG)
+                # We need to control the exact format:
+                console.setFormatter(logging.Formatter("%(message)s"))
+                self.logger.addHandler(console)
+                self.logger.propagate = False
+            if set_logger_level is not None:
+                self.logger.setLevel(set_logger_level)
+        else:
+            self.logger = logger
+
+    def __call__(self, environ, start_response):
+        start = time.localtime()
+        req_uri = urllib.parse.quote(
+            environ.get("SCRIPT_NAME", "") + environ.get("PATH_INFO", "")
+        )
+        if environ.get("QUERY_STRING"):
+            req_uri += "?" + environ["QUERY_STRING"]
+        method = environ["REQUEST_METHOD"]
+
+        def replacement_start_response(status, headers, exc_info=None):
+            # @@: Ideally we would count the bytes going by if no
+            # content-length header was provided; but that does add
+            # some overhead, so at least for now we'll be lazy.
+            bytes = None
+            for name, value in headers:
+                if name.lower() == "content-length":
+                    bytes = value
+            self.write_log(environ, method, req_uri, start, status, bytes)
+            return start_response(status, headers)
+
+        return self.application(environ, replacement_start_response)
+
+    def write_log(self, environ, method, req_uri, start, status, bytes):
+        if bytes is None:
+            bytes = "-"
+        if time.daylight:
+            offset = time.altzone / 60 / 60 * -100
+        else:
+            offset = time.timezone / 60 / 60 * -100
+        if offset >= 0:
+            offset = "+%0.4d" % (offset)
+        elif offset < 0:
+            offset = "%0.4d" % (offset)
+        remote_addr = "-"
+        if environ.get("HTTP_X_FORWARDED_FOR"):
+            remote_addr = environ["HTTP_X_FORWARDED_FOR"]
+        elif environ.get("REMOTE_ADDR"):
+            remote_addr = environ["REMOTE_ADDR"]
+        d = {
+            "REMOTE_ADDR": remote_addr,
+            "REMOTE_USER": environ.get("REMOTE_USER") or "-",
+            "REQUEST_METHOD": method,
+            "REQUEST_URI": req_uri,
+            "HTTP_VERSION": environ.get("SERVER_PROTOCOL"),
+            "time": time.strftime("%d/%b/%Y:%H:%M:%S ", start) + offset,
+            "status": status.split(None, 1)[0],
+            "bytes": bytes,
+            "HTTP_REFERER": environ.get("HTTP_REFERER", "-"),
+            "HTTP_USER_AGENT": environ.get("HTTP_USER_AGENT", "-"),
+        }
+        message = self.format % d
+        self.logger.log(self.logging_level, message)
diff --git a/pyproject.toml b/pyproject.toml
@@ -26,6 +26,7 @@ dependencies = [
     "Flask-Limiter>=2.9.0",
     "Flask-SQLAlchemy",
     "marshmallow>=3.13.0",
+    "waitress",
     "webargs",
     "SQLAlchemy",
     "pdf2image",
