From f340432827bb7dffc86fdb792785141a5c5eb288 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Tue, 13 Aug 2024 23:06:11 +0200 Subject: [PATCH 1/3] style: argparse inheritance --- gunicorn/config.py | 296 +++++++++++++++++---------------------------- 1 file changed, 111 insertions(+), 185 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 402a26b68..c19ba3ab0 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -4,6 +4,7 @@ # Please remember to run "make -C docs html" after update "desc" attributes. +from functools import reduce import argparse import copy import grp @@ -238,9 +239,24 @@ class SettingMeta(type): def __new__(cls, name, bases, attrs): super_new = super().__new__ parents = [b for b in bases if isinstance(b, SettingMeta)] - if not parents: + + # creating parent class, return without registering known setting + if "desc" not in attrs: return super_new(cls, name, bases, attrs) + # creating new leaf class, register setting + # FIXME: put idiomatic expression for inheritance in metaclass here + parent_attrs = [ + {k: v for (k, v) in vars(b).items() if not k.startswith("__")} + for b in bases[::-1] # backwards: last in class hierarchy wins + ] + + # Python 3.9 permits dict() | dict() => operator.or_ suffices + def dict_or(a, b): + c = a.copy() + c.update(b) + return c + attrs = reduce(dict_or, parent_attrs + [attrs, ]) attrs["order"] = len(KNOWN_SETTINGS) attrs["validator"] = staticmethod(attrs["validator"]) @@ -255,20 +271,21 @@ def fmt_desc(cls, desc): setattr(cls, "short", desc.splitlines()[0]) -class Setting: +class BaseSetting: name = None value = None section = None cli = None validator = None - type = None + type = str meta = None - action = None + action = "store" default = None short = None desc = None nargs = None const = None + hidden_in_help = False def __init__(self): if self.default is not None: @@ -284,10 +301,10 @@ def add_option(self, parser): kwargs = { "dest": self.name, - "action": self.action or "store", - "type": self.type or str, + "action": self.action, + "type": self.type, "default": None, - "help": help_txt + "help": argparse.SUPPRESS if self.hidden_in_help else help_txt, } if self.meta is not None: @@ -329,7 +346,16 @@ def __repr__(self): ) -Setting = SettingMeta('Setting', (Setting,), {}) +class Setting(BaseSetting, metaclass=SettingMeta): + pass + + +class Unstable: + pass # likely: move to separate document + + +class Deprecated(Unstable): + hidden_in_help = True def validate_bool(val): @@ -547,6 +573,31 @@ def get_default_config_file(): return None +class PosIntSetting(Setting): + meta = "INT" + validator = validate_pos_int + type = int + + +class HexPosIntSetting(Setting): + meta = "INT" + validator = validate_pos_int + type = auto_int + + +class BoolSetting(Setting): + validator = validate_bool + default = False + action = 'store_true' + + +class HookSetting(Setting): + # typically; not defined here to keep # of args in leaf: + # validator = validate_callable(-1) + section = "Server Hooks" + type = callable + + class ConfigFile(Setting): name = "config" section = "Config File" @@ -619,13 +670,10 @@ class Bind(Setting): """ -class Backlog(Setting): +class Backlog(PosIntSetting): name = "backlog" section = "Server Socket" cli = ["--backlog"] - meta = "INT" - validator = validate_pos_int - type = int default = 2048 desc = """\ The maximum number of pending connections. @@ -639,13 +687,10 @@ class Backlog(Setting): """ -class Workers(Setting): +class Workers(PosIntSetting): name = "workers" section = "Worker Processes" cli = ["-w", "--workers"] - meta = "INT" - validator = validate_pos_int - type = int default = int(os.environ.get("WEB_CONCURRENCY", 1)) desc = """\ The number of worker processes for handling requests. @@ -694,13 +739,10 @@ class WorkerClass(Setting): """ -class WorkerThreads(Setting): +class WorkerThreads(PosIntSetting): name = "threads" section = "Worker Processes" cli = ["--threads"] - meta = "INT" - validator = validate_pos_int - type = int default = 1 desc = """\ The number of worker threads for handling requests. @@ -722,13 +764,10 @@ class WorkerThreads(Setting): """ -class WorkerConnections(Setting): +class WorkerConnections(PosIntSetting): name = "worker_connections" section = "Worker Processes" cli = ["--worker-connections"] - meta = "INT" - validator = validate_pos_int - type = int default = 1000 desc = """\ The maximum number of simultaneous clients. @@ -737,13 +776,10 @@ class WorkerConnections(Setting): """ -class MaxRequests(Setting): +class MaxRequests(PosIntSetting): name = "max_requests" section = "Worker Processes" cli = ["--max-requests"] - meta = "INT" - validator = validate_pos_int - type = int default = 0 desc = """\ The maximum number of requests a worker will process before restarting. @@ -757,13 +793,10 @@ class MaxRequests(Setting): """ -class MaxRequestsJitter(Setting): +class MaxRequestsJitter(PosIntSetting): name = "max_requests_jitter" section = "Worker Processes" cli = ["--max-requests-jitter"] - meta = "INT" - validator = validate_pos_int - type = int default = 0 desc = """\ The maximum jitter to add to the *max_requests* setting. @@ -776,13 +809,10 @@ class MaxRequestsJitter(Setting): """ -class Timeout(Setting): +class Timeout(PosIntSetting): name = "timeout" section = "Worker Processes" cli = ["-t", "--timeout"] - meta = "INT" - validator = validate_pos_int - type = int default = 30 desc = """\ Workers silent for more than this many seconds are killed and restarted. @@ -798,13 +828,10 @@ class Timeout(Setting): """ -class GracefulTimeout(Setting): +class GracefulTimeout(PosIntSetting): name = "graceful_timeout" section = "Worker Processes" cli = ["--graceful-timeout"] - meta = "INT" - validator = validate_pos_int - type = int default = 30 desc = """\ Timeout for graceful workers restart. @@ -815,13 +842,10 @@ class GracefulTimeout(Setting): """ -class Keepalive(Setting): +class Keepalive(PosIntSetting): name = "keepalive" section = "Worker Processes" cli = ["--keep-alive"] - meta = "INT" - validator = validate_pos_int - type = int default = 2 desc = """\ The number of seconds to wait for requests on a Keep-Alive connection. @@ -837,13 +861,10 @@ class Keepalive(Setting): """ -class LimitRequestLine(Setting): +class LimitRequestLine(PosIntSetting): name = "limit_request_line" section = "Security" cli = ["--limit-request-line"] - meta = "INT" - validator = validate_pos_int - type = int default = 4094 desc = """\ The maximum size of HTTP request line in bytes. @@ -861,13 +882,10 @@ class LimitRequestLine(Setting): """ -class LimitRequestFields(Setting): +class LimitRequestFields(PosIntSetting): name = "limit_request_fields" section = "Security" cli = ["--limit-request-fields"] - meta = "INT" - validator = validate_pos_int - type = int default = 100 desc = """\ Limit the number of HTTP headers fields in a request. @@ -879,13 +897,10 @@ class LimitRequestFields(Setting): """ -class LimitRequestFieldSize(Setting): +class LimitRequestFieldSize(PosIntSetting): name = "limit_request_field_size" section = "Security" cli = ["--limit-request-field_size"] - meta = "INT" - validator = validate_pos_int - type = int default = 8190 desc = """\ Limit the allowed size of an HTTP request header field. @@ -899,13 +914,10 @@ class LimitRequestFieldSize(Setting): """ -class Reload(Setting): +class Reload(BoolSetting): name = "reload" section = 'Debugging' cli = ['--reload'] - validator = validate_bool - action = 'store_true' - default = False desc = '''\ Restart workers when code changes. @@ -963,13 +975,10 @@ class ReloadExtraFiles(Setting): """ -class Spew(Setting): +class Spew(BoolSetting): name = "spew" section = "Debugging" cli = ["--spew"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Install a trace function that spews every line executed by the server. @@ -977,38 +986,29 @@ class Spew(Setting): """ -class ConfigCheck(Setting): +class ConfigCheck(BoolSetting): name = "check_config" section = "Debugging" cli = ["--check-config"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Check the configuration and exit. The exit status is 0 if the configuration is correct, and 1 if the configuration is incorrect. """ -class PrintConfig(Setting): +class PrintConfig(BoolSetting): name = "print_config" section = "Debugging" cli = ["--print-config"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Print the configuration settings as fully resolved. Implies :ref:`check-config`. """ -class PreloadApp(Setting): +class PreloadApp(BoolSetting): name = "preload_app" section = "Server Mechanics" cli = ["--preload"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Load application code before the worker processes are forked. @@ -1025,6 +1025,7 @@ class Sendfile(Setting): cli = ["--no-sendfile"] validator = validate_bool action = "store_const" + # FIXME: how should the default look in docs? const = False desc = """\ @@ -1042,13 +1043,10 @@ class Sendfile(Setting): """ -class ReusePort(Setting): +class ReusePort(BoolSetting): name = "reuse_port" section = "Server Mechanics" cli = ["--reuse-port"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Set the ``SO_REUSEPORT`` flag on the listening socket. @@ -1069,13 +1067,10 @@ class Chdir(Setting): """ -class Daemon(Setting): +class Daemon(BoolSetting): name = "daemon" section = "Server Mechanics" cli = ["-D", "--daemon"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Daemonize the Gunicorn process. @@ -1182,13 +1177,10 @@ class Group(Setting): """ -class Umask(Setting): +class Umask(HexPosIntSetting): name = "umask" section = "Server Mechanics" cli = ["-m", "--umask"] - meta = "INT" - validator = validate_pos_int - type = auto_int default = 0 desc = """\ A bit mask for the file mode on files written by Gunicorn. @@ -1202,13 +1194,10 @@ class Umask(Setting): """ -class Initgroups(Setting): +class Initgroups(BoolSetting): name = "initgroups" section = "Server Mechanics" cli = ["--initgroups"] - validator = validate_bool - action = 'store_true' - default = False desc = """\ If true, set the worker process's group access list with all of the @@ -1373,13 +1362,10 @@ class AccessLog(Setting): """ -class DisableRedirectAccessToSyslog(Setting): +class DisableRedirectAccessToSyslog(BoolSetting): name = "disable_redirect_access_to_syslog" section = "Logging" cli = ["--disable-redirect-access-to-syslog"] - validator = validate_bool - action = 'store_true' - default = False desc = """\ Disable redirect access logs to syslog. @@ -1469,13 +1455,10 @@ class Loglevel(Setting): """ -class CaptureOutput(Setting): +class CaptureOutput(BoolSetting): name = "capture_output" section = "Logging" cli = ["--capture-output"] - validator = validate_bool - action = 'store_true' - default = False desc = """\ Redirect stdout/stderr to specified file in :ref:`errorlog`. @@ -1582,13 +1565,10 @@ class SyslogTo(Setting): """ -class Syslog(Setting): +class Syslog(BoolSetting): name = "syslog" section = "Logging" cli = ["--log-syslog"] - validator = validate_bool - action = 'store_true' - default = False desc = """\ Send *Gunicorn* logs to syslog. @@ -1625,13 +1605,10 @@ class SyslogFacility(Setting): """ -class EnableStdioInheritance(Setting): +class EnableStdioInheritance(BoolSetting): name = "enable_stdio_inheritance" section = "Logging" cli = ["-R", "--enable-stdio-inheritance"] - validator = validate_bool - default = False - action = "store_true" desc = """\ Enable stdio inheritance. @@ -1754,11 +1731,9 @@ class Paste(Setting): """ -class OnStarting(Setting): +class OnStarting(HookSetting): name = "on_starting" - section = "Server Hooks" validator = validate_callable(1) - type = callable def on_starting(server): pass @@ -1770,11 +1745,9 @@ def on_starting(server): """ -class OnReload(Setting): +class OnReload(HookSetting): name = "on_reload" - section = "Server Hooks" validator = validate_callable(1) - type = callable def on_reload(server): pass @@ -1786,11 +1759,9 @@ def on_reload(server): """ -class WhenReady(Setting): +class WhenReady(HookSetting): name = "when_ready" - section = "Server Hooks" validator = validate_callable(1) - type = callable def when_ready(server): pass @@ -1802,11 +1773,9 @@ def when_ready(server): """ -class Prefork(Setting): +class Prefork(HookSetting): name = "pre_fork" - section = "Server Hooks" validator = validate_callable(2) - type = callable def pre_fork(server, worker): pass @@ -1819,11 +1788,9 @@ def pre_fork(server, worker): """ -class Postfork(Setting): +class Postfork(HookSetting): name = "post_fork" - section = "Server Hooks" validator = validate_callable(2) - type = callable def post_fork(server, worker): pass @@ -1836,11 +1803,9 @@ def post_fork(server, worker): """ -class PostWorkerInit(Setting): +class PostWorkerInit(HookSetting): name = "post_worker_init" - section = "Server Hooks" validator = validate_callable(1) - type = callable def post_worker_init(worker): pass @@ -1854,11 +1819,9 @@ def post_worker_init(worker): """ -class WorkerInt(Setting): +class WorkerInt(HookSetting): name = "worker_int" - section = "Server Hooks" validator = validate_callable(1) - type = callable def worker_int(worker): pass @@ -1872,11 +1835,9 @@ def worker_int(worker): """ -class WorkerAbort(Setting): +class WorkerAbort(HookSetting): name = "worker_abort" - section = "Server Hooks" validator = validate_callable(1) - type = callable def worker_abort(worker): pass @@ -1892,11 +1853,9 @@ def worker_abort(worker): """ -class PreExec(Setting): +class PreExec(HookSetting): name = "pre_exec" - section = "Server Hooks" validator = validate_callable(1) - type = callable def pre_exec(server): pass @@ -1908,11 +1867,9 @@ def pre_exec(server): """ -class PreRequest(Setting): +class PreRequest(HookSetting): name = "pre_request" - section = "Server Hooks" validator = validate_callable(2) - type = callable def pre_request(worker, req): worker.log.debug("%s %s", req.method, req.path) @@ -1942,11 +1899,9 @@ def post_request(worker, req, environ, resp): """ -class ChildExit(Setting): +class ChildExit(HookSetting): name = "child_exit" - section = "Server Hooks" validator = validate_callable(2) - type = callable def child_exit(server, worker): pass @@ -1961,11 +1916,9 @@ def child_exit(server, worker): """ -class WorkerExit(Setting): +class WorkerExit(HookSetting): name = "worker_exit" - section = "Server Hooks" validator = validate_callable(2) - type = callable def worker_exit(server, worker): pass @@ -1978,11 +1931,9 @@ def worker_exit(server, worker): """ -class NumWorkersChanged(Setting): +class NumWorkersChanged(HookSetting): name = "nworkers_changed" - section = "Server Hooks" validator = validate_callable(3) - type = callable def nworkers_changed(server, new_value, old_value): pass @@ -1998,9 +1949,8 @@ def nworkers_changed(server, new_value, old_value): """ -class OnExit(Setting): +class OnExit(HookSetting): name = "on_exit" - section = "Server Hooks" validator = validate_callable(1) def on_exit(server): @@ -2014,11 +1964,9 @@ def on_exit(server): """ -class NewSSLContext(Setting): +class NewSSLContext(HookSetting): name = "ssl_context" - section = "Server Hooks" validator = validate_callable(2) - type = callable def ssl_context(config, default_ssl_context_factory): return default_ssl_context_factory() @@ -2049,13 +1997,10 @@ def ssl_context(conf, default_ssl_context_factory): """ -class ProxyProtocol(Setting): +class ProxyProtocol(BoolSetting): name = "proxy_protocol" section = "Server Mechanics" cli = ["--proxy-protocol"] - validator = validate_bool - default = False - action = "store_true" desc = """\ Enable detect PROXY protocol (PROXY mode). @@ -2120,7 +2065,7 @@ class CertFile(Setting): """ -class SSLVersion(Setting): +class SSLVersion(Setting, Deprecated): name = "ssl_version" section = "SSL" cli = ["--ssl-version"] @@ -2196,25 +2141,20 @@ class CACerts(Setting): """ -class SuppressRaggedEOFs(Setting): +class SuppressRaggedEOFs(BoolSetting): name = "suppress_ragged_eofs" section = "SSL" cli = ["--suppress-ragged-eofs"] - action = "store_true" - default = True - validator = validate_bool + default = True # (sic!) desc = """\ Suppress ragged EOFs (see stdlib ssl module's) """ -class DoHandshakeOnConnect(Setting): +class DoHandshakeOnConnect(BoolSetting): name = "do_handshake_on_connect" section = "SSL" cli = ["--do-handshake-on-connect"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Whether to perform SSL handshake on socket connect (see stdlib ssl module's) """ @@ -2266,13 +2206,10 @@ class PasteGlobalConf(Setting): """ -class PermitObsoleteFolding(Setting): +class PermitObsoleteFolding(BoolSetting): name = "permit_obsolete_folding" section = "Server Mechanics" cli = ["--permit-obsolete-folding"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Permit requests employing obsolete HTTP line folding mechanism @@ -2287,13 +2224,10 @@ class PermitObsoleteFolding(Setting): """ -class StripHeaderSpaces(Setting): +class StripHeaderSpaces(BoolSetting, Deprecated): name = "strip_header_spaces" section = "Server Mechanics" cli = ["--strip-header-spaces"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Strip spaces present between the header name and the the ``:``. @@ -2306,13 +2240,11 @@ class StripHeaderSpaces(Setting): """ -class PermitUnconventionalHTTPMethod(Setting): +class PermitUnconventionalHTTPMethod(BoolSetting, Unstable): name = "permit_unconventional_http_method" section = "Server Mechanics" cli = ["--permit-unconventional-http-method"] - validator = validate_bool - action = "store_true" - default = False + # when removed or changed: consider making True the new default desc = """\ Permit HTTP methods not matching conventions, such as IANA registration guidelines @@ -2332,13 +2264,10 @@ class PermitUnconventionalHTTPMethod(Setting): """ -class PermitUnconventionalHTTPVersion(Setting): +class PermitUnconventionalHTTPVersion(BoolSetting, Unstable): name = "permit_unconventional_http_version" section = "Server Mechanics" cli = ["--permit-unconventional-http-version"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Permit HTTP version not matching conventions of 2023 @@ -2353,13 +2282,10 @@ class PermitUnconventionalHTTPVersion(Setting): """ -class CasefoldHTTPMethod(Setting): +class CasefoldHTTPMethod(BoolSetting, Deprecated): name = "casefold_http_method" section = "Server Mechanics" cli = ["--casefold-http-method"] - validator = validate_bool - action = "store_true" - default = False desc = """\ Transform received HTTP methods to uppercase From 353927eec8c31eacf1f4423f0274b43f14a8fd80 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Tue, 13 Aug 2024 23:19:52 +0200 Subject: [PATCH 2/3] style: unused code --- gunicorn/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index c19ba3ab0..36d5268eb 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -238,7 +238,6 @@ def paste_global_conf(self): class SettingMeta(type): def __new__(cls, name, bases, attrs): super_new = super().__new__ - parents = [b for b in bases if isinstance(b, SettingMeta)] # creating parent class, return without registering known setting if "desc" not in attrs: From 60beebb1abb5b96792f3aae4ed92947c10d80a87 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 21 Mar 2025 02:15:09 +0100 Subject: [PATCH 3/3] mark tmp_upload_dir as deprecated removed in 27cc404f1f523746dfd5866f7c058fd006e180d4 --- docs/source/settings.rst | 6 +----- gunicorn/config.py | 8 ++------ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index e1e91fa76..dab872c2d 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -1193,11 +1193,7 @@ group id. Directory to store temporary request data as they are read. -This may disappear in the near future. - -This path should be writable by the process permissions set for Gunicorn -workers. If not specified, Gunicorn will choose a system generated -temporary directory. +Unused, and may be removed from a future Gunicorn version. .. _secure-scheme-headers: diff --git a/gunicorn/config.py b/gunicorn/config.py index 36d5268eb..5a9e3266f 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1207,7 +1207,7 @@ class Initgroups(BoolSetting): """ -class TmpUploadDir(Setting): +class TmpUploadDir(Setting, Deprecated): name = "tmp_upload_dir" section = "Server Mechanics" meta = "DIR" @@ -1216,11 +1216,7 @@ class TmpUploadDir(Setting): desc = """\ Directory to store temporary request data as they are read. - This may disappear in the near future. - - This path should be writable by the process permissions set for Gunicorn - workers. If not specified, Gunicorn will choose a system generated - temporary directory. + Unused, and may be removed from a future Gunicorn version. """