diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index 6d2d4661db5..3dac3906ccc 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -88,8 +88,8 @@ b0f434f64105bd61ab0f6867b3f681b97fa02b4fb809ac538db382d031f0e609 data/xml/paylo a2a2d3f8bf506f27ab0847ad4daa1fc41ca781dd58b70d2d9ac1360cf8151260 data/xml/queries.xml abb6261b1c531ad2ee3ada8184c76bcdc38732558d11a8e519f36fcc95325f7e doc/AUTHORS ce20a4b452f24a97fde7ec9ed816feee12ac148e1fde5f1722772cc866b12740 doc/CHANGELOG.md -7af515e3ad13fb7e9cfa4debc8ec879758c0cfbe67642b760172178cda9cf5cb doc/THANKS.md -5112f71069f35d4b3737791ca680f2815f0c42fdf5c9bedff7654dde8372327f doc/THIRD-PARTY.md +c8d5733111c6d1e387904bc14e98815f98f816f6e73f6a664de24c0f1d331d9b doc/THANKS.md +d7e38b213c70fe519fff2e06a9fd0dcfb1d8bed7787e37916cd14faaf002e167 doc/THIRD-PARTY.md 25012296e8484ea04f7d2368ac9bdbcded4e42dbc5e3373d59c2bb3e950be0b8 doc/translations/README-ar-AR.md c25f7d7f0cc5e13db71994d2b34ada4965e06c87778f1d6c1a103063d25e2c89 doc/translations/README-bg-BG.md e85c82df1a312d93cd282520388c70ecb48bfe8692644fe8dbbf7d43244cda41 doc/translations/README-bn-BD.md @@ -164,7 +164,7 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264 extra/shutils/ b8411d1035bb49b073476404e61e1be7f4c61e205057730e2f7880beadcd5f60 lib/controller/action.py e376093d4f6e42ee38b050af329179df9c1c136b7667b2f1cb559f5d4b69ebd9 lib/controller/checks.py 430475857a37fd997e73a47d7485c5dd4aa0985ef32c5a46b5e7bff01749ba66 lib/controller/controller.py -1ecbca13afdc7c2bc8dc215c5d7fca453bf836dbe3ca377609750bfbc4874a85 lib/controller/handler.py +56e03690c1b783699c9f30cb2f8cc743d3716aba8137e6b253b21d1dd31a4314 lib/controller/handler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/controller/__init__.py 2a96190ced25d8929861b13866101812fcadf5cac23dd1dd4b29b1a915918769 lib/core/agent.py 1da4ec9cd9b67c8b54e4a3d314f8237d58778d8f3a00bc26a1e0540294dca30f lib/core/bigarray.py @@ -189,7 +189,7 @@ e18c0c2c5a57924a623792a48bfd36e98d9bc085f6db61a95fc0dc8a3bcedc0c lib/core/decor 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3 lib/core/replication.py 3574639db4942d16a2dc0a2f04bb7c0913c40c3862b54d34c44075a760e0c194 lib/core/revision.py 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py -c0f848d501c33ae35d0372c7d70ce9bde176691b6962ae94e2d753c2fff17543 lib/core/settings.py +4267b95315e5351fa06aa27c883e187186adc3709e50acedf10079fd611dbb8d lib/core/settings.py cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82 lib/core/shell.py bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725 lib/core/subprocessng.py d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038 lib/core/target.py @@ -561,9 +561,9 @@ dcb7a5584390f1604adff075c94139dd23711f2f516b68683ec4208dd0a00fda tamper/version ce1b6bf8f296de27014d6f21aa8b3df9469d418740cd31c93d1f5e36d6c509cf tamper/xforwardedfor.py 55eaefc664bd8598329d535370612351ec8443c52465f0a37172ea46a97c458a thirdparty/ansistrm/ansistrm.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/ansistrm/__init__.py -dfb8a36f58a3ae72c34d6a350830857c88ff8938fe256af585d5c9c63040c5b2 thirdparty/beautifulsoup/beautifulsoup.py +f597b49ef445bfbfb8f98d1f1a08dcfe4810de5769c0abfab7cdce4eebbfcae7 thirdparty/beautifulsoup/beautifulsoup.py 7d62c59f787f987cbce0de5375f604da8de0ba01742842fb2b3d12fcb92fcb63 thirdparty/beautifulsoup/__init__.py -0915f7e3d0025f81a2883cd958813470a4be661744d7fffa46848b45506b951a thirdparty/bottle/bottle.py +f862301288d2ba2f913860bb901cd5197e72c0461e3330164f90375f713b8199 thirdparty/bottle/bottle.py 9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2 thirdparty/bottle/__init__.py 0ffccae46cb3a15b117acd0790b2738a5b45417d1b2822ceac57bdff10ef3bff thirdparty/chardet/big5freq.py 901c476dd7ad0693deef1ae56fe7bdf748a8b7ae20fde1922dddf6941eff8773 thirdparty/chardet/big5prober.py @@ -627,14 +627,12 @@ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/mul 2574a2027b4a63214bad8bd71f28cac66b5748159bf16d63eb2a3e933985b0a5 thirdparty/multipart/multipartpost.py ef70b88cc969a3e259868f163ad822832f846196e3f7d7eccb84958c80b7f696 thirdparty/odict/__init__.py 9a8186aeb9553407f475f59d1fab0346ceab692cf4a378c15acd411f271c8fdb thirdparty/odict/ordereddict.py -691ae693e3a33dd730930492ff9e7e3bdec45e90e3a607b869a37ecd0354c2d8 thirdparty/prettyprint/__init__.py -8df6e8c60eac4c83b1bf8c4e0e0276a4caa3c5f0ca57bc6a2116f31f19d3c33f thirdparty/prettyprint/prettyprint.py 3739db672154ad4dfa05c9ac298b0440f3f1500c6a3697c2b8ac759479426b84 thirdparty/pydes/__init__.py 4c9d2c630064018575611179471191914299992d018efdc861a7109f3ec7de5e thirdparty/pydes/pyDes.py c51c91f703d3d4b3696c923cb5fec213e05e75d9215393befac7f2fa6a3904df thirdparty/six/__init__.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/socks/__init__.py 7027e214e014eb78b7adcc1ceda5aca713a79fc4f6a0c52c9da5b3e707e6ffe9 thirdparty/socks/LICENSE -57dba7460c09b7922df68b981e824135f1a6306180ba4c107b626e3232513eff thirdparty/socks/socks.py +56ae8fb03a5cf34cc5babb59f8c2c3bb20388a04f94491f6847989428ce49b82 thirdparty/socks/socks.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/termcolor/__init__.py b14474d467c70f5fe6cb8ed624f79d881c04fe6aeb7d406455da624fe8b3c0df thirdparty/termcolor/termcolor.py 4db695470f664b0d7cd5e6b9f3c94c8d811c4c550f37f17ed7bdab61bc3bdefc thirdparty/wininetpton/__init__.py diff --git a/doc/THANKS.md b/doc/THANKS.md index 36d8572620d..62d4ba136cf 100644 --- a/doc/THANKS.md +++ b/doc/THANKS.md @@ -194,9 +194,6 @@ David Guimaraes, * for reporting considerable amount of bugs * for suggesting several features -Chris Hall, -* for coding the prettyprint.py library - Tate Hansen, * for donating to sqlmap development diff --git a/doc/THIRD-PARTY.md b/doc/THIRD-PARTY.md index f2aea92a599..03c0c01e8f4 100644 --- a/doc/THIRD-PARTY.md +++ b/doc/THIRD-PARTY.md @@ -15,8 +15,6 @@ This file lists bundled packages and their associated licensing terms. Copyright (C) 2013, Jonathan Hartley. * The `Fcrypt` library located under `thirdparty/fcrypt/`. Copyright (C) 2000, 2001, 2004 Carey Evans. -* The `PrettyPrint` library located under `thirdparty/prettyprint/`. - Copyright (C) 2010, Chris Hall. * The `SocksiPy` library located under `thirdparty/socks/`. Copyright (C) 2006, Dan-Haim. @@ -271,7 +269,7 @@ be bound by the terms and conditions of this License Agreement. # MIT * The `bottle` web framework library located under `thirdparty/bottle/`. - Copyright (C) 2018, Marcel Hellkamp. + Copyright (C) 2024, Marcel Hellkamp. * The `identYwaf` library located under `thirdparty/identywaf/`. Copyright (C) 2019-2021, Miroslav Stampar. * The `ordereddict` library located under `thirdparty/odict/`. diff --git a/lib/controller/handler.py b/lib/controller/handler.py index e6736680bae..a2ea41f25bf 100644 --- a/lib/controller/handler.py +++ b/lib/controller/handler.py @@ -44,63 +44,34 @@ from lib.core.settings import SNOWFLAKE_ALIASES from lib.utils.sqlalchemy import SQLAlchemy -from plugins.dbms.access.connector import Connector as AccessConn from plugins.dbms.access import AccessMap -from plugins.dbms.altibase.connector import Connector as AltibaseConn from plugins.dbms.altibase import AltibaseMap -from plugins.dbms.cache.connector import Connector as CacheConn from plugins.dbms.cache import CacheMap -from plugins.dbms.clickhouse.connector import Connector as ClickHouseConn from plugins.dbms.clickhouse import ClickHouseMap -from plugins.dbms.cratedb.connector import Connector as CrateDBConn from plugins.dbms.cratedb import CrateDBMap -from plugins.dbms.cubrid.connector import Connector as CubridConn from plugins.dbms.cubrid import CubridMap -from plugins.dbms.db2.connector import Connector as DB2Conn from plugins.dbms.db2 import DB2Map -from plugins.dbms.derby.connector import Connector as DerbyConn from plugins.dbms.derby import DerbyMap -from plugins.dbms.extremedb.connector import Connector as ExtremeDBConn from plugins.dbms.extremedb import ExtremeDBMap -from plugins.dbms.firebird.connector import Connector as FirebirdConn from plugins.dbms.firebird import FirebirdMap -from plugins.dbms.frontbase.connector import Connector as FrontBaseConn from plugins.dbms.frontbase import FrontBaseMap -from plugins.dbms.h2.connector import Connector as H2Conn from plugins.dbms.h2 import H2Map -from plugins.dbms.hsqldb.connector import Connector as HSQLDBConn from plugins.dbms.hsqldb import HSQLDBMap -from plugins.dbms.informix.connector import Connector as InformixConn from plugins.dbms.informix import InformixMap -from plugins.dbms.maxdb.connector import Connector as MaxDBConn from plugins.dbms.maxdb import MaxDBMap -from plugins.dbms.mckoi.connector import Connector as MckoiConn from plugins.dbms.mckoi import MckoiMap -from plugins.dbms.mimersql.connector import Connector as MimerSQLConn from plugins.dbms.mimersql import MimerSQLMap -from plugins.dbms.monetdb.connector import Connector as MonetDBConn from plugins.dbms.monetdb import MonetDBMap -from plugins.dbms.mssqlserver.connector import Connector as MSSQLServerConn from plugins.dbms.mssqlserver import MSSQLServerMap -from plugins.dbms.mysql.connector import Connector as MySQLConn from plugins.dbms.mysql import MySQLMap -from plugins.dbms.oracle.connector import Connector as OracleConn from plugins.dbms.oracle import OracleMap -from plugins.dbms.postgresql.connector import Connector as PostgreSQLConn from plugins.dbms.postgresql import PostgreSQLMap -from plugins.dbms.presto.connector import Connector as PrestoConn from plugins.dbms.presto import PrestoMap -from plugins.dbms.raima.connector import Connector as RaimaConn from plugins.dbms.raima import RaimaMap -from plugins.dbms.sqlite.connector import Connector as SQLiteConn from plugins.dbms.sqlite import SQLiteMap -from plugins.dbms.sybase.connector import Connector as SybaseConn from plugins.dbms.sybase import SybaseMap -from plugins.dbms.vertica.connector import Connector as VerticaConn from plugins.dbms.vertica import VerticaMap -from plugins.dbms.virtuoso.connector import Connector as VirtuosoConn from plugins.dbms.virtuoso import VirtuosoMap -from plugins.dbms.snowflake.connector import Connector as SnowflakeConn from plugins.dbms.snowflake import SnowflakeMap def setHandler(): @@ -110,36 +81,35 @@ def setHandler(): """ items = [ - (DBMS.SNOWFLAKE, SNOWFLAKE_ALIASES, SnowflakeMap, SnowflakeConn), - (DBMS.MYSQL, MYSQL_ALIASES, MySQLMap, MySQLConn), - (DBMS.ORACLE, ORACLE_ALIASES, OracleMap, OracleConn), - (DBMS.PGSQL, PGSQL_ALIASES, PostgreSQLMap, PostgreSQLConn), - (DBMS.MSSQL, MSSQL_ALIASES, MSSQLServerMap, MSSQLServerConn), - (DBMS.SQLITE, SQLITE_ALIASES, SQLiteMap, SQLiteConn), - (DBMS.ACCESS, ACCESS_ALIASES, AccessMap, AccessConn), - (DBMS.FIREBIRD, FIREBIRD_ALIASES, FirebirdMap, FirebirdConn), - (DBMS.MAXDB, MAXDB_ALIASES, MaxDBMap, MaxDBConn), - (DBMS.SYBASE, SYBASE_ALIASES, SybaseMap, SybaseConn), - (DBMS.DB2, DB2_ALIASES, DB2Map, DB2Conn), - (DBMS.HSQLDB, HSQLDB_ALIASES, HSQLDBMap, HSQLDBConn), - (DBMS.H2, H2_ALIASES, H2Map, H2Conn), - (DBMS.INFORMIX, INFORMIX_ALIASES, InformixMap, InformixConn), - (DBMS.MONETDB, MONETDB_ALIASES, MonetDBMap, MonetDBConn), - (DBMS.DERBY, DERBY_ALIASES, DerbyMap, DerbyConn), - (DBMS.VERTICA, VERTICA_ALIASES, VerticaMap, VerticaConn), - (DBMS.MCKOI, MCKOI_ALIASES, MckoiMap, MckoiConn), - (DBMS.PRESTO, PRESTO_ALIASES, PrestoMap, PrestoConn), - (DBMS.ALTIBASE, ALTIBASE_ALIASES, AltibaseMap, AltibaseConn), - (DBMS.MIMERSQL, MIMERSQL_ALIASES, MimerSQLMap, MimerSQLConn), - (DBMS.CLICKHOUSE, CLICKHOUSE_ALIASES, ClickHouseMap, ClickHouseConn), - (DBMS.CRATEDB, CRATEDB_ALIASES, CrateDBMap, CrateDBConn), - (DBMS.CUBRID, CUBRID_ALIASES, CubridMap, CubridConn), - (DBMS.CACHE, CACHE_ALIASES, CacheMap, CacheConn), - (DBMS.EXTREMEDB, EXTREMEDB_ALIASES, ExtremeDBMap, ExtremeDBConn), - (DBMS.FRONTBASE, FRONTBASE_ALIASES, FrontBaseMap, FrontBaseConn), - (DBMS.RAIMA, RAIMA_ALIASES, RaimaMap, RaimaConn), - (DBMS.VIRTUOSO, VIRTUOSO_ALIASES, VirtuosoMap, VirtuosoConn), - # TODO: put snowflake stuff on this line + (DBMS.MYSQL, MYSQL_ALIASES, MySQLMap, "plugins.dbms.mysql.connector"), + (DBMS.ORACLE, ORACLE_ALIASES, OracleMap, "plugins.dbms.oracle.connector"), + (DBMS.PGSQL, PGSQL_ALIASES, PostgreSQLMap, "plugins.dbms.postgresql.connector"), + (DBMS.MSSQL, MSSQL_ALIASES, MSSQLServerMap, "plugins.dbms.mssqlserver.connector"), + (DBMS.SQLITE, SQLITE_ALIASES, SQLiteMap, "plugins.dbms.sqlite.connector"), + (DBMS.ACCESS, ACCESS_ALIASES, AccessMap, "plugins.dbms.access.connector"), + (DBMS.FIREBIRD, FIREBIRD_ALIASES, FirebirdMap, "plugins.dbms.firebird.connector"), + (DBMS.MAXDB, MAXDB_ALIASES, MaxDBMap, "plugins.dbms.maxdb.connector"), + (DBMS.SYBASE, SYBASE_ALIASES, SybaseMap, "plugins.dbms.sybase.connector"), + (DBMS.DB2, DB2_ALIASES, DB2Map, "plugins.dbms.db2.connector"), + (DBMS.HSQLDB, HSQLDB_ALIASES, HSQLDBMap, "plugins.dbms.hsqldb.connector"), + (DBMS.H2, H2_ALIASES, H2Map, "plugins.dbms.h2.connector"), + (DBMS.INFORMIX, INFORMIX_ALIASES, InformixMap, "plugins.dbms.informix.connector"), + (DBMS.MONETDB, MONETDB_ALIASES, MonetDBMap, "plugins.dbms.monetdb.connector"), + (DBMS.DERBY, DERBY_ALIASES, DerbyMap, "plugins.dbms.derby.connector"), + (DBMS.VERTICA, VERTICA_ALIASES, VerticaMap, "plugins.dbms.vertica.connector"), + (DBMS.MCKOI, MCKOI_ALIASES, MckoiMap, "plugins.dbms.mckoi.connector"), + (DBMS.PRESTO, PRESTO_ALIASES, PrestoMap, "plugins.dbms.presto.connector"), + (DBMS.ALTIBASE, ALTIBASE_ALIASES, AltibaseMap, "plugins.dbms.altibase.connector"), + (DBMS.MIMERSQL, MIMERSQL_ALIASES, MimerSQLMap, "plugins.dbms.mimersql.connector"), + (DBMS.CLICKHOUSE, CLICKHOUSE_ALIASES, ClickHouseMap, "plugins.dbms.clickhouse.connector"), + (DBMS.CRATEDB, CRATEDB_ALIASES, CrateDBMap, "plugins.dbms.cratedb.connector"), + (DBMS.CUBRID, CUBRID_ALIASES, CubridMap, "plugins.dbms.cubrid.connector"), + (DBMS.CACHE, CACHE_ALIASES, CacheMap, "plugins.dbms.cache.connector"), + (DBMS.EXTREMEDB, EXTREMEDB_ALIASES, ExtremeDBMap, "plugins.dbms.extremedb.connector"), + (DBMS.FRONTBASE, FRONTBASE_ALIASES, FrontBaseMap, "plugins.dbms.frontbase.connector"), + (DBMS.RAIMA, RAIMA_ALIASES, RaimaMap, "plugins.dbms.raima.connector"), + (DBMS.VIRTUOSO, VIRTUOSO_ALIASES, VirtuosoMap, "plugins.dbms.virtuoso.connector"), + (DBMS.SNOWFLAKE, SNOWFLAKE_ALIASES, SnowflakeMap, "plugins.dbms.snowflake.connector"), ] _ = max(_ if (conf.get("dbms") or Backend.getIdentifiedDbms() or kb.heuristicExtendedDbms or "").lower() in _[1] else () for _ in items) @@ -147,7 +117,7 @@ def setHandler(): items.remove(_) items.insert(0, _) - for dbms, aliases, Handler, Connector in items: + for dbms, aliases, Handler, connector in items: if conf.forceDbms: if conf.forceDbms.lower() not in aliases: continue @@ -159,9 +129,12 @@ def setHandler(): continue handler = Handler() - conf.dbmsConnector = Connector() + conf.dbmsConnector = None if conf.direct: + _ = __import__(connector, fromlist=['Connector']) + conf.dbmsConnector = _.Connector() + exception = None dialect = DBMS_DICT[dbms][3] diff --git a/lib/core/settings.py b/lib/core/settings.py index 87e67878cc3..1b48995cfd8 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -19,7 +19,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.1.41" +VERSION = "1.10.1.47" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/thirdparty/beautifulsoup/beautifulsoup.py b/thirdparty/beautifulsoup/beautifulsoup.py index 7401def4117..849956bb0e2 100644 --- a/thirdparty/beautifulsoup/beautifulsoup.py +++ b/thirdparty/beautifulsoup/beautifulsoup.py @@ -126,7 +126,7 @@ def _match_css_class(str): """Build a RE to match the given CSS class.""" - return re.compile(r"(^|.*\s)%s($|\s)" % str) + return re.compile(r"(^|.*\s)%s($|\s)" % re.escape(str)) # First, the classes that represent markup elements. @@ -490,7 +490,7 @@ def __unicode__(self): def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): # Substitute outgoing XML entities. data = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, self) - if encoding: + if encoding and sys.version_info < (3, 0): return data.encode(encoding) else: return data diff --git a/thirdparty/bottle/bottle.py b/thirdparty/bottle/bottle.py index 9df46294b37..e0b3185d27d 100644 --- a/thirdparty/bottle/bottle.py +++ b/thirdparty/bottle/bottle.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function """ Bottle is a fast and simple micro-framework for small web applications. It offers request dispatching (Routes) with URL parameter support, templates, @@ -9,15 +10,14 @@ Homepage and documentation: http://bottlepy.org/ -Copyright (c) 2009-2018, Marcel Hellkamp. +Copyright (c) 2009-2024, Marcel Hellkamp. License: MIT (see LICENSE for details) """ -from __future__ import print_function import sys __author__ = 'Marcel Hellkamp' -__version__ = '0.13-dev' +__version__ = '0.13.4' __license__ = 'MIT' ############################################################################### @@ -94,12 +94,13 @@ def _cli_patch(cli_args): # pragma: no coverage from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote urlunquote = functools.partial(urlunquote, encoding='latin1') from http.cookies import SimpleCookie, Morsel, CookieError - from collections import defaultdict from collections.abc import MutableMapping as DictMixin from types import ModuleType as new_module import pickle from io import BytesIO import configparser + from datetime import timezone + UTC = timezone.utc # getfullargspec was deprecated in 3.5 and un-deprecated in 3.6 # getargspec was deprecated in 3.0 and removed in 3.11 from inspect import getfullargspec @@ -117,6 +118,7 @@ def getargspec(func): def _raise(*a): raise a[0](a[1]).with_traceback(a[2]) else: # 2.x + warnings.warn("Python 2 support will be dropped in Bottle 0.14", DeprecationWarning) import httplib import thread from urlparse import urljoin, SplitResult as UrlSplitResult @@ -127,11 +129,19 @@ def _raise(*a): from imp import new_module from StringIO import StringIO as BytesIO import ConfigParser as configparser - from collections import MutableMapping as DictMixin, defaultdict + from collections import MutableMapping as DictMixin from inspect import getargspec + from datetime import tzinfo + + class _UTC(tzinfo): + def utcoffset(self, dt): return timedelta(0) + def tzname(self, dt): return "UTC" + def dst(self, dt): return timedelta(0) + UTC = _UTC() unicode = unicode json_loads = json_lds + exec(compile('def _raise(*a): raise a[0], a[1], a[2]', '', 'exec')) # Some helpers for string/byte handling @@ -168,13 +178,13 @@ def update_wrapper(wrapper, wrapped, *a, **ka): # And yes, I know PEP-8, but sometimes a lower-case classname makes more sense. -def depr(major, minor, cause, fix): +def depr(major, minor, cause, fix, stacklevel=3): text = "Warning: Use of deprecated feature or API. (Deprecated in Bottle-%d.%d)\n"\ "Cause: %s\n"\ "Fix: %s\n" % (major, minor, cause, fix) if DEBUG == 'strict': raise DeprecationWarning(text) - warnings.warn(text, DeprecationWarning, stacklevel=3) + warnings.warn(text, DeprecationWarning, stacklevel=stacklevel) return DeprecationWarning(text) @@ -340,7 +350,8 @@ def _itertokens(self, rule): g = match.groups() if g[2] is not None: depr(0, 13, "Use of old route syntax.", - "Use instead of :name in routes.") + "Use instead of :name in routes.", + stacklevel=4) if len(g[0]) % 2: # Escaped wildcard prefix += match.group(0)[len(g[0]):] offset = match.end() @@ -417,7 +428,7 @@ def getargs(path): if (flatpat, method) in self._groups: if DEBUG: msg = 'Route <%s %s> overwrites a previously defined route' - warnings.warn(msg % (method, rule), RuntimeWarning) + warnings.warn(msg % (method, rule), RuntimeWarning, stacklevel=3) self.dyna_routes[method][ self._groups[flatpat, method]] = whole_rule else: @@ -562,18 +573,17 @@ def get_undecorated_callback(self): """ Return the callback. If the callback is a decorated function, try to recover the original function. """ func = self.callback - func = getattr(func, '__func__' if py3k else 'im_func', func) - closure_attr = '__closure__' if py3k else 'func_closure' - while hasattr(func, closure_attr) and getattr(func, closure_attr): - attributes = getattr(func, closure_attr) - func = attributes[0].cell_contents - - # in case of decorators with multiple arguments - if not isinstance(func, FunctionType): - # pick first FunctionType instance from multiple arguments - func = filter(lambda x: isinstance(x, FunctionType), - map(lambda x: x.cell_contents, attributes)) - func = list(func)[0] # py3 support + while True: + if getattr(func, '__wrapped__', False): + func = func.__wrapped__ + elif getattr(func, '__func__', False): + func = func.__func__ + elif getattr(func, '__closure__', False): + cells_values = (cell.cell_contents for cell in func.__closure__) + isfunc = lambda x: isinstance(x, FunctionType) or hasattr(x, '__call__') + func = next(iter(filter(isfunc, cells_values)), func) + else: + break return func def get_callback_args(self): @@ -592,7 +602,9 @@ def get_config(self, key, default=None): def __repr__(self): cb = self.get_undecorated_callback() - return '<%s %s -> %s:%s>' % (self.method, self.rule, cb.__module__, cb.__name__) + return '<%s %s -> %s:%s>' % ( + self.method, self.rule, cb.__module__, getattr(cb, '__name__', '__call__') + ) ############################################################################### # Application Object ########################################################### @@ -1131,406 +1143,12 @@ def __exit__(self, exc_type, exc_value, traceback): def __setattr__(self, name, value): if name in self.__dict__: raise AttributeError("Attribute %s already defined. Plugin conflict?" % name) - self.__dict__[name] = value - + object.__setattr__(self, name, value) ############################################################################### # HTTP and WSGI Tools ########################################################## ############################################################################### -# Multipart parsing stuff - -class StopMarkupException(BottleException): - pass - - -HYPHEN = tob('-') -CR = tob('\r') -LF = tob('\n') -CRLF = CR + LF -LFCRLF = LF + CR + LF -HYPHENx2 = HYPHEN * 2 -CRLFx2 = CRLF * 2 -CRLF_LEN = len(CRLF) -CRLFx2_LEN = len(CRLFx2) - -MULTIPART_BOUNDARY_PATT = re.compile(r'^multipart/.+?boundary=(.+?)(;|$)') - -class MPHeadersEaeter: - end_headers_patt = re.compile(tob(r'(\r\n\r\n)|(\r(\n\r?)?)$')) - - def __init__(self): - self.headers_end_expected = None - self.eat_meth = self._eat_first_crlf_or_last_hyphens - self._meth_map = { - CR: self._eat_lf, - HYPHEN: self._eat_last_hyphen - } - self.stopped = False - - def eat(self, chunk, base): - pos = self.eat_meth(chunk, base) - if pos is None: return - if self.eat_meth != self._eat_headers: - if self.stopped: - raise StopMarkupException() - base = pos - self.eat_meth = self._eat_headers - return self.eat(chunk, base) - # found headers section end, reset eater - self.eat_meth = self._eat_first_crlf_or_last_hyphens - return pos - - def _eat_last_hyphen(self, chunk, base): - chunk_start = chunk[base: base + 2] - if not chunk_start: return - if chunk_start == HYPHEN: - self.stopped = True - return base + 1 - raise HTTPError(422, 'Last hyphen was expected, got (first 2 symbols slice): %s' % chunk_start) - - def _eat_lf(self, chunk, base): - chunk_start = chunk[base: base + 1] - if not chunk_start: return - if chunk_start == LF: return base + 1 - invalid_sequence = CR + chunk_start - raise HTTPError(422, 'Malformed headers, found invalid sequence: %s' % invalid_sequence) - - def _eat_first_crlf_or_last_hyphens(self, chunk, base): - chunk_start = chunk[base: base + 2] - if not chunk_start: return - if chunk_start == CRLF: return base + 2 - if len(chunk_start) == 1: - self.eat_meth = self._meth_map.get(chunk_start) - elif chunk_start == HYPHENx2: - self.stopped = True - return base + 2 - if self.eat_meth is None: - raise HTTPError(422, 'Malformed headers, invalid section start: %s' % chunk_start) - - def _eat_headers(self, chunk, base): - expected = self.headers_end_expected - if expected is not None: - expected_len = len(expected) - chunk_start = chunk[base:expected_len] - if chunk_start == expected: - self.headers_end_expected = None - return base + expected_len - CRLFx2_LEN - chunk_start_len = len(chunk_start) - if not chunk_start_len: return - if chunk_start_len < expected_len: - if expected.startswith(chunk_start): - self.headers_end_expected = expected[chunk_start_len:] - return - self.headers_end_expected = None - if expected == LF: # we saw CRLFCR - invalid_sequence = CR + chunk_start[0:1] - # NOTE we don not catch all CRLF-malformed errors, but only obvious ones - # to stop doing useless work - raise HTTPError(422, 'Malformed headers, found invalid sequence: %s' % invalid_sequence) - else: - assert expected_len >= 2 # (CR)LFCRLF or (CRLF)CRLF - self.headers_end_expected = None - assert self.headers_end_expected is None - s = self.end_headers_patt.search(chunk, base) - if s is None: return - end_found = s.start(1) - if end_found >= 0: return end_found - end_head = s.group(2) - if end_head is not None: - self.headers_end_expected = CRLFx2[len(end_head):] - - -class MPBodyMarkup: - def __init__(self, boundary): - self.markups = [] - self.error = None - if CR in boundary: - raise HTTPError(422, 'The `CR` must not be in the boundary: %s' % boundary) - boundary = HYPHENx2 + boundary - self.boundary = boundary - token = CRLF + boundary - self.tlen = len(token) - self.token = token - self.trest = self.trest_len = None - self.abspos = 0 - self.abs_start_section = 0 - self.headers_eater = MPHeadersEaeter() - self.cur_meth = self._eat_start_boundary - self._eat_headers = self.headers_eater.eat - self.stopped = False - self.idx = idx = defaultdict(list) # 1-based indices for each token symbol - for i, c in enumerate(token, start=1): - idx[c].append([i, token[:i]]) - - def _match_tail(self, s, start, end): - idxs = self.idx.get(s[end - 1]) - if idxs is None: return - slen = end - start - assert slen <= self.tlen - for i, thead in idxs: # idxs is 1-based index - search_pos = slen - i - if search_pos < 0: return - if s[start + search_pos:end] == thead: return i # if s_tail == token_head - - def _iter_markup(self, chunk): - if self.stopped: - raise StopMarkupException() - cur_meth = self.cur_meth - abs_start_section = self.abs_start_section - start_next_sec = 0 - skip_start = 0 - tlen = self.tlen - eat_data, eat_headers = self._eat_data, self._eat_headers - while True: - try: - end_section = cur_meth(chunk, start_next_sec) - except StopMarkupException: - self.stopped = True - return - if end_section is None: break - if cur_meth == eat_headers: - sec_name = 'headers' - start_next_sec = end_section + CRLFx2_LEN - cur_meth = eat_data - skip_start = 0 - elif cur_meth == eat_data: - sec_name = 'data' - start_next_sec = end_section + tlen - skip_start = CRLF_LEN - cur_meth = eat_headers - else: - assert cur_meth == self._eat_start_boundary - sec_name = 'data' - start_next_sec = end_section + tlen - skip_start = CRLF_LEN - cur_meth = eat_headers - - # if the body starts with a hyphen, - # we will have a negative abs_end_section equal to the length of the CRLF - abs_end_section = self.abspos + end_section - if abs_end_section < 0: - assert abs_end_section == -CRLF_LEN - end_section = -self.abspos - yield sec_name, (abs_start_section, self.abspos + end_section) - abs_start_section = self.abspos + start_next_sec + skip_start - self.abspos += len(chunk) - self.cur_meth = cur_meth - self.abs_start_section = abs_start_section - - def _eat_start_boundary(self, chunk, base): - if self.trest is None: - chunk_start = chunk[base: base + 1] - if not chunk_start: return - if chunk_start == CR: return self._eat_data(chunk, base) - boundary = self.boundary - if chunk.startswith(boundary): return base - CRLF_LEN - if chunk_start != boundary[:1]: - raise HTTPError( - 422, 'Invalid multipart/formdata body start, expected hyphen or CR, got: %s' % chunk_start) - self.trest = boundary - self.trest_len = len(boundary) - end_section = self._eat_data(chunk, base) - if end_section is not None: return end_section - - def _eat_data(self, chunk, base): - chunk_len = len(chunk) - token, tlen, trest, trest_len = self.token, self.tlen, self.trest, self.trest_len - start = base - match_tail = self._match_tail - part = None - while True: - end = start + tlen - if end > chunk_len: - part = chunk[start:] - break - if trest is not None: - if chunk[start:start + trest_len] == trest: - data_end = start + trest_len - tlen - self.trest_len = self.trest = None - return data_end - else: - trest_len = trest = None - matched_len = match_tail(chunk, start, end) - if matched_len is not None: - if matched_len == tlen: - self.trest_len = self.trest = None - return start - else: - trest_len, trest = tlen - matched_len, token[matched_len:] - start += tlen - # process the tail of the chunk - if part: - part_len = len(part) - if trest is not None: - if part_len < trest_len: - if trest.startswith(part): - trest_len -= part_len - trest = trest[part_len:] - part = None - else: - trest_len = trest = None - else: - if part.startswith(trest): - data_end = start + trest_len - tlen - self.trest_len = self.trest = None - return data_end - trest_len = trest = None - - if part is not None: - assert trest is None - matched_len = match_tail(part, 0, part_len) - if matched_len is not None: - trest_len, trest = tlen - matched_len, token[matched_len:] - self.trest_len, self.trest = trest_len, trest - - def _parse(self, chunk): - for name, start_end in self._iter_markup(chunk): - self.markups.append([name, start_end]) - - def parse(self, chunk): - if self.error is not None: return - try: - self._parse(chunk) - except Exception as exc: - self.error = exc - - -class MPBytesIOProxy: - def __init__(self, src, start, end): - self._src = src - self._st = start - self._end = end - self._pos = start - - def tell(self): - return self._pos - self._st - - def seek(self, pos): - if pos < 0: pos = 0 - self._pos = min(self._st + pos, self._end) - - def read(self, sz=None): - max_sz = self._end - self._pos - if max_sz <= 0: - return tob('') - if sz is not None and sz > 0: - sz = min(sz, max_sz) - else: - sz = max_sz - self._src.seek(self._pos) - self._pos += sz - return self._src.read(sz) - - def writable(self): - return False - - def fileno(self): - raise OSError('Not supported') - - def closed(self): - return self._src.closed() - - def close(self): - pass - - -class MPHeader: - def __init__(self, name, value, options): - self.name = name - self.value = value - self.options = options - - -class MPFieldStorage: - - _patt = re.compile(tonat('(.+?)(=(.+?))?(;|$)')) - - def __init__(self): - self.name = None - self.value = None - self.filename = None - self.file = None - self.ctype = None - self.headers = {} - - def read(self, src, headers_section, data_section, max_read): - start, end = headers_section - sz = end - start - has_read = sz - if has_read > max_read: - raise HTTPError(413, 'Request entity too large') - src.seek(start) - headers_raw = tonat(src.read(sz)) - for header_raw in headers_raw.splitlines(): - header = self.parse_header(header_raw) - self.headers[header.name] = header - if header.name == 'Content-Disposition': - self.name = header.options['name'] - self.filename = header.options.get('filename') - elif header.name == 'Content-Type': - self.ctype = header.value - if self.name is None: - raise HTTPError(422, 'Noname field found while parsing multipart/formdata body: %s' % header_raw) - if self.filename is not None: - self.file = MPBytesIOProxy(src, *data_section) - else: - start, end = data_section - sz = end - start - if sz: - has_read += sz - if has_read > max_read: - raise HTTPError(413, 'Request entity too large') - src.seek(start) - self.value = tonat(src.read(sz)) - else: - self.value = '' - return has_read - - @classmethod - def parse_header(cls, s): - htype, rest = s.split(':', 1) - opt_iter = cls._patt.finditer(rest) - hvalue = next(opt_iter).group(1).strip() - dct = {} - for it in opt_iter: - k = it.group(1).strip() - v = it.group(3) - if v is not None: - v = v.strip('"') - dct[k.lower()] = v - return MPHeader(name=htype, value=hvalue, options=dct) - - @classmethod - def iter_items(cls, src, markup, max_read): - iter_markup = iter(markup) - # check & skip empty data (body should start from empty data) - null_data = next(iter_markup, None) - if null_data is None: return - sec_name, [start, end] = null_data - assert sec_name == 'data' - if end > 0: - raise HTTPError( - 422, 'Malformed multipart/formdata, unexpected data before the first boundary at: [%d:%d]' - % (start, end)) - headers = next(iter_markup, None) - data = next(iter_markup, None) - while headers: - sec_name, headers_slice = headers - assert sec_name == 'headers' - if not data: - raise HTTPError( - 422, 'Malformed multipart/formdata, no data found for the field at: [%d:%d]' - % tuple(headers_slice)) - sec_name, data_slice = data - assert sec_name == 'data' - field = cls() - has_read = field.read(src, headers_slice, data_slice, max_read=max_read) - max_read -= has_read - yield field - headers = next(iter_markup, None) - data = next(iter_markup, None) - class BaseRequest(object): """ A wrapper for WSGI environment dictionaries that adds a lot of @@ -1720,10 +1338,6 @@ def _iter_chunked(read, bufsize): @DictProperty('environ', 'bottle.request.body', read_only=True) def _body(self): - mp_markup = None - mp_boundary_match = MULTIPART_BOUNDARY_PATT.match(self.environ.get('CONTENT_TYPE', '')) - if mp_boundary_match is not None: - mp_markup = MPBodyMarkup(tob(mp_boundary_match.group(1))) try: read_func = self.environ['wsgi.input'].read except KeyError: @@ -1733,15 +1347,12 @@ def _body(self): body, body_size, is_temp_file = BytesIO(), 0, False for part in body_iter(read_func, self.MEMFILE_MAX): body.write(part) - if mp_markup is not None: - mp_markup.parse(part) body_size += len(part) if not is_temp_file and body_size > self.MEMFILE_MAX: body, tmp = NamedTemporaryFile(mode='w+b'), body body.write(tmp.getvalue()) del tmp is_temp_file = True - body.multipart_markup = mp_markup self.environ['wsgi.input'] = body body.seek(0) return body @@ -1779,31 +1390,35 @@ def chunked(self): def POST(self): """ The values of :attr:`forms` and :attr:`files` combined into a single :class:`FormsDict`. Values are either strings (form values) or - instances of :class:`MPBytesIOProxy` (file uploads). + instances of :class:`FileUpload`. """ post = FormsDict() + content_type = self.environ.get('CONTENT_TYPE', '') + content_type, options = _parse_http_header(content_type)[0] # We default to application/x-www-form-urlencoded for everything that # is not multipart and take the fast path (also: 3.1 workaround) - if not self.content_type.startswith('multipart/'): + if not content_type.startswith('multipart/'): body = tonat(self._get_body_string(self.MEMFILE_MAX), 'latin1') for key, value in _parse_qsl(body): post[key] = value return post - if py3k: - post.recode_unicode = False - body = self.body - markup = body.multipart_markup - if markup is None: - raise HTTPError(400, '`boundary` required for mutlipart content') - elif markup.error is not None: - raise markup.error - for item in MPFieldStorage.iter_items(body, markup.markups, self.MEMFILE_MAX): - if item.filename is None: - post[item.name] = item.value + post.recode_unicode = False + charset = options.get("charset", "utf8") + boundary = options.get("boundary") + if not boundary: + raise MultipartError("Invalid content type header, missing boundary") + parser = _MultipartParser(self.body, boundary, self.content_length, + mem_limit=self.MEMFILE_MAX, memfile_limit=self.MEMFILE_MAX, + charset=charset) + + for part in parser.parse(): + if not part.filename and part.is_buffered(): + post[part.name] = tonat(part.value, 'utf8') else: - post[item.name] = FileUpload(item.file, item.name, - item.filename, item.headers) + post[part.name] = FileUpload(part.file, part.name, + part.filename, part.headerlist) + return post @property @@ -1974,6 +1589,7 @@ def __getattr__(self, name): raise AttributeError('Attribute %r not defined.' % name) def __setattr__(self, name, value): + """ Define new attributes that are local to the bound request environment. """ if name == 'environ': return object.__setattr__(self, name, value) key = 'bottle.request.ext.%s' % name if hasattr(self, name): @@ -2024,14 +1640,6 @@ class BaseResponse(object): This class does support dict-like case-insensitive item-access to headers, but is NOT a dict. Most notably, iterating over a response yields parts of the body and not the headers. - - :param body: The response body as one of the supported types. - :param status: Either an HTTP status code (e.g. 200) or a status line - including the reason phrase (e.g. '200 OK'). - :param headers: A dictionary or a list of name-value pairs. - - Additional keyword arguments are added to the list of headers. - Underscores in the header name are replaced with dashes. """ default_status = 200 @@ -2047,6 +1655,16 @@ class BaseResponse(object): } def __init__(self, body='', status=None, headers=None, **more_headers): + """ Create a new response object. + + :param body: The response body as one of the supported types. + :param status: Either an HTTP status code (e.g. 200) or a status line + including the reason phrase (e.g. '200 OK'). + :param headers: A dictionary or a list of name-value pairs. + + Additional keyword arguments are added to the list of headers. + Underscores in the header name are replaced with dashes. + """ self._cookies = None self._headers = {} self.body = body @@ -2185,7 +1803,7 @@ def headerlist(self): content_length = HeaderProperty('Content-Length', reader=int, default=-1) expires = HeaderProperty( 'Expires', - reader=lambda x: datetime.utcfromtimestamp(parse_date(x)), + reader=lambda x: datetime.fromtimestamp(parse_date(x), UTC), writer=lambda x: http_date(x)) @property @@ -2337,10 +1955,18 @@ class LocalResponse(BaseResponse): class HTTPResponse(Response, BottleException): + """ A subclass of :class:`Response` that can be raised or returned from request + handlers to short-curcuit request processing and override changes made to the + global :data:`request` object. This bypasses error handlers, even if the status + code indicates an error. Return or raise :class:`HTTPError` to trigger error + handlers. + """ + def __init__(self, body='', status=None, headers=None, **more_headers): super(HTTPResponse, self).__init__(body, status, headers, **more_headers) def apply(self, other): + """ Copy the state of this response to a different :class:`Response` object. """ other._status_code = self._status_code other._status_line = self._status_line other._headers = self._headers @@ -2349,6 +1975,8 @@ def apply(self, other): class HTTPError(HTTPResponse): + """ A subclass of :class:`HTTPResponse` that triggers error handlers. """ + default_status = 500 def __init__(self, @@ -2460,6 +2088,12 @@ def find_module(self, fullname, path=None): if fullname.rsplit('.', 1)[0] != self.name: return return self + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass # This probably breaks importlib.reload() :/ + def load_module(self, fullname): if fullname in sys.modules: return sys.modules[fullname] modname = fullname.rsplit('.', 1)[1] @@ -2725,10 +2359,10 @@ def __contains__(self, key): class ConfigDict(dict): """ A dict-like configuration storage with additional support for - namespaces, validators, meta-data, overlays and more. + namespaces, validators, meta-data and overlays. - This dict-like class is heavily optimized for read access. All read-only - methods as well as item access should be as fast as the built-in dict. + This dict-like class is heavily optimized for read access. + Read-only methods and item access should be as fast as a native dict. """ __slots__ = ('_meta', '_change_listener', '_overlays', '_virtual_keys', '_source', '__weakref__') @@ -2743,29 +2377,19 @@ def __init__(self): #: Keys of values copied from the source (values we do not own) self._virtual_keys = set() - def load_module(self, path, squash=True): + def load_module(self, name, squash=True): """Load values from a Python module. - Example modue ``config.py``:: + Import a python module by name and add all upper-case module-level + variables to this config dict. - DEBUG = True - SQLITE = { - "db": ":memory:" - } - - - >>> c = ConfigDict() - >>> c.load_module('config') - {DEBUG: True, 'SQLITE.DB': 'memory'} - >>> c.load_module("config", False) - {'DEBUG': True, 'SQLITE': {'DB': 'memory'}} - - :param squash: If true (default), dictionary values are assumed to - represent namespaces (see :meth:`load_dict`). + :param name: Module name to import and load. + :param squash: If true (default), nested dicts are assumed to + represent namespaces and flattened (see :meth:`load_dict`). """ - config_obj = load(path) - obj = {key: getattr(config_obj, key) for key in dir(config_obj) - if key.isupper()} + config_obj = load(name) + obj = {key: getattr(config_obj, key) + for key in dir(config_obj) if key.isupper()} if squash: self.load_dict(obj) @@ -2774,28 +2398,15 @@ def load_module(self, path, squash=True): return self def load_config(self, filename, **options): - """ Load values from an ``*.ini`` style config file. - - A configuration file consists of sections, each led by a - ``[section]`` header, followed by key/value entries separated by - either ``=`` or ``:``. Section names and keys are case-insensitive. - Leading and trailing whitespace is removed from keys and values. - Values can be omitted, in which case the key/value delimiter may - also be left out. Values can also span multiple lines, as long as - they are indented deeper than the first line of the value. Commands - are prefixed by ``#`` or ``;`` and may only appear on their own on - an otherwise empty line. - - Both section and key names may contain dots (``.``) as namespace - separators. The actual configuration parameter name is constructed - by joining section name and key name together and converting to - lower case. - - The special sections ``bottle`` and ``ROOT`` refer to the root - namespace and the ``DEFAULT`` section defines default values for all - other sections. + """ Load values from ``*.ini`` style config files using configparser. + + INI style sections (e.g. ``[section]``) are used as namespace for + all keys within that section. Both section and key names may contain + dots as namespace separators and are converted to lower-case. - With Python 3, extended string interpolation is enabled. + The special sections ``[bottle]`` and ``[ROOT]`` refer to the root + namespace and the ``[DEFAULT]`` section defines default values for all + other sections. :param filename: The path of a config file, or a list of paths. :param options: All keyword parameters are passed to the underlying @@ -2849,7 +2460,7 @@ def update(self, *a, **ka): for key, value in dict(*a, **ka).items(): self[prefix + key] = value - def setdefault(self, key, value): + def setdefault(self, key, value=None): if key not in self: self[key] = value return self[key] @@ -2887,8 +2498,7 @@ def __delitem__(self, key): overlay._delete_virtual(key) def _set_virtual(self, key, value): - """ Recursively set or update virtual keys. Do nothing if non-virtual - value is present. """ + """ Recursively set or update virtual keys. """ if key in self and key not in self._virtual_keys: return # Do nothing for non-virtual keys. @@ -2900,8 +2510,7 @@ def _set_virtual(self, key, value): overlay._set_virtual(key, value) def _delete_virtual(self, key): - """ Recursively delete virtual entry. Do nothing if key is not virtual. - """ + """ Recursively delete virtual entry. """ if key not in self._virtual_keys: return # Do nothing for non-virtual keys. @@ -2926,7 +2535,10 @@ def meta_get(self, key, metafield, default=None): return self._meta.get(key, {}).get(metafield, default) def meta_set(self, key, metafield, value): - """ Set the meta field for a key to a new value. """ + """ Set the meta field for a key to a new value. + + Meta-fields are shared between all members of an overlay tree. + """ self._meta.setdefault(key, {})[metafield] = value def meta_list(self, key): @@ -3127,7 +2739,7 @@ def open(self, name, mode='r', *args, **kwargs): class FileUpload(object): def __init__(self, fileobj, name, filename, headers=None): - """ Wrapper for file uploads. """ + """ Wrapper for a single file uploaded via ``multipart/form-data``. """ #: Open file(-like) object (BytesIO buffer or temporary file) self.file = fileobj #: Name of the upload form field @@ -3259,12 +2871,12 @@ def static_file(filename, root, ``If-None-Match``) are answered with ``304 Not Modified`` whenever possible. ``HEAD`` and ``Range`` requests (used by download managers to check or continue partial downloads) are also handled automatically. - """ root = os.path.join(os.path.abspath(root), '') filename = os.path.abspath(os.path.join(root, filename.strip('/\\'))) headers = headers.copy() if headers else {} + getenv = request.environ.get if not filename.startswith(root): return HTTPError(403, "Access denied.") @@ -3274,31 +2886,32 @@ def static_file(filename, root, return HTTPError(403, "You do not have permission to access this file.") if mimetype is True: - if download and download is not True: - mimetype, encoding = mimetypes.guess_type(download) - else: - mimetype, encoding = mimetypes.guess_type(filename) - if encoding: - headers['Content-Encoding'] = encoding + name = download if isinstance(download, str) else filename + mimetype, encoding = mimetypes.guess_type(name) + if encoding == 'gzip': + mimetype = 'application/gzip' + elif encoding: # e.g. bzip2 -> application/x-bzip2 + mimetype = 'application/x-' + encoding + + if charset and mimetype and 'charset=' not in mimetype \ + and (mimetype[:5] == 'text/' or mimetype == 'application/javascript'): + mimetype += '; charset=%s' % charset if mimetype: - if (mimetype[:5] == 'text/' or mimetype == 'application/javascript')\ - and charset and 'charset' not in mimetype: - mimetype += '; charset=%s' % charset headers['Content-Type'] = mimetype + if download is True: + download = os.path.basename(filename) + if download: - download = os.path.basename(filename if download is True else download) + download = download.replace('"','') headers['Content-Disposition'] = 'attachment; filename="%s"' % download stats = os.stat(filename) headers['Content-Length'] = clen = stats.st_size - headers['Last-Modified'] = email.utils.formatdate(stats.st_mtime, - usegmt=True) + headers['Last-Modified'] = email.utils.formatdate(stats.st_mtime, usegmt=True) headers['Date'] = email.utils.formatdate(time.time(), usegmt=True) - getenv = request.environ.get - if etag is None: etag = '%d:%d:%d:%d:%s' % (stats.st_dev, stats.st_ino, stats.st_mtime, clen, filename) @@ -3416,7 +3029,7 @@ def _parse_http_header(h): values.append((parts[0].strip(), {})) for attr in parts[1:]: name, value = attr.split('=', 1) - values[-1][1][name.strip()] = value.strip() + values[-1][1][name.strip().lower()] = value.strip() else: lop, key, attrs = ',', None, {} for quoted, plain, tok in _hsplit(h): @@ -3428,9 +3041,9 @@ def _parse_http_header(h): if tok == '=': key = value else: - attrs[value] = '' + attrs[value.strip().lower()] = '' elif lop == '=' and key: - attrs[key] = value + attrs[key.strip().lower()] = value key = None lop = tok return values @@ -3595,6 +3208,255 @@ def wrapper(*a, **ka): uninstall = make_default_app_wrapper('uninstall') url = make_default_app_wrapper('get_url') + +############################################################################### +# Multipart Handling ########################################################### +############################################################################### +# cgi.FieldStorage was deprecated in Python 3.11 and removed in 3.13 +# This implementation is based on https://github.com/defnull/multipart/ + + +class MultipartError(HTTPError): + def __init__(self, msg): + HTTPError.__init__(self, 400, "MultipartError: " + msg) + + +class _MultipartParser(object): + def __init__( + self, + stream, + boundary, + content_length=-1, + disk_limit=2 ** 30, + mem_limit=2 ** 20, + memfile_limit=2 ** 18, + buffer_size=2 ** 16, + charset="latin1", + ): + self.stream = stream + self.boundary = boundary + self.content_length = content_length + self.disk_limit = disk_limit + self.memfile_limit = memfile_limit + self.mem_limit = min(mem_limit, self.disk_limit) + self.buffer_size = min(buffer_size, self.mem_limit) + self.charset = charset + + if not boundary: + raise MultipartError("No boundary.") + + if self.buffer_size - 6 < len(boundary): # "--boundary--\r\n" + raise MultipartError("Boundary does not fit into buffer_size.") + + def _lineiter(self): + """ Iterate over a binary file-like object (crlf terminated) line by + line. Each line is returned as a (line, crlf) tuple. Lines larger + than buffer_size are split into chunks where all but the last chunk + has an empty string instead of crlf. Maximum chunk size is twice the + buffer size. + """ + + read = self.stream.read + maxread, maxbuf = self.content_length, self.buffer_size + partial = b"" # Contains the last (partial) line + + while True: + chunk = read(maxbuf if maxread < 0 else min(maxbuf, maxread)) + maxread -= len(chunk) + if not chunk: + if partial: + yield partial, b'' + break + + if partial: + chunk = partial + chunk + + scanpos = 0 + while True: + i = chunk.find(b'\r\n', scanpos) + if i >= 0: + yield chunk[scanpos:i], b'\r\n' + scanpos = i + 2 + else: # CRLF not found + partial = chunk[scanpos:] if scanpos else chunk + break + + if len(partial) > maxbuf: + yield partial[:-1], b"" + partial = partial[-1:] + + def parse(self): + """ Return a MultiPart iterator. Can only be called once. """ + + lines, line = self._lineiter(), "" + separator = b"--" + tob(self.boundary) + terminator = separator + b"--" + mem_used, disk_used = 0, 0 # Track used resources to prevent DoS + is_tail = False # True if the last line was incomplete (cutted) + + # Consume first boundary. Ignore any preamble, as required by RFC + # 2046, section 5.1.1. + for line, nl in lines: + if line in (separator, terminator): + break + else: + raise MultipartError("Stream does not contain boundary") + + # First line is termainating boundary -> empty multipart stream + if line == terminator: + for _ in lines: + raise MultipartError("Found data after empty multipart stream") + return + + part_options = { + "buffer_size": self.buffer_size, + "memfile_limit": self.memfile_limit, + "charset": self.charset, + } + part = _MultipartPart(**part_options) + + for line, nl in lines: + if not is_tail and (line == separator or line == terminator): + part.finish() + if part.is_buffered(): + mem_used += part.size + else: + disk_used += part.size + yield part + if line == terminator: + break + part = _MultipartPart(**part_options) + else: + is_tail = not nl # The next line continues this one + try: + part.feed(line, nl) + if part.is_buffered(): + if part.size + mem_used > self.mem_limit: + raise MultipartError("Memory limit reached.") + elif part.size + disk_used > self.disk_limit: + raise MultipartError("Disk limit reached.") + except MultipartError: + part.close() + raise + else: + part.close() + + if line != terminator: + raise MultipartError("Unexpected end of multipart stream.") + + +class _MultipartPart(object): + def __init__(self, buffer_size=2 ** 16, memfile_limit=2 ** 18, charset="latin1"): + self.headerlist = [] + self.headers = None + self.file = False + self.size = 0 + self._buf = b"" + self.disposition = None + self.name = None + self.filename = None + self.content_type = None + self.charset = charset + self.memfile_limit = memfile_limit + self.buffer_size = buffer_size + + def feed(self, line, nl=""): + if self.file: + return self.write_body(line, nl) + return self.write_header(line, nl) + + def write_header(self, line, nl): + line = line.decode(self.charset) + + if not nl: + raise MultipartError("Unexpected end of line in header.") + + if not line.strip(): # blank line -> end of header segment + self.finish_header() + elif line[0] in " \t" and self.headerlist: + name, value = self.headerlist.pop() + self.headerlist.append((name, value + line.strip())) + else: + if ":" not in line: + raise MultipartError("Syntax error in header: No colon.") + + name, value = line.split(":", 1) + self.headerlist.append((name.strip(), value.strip())) + + def write_body(self, line, nl): + if not line and not nl: + return # This does not even flush the buffer + + self.size += len(line) + len(self._buf) + self.file.write(self._buf + line) + self._buf = nl + + if self.content_length > 0 and self.size > self.content_length: + raise MultipartError("Size of body exceeds Content-Length header.") + + if self.size > self.memfile_limit and isinstance(self.file, BytesIO): + self.file, old = NamedTemporaryFile(mode="w+b"), self.file + old.seek(0) + + copied, maxcopy, chunksize = 0, self.size, self.buffer_size + read, write = old.read, self.file.write + while copied < maxcopy: + chunk = read(min(chunksize, maxcopy - copied)) + write(chunk) + copied += len(chunk) + + def finish_header(self): + self.file = BytesIO() + self.headers = HeaderDict(self.headerlist) + content_disposition = self.headers.get("Content-Disposition") + content_type = self.headers.get("Content-Type") + + if not content_disposition: + raise MultipartError("Content-Disposition header is missing.") + + self.disposition, self.options = _parse_http_header(content_disposition)[0] + self.name = self.options.get("name") + if "filename" in self.options: + self.filename = self.options.get("filename") + if self.filename[1:3] == ":\\" or self.filename[:2] == "\\\\": + self.filename = self.filename.split("\\")[-1] # ie6 bug + + self.content_type, options = _parse_http_header(content_type)[0] if content_type else (None, {}) + self.charset = options.get("charset") or self.charset + + self.content_length = int(self.headers.get("Content-Length", "-1")) + + def finish(self): + if not self.file: + raise MultipartError("Incomplete part: Header section not closed.") + self.file.seek(0) + + def is_buffered(self): + """ Return true if the data is fully buffered in memory.""" + return isinstance(self.file, BytesIO) + + @property + def value(self): + """ Data decoded with the specified charset """ + + return self.raw.decode(self.charset) + + @property + def raw(self): + """ Data without decoding """ + pos = self.file.tell() + self.file.seek(0) + + try: + return self.file.read() + finally: + self.file.seek(pos) + + def close(self): + if self.file: + self.file.close() + self.file = False + ############################################################################### # Server Adapter ############################################################### ############################################################################### @@ -4811,5 +4673,9 @@ def _cli_error(cli_msg): config=config) -if __name__ == '__main__': # pragma: no coverage +def main(): _main(sys.argv) + + +if __name__ == '__main__': # pragma: no coverage + main() diff --git a/thirdparty/prettyprint/__init__.py b/thirdparty/prettyprint/__init__.py deleted file mode 100644 index 1f9e1434354..00000000000 --- a/thirdparty/prettyprint/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python - -#Copyright (c) 2010, Chris Hall -#All rights reserved. - -#Redistribution and use in source and binary forms, with or without modification, -#are permitted provided that the following conditions are met: - -#* Redistributions of source code must retain the above copyright notice, -#this list of conditions and the following disclaimer. -#* Redistributions in binary form must reproduce the above copyright notice, -#this list of conditions and the following disclaimer in the documentation -#and/or other materials provided with the distribution. - -#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -#DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -#ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -#(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -#ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -#SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -pass diff --git a/thirdparty/prettyprint/prettyprint.py b/thirdparty/prettyprint/prettyprint.py deleted file mode 100644 index 586d808114a..00000000000 --- a/thirdparty/prettyprint/prettyprint.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python - -#Copyright (c) 2010, Chris Hall -#All rights reserved. - -#Redistribution and use in source and binary forms, with or without modification, -#are permitted provided that the following conditions are met: - -#* Redistributions of source code must retain the above copyright notice, -#this list of conditions and the following disclaimer. -#* Redistributions in binary form must reproduce the above copyright notice, -#this list of conditions and the following disclaimer in the documentation -#and/or other materials provided with the distribution. - -#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -#DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -#ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -#(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -#ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -#SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from xml.dom import minidom -from xml.dom import Node - -def format(text): - doc = minidom.parseString(text) - root = doc.childNodes[0] - return root.toprettyxml(indent=' ') - -def formatXML(doc, encoding=None): - root = doc.childNodes[0] - return root.toprettyxml(indent=' ', encoding=encoding) - -def _patch_minidom(): - minidom.Text.writexml = _writexml_text - minidom.Element.writexml = _writexml_element - minidom.Node.toprettyxml = _toprettyxml_node - -def _collapse(node): - for child in node.childNodes: - if child.nodeType == Node.TEXT_NODE and len(child.data.strip()) == 0: - child.data = '' - else: - _collapse(child) - -def _writexml_text(self, writer, indent="", addindent="", newl=""): - minidom._write_data(writer, "%s"%(self.data.strip())) - -def _writexml_element(self, writer, indent="", addindent="", newl=""): - # indent = current indentation - # addindent = indentation to add to higher levels - # newl = newline string - writer.write(indent+"<" + self.tagName) - - attrs = self._get_attributes() - a_names = attrs.keys() - a_names.sort() - - for a_name in a_names: - writer.write(" %s=\"" % a_name) - minidom._write_data(writer, attrs[a_name].value) - writer.write("\"") - if self.childNodes: - if self.childNodes[0].nodeType == Node.TEXT_NODE and len(self.childNodes[0].data) > 0: - writer.write(">") - else: - writer.write(">%s"%(newl)) - for node in self.childNodes: - node.writexml(writer,indent+addindent,addindent,newl) - if self.childNodes[-1].nodeType == Node.TEXT_NODE and len(self.childNodes[0].data) > 0: - writer.write("%s" % (self.tagName,newl)) - else: - writer.write("%s%s" % (indent,self.tagName,newl)) - else: - writer.write("/>%s"%(newl)) - -def _toprettyxml_node(self, indent="\t", newl="\n", encoding = None): - _collapse(self) - # indent = the indentation string to prepend, per level - # newl = the newline string to append - writer = minidom._get_StringIO() - if encoding is not None: - import codecs - # Can't use codecs.getwriter to preserve 2.0 compatibility - writer = codecs.lookup(encoding)[3](writer) - if self.nodeType == Node.DOCUMENT_NODE: - # Can pass encoding only to document, to put it into XML header - self.writexml(writer, "", indent, newl, encoding) - else: - self.writexml(writer, "", indent, newl) - return writer.getvalue() - -_patch_minidom() diff --git a/thirdparty/socks/socks.py b/thirdparty/socks/socks.py index d9907e7ac5b..065f90e0869 100644 --- a/thirdparty/socks/socks.py +++ b/thirdparty/socks/socks.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """SocksiPy - Python SOCKS module. -Version 1.00 +Version 1.01 Copyright 2006 Dan-Haim. All rights reserved. @@ -44,6 +44,7 @@ """ +import functools import socket import struct @@ -107,8 +108,29 @@ def wrapmodule(module): This will only work on modules that import socket directly into the namespace; most of the Python Standard Library falls into this category. """ - if _defaultproxy != None: - module.socket.socket = socksocket + if _defaultproxy is not None: + _orig_socket_ctor = _orgsocket + + @functools.wraps(_orig_socket_ctor) + def guarded_socket(*args, **kwargs): + # socket.socket([family[, type[, proto]]]) + family = args[0] if len(args) > 0 else kwargs.get("family", socket.AF_INET) + stype = args[1] if len(args) > 1 else kwargs.get("type", socket.SOCK_STREAM) + + # Normalize socket type by stripping flags (Py3.3+ may OR these in) + flags = 0 + flags |= getattr(socket, "SOCK_CLOEXEC", 0) + flags |= getattr(socket, "SOCK_NONBLOCK", 0) + base_type = stype & ~flags + + if family in (socket.AF_INET, getattr(socket, "AF_INET6", socket.AF_INET)) and base_type == socket.SOCK_STREAM: + return socksocket(*args, **kwargs) + + # Fallback: don't proxy AF_UNIX / raw / etc. + return _orig_socket_ctor(*args, **kwargs) + + module.socket.socket = guarded_socket + if _defaultproxy[0] == PROXY_TYPE_SOCKS4: # Note: unable to prevent DNS leakage in SOCKS4 (Reference: https://security.stackexchange.com/a/171280) pass