diff --git a/poetry.lock b/poetry.lock index 3cac9a84..43404a83 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aniso8601" version = "9.0.1" description = "A library for parsing ISO 8601 strings." -category = "main" optional = false python-versions = "*" files = [ @@ -19,7 +18,6 @@ dev = ["black", "coverage", "isort", "pre-commit", "pyenchant", "pylint"] name = "ansicolors" version = "1.1.8" description = "ANSI colors for Python" -category = "main" optional = false python-versions = "*" files = [ @@ -31,7 +29,6 @@ files = [ name = "argparse" version = "1.4.0" description = "Python command-line parsing library" -category = "main" optional = false python-versions = "*" files = [ @@ -43,7 +40,6 @@ files = [ name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -55,7 +51,6 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -74,7 +69,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "bcrypt" version = "4.0.1" description = "Modern password hashing for your software and your servers" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -109,7 +103,6 @@ typecheck = ["mypy"] name = "black" version = "22.10.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -154,7 +147,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "boto3" version = "1.26.115" description = "The AWS SDK for Python" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -174,7 +166,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.115" description = "Low-level, data-driven core of boto 3." -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -194,7 +185,6 @@ crt = ["awscrt (==0.16.9)"] name = "build" version = "0.10.0" description = "A simple, correct Python build frontend" -category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -218,7 +208,6 @@ virtualenv = ["virtualenv (>=20.0.35)"] name = "cachecontrol" version = "0.12.11" description = "httplib2 caching for requests" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -239,7 +228,6 @@ redis = ["redis (>=2.10.5)"] name = "certifi" version = "2021.10.8" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = "*" files = [ @@ -251,7 +239,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -328,7 +315,6 @@ pycparser = "*" name = "chardet" version = "5.1.0" description = "Universal encoding detector for Python 3" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -340,7 +326,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -425,7 +410,6 @@ files = [ name = "cleo" version = "2.0.1" description = "Cleo allows you to create beautiful and testable command-line interfaces." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -441,7 +425,6 @@ rapidfuzz = ">=2.2.0,<3.0.0" name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -456,7 +439,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -468,7 +450,6 @@ files = [ name = "contourpy" version = "1.0.7" description = "Python library for calculating contours of 2D quadrilateral grids" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -543,7 +524,6 @@ test-no-images = ["pytest"] name = "coverage" version = "7.2.3" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -610,7 +590,6 @@ toml = ["tomli"] name = "crashtest" version = "0.4.1" description = "Manage Python errors with ease" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -622,7 +601,6 @@ files = [ name = "cryptography" version = "40.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -664,7 +642,6 @@ tox = ["tox"] name = "cycler" version = "0.11.0" description = "Composable style cycles" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -676,7 +653,6 @@ files = [ name = "daemonize" version = "2.5.0" description = "Library to enable your code run as a daemon process on Unix-like systems." -category = "main" optional = false python-versions = "*" files = [ @@ -688,7 +664,6 @@ files = [ name = "dataproperty" version = "0.55.0" description = "Python library for extract property from data." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -708,7 +683,6 @@ test = ["pytest (>=6.0.1)", "pytest-md-report (>=0.1)", "termcolor"] name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -720,7 +694,6 @@ files = [ name = "deprecated" version = "1.2.13" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -738,7 +711,6 @@ dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -748,30 +720,30 @@ files = [ [[package]] name = "docker" -version = "5.0.3" +version = "7.1.0" description = "A Python library for the Docker Engine API." -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "docker-5.0.3-py2.py3-none-any.whl", hash = "sha256:7a79bb439e3df59d0a72621775d600bc8bc8b422d285824cb37103eab91d1ce0"}, - {file = "docker-5.0.3.tar.gz", hash = "sha256:d916a26b62970e7c2f554110ed6af04c7ccff8e9f81ad17d0d40c75637e227fb"}, + {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, + {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, ] [package.dependencies] -pywin32 = {version = "227", markers = "sys_platform == \"win32\""} -requests = ">=2.14.2,<2.18.0 || >2.18.0" -websocket-client = ">=0.32.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" [package.extras] -ssh = ["paramiko (>=2.4.2)"] -tls = ["cryptography (>=3.4.7)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] +dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"] +docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "dominate" version = "2.7.0" description = "Dominate is a Python library for creating and manipulating HTML documents using an elegant DOM API." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -783,7 +755,6 @@ files = [ name = "dulwich" version = "0.21.3" description = "Python Git Library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -858,7 +829,6 @@ pgp = ["gpg"] name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -873,7 +843,6 @@ test = ["pytest (>=6)"] name = "filelock" version = "3.11.0" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -889,7 +858,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "p name = "flake8" version = "6.0.0" description = "the modular source code checker: pep8 pyflakes and co" -category = "dev" optional = false python-versions = ">=3.8.1" files = [ @@ -906,7 +874,6 @@ pyflakes = ">=3.0.0,<3.1.0" name = "flask" version = "2.2.3" description = "A simple framework for building complex web applications." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -929,7 +896,6 @@ dotenv = ["python-dotenv"] name = "flask-httpauth" version = "4.7.0" description = "HTTP authentication for Flask routes" -category = "main" optional = false python-versions = "*" files = [ @@ -944,7 +910,6 @@ flask = "*" name = "flask-restx" version = "0.5.1" description = "Fully featured framework for fast, easy and documented API development with Flask" -category = "main" optional = false python-versions = "*" files = [ @@ -969,7 +934,6 @@ test = ["Faker (==2.0.0)", "blinker", "invoke (==1.3.0)", "mock (==3.0.5)", "oss name = "fonttools" version = "4.39.3" description = "Tools to manipulate font files" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -995,7 +959,6 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1010,7 +973,6 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1025,7 +987,6 @@ gitdb = ">=4.0.1,<5" name = "html5lib" version = "1.1" description = "HTML parser based on the WHATWG HTML specification" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1047,7 +1008,6 @@ lxml = ["lxml"] name = "humanize" version = "2.6.0" description = "Python humanize utilities" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1062,7 +1022,6 @@ tests = ["freezegun", "pytest", "pytest-cov"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1074,7 +1033,6 @@ files = [ name = "importlib-metadata" version = "6.4.1" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1094,7 +1052,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1113,7 +1070,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1125,7 +1081,6 @@ files = [ name = "installer" version = "0.7.0" description = "A library for installing Python wheels." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1137,7 +1092,6 @@ files = [ name = "itsdangerous" version = "2.1.2" description = "Safely pass data to untrusted environments and back." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1149,7 +1103,6 @@ files = [ name = "jaraco-classes" version = "3.2.3" description = "Utility functions for Python class constructs" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1168,7 +1121,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "jeepney" version = "0.8.0" description = "Low-level, pure Python DBus protocol wrapper." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1184,7 +1136,6 @@ trio = ["async_generator", "trio"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1202,7 +1153,6 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1214,7 +1164,6 @@ files = [ name = "jsonpath-ng" version = "1.5.3" description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." -category = "main" optional = false python-versions = "*" files = [ @@ -1232,7 +1181,6 @@ six = "*" name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1254,7 +1202,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "keyring" version = "23.13.1" description = "Store and access your passwords safely." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1279,7 +1226,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "kiwisolver" version = "1.4.4" description = "A fast implementation of the Cassowary constraint solver" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1357,7 +1303,6 @@ files = [ name = "lockfile" version = "0.12.2" description = "Platform-independent file locking module" -category = "dev" optional = false python-versions = "*" files = [ @@ -1369,7 +1314,6 @@ files = [ name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1429,7 +1373,6 @@ files = [ name = "marshmallow" version = "3.19.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1450,7 +1393,6 @@ tests = ["pytest", "pytz", "simplejson"] name = "matplotlib" version = "3.7.1" description = "Python plotting package" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1513,7 +1455,6 @@ python-dateutil = ">=2.7" name = "mbstrdecoder" version = "1.1.2" description = "multi-byte character string decoder" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1531,7 +1472,6 @@ test = ["Faker (>=1.0.2)", "pytest (>=6.0.1)", "pytest-md-report (>=0.1)"] name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1543,7 +1483,6 @@ files = [ name = "more-itertools" version = "9.1.0" description = "More routines for operating on iterables, beyond itertools" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1555,7 +1494,6 @@ files = [ name = "msgpack" version = "1.0.5" description = "MessagePack serializer" -category = "dev" optional = false python-versions = "*" files = [ @@ -1628,7 +1566,6 @@ files = [ name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1640,7 +1577,6 @@ files = [ name = "node-semver" version = "0.8.1" description = "port of node-semver" -category = "main" optional = false python-versions = "*" files = [ @@ -1655,7 +1591,6 @@ testing = ["pytest"] name = "numpy" version = "1.24.2" description = "Fundamental package for array computing in Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1693,7 +1628,6 @@ files = [ name = "oyaml" version = "1.0" description = "Ordered YAML: drop-in replacement for PyYAML which preserves dict ordering" -category = "main" optional = false python-versions = "*" files = [ @@ -1708,7 +1642,6 @@ pyyaml = "*" name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1720,7 +1653,6 @@ files = [ name = "pandas" version = "1.5.3" description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1756,8 +1688,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.1" pytz = ">=2020.1" @@ -1769,7 +1701,6 @@ test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] name = "paramiko" version = "2.12.0" description = "SSH2 protocol library" -category = "main" optional = false python-versions = "*" files = [ @@ -1793,7 +1724,6 @@ invoke = ["invoke (>=1.3)"] name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1805,7 +1735,6 @@ files = [ name = "pathvalidate" version = "2.5.2" description = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1820,7 +1749,6 @@ test = ["allpairspy", "click", "faker", "pytest (>=6.0.1)", "pytest-discord (>=0 name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." -category = "dev" optional = false python-versions = "*" files = [ @@ -1835,7 +1763,6 @@ ptyprocess = ">=0.5" name = "pillow" version = "9.5.0" description = "Python Imaging Library (Fork)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1915,7 +1842,6 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pkginfo" version = "1.9.6" description = "Query metadata from sdists / bdists / installed packages." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1930,7 +1856,6 @@ testing = ["pytest", "pytest-cov"] name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1942,7 +1867,6 @@ files = [ name = "platformdirs" version = "2.6.2" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1958,7 +1882,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest- name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1974,7 +1897,6 @@ testing = ["pytest", "pytest-benchmark"] name = "ply" version = "3.11" description = "Python Lex & Yacc" -category = "main" optional = false python-versions = "*" files = [ @@ -1986,7 +1908,6 @@ files = [ name = "poetry" version = "1.4.2" description = "Python dependency management and packaging made easy." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -2022,8 +1943,8 @@ tomlkit = ">=0.11.1,<0.11.2 || >0.11.2,<0.11.3 || >0.11.3,<1.0.0" trove-classifiers = ">=2022.5.19" urllib3 = ">=1.26.0,<2.0.0" virtualenv = [ - {version = ">=20.4.3,<20.4.5 || >20.4.5,<20.4.6 || >20.4.6,<21.0.0", markers = "sys_platform != \"win32\" or python_version != \"3.9\""}, {version = ">=20.4.3,<20.4.5 || >20.4.5,<20.4.6 || >20.4.6,<20.16.6", markers = "sys_platform == \"win32\" and python_version == \"3.9\""}, + {version = ">=20.4.3,<20.4.5 || >20.4.5,<20.4.6 || >20.4.6,<21.0.0", markers = "sys_platform != \"win32\" or python_version != \"3.9\""}, ] xattr = {version = ">=0.10.0,<0.11.0", markers = "sys_platform == \"darwin\""} @@ -2031,7 +1952,6 @@ xattr = {version = ">=0.10.0,<0.11.0", markers = "sys_platform == \"darwin\""} name = "poetry-core" version = "1.5.2" description = "Poetry PEP 517 Build Backend" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -2043,7 +1963,6 @@ files = [ name = "poetry-plugin-export" version = "1.3.1" description = "Poetry plugin to export the dependencies to various formats" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -2059,7 +1978,6 @@ poetry-core = ">=1.3.0,<2.0.0" name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2086,7 +2004,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" -category = "dev" optional = false python-versions = "*" files = [ @@ -2098,7 +2015,6 @@ files = [ name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2110,7 +2026,6 @@ files = [ name = "py-cpuinfo" version = "5.0.0" description = "Get CPU info with pure Python 2 & 3" -category = "main" optional = false python-versions = "*" files = [ @@ -2121,7 +2036,6 @@ files = [ name = "pycodestyle" version = "2.10.0" description = "Python style guide checker" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2133,7 +2047,6 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2145,7 +2058,6 @@ files = [ name = "pyflakes" version = "3.0.1" description = "passive checker of Python programs" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2157,7 +2069,6 @@ files = [ name = "pygithub" version = "1.58.1" description = "Use the full Github API v3" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2175,7 +2086,6 @@ requests = ">=2.14.0" name = "pyjwt" version = "2.6.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2196,7 +2106,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pynacl" version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2223,7 +2132,6 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -2238,7 +2146,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyproject-hooks" version = "1.0.0" description = "Wrappers to call pyproject.toml-based build backend hooks." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2253,7 +2160,6 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2290,7 +2196,6 @@ files = [ name = "pysftp" version = "0.2.9" description = "A friendly face on SFTP" -category = "main" optional = false python-versions = "*" files = [ @@ -2304,7 +2209,6 @@ paramiko = ">=1.17" name = "pytablewriter" version = "0.64.2" description = "pytablewriter is a Python library to write a table in various formats: AsciiDoc / CSV / Elasticsearch / HTML / JavaScript / JSON / LaTeX / LDJSON / LTSV / Markdown / MediaWiki / NumPy / Excel / Pandas / Python / reStructuredText / SQLite / TOML / TSV / YAML." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2341,7 +2245,6 @@ yaml = ["PyYAML (>=3.11,<7)"] name = "pytest" version = "7.3.1" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2364,7 +2267,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-cov" version = "4.0.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2383,7 +2285,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2398,7 +2299,6 @@ six = ">=1.5" name = "python-terraform" version = "0.10.1" description = "This is a python module provide a wrapper of terraform command line tool" -category = "main" optional = false python-versions = "*" files = [ @@ -2409,7 +2309,6 @@ files = [ name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -2419,31 +2318,31 @@ files = [ [[package]] name = "pywin32" -version = "227" +version = "306" description = "Python for Window Extensions" -category = "main" optional = false python-versions = "*" files = [ - {file = "pywin32-227-cp27-cp27m-win32.whl", hash = "sha256:371fcc39416d736401f0274dd64c2302728c9e034808e37381b5e1b22be4a6b0"}, - {file = "pywin32-227-cp27-cp27m-win_amd64.whl", hash = "sha256:4cdad3e84191194ea6d0dd1b1b9bdda574ff563177d2adf2b4efec2a244fa116"}, - {file = "pywin32-227-cp35-cp35m-win32.whl", hash = "sha256:f4c5be1a293bae0076d93c88f37ee8da68136744588bc5e2be2f299a34ceb7aa"}, - {file = "pywin32-227-cp35-cp35m-win_amd64.whl", hash = "sha256:a929a4af626e530383a579431b70e512e736e9588106715215bf685a3ea508d4"}, - {file = "pywin32-227-cp36-cp36m-win32.whl", hash = "sha256:300a2db938e98c3e7e2093e4491439e62287d0d493fe07cce110db070b54c0be"}, - {file = "pywin32-227-cp36-cp36m-win_amd64.whl", hash = "sha256:9b31e009564fb95db160f154e2aa195ed66bcc4c058ed72850d047141b36f3a2"}, - {file = "pywin32-227-cp37-cp37m-win32.whl", hash = "sha256:47a3c7551376a865dd8d095a98deba954a98f326c6fe3c72d8726ca6e6b15507"}, - {file = "pywin32-227-cp37-cp37m-win_amd64.whl", hash = "sha256:31f88a89139cb2adc40f8f0e65ee56a8c585f629974f9e07622ba80199057511"}, - {file = "pywin32-227-cp38-cp38-win32.whl", hash = "sha256:7f18199fbf29ca99dff10e1f09451582ae9e372a892ff03a28528a24d55875bc"}, - {file = "pywin32-227-cp38-cp38-win_amd64.whl", hash = "sha256:7c1ae32c489dc012930787f06244426f8356e129184a02c25aef163917ce158e"}, - {file = "pywin32-227-cp39-cp39-win32.whl", hash = "sha256:c054c52ba46e7eb6b7d7dfae4dbd987a1bb48ee86debe3f245a2884ece46e295"}, - {file = "pywin32-227-cp39-cp39-win_amd64.whl", hash = "sha256:f27cec5e7f588c3d1051651830ecc00294f90728d19c3bf6916e6dba93ea357c"}, + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, ] [[package]] name = "pywin32-ctypes" version = "0.2.0" description = "" -category = "dev" optional = false python-versions = "*" files = [ @@ -2455,7 +2354,6 @@ files = [ name = "pyworkflow" version = "0.0.2" description = "A lightweight parallel task engine" -category = "main" optional = false python-versions = "*" files = [ @@ -2466,7 +2364,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2516,7 +2413,6 @@ files = [ name = "rapidfuzz" version = "2.15.1" description = "rapid fuzzy string matching" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2621,7 +2517,6 @@ full = ["numpy"] name = "redis" version = "4.5.4" description = "Python client for Redis database and key-value store" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2640,7 +2535,6 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" name = "redisbench-admin" version = "0.9.55" description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )." -category = "main" optional = false python-versions = ">=3.7.0,<4.0.0" files = [ @@ -2685,7 +2579,6 @@ wget = ">=3.2,<4.0" name = "requests" version = "2.28.2" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7, <4" files = [ @@ -2707,7 +2600,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-toolbelt" version = "0.10.1" description = "A utility belt for advanced users of python-requests" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2722,7 +2614,6 @@ requests = ">=2.0.1,<3.0.0" name = "s3transfer" version = "0.6.0" description = "An Amazon S3 Transfer Manager" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -2740,7 +2631,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "secretstorage" version = "3.3.3" description = "Python bindings to FreeDesktop.org Secret Service API" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2756,7 +2646,6 @@ jeepney = ">=0.6" name = "semver" version = "2.13.0" description = "Python helper for Semantic Versioning (http://semver.org/)" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2768,7 +2657,6 @@ files = [ name = "setuptools" version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2785,7 +2673,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "shellingham" version = "1.5.0.post1" description = "Tool to Detect Surrounding Shell" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2797,7 +2684,6 @@ files = [ name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2809,7 +2695,6 @@ files = [ name = "slack-bolt" version = "1.17.2" description = "The Bolt Framework for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2831,7 +2716,6 @@ testing-without-asyncio = ["Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werk name = "slack-sdk" version = "3.21.2" description = "The Slack API Platform SDK for Python" -category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -2847,7 +2731,6 @@ testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "We name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2859,7 +2742,6 @@ files = [ name = "sshtunnel" version = "0.4.0" description = "Pure python SSH tunnels" -category = "main" optional = false python-versions = "*" files = [ @@ -2879,7 +2761,6 @@ test = ["tox (>=1.8.1)"] name = "tabledata" version = "1.3.1" description = "tabledata is a Python library to represent tabular data. Used for pytablewriter/pytablereader/SimpleSQLite/etc." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2899,7 +2780,6 @@ test = ["pytablewriter (>=0.46)", "pytest"] name = "tcolorpy" version = "0.1.2" description = "tcolopy is a Python library to apply true color for terminal text." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2914,7 +2794,6 @@ test = ["pytest", "pytest-md-report (>=0.1)"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2926,7 +2805,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2938,7 +2816,6 @@ files = [ name = "tomlkit" version = "0.11.7" description = "Style preserving TOML library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2950,7 +2827,6 @@ files = [ name = "tox" version = "3.28.0" description = "tox is a generic virtualenv management and test command line tool" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2972,48 +2848,30 @@ virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2, docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] -[[package]] -name = "tox-docker" -version = "3.1.0" -description = "Launch a docker instance around test runs" -category = "dev" -optional = false -python-versions = "*" -files = [ - {file = "tox-docker-3.1.0.tar.gz", hash = "sha256:e1d7d60254788a2c1956d6cdfbe2a6418ed0c18c7cf2486fd484079fc84d832c"}, - {file = "tox_docker-3.1.0-py2.py3-none-any.whl", hash = "sha256:3080c436f7fdfb5a7446215aee620638489ccd7abf812d38b6fb860337efd6f2"}, -] - -[package.dependencies] -docker = ">=2.3.0,<6.0" -tox = ">=3.0.0,<4.0" - [[package]] name = "tox-poetry-installer" -version = "0.10.2" +version = "0.10.3" description = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "tox_poetry_installer-0.10.2-py3-none-any.whl", hash = "sha256:7bf7455be5f3565951a0c4f86f3a0fe293725e05139fe777a0feff8730312b69"}, - {file = "tox_poetry_installer-0.10.2.tar.gz", hash = "sha256:07e3d8cb63066b6c8d7cfa332534b9b78ac559ce8171374b6b9ec9cc2273b9ca"}, + {file = "tox_poetry_installer-0.10.3-py3-none-any.whl", hash = "sha256:8580caf05f1ac83a7a938e449cfcb220cadaf6b087e9581ae837d782804ad4bc"}, + {file = "tox_poetry_installer-0.10.3.tar.gz", hash = "sha256:58af36082293b87e3488184f46ccf6f288c52c90e802066f53d3209c0f1c6226"}, ] [package.dependencies] cleo = {version = ">=1.0,<3.0", optional = true, markers = "extra == \"poetry\""} -poetry = {version = ">=1.2.0,<2.0.0", optional = true, markers = "extra == \"poetry\""} +poetry = {version = ">=1.2.0,<1.5.0", optional = true, markers = "extra == \"poetry\""} poetry-core = ">=1.1.0,<2.0.0" tox = ">=3.8.0,<4.0.0" [package.extras] -poetry = ["cleo (>=1.0,<3.0)", "poetry (>=1.2.0,<2.0.0)"] +poetry = ["cleo (>=1.0,<3.0)", "poetry (>=1.2.0,<1.5.0)"] [[package]] name = "tqdm" version = "4.65.0" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3034,7 +2892,6 @@ telegram = ["requests"] name = "trove-classifiers" version = "2023.3.9" description = "Canonical source for classifiers on PyPI (pypi.org)." -category = "dev" optional = false python-versions = "*" files = [ @@ -3046,7 +2903,6 @@ files = [ name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3080,7 +2936,6 @@ files = [ name = "typepy" version = "1.3.0" description = "typepy is a Python library for variable type checker/validator/converter at a run time." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3102,7 +2957,6 @@ test = ["packaging", "pytest (>=6.0.1)", "python-dateutil (>=2.8.0,<3.0.0)", "py name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3114,7 +2968,6 @@ files = [ name = "urllib3" version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -3131,7 +2984,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "virtualenv" version = "20.16.5" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -3152,7 +3004,6 @@ testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7 name = "virtualenv" version = "20.21.0" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3173,7 +3024,6 @@ test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess name = "watchdog" version = "2.3.1" description = "Filesystem events monitoring" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3214,7 +3064,6 @@ watchmedo = ["PyYAML (>=3.10)"] name = "webencodings" version = "0.5.1" description = "Character encoding aliases for legacy web content" -category = "dev" optional = false python-versions = "*" files = [ @@ -3222,28 +3071,10 @@ files = [ {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, ] -[[package]] -name = "websocket-client" -version = "1.5.1" -description = "WebSocket client for Python with low level API options" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "websocket-client-1.5.1.tar.gz", hash = "sha256:3f09e6d8230892547132177f575a4e3e73cfdf06526e20cc02aa1c3b47184d40"}, - {file = "websocket_client-1.5.1-py3-none-any.whl", hash = "sha256:cdf5877568b7e83aa7cf2244ab56a3213de587bbe0ce9d8b9600fc77b455d89e"}, -] - -[package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - [[package]] name = "werkzeug" version = "2.2.3" description = "The comprehensive WSGI web application library." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3261,7 +3092,6 @@ watchdog = ["watchdog"] name = "wget" version = "3.2" description = "pure python download utility" -category = "main" optional = false python-versions = "*" files = [ @@ -3272,7 +3102,6 @@ files = [ name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -3357,7 +3186,6 @@ files = [ name = "xattr" version = "0.10.1" description = "Python wrapper for extended filesystem attributes" -category = "dev" optional = false python-versions = "*" files = [ @@ -3442,7 +3270,6 @@ cffi = ">=1.0" name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3457,4 +3284,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "d439361509cde178c3bab7321ee6ddcf6c57df92f4837fa468e67747c14cb93f" +content-hash = "4e598701e87cfb5ff1e54e2d05072f74b33fb9a068eb9db0deaa29f03d0320dc" diff --git a/pyproject.toml b/pyproject.toml index 060092ee..d1a41daa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.75" +version = "0.1.205" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" @@ -14,7 +14,7 @@ marshmallow = "^3.12.2" argparse = "^1.4.0" Flask-HTTPAuth = "^4.4.0" PyYAML = "^6.0" -docker = "^5.0.0" +docker = "^7.1.0" redisbench-admin = "^0.9.23" psutil = "^5.9.4" PyGithub = "^1.55" @@ -29,8 +29,7 @@ pytest = "^7.2.0" pytest-cov = "^4.0.0" black = "22.10.0" flake8 = "^6.0.0" -tox-poetry-installer = {extras = ["poetry"], version = "^0.10.2"} -tox-docker = {extras = ["poetry"], version = "^3.1.0"} +tox-poetry-installer = {extras = ["poetry"], version = "^0.10.3"} [build-system] requires = ["poetry_core>=1.0.0"] diff --git a/redis_benchmarks_specification/__api__/app.py b/redis_benchmarks_specification/__api__/app.py index 69f62490..facbe03a 100644 --- a/redis_benchmarks_specification/__api__/app.py +++ b/redis_benchmarks_specification/__api__/app.py @@ -86,11 +86,13 @@ def base(): event_type = "Ignored event from webhook" use_event = False # Pull request labeled + pull_request_number = None trigger_label = PULL_REQUEST_TRIGGER_LABEL if "pull_request" in request_data: action = request_data["action"] if should_action(action): pull_request_dict = request_data["pull_request"] + head_dict = pull_request_dict["head"] repo_dict = head_dict["repo"] labels = [] @@ -107,9 +109,8 @@ def base(): label_name = label["name"] if trigger_label == label_name: use_event = True - event_type = "Pull request labeled with '{}'".format( - trigger_label - ) + pull_request_number = request_data["number"] + event_type = f"Pull request #{pull_request_number} labeled with '{trigger_label}'" detected_label = True if detected_label is False: app.logger.info( @@ -161,6 +162,8 @@ def base(): "gh_repo": gh_repo, "gh_org": gh_org, } + if pull_request_number is not None: + fields_after["pull_request"] = pull_request_number app.logger.info( "Using event {} to trigger benchmark. final fields: {}".format( event_type, fields_after @@ -174,7 +177,6 @@ def base(): event_type, response_data ) ) - else: app.logger.info( "{}. input json was: {}".format(event_type, request_data) diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index 75811e1d..27bf365c 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -1,4 +1,5 @@ import argparse +import datetime import io import json import logging @@ -31,11 +32,20 @@ REDIS_SOCKET_TIMEOUT, REDIS_BINS_EXPIRE_SECS, ) +from redis_benchmarks_specification.__common__.github import ( + check_github_available_and_actionable, + generate_build_finished_pr_comment, + update_comment_if_needed, + create_new_pr_comment, + generate_build_started_pr_comment, +) from redis_benchmarks_specification.__common__.package import ( populate_with_poetry_data, get_version_string, ) +PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None) + class ZipFileWithPermissions(ZipFile): def _extract_member(self, member, targetpath, pwd): @@ -80,6 +90,8 @@ def main(): action="store_true", help="Store the docker images in redis keys.", ) + parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN) + parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="") args = parser.parse_args() if args.logname is not None: print("Writting log to {}".format(args.logname)) @@ -138,16 +150,18 @@ def main(): build_spec_image_prefetch(builders_folder, different_build_specs) builder_consumer_group_create(conn) - + if args.github_token is not None: + logging.info("detected a github token. will update as much as possible!!! =)") previous_id = args.consumer_start_id while True: - previous_id, new_builds_count = builder_process_stream( + previous_id, new_builds_count, _ = builder_process_stream( builders_folder, conn, different_build_specs, previous_id, args.docker_air_gap, arch, + args.github_token, ) @@ -172,6 +186,17 @@ def builder_consumer_group_create(conn, id="$"): ) +def check_benchmark_build_comment(comments): + res = False + pos = -1 + for n, comment in enumerate(comments): + body = comment.body + if "CE Performance Automation : step 1 of 2" in body: + res = True + pos = n + return res, pos + + def builder_process_stream( builders_folder, conn, @@ -179,8 +204,11 @@ def builder_process_stream( previous_id, docker_air_gap=False, arch="amd64", + github_token=None, ): new_builds_count = 0 + auto_approve_github_comments = True + build_stream_fields_arr = [] logging.info("Entering blocking read waiting for work.") consumer_name = "{}-proc#{}".format(STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, "1") newTestInfo = conn.xreadgroup( @@ -215,12 +243,58 @@ def builder_process_stream( buffer = conn.get(binary_zip_key) git_timestamp_ms = None use_git_timestamp = False + commit_datetime = "n/a" + if b"commit_datetime" in testDetails: + commit_datetime = testDetails[b"commit_datetime"].decode() + commit_summary = "n/a" + if b"commit_summary" in testDetails: + commit_summary = testDetails[b"commit_summary"].decode() git_branch, git_version = get_branch_version_from_test_details(testDetails) if b"use_git_timestamp" in testDetails: use_git_timestamp = bool(testDetails[b"use_git_timestamp"]) if b"git_timestamp_ms" in testDetails: git_timestamp_ms = int(testDetails[b"git_timestamp_ms"].decode()) + tests_regexp = ".*" + if b"tests_regexp" in testDetails: + tests_regexp = testDetails[b"tests_regexp"].decode() + tests_priority_upper_limit = 10000 + if b"tests_priority_upper_limit" in testDetails: + tests_priority_upper_limit = int( + testDetails[b"tests_priority_upper_limit"].decode() + ) + tests_priority_lower_limit = 0 + if b"tests_priority_lower_limit" in testDetails: + tests_priority_lower_limit = int( + testDetails[b"tests_priority_lower_limit"].decode() + ) + tests_groups_regexp = ".*" + if b"tests_groups_regexp" in testDetails: + tests_groups_regexp = testDetails[b"tests_groups_regexp"].decode() + + # github updates + is_actionable_pr = False + contains_regression_comment = False + github_pr = None + old_regression_comment_body = "" + pr_link = "" + regression_comment = "" + pull_request = None + if b"pull_request" in testDetails: + pull_request = testDetails[b"pull_request"].decode() + logging.info(f"Detected PR info in builder. PR: {pull_request}") + verbose = True + fn = check_benchmark_build_comment + ( + contains_regression_comment, + github_pr, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + ) = check_github_available_and_actionable( + fn, github_token, pull_request, "redis", "redis", verbose + ) for build_spec in different_build_specs: build_config, id = get_build_config(builders_folder + "/" + build_spec) build_config_metadata = get_build_config_metadata(build_config) @@ -306,9 +380,41 @@ def builder_process_stream( "redis-server", build_vars_str, ) + build_start_datetime = datetime.datetime.utcnow() logging.info( - "Using the following build command {}".format(build_command) + "Using the following build command {}.".format(build_command) ) + if is_actionable_pr: + logging.info( + f"updating on github we'll start the build at {build_start_datetime}" + ) + comment_body = generate_build_started_pr_comment( + build_start_datetime, + commit_datetime, + commit_summary, + git_branch, + git_hash, + tests_groups_regexp, + tests_priority_lower_limit, + tests_priority_upper_limit, + tests_regexp, + ) + if contains_regression_comment: + update_comment_if_needed( + auto_approve_github_comments, + comment_body, + old_regression_comment_body, + regression_comment, + verbose, + ) + else: + regression_comment = create_new_pr_comment( + auto_approve_github_comments, + comment_body, + github_pr, + pr_link, + ) + docker_client.containers.run( image=build_image, volumes={ @@ -319,6 +425,10 @@ def builder_process_stream( working_dir="/mnt/redis/", command=build_command, ) + build_end_datetime = datetime.datetime.utcnow() + build_duration = build_end_datetime - build_start_datetime + build_duration_secs = build_duration.total_seconds() + build_stream_fields = { "id": id, "git_hash": git_hash, @@ -333,7 +443,13 @@ def builder_process_stream( "build_command": build_command, "metadata": json.dumps(build_config_metadata), "build_artifacts": ",".join(build_artifacts), + "tests_regexp": tests_regexp, + "tests_priority_upper_limit": tests_priority_upper_limit, + "tests_priority_lower_limit": tests_priority_lower_limit, + "tests_groups_regexp": tests_groups_regexp, } + if pull_request is not None: + build_stream_fields["pull_request"] = pull_request if git_branch is not None: build_stream_fields["git_branch"] = git_branch if git_version is not None: @@ -356,16 +472,61 @@ def builder_process_stream( if b"platform" in testDetails: build_stream_fields["platform"] = testDetails[b"platform"] if result is True: - stream_id = conn.xadd( + benchmark_stream_id = conn.xadd( STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields ) logging.info( "sucessfully built build variant {} for redis git_sha {}. Stream id: {}".format( - id, git_hash, stream_id + id, git_hash, benchmark_stream_id ) ) + streamId_decoded = streamId.decode() + benchmark_stream_id_decoded = benchmark_stream_id.decode() + builder_list_completed = ( + f"builder:{streamId_decoded}:builds_completed" + ) + conn.lpush(builder_list_completed, benchmark_stream_id_decoded) + conn.expire(builder_list_completed, REDIS_BINS_EXPIRE_SECS) + logging.info( + f"Adding information of build->benchmark stream info in list {builder_list_completed}. Adding benchmark stream id: {benchmark_stream_id_decoded}" + ) + benchmark_stream_ids = [benchmark_stream_id_decoded] + + if is_actionable_pr: + logging.info( + f"updating on github that the build finished after {build_duration_secs} seconds" + ) + comment_body = generate_build_finished_pr_comment( + benchmark_stream_ids, + commit_datetime, + commit_summary, + git_branch, + git_hash, + tests_groups_regexp, + tests_priority_lower_limit, + tests_priority_upper_limit, + tests_regexp, + build_start_datetime, + build_duration_secs, + ) + if contains_regression_comment: + update_comment_if_needed( + auto_approve_github_comments, + comment_body, + old_regression_comment_body, + regression_comment, + verbose, + ) + else: + create_new_pr_comment( + auto_approve_github_comments, + comment_body, + github_pr, + pr_link, + ) shutil.rmtree(temporary_dir, ignore_errors=True) new_builds_count = new_builds_count + 1 + build_stream_fields_arr.append(build_stream_fields) ack_reply = conn.xack( STREAM_KEYNAME_GH_EVENTS_COMMIT, STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, @@ -387,7 +548,7 @@ def builder_process_stream( ) else: logging.error("Missing commit information within received message.") - return previous_id, new_builds_count + return previous_id, new_builds_count, build_stream_fields_arr def build_spec_image_prefetch(builders_folder, different_build_specs): diff --git a/redis_benchmarks_specification/__cli__/args.py b/redis_benchmarks_specification/__cli__/args.py index adc94c2c..a45a8cb2 100644 --- a/redis_benchmarks_specification/__cli__/args.py +++ b/redis_benchmarks_specification/__cli__/args.py @@ -4,7 +4,7 @@ # All rights reserved. # import datetime - +import os from redis_benchmarks_specification.__common__.env import ( GH_REDIS_SERVER_HOST, @@ -18,9 +18,10 @@ from redisbench_admin.run.common import get_start_time_vars START_TIME_NOW_UTC, _, _ = get_start_time_vars() -START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7) +START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=90) CLI_TOOL_STATS = "stats" CLI_TOOL_TRIGGER = "trigger" +PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None) def spec_cli_args(parser): @@ -30,6 +31,30 @@ def spec_cli_args(parser): default=SPECS_PATH_TEST_SUITES, help="Test suites folder, containing the different test variations", ) + parser.add_argument( + "--tests-regexp", + type=str, + default=".*", + help="Interpret PATTERN as a regular expression to filter test names", + ) + parser.add_argument( + "--tests-groups-regexp", + type=str, + default=".*", + help="Interpret PATTERN as a regular expression to filter test group names", + ) + parser.add_argument( + "--tests-priority-lower-limit", + type=int, + default=0, + help="Run a subset of the tests based uppon a preset priority. By default runs all tests.", + ) + parser.add_argument( + "--tests-priority-upper-limit", + type=int, + default=100000, + help="Run a subset of the tests based uppon a preset priority. By default runs all tests.", + ) parser.add_argument( "--defaults_filename", type=str, @@ -45,6 +70,15 @@ def spec_cli_args(parser): action="store_true", help="Include modules statistics on commandstats.", ) + parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN) + parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="") + parser.add_argument( + "--auto-approve", + required=False, + default=False, + action="store_true", + help="Skip interactive approval of changes to github before applying.", + ) parser.add_argument("--summary-csv", type=str, default="") parser.add_argument("--group-csv", type=str, default="") parser.add_argument("--commands-json-file", type=str, default="./commands.json") @@ -129,8 +163,8 @@ def spec_cli_args(parser): parser.add_argument( "--last_n", type=int, - default=-1, - help="Use the last N samples. by default will use all available values", + default=1, + help="Use the last N samples. by default will use last commit", ) parser.add_argument( "--platform", @@ -138,4 +172,16 @@ def spec_cli_args(parser): default="", help="Only trigger tests on the specified platform.", ) + parser.add_argument( + "--wait-build", + default=False, + action="store_true", + help="Wait for build to be finished.", + ) + parser.add_argument( + "--wait-build-timeout", + type=int, + default=-1, + help="Wait x sections for build. If -1, waits forever.", + ) return parser diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 08809b69..62b41a52 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -16,8 +16,17 @@ import packaging import redis from packaging import version +import time +from github import Github +from redis_benchmarks_specification.__common__.github import ( + update_comment_if_needed, + create_new_pr_comment, + check_github_available_and_actionable, + generate_build_finished_pr_comment, +) + from redis_benchmarks_specification.__cli__.args import spec_cli_args from redis_benchmarks_specification.__cli__.stats import ( generate_stats_cli_command_logic, @@ -26,7 +35,11 @@ get_commit_dict_from_sha, request_build_from_commit_info, ) -from redis_benchmarks_specification.__common__.env import REDIS_BINS_EXPIRE_SECS +from redis_benchmarks_specification.__common__.env import ( + REDIS_BINS_EXPIRE_SECS, + STREAM_KEYNAME_GH_EVENTS_COMMIT, + STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, +) from redis_benchmarks_specification.__common__.package import ( get_version_string, populate_with_poetry_data, @@ -160,6 +173,28 @@ def get_repo(args): return redisDirPath, cleanUp +def check_benchmark_run_comment(comments): + res = False + pos = -1 + for n, comment in enumerate(comments): + body = comment.body + if "CE Performance Automation" in body and "Triggered a benchmark" in body: + res = True + pos = n + return res, pos + + +def check_benchmark_build_comment(comments): + res = False + pos = -1 + for n, comment in enumerate(comments): + body = comment.body + if "CE Performance Automation : step 1 of 2" in body: + res = True + pos = n + return res, pos + + def trigger_tests_cli_command_logic(args, project_name, project_version): logging.info( "Using: {project_name} {project_version}".format( @@ -171,6 +206,23 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): logging.error("You must specify either --use-tags or --use-branch flag") sys.exit(1) + github_token = args.github_token + pull_request = args.pull_request + verbose = True + auto_approve = args.auto_approve + + fn = check_benchmark_build_comment + ( + contains_regression_comment, + github_pr, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + ) = check_github_available_and_actionable( + fn, github_token, pull_request, "redis", "redis", verbose + ) + redisDirPath, cleanUp = get_repo(args) repo = git.Repo(redisDirPath) @@ -228,62 +280,182 @@ def trigger_tests_cli_command_logic(args, project_name, project_version): ) filtered_hash_commits.append(cdict) - if args.dry_run is False: - conn = redis.StrictRedis( - host=args.redis_host, - port=args.redis_port, - password=args.redis_pass, - username=args.redis_user, - decode_responses=False, + logging.info( + "Checking connection to redis with user: {}, host: {}, port: {}".format( + args.redis_user, + args.redis_host, + args.redis_port, ) - - for rep in range(0, 1): - for cdict in filtered_hash_commits: - ( - result, - error_msg, - commit_dict, - _, - binary_key, - binary_value, - ) = get_commit_dict_from_sha( - cdict["git_hash"], - args.gh_org, - args.gh_repo, - cdict, - True, - args.gh_token, + ) + conn = redis.StrictRedis( + host=args.redis_host, + port=args.redis_port, + password=args.redis_pass, + username=args.redis_user, + decode_responses=False, + ) + conn.ping() + for rep in range(0, 1): + for cdict in filtered_hash_commits: + ( + result, + error_msg, + commit_dict, + _, + binary_key, + binary_value, + ) = get_commit_dict_from_sha( + cdict["git_hash"], + args.gh_org, + args.gh_repo, + cdict, + True, + args.gh_token, + ) + if args.platform: + commit_dict["platform"] = args.platform + tests_priority_upper_limit = args.tests_priority_upper_limit + tests_priority_lower_limit = args.tests_priority_lower_limit + tests_regexp = args.tests_regexp + tests_groups_regexp = args.tests_groups_regexp + commit_dict["tests_priority_upper_limit"] = tests_priority_upper_limit + commit_dict["tests_priority_lower_limit"] = tests_priority_lower_limit + commit_dict["tests_regexp"] = tests_regexp + commit_dict["tests_groups_regexp"] = tests_groups_regexp + if pull_request is not None: + logging.info( + f"Have a pull request info to include in build request {pull_request}" ) - if args.platform: - commit_dict["platform"] = args.platform - if result is True: - stream_id = "n/a" - if args.dry_run is False: - ( - result, - reply_fields, - error_msg, - ) = request_build_from_commit_info( - conn, - commit_dict, - {}, - binary_key, - binary_value, - REDIS_BINS_EXPIRE_SECS, - ) - stream_id = reply_fields["id"] + commit_dict["pull_request"] = pull_request + git_hash = cdict["git_hash"] + git_branch = "n/a" + if "git_branch" in cdict: + git_branch = cdict["git_branch"] + commit_datetime = cdict["commit_datetime"] + commit_summary = cdict["commit_summary"] + if result is True: + stream_id = "n/a" + if args.dry_run is False: + (result, reply_fields, error_msg,) = request_build_from_commit_info( + conn, + commit_dict, + {}, + binary_key, + binary_value, + REDIS_BINS_EXPIRE_SECS, + ) + stream_id = reply_fields["id"] logging.info( - "Successfully requested a build for commit: {}. Date: {} Request stream id: {}.".format( + "Successfully requested a build for commit: {}. Date: {} Request stream id: {}. full commited info: {}. Reply fields: {}".format( cdict["git_hash"], cdict["commit_datetime"], stream_id, + commit_dict, + reply_fields, ) ) - else: - logging.error(error_msg) - else: - logging.info("Skipping actual work trigger ( dry-run )") + if args.wait_build is True: + build_start_datetime = datetime.datetime.utcnow() + decoded_stream_id = stream_id.decode() + builder_list_streams = ( + f"builder:{decoded_stream_id}:builds_completed" + ) + len_list = 0 + stream_ack = False + sleep_secs = 10 + benchmark_stream_ids = [] + while len_list == 0 or stream_ack is False: + + logging.info( + f"checking benchmark streams info in key: {builder_list_streams}" + ) + benchmark_stream_ids = conn.lrange( + builder_list_streams, 0, -1 + ) + len_list = len(benchmark_stream_ids) + logging.info( + f"There is a total of {len_list} already build benchmark stream ids for this build: {benchmark_stream_ids}" + ) + + if len_list > 0: + pending_build_streams = conn.xpending_range( + STREAM_KEYNAME_GH_EVENTS_COMMIT, + STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, + "-", + "+", + 1000, + ) + len_pending = len(pending_build_streams) + logging.info( + f"There is a total of {len_pending} pending builds for stream {STREAM_KEYNAME_GH_EVENTS_COMMIT} and cg {STREAM_GH_EVENTS_COMMIT_BUILDERS_CG}. Checking for stream id: {stream_id}" + ) + found_id = False + for pending_try in pending_build_streams: + logging.info(f"pending entry: {pending_try}") + pending_id = pending_try["message_id"] + if stream_id == pending_id: + found_id = True + logging.info( + f"Found the stream id {stream_id} as part of pending entry list. Waiting for it to be ack." + ) + + if found_id is True: + logging.info( + f"Sleeping for {sleep_secs} before checking pending list again." + ) + time.sleep(sleep_secs) + else: + stream_ack = True + else: + logging.info( + f"Sleeping for {sleep_secs} before checking builds again." + ) + time.sleep(sleep_secs) + logging.info( + f"FINAL total of {len_list} already build benchmark stream ids for this build: {benchmark_stream_ids}" + ) + build_end_datetime = datetime.datetime.utcnow() + build_duration = build_end_datetime - build_start_datetime + build_duration_secs = build_duration.total_seconds() + + comment_body = generate_build_finished_pr_comment( + benchmark_stream_ids, + commit_datetime, + commit_summary, + git_branch, + git_hash, + tests_groups_regexp, + tests_priority_lower_limit, + tests_priority_upper_limit, + tests_regexp, + build_start_datetime, + build_duration_secs, + ) + if is_actionable_pr: + if contains_regression_comment: + update_comment_if_needed( + auto_approve, + comment_body, + old_regression_comment_body, + regression_comment, + verbose, + ) + else: + create_new_pr_comment( + auto_approve, comment_body, github_pr, pr_link + ) + + else: + logging.info( + "DRY-RUN: build for commit: {}. Date: {} Full commited info: {}".format( + cdict["git_hash"], + cdict["commit_datetime"], + commit_dict, + ) + ) + else: + logging.error(error_msg) if cleanUp is True: logging.info("Removing temporary redis dir {}.".format(redisDirPath)) shutil.rmtree(redisDirPath) diff --git a/redis_benchmarks_specification/__common__/github.py b/redis_benchmarks_specification/__common__/github.py new file mode 100644 index 00000000..1b387a31 --- /dev/null +++ b/redis_benchmarks_specification/__common__/github.py @@ -0,0 +1,279 @@ +import logging +from github import Github + + +def check_regression_comment(comments): + res = False + pos = -1 + for n, comment in enumerate(comments): + body = comment.body + if "Comparison between" in body and "Time Period from" in body: + res = True + pos = n + return res, pos + + +def generate_build_started_pr_comment( + build_datetime, + commit_datetime, + commit_summary, + git_branch, + git_hash, + tests_groups_regexp, + tests_priority_lower_limit, + tests_priority_upper_limit, + tests_regexp, +): + comment_body = ( + f"### CE Performance Automation : step 1 of 2 (build) STARTING...\n\n" + ) + comment_body += ( + "This comment was automatically generated given a benchmark was triggered.\n" + ) + comment_body += f"Started building at {build_datetime}\n" + comment_body += "You can check each build/benchmark progress in grafana:\n" + if not isinstance(git_hash, str): + git_hash = git_hash.decode() + comment_body += f" - git hash: {git_hash}\n" + comment_body += f" - git branch: {git_branch}\n" + comment_body += f" - commit date and time: {commit_datetime}\n" + comment_body += f" - commit summary: {commit_summary}\n" + comment_body += f" - test filters:\n" + comment_body += ( + f" - command priority lower limit: {tests_priority_lower_limit}\n" + ) + comment_body += ( + f" - command priority upper limit: {tests_priority_upper_limit}\n" + ) + comment_body += f" - test name regex: {tests_regexp}\n" + comment_body += f" - command group regex: {tests_groups_regexp}\n\n" + return comment_body + + +def generate_build_finished_pr_comment( + benchmark_stream_ids, + commit_datetime, + commit_summary, + git_branch, + git_hash, + tests_groups_regexp, + tests_priority_lower_limit, + tests_priority_upper_limit, + tests_regexp, + build_start_datetime, + build_duration_seconds, +): + build_duration_seconds = int(build_duration_seconds) + comment_body = f"### CE Performance Automation : step 1 of 2 (build) DONE.\n\n" + comment_body += ( + "This comment was automatically generated given a benchmark was triggered.\n" + ) + comment_body += f"Started building at {build_start_datetime} and took {build_duration_seconds} seconds.\n" + comment_body += "You can check each build/benchmark progress in grafana:\n" + if not isinstance(git_hash, str): + git_hash = git_hash.decode() + comment_body += f" - git hash: {git_hash}\n" + comment_body += f" - git branch: {git_branch}\n" + comment_body += f" - commit date and time: {commit_datetime}\n" + comment_body += f" - commit summary: {commit_summary}\n" + comment_body += f" - test filters:\n" + comment_body += ( + f" - command priority lower limit: {tests_priority_lower_limit}\n" + ) + comment_body += ( + f" - command priority upper limit: {tests_priority_upper_limit}\n" + ) + comment_body += f" - test name regex: {tests_regexp}\n" + comment_body += f" - command group regex: {tests_groups_regexp}\n\n" + for benchmark_stream_id in benchmark_stream_ids: + if not isinstance(benchmark_stream_id, str): + benchmark_stream_id = benchmark_stream_id.decode() + grafana_benchmark_status_link = f"https://benchmarksredisio.grafana.net/d/edsxdsrbexhc0f/ce-benchmark-run-status?orgId=1&var-benchmark_work_stream={benchmark_stream_id}" + print("=============================================================") + print(f"Check benchmark run status in: {grafana_benchmark_status_link}") + comment_body += f"You can check a comparison in detail via the [grafana link]({grafana_benchmark_status_link})" + return comment_body + + +def check_github_available_and_actionable( + fn, github_token, pull_request, tf_github_org, tf_github_repo, verbose +): + # using an access token + is_actionable_pr = False + contains_regression_comment = False + regression_comment = None + github_pr = None + old_regression_comment_body = "" + pr_link = None + if github_token is not None: + logging.info("Detected github token") + g = Github(github_token) + if pull_request is not None and pull_request != "": + pull_request_n = int(pull_request) + github_pr = ( + g.get_user(tf_github_org) + .get_repo(tf_github_repo) + .get_issue(pull_request_n) + ) + comments = github_pr.get_comments() + pr_link = github_pr.html_url + logging.info("Working on github PR already: {}".format(pr_link)) + is_actionable_pr = True + contains_regression_comment, pos = fn(comments) + if contains_regression_comment: + regression_comment = comments[pos] + old_regression_comment_body = regression_comment.body + logging.info( + "Already contains PR comment. Link: {}".format( + regression_comment.html_url + ) + ) + if verbose: + logging.info("Printing old PR comment:") + print("".join(["-" for x in range(1, 80)])) + print(regression_comment.body) + print("".join(["-" for x in range(1, 80)])) + else: + logging.info("Does not contain PR comment") + logging.info( + f"contains_regression_comment: {contains_regression_comment}, is_actionable_pr: {is_actionable_pr}, pr_link: {pr_link}" + ) + + return ( + contains_regression_comment, + github_pr, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + ) + + +def create_new_pr_comment(auto_approve, comment_body, github_pr, pr_link): + regression_comment = None + user_input = "n" + if auto_approve: + print("auto approving...") + else: + user_input = input("Do you wish to add a comment in {} (y/n): ".format(pr_link)) + if user_input.lower() == "y" or auto_approve: + print("creating an comment in PR {}".format(pr_link)) + regression_comment = github_pr.create_comment(comment_body) + html_url = regression_comment.html_url + print("created comment. Access it via {}".format(html_url)) + return regression_comment + + +def update_comment_if_needed( + auto_approve, comment_body, old_regression_comment_body, regression_comment, verbose +): + same_comment = False + user_input = "n" + if comment_body == old_regression_comment_body: + logging.info( + "The old github comment is the same as the new comment. skipping..." + ) + same_comment = True + else: + logging.info( + "The old github comment is different from the new comment. updating it..." + ) + comment_body_arr = comment_body.split("\n") + old_regression_comment_body_arr = old_regression_comment_body.split("\n") + if verbose: + DF = [ + x for x in comment_body_arr if x not in old_regression_comment_body_arr + ] + print("---------------------") + print(DF) + print("---------------------") + if same_comment is False: + if auto_approve: + print("auto approving...") + else: + user_input = input( + "Do you wish to update the comment {} (y/n): ".format( + regression_comment.html_url + ) + ) + if user_input.lower() == "y" or auto_approve: + print("Updating comment {}".format(regression_comment.html_url)) + regression_comment.edit(comment_body) + html_url = regression_comment.html_url + print( + "Updated comment. Access it via {}".format(regression_comment.html_url) + ) + + +def check_benchmark_build_comment(comments): + res = False + pos = -1 + for n, comment in enumerate(comments): + body = comment.body + if "CE Performance Automation : step" in body: + res = True + pos = n + return res, pos + + +def check_benchmark_running_comment(comments): + res = False + pos = -1 + for n, comment in enumerate(comments): + body = comment.body + if "CE Performance Automation : step" in body: + res = True + pos = n + return res, pos + + +def markdown_progress_bar(current, total, bar_length=40): + progress = 1.0 + if total > 0: + progress = current / total + block = int(round(bar_length * progress)) + bar = "#" * block + "-" * (bar_length - block) + percentage = round(progress * 100, 2) + return f"[{bar}] {percentage}%" + + +def generate_benchmark_started_pr_comment( + benchmark_stream_id, + total_pending, + total_benchmarks, + total_failed, + benchmark_suite_start_datetime, + benchmark_suite_duration_secs, +): + comment_body = "### CE Performance Automation : step 2 of 2 (benchmark) " + if total_pending > 0: + comment_body += "RUNNING...\n\n" + else: + comment_body += "FINISHED.\n\n" + + comment_body += ( + "This comment was automatically generated given a benchmark was triggered.\n\n" + ) + + comment_body += f"Started benchmark suite at {benchmark_suite_start_datetime} and took {benchmark_suite_duration_secs} seconds " + if total_pending > 0: + comment_body += "up until now.\n" + else: + comment_body += "to finish.\n" + + completed = total_benchmarks - total_pending + successful = completed - total_failed + comment_body += ( + f"Status: {markdown_progress_bar(completed,total_benchmarks,80)} completed.\n\n" + ) + comment_body += f"In total will run {total_benchmarks} benchmarks.\n" + comment_body += f" - {total_pending} pending.\n" + comment_body += f" - {completed} completed:\n" + comment_body += f" - {successful} successful.\n" + comment_body += f" - {total_failed} failed.\n" + + if not isinstance(benchmark_stream_id, str): + benchmark_stream_id = benchmark_stream_id.decode() + grafana_benchmark_status_link = f"https://benchmarksredisio.grafana.net/d/edsxdsrbexhc0f/ce-benchmark-run-status?orgId=1&var-benchmark_work_stream={benchmark_stream_id}" + comment_body += f"You can check a the status in detail via the [grafana link]({grafana_benchmark_status_link})" + return comment_body diff --git a/redis_benchmarks_specification/__compare__/compare.py b/redis_benchmarks_specification/__compare__/compare.py index e6c8712a..6cb14460 100644 --- a/redis_benchmarks_specification/__compare__/compare.py +++ b/redis_benchmarks_specification/__compare__/compare.py @@ -14,9 +14,14 @@ import datetime as dt import os from tqdm import tqdm -from github import Github -from slack_sdk.webhook import WebhookClient import argparse + +from redis_benchmarks_specification.__common__.github import ( + update_comment_if_needed, + create_new_pr_comment, + check_github_available_and_actionable, + check_regression_comment, +) from redis_benchmarks_specification.__compare__.args import create_compare_arguments @@ -161,45 +166,9 @@ def compare_command_logic(args, project_name, project_version): username=args.redistimeseries_user, ) rts.ping() - default_baseline_branch = None - default_metrics_str = "" - if args.defaults_filename != "" and os.path.exists(args.defaults_filename): - logging.info( - "Loading configuration from defaults file: {}".format( - args.defaults_filename - ) - ) - with open(args.defaults_filename) as yaml_fd: - defaults_dict = yaml.safe_load(yaml_fd) - if "exporter" in defaults_dict: - exporter_dict = defaults_dict["exporter"] - if "comparison" in exporter_dict: - comparison_dict = exporter_dict["comparison"] - if "metrics" in comparison_dict: - metrics = comparison_dict["metrics"] - logging.info("Detected defaults metrics info. reading metrics") - default_metrics = [] - - for metric in metrics: - if metric.startswith("$."): - metric = metric[2:] - logging.info("Will use metric: {}".format(metric)) - default_metrics.append(metric) - if len(default_metrics) == 1: - default_metrics_str = default_metrics[0] - if len(default_metrics) > 1: - default_metrics_str = "({})".format( - ",".join(default_metrics) - ) - logging.info("Default metrics: {}".format(default_metrics_str)) - - if "baseline-branch" in comparison_dict: - default_baseline_branch = comparison_dict["baseline-branch"] - logging.info( - "Detected baseline branch in defaults file. {}".format( - default_baseline_branch - ) - ) + default_baseline_branch, default_metrics_str = extract_default_branch_and_metric( + args.defaults_filename + ) tf_github_org = args.github_org tf_github_repo = args.github_repo @@ -277,14 +246,7 @@ def compare_command_logic(args, project_name, project_version): auto_approve = args.auto_approve running_platform = args.running_platform grafana_base_dashboard = args.grafana_base_dashboard - # using an access token - is_actionable_pr = False - contains_regression_comment = False - regression_comment = None - github_pr = None - # slack related - webhook_notifications_active = False - webhook_client_slack = None + if running_platform is not None: logging.info( "Using platform named: {} to do the comparison.\n\n".format( @@ -292,56 +254,18 @@ def compare_command_logic(args, project_name, project_version): ) ) - old_regression_comment_body = "" - if github_token is not None: - logging.info("Detected github token") - g = Github(github_token) - if pull_request is not None and pull_request != "": - pull_request_n = int(pull_request) - github_pr = ( - g.get_user(tf_github_org) - .get_repo(tf_github_repo) - .get_issue(pull_request_n) - ) - comments = github_pr.get_comments() - pr_link = github_pr.html_url - logging.info("Working on github PR already: {}".format(pr_link)) - is_actionable_pr = True - contains_regression_comment, pos = check_regression_comment(comments) - if contains_regression_comment: - regression_comment = comments[pos] - old_regression_comment_body = regression_comment.body - logging.info( - "Already contains regression comment. Link: {}".format( - regression_comment.html_url - ) - ) - if verbose: - logging.info("Printing old regression comment:") - print("".join(["-" for x in range(1, 80)])) - print(regression_comment.body) - print("".join(["-" for x in range(1, 80)])) - else: - logging.info("Does not contain regression comment") - - grafana_dashboards_uids = { - "redisgraph": "SH9_rQYGz", - "redisbloom": "q4-5sRR7k", - "redisearch": "3Ejv2wZnk", - "redisjson": "UErSC0jGk", - "redistimeseries": "2WMw61UGz", - } - uid = None - if tf_github_repo.lower() in grafana_dashboards_uids: - uid = grafana_dashboards_uids[tf_github_repo.lower()] - grafana_link_base = None - if uid is not None: - grafana_link_base = "{}/{}".format(grafana_base_dashboard, uid) - logging.info( - "There is a grafana dashboard for this repo. Base link: {}".format( - grafana_link_base - ) - ) + fn = check_regression_comment + ( + contains_regression_comment, + github_pr, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + ) = check_github_available_and_actionable( + fn, github_token, pull_request, tf_github_org, tf_github_repo, verbose + ) + grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks" ( detected_regressions, @@ -381,7 +305,71 @@ def compare_command_logic(args, project_name, project_version): use_metric_context_path, running_platform, ) - comment_body = "" + prepare_regression_comment( + auto_approve, + baseline_branch, + baseline_tag, + comparison_branch, + comparison_tag, + contains_regression_comment, + github_pr, + grafana_link_base, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + rts, + running_platform, + table_output, + tf_github_org, + tf_github_repo, + tf_triggering_env, + total_comparison_points, + total_improvements, + total_regressions, + total_stable, + total_unstable, + verbose, + args.regressions_percent_lower_limit, + ) + return ( + detected_regressions, + "", + total_improvements, + total_regressions, + total_stable, + total_unstable, + total_comparison_points, + ) + + +def prepare_regression_comment( + auto_approve, + baseline_branch, + baseline_tag, + comparison_branch, + comparison_tag, + contains_regression_comment, + github_pr, + grafana_link_base, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + rts, + running_platform, + table_output, + tf_github_org, + tf_github_repo, + tf_triggering_env, + total_comparison_points, + total_improvements, + total_regressions, + total_stable, + total_unstable, + verbose, + regressions_percent_lower_limit, +): if total_comparison_points > 0: comment_body = "### Automated performance analysis summary\n\n" comment_body += "This comment was automatically generated given there is performance data available.\n\n" @@ -409,7 +397,7 @@ def compare_command_logic(args, project_name, project_version): ) if total_regressions > 0: comparison_summary += "- Detected a total of {} regressions bellow the regression water line {}.\n".format( - total_regressions, args.regressions_percent_lower_limit + total_regressions, regressions_percent_lower_limit ) comment_body += comparison_summary @@ -453,99 +441,61 @@ def compare_command_logic(args, project_name, project_version): zset_project_pull_request, comparison_branch, res ) ) - user_input = "n" - html_url = "n/a" - regression_count = len(detected_regressions) - ( - baseline_str, - by_str_baseline, - comparison_str, - by_str_comparison, - ) = get_by_strings( - baseline_branch, - comparison_branch, - baseline_tag, - comparison_tag, - ) if contains_regression_comment: - same_comment = False - if comment_body == old_regression_comment_body: - logging.info( - "The old regression comment is the same as the new comment. skipping..." - ) - same_comment = True - else: - logging.info( - "The old regression comment is different from the new comment. updating it..." - ) - comment_body_arr = comment_body.split("\n") - old_regression_comment_body_arr = old_regression_comment_body.split( - "\n" - ) - if verbose: - DF = [ - x - for x in comment_body_arr - if x not in old_regression_comment_body_arr - ] - print("---------------------") - print(DF) - print("---------------------") - if same_comment is False: - if auto_approve: - print("auto approving...") - else: - user_input = input( - "Do you wish to update the comment {} (y/n): ".format( - regression_comment.html_url - ) - ) - if user_input.lower() == "y" or auto_approve: - print("Updating comment {}".format(regression_comment.html_url)) - regression_comment.edit(comment_body) - html_url = regression_comment.html_url - print( - "Updated comment. Access it via {}".format( - regression_comment.html_url - ) - ) - + update_comment_if_needed( + auto_approve, + comment_body, + old_regression_comment_body, + regression_comment, + verbose, + ) else: - if auto_approve: - print("auto approving...") - else: - user_input = input( - "Do you wish to add a comment in {} (y/n): ".format(pr_link) - ) - if user_input.lower() == "y" or auto_approve: - print("creating an comment in PR {}".format(pr_link)) - regression_comment = github_pr.create_comment(comment_body) - html_url = regression_comment.html_url - print("created comment. Access it via {}".format(html_url)) + create_new_pr_comment(auto_approve, comment_body, github_pr, pr_link) else: logging.error("There was no comparison points to produce a table...") - return ( - detected_regressions, - comment_body, - total_improvements, - total_regressions, - total_stable, - total_unstable, - total_comparison_points, - ) -def check_regression_comment(comments): - res = False - pos = -1 - for n, comment in enumerate(comments): - body = comment.body - if "Comparison between" in body and "Time Period from" in body: - res = True - pos = n - return res, pos +def extract_default_branch_and_metric(defaults_filename): + default_baseline_branch = "unstable" + default_metrics_str = "" + if defaults_filename != "" and os.path.exists(defaults_filename): + logging.info( + "Loading configuration from defaults file: {}".format(defaults_filename) + ) + with open(defaults_filename) as yaml_fd: + defaults_dict = yaml.safe_load(yaml_fd) + if "exporter" in defaults_dict: + exporter_dict = defaults_dict["exporter"] + if "comparison" in exporter_dict: + comparison_dict = exporter_dict["comparison"] + if "metrics" in comparison_dict: + metrics = comparison_dict["metrics"] + logging.info("Detected defaults metrics info. reading metrics") + default_metrics = [] + + for metric in metrics: + if metric.startswith("$."): + metric = metric[2:] + logging.info("Will use metric: {}".format(metric)) + default_metrics.append(metric) + if len(default_metrics) == 1: + default_metrics_str = default_metrics[0] + if len(default_metrics) > 1: + default_metrics_str = "({})".format( + ",".join(default_metrics) + ) + logging.info("Default metrics: {}".format(default_metrics_str)) + + if "baseline-branch" in comparison_dict: + default_baseline_branch = comparison_dict["baseline-branch"] + logging.info( + "Detected baseline branch in defaults file. {}".format( + default_baseline_branch + ) + ) + return default_baseline_branch, default_metrics_str def compute_regression_table( @@ -801,7 +751,6 @@ def from_rts_to_regression_table( total_comparison_points = 0 noise_waterline = 3 progress = tqdm(unit="benchmark time-series", total=len(test_names)) - at_comparison = 0 for test_name in test_names: multi_value_baseline = check_multi_value_filter(baseline_str) multi_value_comparison = check_multi_value_filter(comparison_str) @@ -1066,7 +1015,8 @@ def get_test_names_from_db(rts, tags_regex_string, test_names, used_key): test_names.sort() final_test_names = [] for test_name in test_names: - test_name = test_name.decode() + if not isinstance(test_name, str): + test_name = test_name.decode() match_obj = re.search(tags_regex_string, test_name) if match_obj is not None: final_test_names.append(test_name) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/args.py b/redis_benchmarks_specification/__self_contained_coordinator__/args.py index 0b9fe6f4..7a1b3cbf 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/args.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/args.py @@ -1,5 +1,6 @@ import argparse - +import datetime +import os from redis_benchmarks_specification.__common__.env import ( MACHINE_CPU_COUNT, SPECS_PATH_SETUPS, @@ -19,6 +20,12 @@ PROFILERS_DEFAULT, ALLOWED_PROFILERS, ) +from redis_benchmarks_specification.__compare__.args import ( + START_TIME_NOW_UTC, + START_TIME_LAST_SIX_MONTHS_UTC, +) + +PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None) def create_self_contained_coordinator_args(project_name): @@ -30,6 +37,7 @@ def create_self_contained_coordinator_args(project_name): parser.add_argument("--event_stream_port", type=int, default=GH_REDIS_SERVER_PORT) parser.add_argument("--event_stream_pass", type=str, default=GH_REDIS_SERVER_AUTH) parser.add_argument("--event_stream_user", type=str, default=GH_REDIS_SERVER_USER) + parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN) parser.add_argument( "--cpu-count", type=int, @@ -150,4 +158,16 @@ def create_self_contained_coordinator_args(project_name): parser.add_argument( "--arch", type=str, default="amd64", help="arch to build artifacts" ) + parser.add_argument( + "--tests-priority-lower-limit", + type=int, + default=0, + help="Run a subset of the tests based uppon a preset priority. By default runs all tests.", + ) + parser.add_argument( + "--tests-priority-upper-limit", + type=int, + default=100000, + help="Run a subset of the tests based uppon a preset priority. By default runs all tests.", + ) return parser diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index ae087be1..a937e856 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -5,7 +5,7 @@ import shutil import tempfile import traceback - +import re import docker import redis import os @@ -23,6 +23,15 @@ LOG_LEVEL, REDIS_HEALTH_CHECK_INTERVAL, REDIS_SOCKET_TIMEOUT, + REDIS_BINS_EXPIRE_SECS, +) +from redis_benchmarks_specification.__common__.github import ( + check_github_available_and_actionable, + check_benchmark_running_comment, + update_comment_if_needed, + create_new_pr_comment, + generate_benchmark_started_pr_comment, + check_regression_comment, ) from redis_benchmarks_specification.__common__.package import ( get_version_string, @@ -34,8 +43,14 @@ exporter_datasink_common, execute_init_commands, ) +from redis_benchmarks_specification.__compare__.compare import ( + compute_regression_table, + prepare_regression_comment, + extract_default_branch_and_metric, +) from redis_benchmarks_specification.__runner__.runner import ( print_results_table_stdout, + prepare_memtier_benchmark_parameters, ) from redis_benchmarks_specification.__self_contained_coordinator__.args import ( create_self_contained_coordinator_args, @@ -137,7 +152,7 @@ def main(): ) ) try: - conn = redis.StrictRedis( + gh_event_conn = redis.StrictRedis( host=args.event_stream_host, port=args.event_stream_port, decode_responses=False, # dont decode due to binary archives @@ -147,7 +162,7 @@ def main(): socket_connect_timeout=REDIS_SOCKET_TIMEOUT, socket_keepalive=True, ) - conn.ping() + gh_event_conn.ping() except redis.exceptions.ConnectionError as e: logging.error( "Unable to connect to redis available at: {}:{} to read the event streams".format( @@ -187,7 +202,7 @@ def main(): logging.info("checking build spec requirements") running_platform = args.platform_name - build_runners_consumer_group_create(conn, running_platform) + build_runners_consumer_group_create(gh_event_conn, running_platform) stream_id = None docker_client = docker.from_env() home = str(Path.home()) @@ -196,6 +211,17 @@ def main(): redis_proc_start_port = args.redis_proc_start_port logging.info("Redis Processes start port: {}".format(redis_proc_start_port)) + priority_lower_limit = args.tests_priority_lower_limit + priority_upper_limit = args.tests_priority_upper_limit + + logging.info( + f"Using priority for test filters [{priority_lower_limit},{priority_upper_limit}]" + ) + + default_baseline_branch, default_metrics_str = extract_default_branch_and_metric( + args.defaults_filename + ) + # TODO: confirm we do have enough cores to run the spec # availabe_cpus = args.cpu_count datasink_push_results_redistimeseries = args.datasink_push_results_redistimeseries @@ -218,6 +244,11 @@ def main(): arch = args.arch logging.info("Running for arch: {}".format(arch)) + # Github token + github_token = args.github_token + if github_token is not None: + logging.info("Detected GITHUB token. will push PR comments with updates") + # Docker air gap usage docker_air_gap = args.docker_air_gap if docker_air_gap: @@ -250,7 +281,7 @@ def main(): stream_id = args.consumer_start_id while True: _, stream_id, _, _ = self_contained_coordinator_blocking_read( - conn, + gh_event_conn, datasink_push_results_redistimeseries, docker_client, home, @@ -269,11 +300,16 @@ def main(): override_memtier_test_time, default_metrics, arch, + github_token, + priority_lower_limit, + priority_upper_limit, + default_baseline_branch, + default_metrics_str, ) def self_contained_coordinator_blocking_read( - conn, + github_event_conn, datasink_push_results_redistimeseries, docker_client, home, @@ -289,9 +325,14 @@ def self_contained_coordinator_blocking_read( redis_proc_start_port=6379, consumer_pos=1, docker_air_gap=False, - override_test_time=None, + override_test_time=1, default_metrics=None, arch="amd64", + github_token=None, + priority_lower_limit=0, + priority_upper_limit=10000, + default_baseline_branch="unstable", + default_metrics_str="ALL_STATS.Totals.Ops/sec", ): num_process_streams = 0 num_process_test_suites = 0 @@ -304,7 +345,7 @@ def self_contained_coordinator_blocking_read( get_runners_consumer_group_name(platform_name), consumer_name ) ) - newTestInfo = conn.xreadgroup( + newTestInfo = github_event_conn.xreadgroup( get_runners_consumer_group_name(platform_name), consumer_name, {STREAM_KEYNAME_NEW_BUILD_EVENTS: stream_id}, @@ -319,7 +360,7 @@ def self_contained_coordinator_blocking_read( overall_result, total_test_suite_runs, ) = process_self_contained_coordinator_stream( - conn, + github_event_conn, datasink_push_results_redistimeseries, docker_client, home, @@ -335,14 +376,19 @@ def self_contained_coordinator_blocking_read( redis_proc_start_port, docker_air_gap, "defaults.yml", - None, + override_test_time, default_metrics, arch, + github_token, + priority_lower_limit, + priority_upper_limit, + default_baseline_branch, + default_metrics_str, ) num_process_streams = num_process_streams + 1 num_process_test_suites = num_process_test_suites + total_test_suite_runs if overall_result is True: - ack_reply = conn.xack( + ack_reply = github_event_conn.xack( STREAM_KEYNAME_NEW_BUILD_EVENTS, get_runners_consumer_group_name(platform_name), stream_id, @@ -364,34 +410,35 @@ def self_contained_coordinator_blocking_read( return overall_result, stream_id, num_process_streams, num_process_test_suites -def prepare_memtier_benchmark_parameters( - clientconfig, - full_benchmark_path, - port, - server, - local_benchmark_output_filename, - oss_cluster_api_enabled, -): - benchmark_command = [ - full_benchmark_path, - "--port", - "{}".format(port), - "--server", - "{}".format(server), - "--json-out-file", - local_benchmark_output_filename, - ] - if oss_cluster_api_enabled is True: - benchmark_command.append("--cluster-mode") - benchmark_command_str = " ".join(benchmark_command) - if "arguments" in clientconfig: - benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"] - - return None, benchmark_command_str +# +# def prepare_memtier_benchmark_parameters( +# clientconfig, +# full_benchmark_path, +# port, +# server, +# local_benchmark_output_filename, +# oss_cluster_api_enabled, +# ): +# benchmark_command = [ +# full_benchmark_path, +# "--port", +# "{}".format(port), +# "--server", +# "{}".format(server), +# "--json-out-file", +# local_benchmark_output_filename, +# ] +# if oss_cluster_api_enabled is True: +# benchmark_command.append("--cluster-mode") +# benchmark_command_str = " ".join(benchmark_command) +# if "arguments" in clientconfig: +# benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"] +# +# return None, benchmark_command_str def process_self_contained_coordinator_stream( - conn, + github_event_conn, datasink_push_results_redistimeseries, docker_client, home, @@ -407,13 +454,27 @@ def process_self_contained_coordinator_stream( redis_proc_start_port=6379, docker_air_gap=False, defaults_filename="defaults.yml", - override_test_time=None, + override_test_time=0, default_metrics=[], arch="amd64", + github_token=None, + priority_lower_limit=0, + priority_upper_limit=10000, + default_baseline_branch="unstable", + default_metrics_str="ALL_STATS.Totals.Ops/sec", ): stream_id = "n/a" overall_result = False total_test_suite_runs = 0 + # github updates + is_actionable_pr = False + contains_benchmark_run_comment = False + github_pr = None + old_benchmark_run_comment_body = "" + pr_link = "" + regression_comment = None + pull_request = None + auto_approve_github = True try: stream_id, testDetails = newTestInfo[0][1][0] stream_id = stream_id.decode() @@ -433,6 +494,56 @@ def process_self_contained_coordinator_stream( run_arch, ) = extract_build_info_from_streamdata(testDetails) + if b"priority_upper_limit" in testDetails: + stream_priority_upper_limit = int( + testDetails[b"priority_upper_limit"].decode() + ) + logging.info( + f"detected a priority_upper_limit definition on the streamdata {stream_priority_upper_limit}. will replace the default upper limit of {priority_upper_limit}" + ) + priority_upper_limit = stream_priority_upper_limit + + if b"priority_lower_limit" in testDetails: + stream_priority_lower_limit = int( + testDetails[b"priority_lower_limit"].decode() + ) + logging.info( + f"detected a priority_lower_limit definition on the streamdata {stream_priority_lower_limit}. will replace the default lower limit of {priority_lower_limit}" + ) + priority_lower_limit = stream_priority_lower_limit + + if b"pull_request" in testDetails: + pull_request = testDetails[b"pull_request"].decode() + logging.info( + f"detected a pull_request definition on the streamdata {pull_request}" + ) + verbose = True + fn = check_benchmark_running_comment + ( + contains_benchmark_run_comment, + github_pr, + is_actionable_pr, + old_benchmark_run_comment_body, + pr_link, + benchmark_run_comment, + ) = check_github_available_and_actionable( + fn, github_token, pull_request, "redis", "redis", verbose + ) + + test_regexp = ".*" + if b"test_regexp" in testDetails: + test_regexp = testDetails[b"test_regexp"] + logging.info( + f"detected a regexp definition on the streamdata {test_regexp}" + ) + + command_groups_regexp = None + if b"tests_groups_regexp" in testDetails: + command_groups_regexp = testDetails[b"tests_groups_regexp"].decode() + logging.info( + f"detected a command groups regexp definition on the streamdata {command_groups_regexp}" + ) + skip_test = False if b"platform" in testDetails: platform = testDetails[b"platform"] @@ -462,29 +573,92 @@ def process_self_contained_coordinator_stream( run_image, airgap_key ) ) - airgap_docker_image_bin = conn.get(airgap_key) + airgap_docker_image_bin = github_event_conn.get(airgap_key) images_loaded = docker_client.images.load(airgap_docker_image_bin) logging.info("Successfully loaded images {}".format(images_loaded)) - for test_file in testsuite_spec_files: - if defaults_filename in test_file: - continue + stream_time_ms = stream_id.split("-")[0] + zset_running_platform_benchmarks = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{running_platform}:zset" + res = github_event_conn.zadd( + zset_running_platform_benchmarks, + {stream_id: stream_time_ms}, + ) + logging.info( + f"Added stream with id {stream_id} to zset {zset_running_platform_benchmarks}" + ) + + stream_test_list_pending = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_pending" + stream_test_list_running = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_running" + stream_test_list_failed = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_failed" + stream_test_list_completed = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_completed" + + filtered_test_files = filter_test_files( + defaults_filename, + priority_lower_limit, + priority_upper_limit, + test_regexp, + testsuite_spec_files, + command_groups_regexp, + ) + + for test_file in filtered_test_files: + with open(test_file, "r") as stream: + ( + _, + benchmark_config, + test_name, + ) = get_final_benchmark_config(None, stream, "") + github_event_conn.lpush(stream_test_list_pending, test_name) + github_event_conn.expire( + stream_test_list_pending, REDIS_BINS_EXPIRE_SECS + ) + logging.info( + f"Added test named {test_name} to the pending test list in key {stream_test_list_pending}" + ) + + pending_tests = len(filtered_test_files) + failed_tests = 0 + benchmark_suite_start_datetime = datetime.datetime.utcnow() + # update on github if needed + if is_actionable_pr: + comment_body = generate_benchmark_started_pr_comment( + stream_id, + pending_tests, + len(filtered_test_files), + failed_tests, + benchmark_suite_start_datetime, + 0, + ) + if contains_benchmark_run_comment: + update_comment_if_needed( + auto_approve_github, + comment_body, + old_benchmark_run_comment_body, + benchmark_run_comment, + verbose, + ) + else: + benchmark_run_comment = create_new_pr_comment( + auto_approve_github, comment_body, github_pr, pr_link + ) + + for test_file in filtered_test_files: redis_containers = [] client_containers = [] - with open(test_file, "r") as stream: ( - result, + _, benchmark_config, test_name, ) = get_final_benchmark_config(None, stream, "") - if result is False: - logging.error( - "Skipping {} given there were errors while calling get_final_benchmark_config()".format( - test_file - ) - ) - continue + github_event_conn.lrem(stream_test_list_pending, 1, test_name) + github_event_conn.lpush(stream_test_list_running, test_name) + github_event_conn.expire( + stream_test_list_running, REDIS_BINS_EXPIRE_SECS + ) + logging.info( + f"Added test named {test_name} to the pending test list in key {stream_test_list_running}" + ) ( _, _, @@ -561,7 +735,10 @@ def process_self_contained_coordinator_stream( ) restore_build_artifacts_from_test_details( - build_artifacts, conn, temporary_dir, testDetails + build_artifacts, + github_event_conn, + temporary_dir, + testDetails, ) mnt_point = "/mnt/redis/" command = generate_standalone_redis_server_args( @@ -679,13 +856,22 @@ def process_self_contained_coordinator_stream( ( _, benchmark_command_str, + arbitrary_command, ) = prepare_memtier_benchmark_parameters( benchmark_config["clientconfig"], full_benchmark_path, redis_proc_start_port, "localhost", + None, local_benchmark_output_filename, - benchmark_tool_workdir, + False, + False, + False, + None, + None, + None, + None, + override_test_time, ) client_container_image = extract_client_container_image( @@ -773,7 +959,7 @@ def process_self_contained_coordinator_stream( tf_github_repo, git_hash, overall_tabular_data_map, - conn, + github_event_conn, setup_name, start_time_ms, start_time_str, @@ -890,50 +1076,81 @@ def process_self_contained_coordinator_stream( ) dataset_load_duration_seconds = 0 + try: + exporter_datasink_common( + benchmark_config, + benchmark_duration_seconds, + build_variant_name, + datapoint_time_ms, + dataset_load_duration_seconds, + datasink_conn, + datasink_push_results_redistimeseries, + git_branch, + git_version, + metadata, + redis_conns, + results_dict, + running_platform, + setup_name, + setup_type, + test_name, + tf_github_org, + tf_github_repo, + tf_triggering_env, + topology_spec_name, + default_metrics, + ) + r.shutdown(save=False) + + except redis.exceptions.ConnectionError as e: + logging.critical( + "Some unexpected exception was caught during metric fetching. Skipping it..." + ) + logging.critical( + f"Exception type: {type(e).__name__}" + ) + logging.critical(f"Exception message: {str(e)}") + logging.critical("Traceback details:") + logging.critical(traceback.format_exc()) + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) - exporter_datasink_common( - benchmark_config, - benchmark_duration_seconds, - build_variant_name, - datapoint_time_ms, - dataset_load_duration_seconds, - datasink_conn, - datasink_push_results_redistimeseries, - git_branch, - git_version, - metadata, - redis_conns, - results_dict, - running_platform, - setup_name, - setup_type, - test_name, - tf_github_org, - tf_github_repo, - tf_triggering_env, - topology_spec_name, - default_metrics, - ) - r.shutdown(save=False) test_result = True total_test_suite_runs = total_test_suite_runs + 1 - except: + except Exception as e: logging.critical( - "Some unexpected exception was caught " - "during local work. Failing test...." + "Some unexpected exception was caught during local work. Failing test...." ) - logging.critical(sys.exc_info()[0]) + logging.critical(f"Exception type: {type(e).__name__}") + logging.critical(f"Exception message: {str(e)}") + logging.critical("Traceback details:") + logging.critical(traceback.format_exc()) print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60) if redis_container is not None: logging.critical("Printing redis container log....") + print("-" * 60) - print( - redis_container.logs(stdout=True, stderr=True) - ) + try: + print( + redis_container.logs( + stdout=True, stderr=True + ) + ) + except docker.errors.NotFound: + logging.info( + "When trying to stop DB container with id {} and image {} it was already stopped".format( + redis_container.id, + redis_container.image, + ) + ) + pass + print("-" * 60) + test_result = False # tear-down logging.info("Tearing down setup") @@ -970,14 +1187,176 @@ def process_self_contained_coordinator_stream( overall_result &= test_result + github_event_conn.lrem(stream_test_list_running, 1, test_name) + github_event_conn.lpush(stream_test_list_completed, test_name) + github_event_conn.expire( + stream_test_list_completed, REDIS_BINS_EXPIRE_SECS + ) + if test_result is False: + github_event_conn.lpush(stream_test_list_failed, test_name) + failed_tests = failed_tests + 1 + logging.warning( + f"updating key {stream_test_list_failed} with the failed test: {test_name}. Total failed tests {failed_tests}." + ) + pending_tests = pending_tests - 1 + + benchmark_suite_end_datetime = datetime.datetime.utcnow() + benchmark_suite_duration = ( + benchmark_suite_end_datetime - benchmark_suite_start_datetime + ) + benchmark_suite_duration_secs = ( + benchmark_suite_duration.total_seconds() + ) + + # update on github if needed + if is_actionable_pr: + comment_body = generate_benchmark_started_pr_comment( + stream_id, + pending_tests, + len(filtered_test_files), + failed_tests, + benchmark_suite_start_datetime, + benchmark_suite_duration_secs, + ) + update_comment_if_needed( + auto_approve_github, + comment_body, + old_benchmark_run_comment_body, + benchmark_run_comment, + verbose, + ) + logging.info( + f"Updated github comment with latest test info {benchmark_run_comment.html_url}" + ) + + ########################### + # regression part + ########################### + fn = check_regression_comment + ( + contains_regression_comment, + github_pr, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + ) = check_github_available_and_actionable( + fn, + github_token, + pull_request, + tf_github_org, + tf_github_repo, + verbose, + ) + logging.info( + f"Preparing regression info for the data available" + ) + print_improvements_only = False + print_regressions_only = False + skip_unstable = False + regressions_percent_lower_limit = 10.0 + simplify_table = False + testname_regex = "" + test = "" + last_n_baseline = 1 + last_n_comparison = 31 + use_metric_context_path = False + baseline_tag = None + baseline_deployment_name = "oss-standalone" + comparison_deployment_name = "oss-standalone" + metric_name = "ALL_STATS.Totals.Ops/sec" + metric_mode = "higher-better" + to_date = datetime.datetime.utcnow() + from_date = to_date - datetime.timedelta(days=180) + baseline_branch = default_baseline_branch + comparison_tag = git_version + comparison_branch = git_branch + to_ts_ms = None + from_ts_ms = None + + ( + detected_regressions, + table_output, + total_improvements, + total_regressions, + total_stable, + total_unstable, + total_comparison_points, + ) = compute_regression_table( + datasink_conn, + tf_github_org, + tf_github_repo, + tf_triggering_env, + metric_name, + comparison_branch, + baseline_branch, + baseline_tag, + comparison_tag, + baseline_deployment_name, + comparison_deployment_name, + print_improvements_only, + print_regressions_only, + skip_unstable, + regressions_percent_lower_limit, + simplify_table, + test, + testname_regex, + verbose, + last_n_baseline, + last_n_comparison, + metric_mode, + from_date, + from_ts_ms, + to_date, + to_ts_ms, + use_metric_context_path, + running_platform, + ) + auto_approve = True + grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks" + + prepare_regression_comment( + auto_approve, + baseline_branch, + baseline_tag, + comparison_branch, + comparison_tag, + contains_regression_comment, + github_pr, + grafana_link_base, + is_actionable_pr, + old_regression_comment_body, + pr_link, + regression_comment, + datasink_conn, + running_platform, + table_output, + tf_github_org, + tf_github_repo, + tf_triggering_env, + total_comparison_points, + total_improvements, + total_regressions, + total_stable, + total_unstable, + verbose, + regressions_percent_lower_limit, + ) + logging.info( + f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}" + ) else: logging.error("Missing commit information within received message.") - except: + + except Exception as e: logging.critical( "Some unexpected exception was caught " "during local work on stream {}. Failing test....".format(stream_id) ) - logging.critical(sys.exc_info()[0]) + logging.critical(f"Exception type: {type(e).__name__}") + logging.critical(f"Exception message: {str(e)}") + logging.critical("Traceback details:") + logging.critical(traceback.format_exc()) print("-" * 60) traceback.print_exc(file=sys.stdout) print("-" * 60) @@ -985,6 +1364,108 @@ def process_self_contained_coordinator_stream( return stream_id, overall_result, total_test_suite_runs +def filter_test_files( + defaults_filename, + priority_lower_limit, + priority_upper_limit, + test_regexp, + testsuite_spec_files, + command_groups_regexp=None, +): + filtered_test_files = [] + for test_file in testsuite_spec_files: + if defaults_filename in test_file: + continue + + if test_regexp != ".*": + logging.info( + "Filtering all tests via a regular expression: {}".format(test_regexp) + ) + tags_regex_string = re.compile(test_regexp) + + match_obj = re.search(tags_regex_string, test_file) + if match_obj is None: + logging.info( + "Skipping {} given it does not match regex {}".format( + test_file, test_regexp + ) + ) + continue + + with open(test_file, "r") as stream: + ( + result, + benchmark_config, + test_name, + ) = get_final_benchmark_config(None, stream, "") + if result is False: + logging.error( + "Skipping {} given there were errors while calling get_final_benchmark_config()".format( + test_file + ) + ) + continue + + if command_groups_regexp is not None: + logging.info( + "Filtering all test command groups via a regular expression: {}".format( + command_groups_regexp + ) + ) + if "tested-groups" in benchmark_config: + command_groups = benchmark_config["tested-groups"] + logging.info( + f"The file {test_file} (test name = {test_name}) contains the following groups: {command_groups}" + ) + groups_regex_string = re.compile(command_groups_regexp) + found = False + for command_group in command_groups: + match_obj = re.search(groups_regex_string, command_group) + if match_obj is not None: + found = True + logging.info(f"found the command group {command_group}") + if found is False: + logging.info( + f"Skipping {test_file} given the following groups: {command_groups} does not match command group regex {command_groups_regexp}" + ) + continue + else: + logging.warning( + f"The file {test_file} (test name = {test_name}) does not contain the property 'tested-groups'. Cannot filter based uppon groups..." + ) + + if "priority" in benchmark_config: + priority = benchmark_config["priority"] + + if priority is not None: + if priority > priority_upper_limit: + logging.warning( + "Skipping test {} giving the priority limit ({}) is above the priority value ({})".format( + test_name, priority_upper_limit, priority + ) + ) + + continue + if priority < priority_lower_limit: + logging.warning( + "Skipping test {} giving the priority limit ({}) is bellow the priority value ({})".format( + test_name, priority_lower_limit, priority + ) + ) + + continue + logging.info( + "Test {} priority ({}) is within the priority limit [{},{}]".format( + test_name, + priority, + priority_lower_limit, + priority_upper_limit, + ) + ) + filtered_test_files.append(test_file) + return filtered_test_files + + def data_prepopulation_step( benchmark_config, benchmark_tool_workdir, diff --git a/redis_benchmarks_specification/test-suites/create-re-string.py b/redis_benchmarks_specification/test-suites/create-re-string.py deleted file mode 100644 index 470a6026..00000000 --- a/redis_benchmarks_specification/test-suites/create-re-string.py +++ /dev/null @@ -1,286 +0,0 @@ -tests = [ - { - "name": "memtier_benchmark-100Kkeys-string-setget50c-20KiB", - "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys", - "check": {"keyspacelen": 100000}, - "command": "--data-size 20000 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=100000 --test-time 180", - "kpis": {}, - "tested-commands": ["setget-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-100Kkeys-string-setget50c-20KiB-pipeline-10", - "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys", - "check": {"keyspacelen": 100000}, - "command": "--pipeline 10 --data-size 20000 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --key-minimum=1 --key-maximum=100000 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-100Kkeys-string-setget200c-20KiB", - "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys", - "check": {"keyspacelen": 100000}, - "command": "--data-size 20000 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=100000 --test-time 180", - "kpis": {}, - "tested-commands": ["setget-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-100Kkeys-string-setget200c-20KiB-pipeline-10", - "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys", - "check": {"keyspacelen": 100000}, - "command": "--pipeline 10 --data-size 20000 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --key-minimum=1 --key-maximum=100000 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-100Kkeys-load-string50c-with-20KiB-values", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 25 -t 2 --hide-histogram", - "kpis": {}, - "tested-commands": ["set-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-100Kkeys-load-string50c-with-20KiB-values-pipeline-10", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--pipeline 10 --data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 25 -t 2 --hide-histogram", - "kpis": {}, - "tested-commands": ["set-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-100Kkeys-load-string200c-with-20KiB-values", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 50 -t 4 --hide-histogram", - "kpis": {}, - "tested-commands": ["set-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-100Kkeys-load-string200c-with-20KiB-values-pipeline-10", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--pipeline 10 --data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 50 -t 4 --hide-histogram", - "kpis": {}, - "tested-commands": ["set-20k"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget50c-100B", - "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--data-size 100 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget50c-1KiB", - "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--data-size 1000 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget50c-100B-pipeline-10", - "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--pipeline 10 --data-size 100 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget50c-1KiB-pipeline-10", - "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--pipeline 10 --data-size 1000 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget200c-100B", - "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--data-size 100 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget200c-1KiB", - "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--data-size 1000 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget200c-100B-pipeline-10", - "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--pipeline 10 --data-size 100 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-setget200c-1KiB-pipeline-10", - "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys", - "check": {"keyspacelen": 1000000}, - "command": "--pipeline 10 --data-size 1000 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180", - "kpis": {}, - "tested-commands": ["setget"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string50c-with-100B-values", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string50c-with-1KiB-values", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string50c-with-100B-values-pipeline-10", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--pipeline 10 --data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string50c-with-1KiB-values-pipeline-10", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--pipeline 10 --data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string200c-with-100B-values", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string200c-with-1KiB-values", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string200c-with-100B-values-pipeline-10", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--pipeline 10 --data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-load-string200c-with-1KiB-values-pipeline-10", - "precommand": "", - "check": {"keyspacelen": 0}, - "command": "--pipeline 10 --data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram", - "kpis": {}, - "tested-commands": ["set"], - "tested-groups": ["string"], - }, - { - "name": "memtier_benchmark-1Mkeys-string-mget-1KiB", - "precommand": "--data-size 1000 --key-minimum=1 --key-maximum 1000000 -n allkeys --ratio=1:0 --key-pattern P:P --hide-histogram -t 2 -c 100", - "check": {"keyspacelen": 1000000}, - "command": ' --command="MGET __key__ memtier-1 memtier-2 memtier-3 memtier-4 memtier-5 memtier-6 memtier-7 memtier-8 memtier-9" --command-key-pattern=R -c 50 -t 2 --hide-histogram --test-time 180', - "kpis": {}, - "tested-commands": ["mget"], - "tested-groups": ["string"], - }, -] - -print(len(tests)) -re_filenames = [x["name"] for x in tests] -re_test_specs = {} -for x in tests: - re_test_specs[x["name"]] = x - -import yaml -import json -import pathlib - - -defaults_filename = "default.yml" -prefix = "memtier_benchmark-" -test_glob = "memtier_*.yml" -files = pathlib.Path().glob(test_glob) -files = [str(x) for x in files] - -base_yaml = yaml.safe_load(open("memtier_benchmark-1Mkeys-string-get-1KiB.yml")) -del base_yaml["description"] -# del base_yaml["clientconfig"]["resources"] -# del base_yaml["build-variants"] -# del base_yaml["priority"] -# del base_yaml["redis-topologies"] -# del base_yaml["tested-commands"] -# del base_yaml["version"] -# del base_yaml["tested-groups"] -# - - -# -# for file in files: -# if defaults_filename in file: -# files.remove(file) - -for re_file in re_filenames: - re_spec = re_test_specs[re_file] - precommand = "" - if "precommand" in re_spec: - precommand = re_spec["precommand"] - - command = "" - if "command" in re_spec: - command = re_spec["command"] - if "dbconfig" in base_yaml: - del base_yaml["dbconfig"] - if precommand != "": - base_yaml["dbconfig"] = {} - base_yaml["dbconfig"]["preload_tool"] = {} - base_yaml["dbconfig"]["preload_tool"][ - "run_image" - ] = "redislabs/memtier_benchmark:edge" - base_yaml["dbconfig"]["preload_tool"]["tool"] = "memtier_benchmark" - base_yaml["dbconfig"]["preload_tool"]["arguments"] = f"{precommand}" - - base_yaml["clientconfig"]["arguments"] = command - base_yaml["name"] = re_file - with open(f"re-string/{re_file}.yml", "w") as outfile: - yaml.dump(base_yaml, outfile) diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-2MB.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-2MB.yml index 67919fe2..f6de6d2e 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-2MB.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-2MB.yml @@ -9,7 +9,7 @@ dbconfig: preload_tool: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: '"--data-size" "2000000" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"' + arguments: '"--data-size" "2000000" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1" "--key-minimum" "1000"' resources: requests: memory: 2g @@ -22,7 +22,7 @@ build-variants: clientconfig: run_image: redislabs/memtier_benchmark:edge tool: memtier_benchmark - arguments: --data-size 2000000 --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180 + arguments: --data-size 2000000 --key-minimum 1 --key-maximum 1000 --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180 resources: requests: cpus: '4' diff --git a/tox.ini b/tox.ini index 48d91bb2..9265ff45 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] isolated_build = True +envlist = integration-tests [tox:.package] # note tox will use the same python version as under what tox is installed to package @@ -9,7 +10,7 @@ basepython = python3 [testenv:integration-tests] deps = -r{toxinidir}/dev_requirements.txt -passenv = TST_BUILDER_X TST_RUNNER_X GH_TOKEN TST_REDIS_DIR +passenv = TST_BUILDER_X,TST_RUNNER_X,GH_TOKEN,TST_REDIS_DIR,DOCKER_HOST,DOCKER_TLS_VERIFY,DOCKER_CERT_PATH commands = black --check redis_benchmarks_specification @@ -18,6 +19,11 @@ commands = coverage run --include=redis_benchmarks_specification/* -m pytest -ra {posargs} coverage report -m + + +[docker] +docker_host = unix:///var/run/docker.sock + docker = datasink db_server diff --git a/utils/tests/test_builder.py b/utils/tests/test_builder.py index 94c54f5d..c326ae61 100644 --- a/utils/tests/test_builder.py +++ b/utils/tests/test_builder.py @@ -63,10 +63,16 @@ def test_commit_schema_to_stream_then_build(): builders_folder = "./redis_benchmarks_specification/setups/builders" different_build_specs = ["gcc:8.5.0-amd64-debian-buster-default.yml"] previous_id = ">" - previous_id, new_builds_count = builder_process_stream( + ( + previous_id, + new_builds_count, + build_stream_fields_arr, + ) = builder_process_stream( builders_folder, conn, different_build_specs, previous_id ) assert new_builds_count == 1 + assert len(build_stream_fields_arr) == 1 + assert len(build_stream_fields_arr[0]["test_regexp"]) == ".*" assert conn.exists(STREAM_KEYNAME_NEW_BUILD_EVENTS) conn.save() @@ -109,7 +115,7 @@ def test_commit_schema_to_stream_then_build_historical_redis(): builders_folder = "./redis_benchmarks_specification/setups/builders" different_build_specs = ["gcc:8.5.0-amd64-debian-buster-default.yml"] previous_id = ">" - previous_id, new_builds_count = builder_process_stream( + previous_id, new_builds_count, _ = builder_process_stream( builders_folder, conn, different_build_specs, previous_id ) assert new_builds_count == 1 diff --git a/utils/tests/test_data/api_builder_common.py b/utils/tests/test_data/api_builder_common.py index 64341ff2..20179300 100644 --- a/utils/tests/test_data/api_builder_common.py +++ b/utils/tests/test_data/api_builder_common.py @@ -37,7 +37,7 @@ def flow_1_and_2_api_builder_checks( builders_folder = "./redis_benchmarks_specification/setups/builders" different_build_specs = ["{}.yml".format(build_spec_name)] previous_id = ">" - previous_id, new_builds_count = builder_process_stream( + previous_id, new_builds_count, _ = builder_process_stream( builders_folder, conn, different_build_specs, previous_id ) assert new_builds_count == 1