From 1b3f2a7143eaf51eb973c53eac4977906458d061 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Mon, 5 Jan 2026 16:36:19 +0000 Subject: [PATCH] Migrate from Pipenv to uv uv has garnered a lot of Python community support recently, consolidating and replacing virtualenv, Pipenv, pip, etc. In addition to consolidating a variety of tools under one interface, it is faster and, at this point, I feel is more recognizable than Pipenv. This commit transitions us from Pipenv to uv. --- CMakeLists.txt | 2 +- ci/docker/deb/Dockerfile | 4 +- ci/docker/yum/Dockerfile | 4 +- doc/CMakeLists.txt | 62 ++-- doc/README.md | 12 +- .../documentation/building.en.rst | 16 +- doc/developer-guide/testing/autests.en.rst | 1 - .../testing/blackbox-testing.en.rst | 272 ------------------ doc/developer-guide/testing/index.en.rst | 1 - doc/{Pipfile => pyproject.toml} | 45 ++- .../txn_box/{Pipfile => pyproject.toml} | 23 +- tests/CMakeLists.txt | 31 +- tests/Pipfile | 62 ---- tests/README.md | 2 +- tests/autest.sh | 2 +- tests/autest.sh.in | 2 +- tests/gold_tests/autest-site/init.cli.ext | 4 +- tests/pyproject.toml | 66 +++++ tests/test-env-check.sh | 15 +- tools/cmake-format.sh | 60 +--- tools/git/pre-commit | 22 +- tools/hrw4u/Makefile | 21 +- tools/hrw4u/README.md | 23 +- tools/hrw4u/bootstrap.sh | 30 +- tools/hrw4u/requirements.txt | 20 -- tools/yapf.sh | 61 +--- 26 files changed, 228 insertions(+), 635 deletions(-) delete mode 100644 doc/developer-guide/testing/blackbox-testing.en.rst rename doc/{Pipfile => pyproject.toml} (52%) rename plugins/experimental/txn_box/{Pipfile => pyproject.toml} (79%) delete mode 100644 tests/Pipfile create mode 100644 tests/pyproject.toml delete mode 100644 tools/hrw4u/requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d318905f93..e0ef131dede 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -610,7 +610,7 @@ find_package(resolv) if(ENABLE_DOCS OR ENABLE_AUTEST) find_package(Python3 REQUIRED) - find_program(PipEnv pipenv REQUIRED) + find_program(UV uv REQUIRED) find_program(NETCAT_PROGRAM nc REQUIRED) endif() diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 4e1398d15b7..85815163e2f 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -64,5 +64,7 @@ RUN apt-get update; apt-get -y dist-upgrade; \ mkdir -p /var/jenkins && chown jenkins.jenkins /var/jenkins # This is for autest stuff, skipping since it's pulling in a *lot* ... -# apt-get -y install python3 httpd-tools procps-ng nmap-ncat pipenv \ +# apt-get -y install python3 httpd-tools procps-ng nmap-ncat \ # python3-virtualenv python3-gunicorn python3-requests python3-httpbin; \ +# Install uv for Python environment management: +# curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 5a160fb24ff..4f53cadea60 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -55,8 +55,10 @@ RUN yum -y update; \ ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff - yum -y install python3 httpd-tools procps-ng nmap-ncat pipenv \ + yum -y install python3 httpd-tools procps-ng nmap-ncat \ python3-virtualenv python3-gunicorn python3-requests python3-httpbin; \ + # Install uv for Python environment management + curl -LsSf https://astral.sh/uv/install.sh | sh; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) yum -y install java openssh-server && ssh-keygen -A; rm -f /run/nologin; \ groupadd -g 665 jenkins && \ diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 7fce45d7120..d7891a440a7 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -74,47 +74,44 @@ foreach(UML ${UML_FILES}) endforeach() add_custom_target(generate_svg_from_uml DEPENDS ${SVG_FILES}) -# Docs are built with python so we need a target to setup pipenv -# Copy Pipfile to build directory and explicitly point pipenv to it -# PIPENV_VENV_IN_PROJECT=1 creates .venv next to the Pipfile location (if not already in a venv) -# PIPENV_VERBOSITY=-1 suppresses warnings -# If user is already in a virtualenv, pipenv will use it; otherwise creates one in build dir -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Pipfile ${CMAKE_CURRENT_BINARY_DIR}/Pipfile COPYONLY) -set(RUNPIPENV PIPENV_PIPFILE=${CMAKE_CURRENT_BINARY_DIR}/Pipfile PIPENV_VENV_IN_PROJECT=1 PIPENV_VERBOSITY=-1 ${PipEnv}) -# Create a marker file to indicate pipenv setup is complete -# This is more reliable than using Pipfile.lock which might already exist +# Docs are built with python so we need a target to setup the virtual environment. +# Copy pyproject.toml to build directory and use uv to manage dependencies. +# uv creates .venv in the project directory by default. +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml ${CMAKE_CURRENT_BINARY_DIR}/pyproject.toml COPYONLY) +set(RUNUV ${UV} --project ${CMAKE_CURRENT_BINARY_DIR}) +# Create a marker file to indicate uv setup is complete. add_custom_command( - OUTPUT .pipenv_installed - COMMAND ${RUNPIPENV} install --python ${Python3_EXECUTABLE} - COMMAND ${CMAKE_COMMAND} -E touch .pipenv_installed - COMMENT "Setup pipenv in build directory (${CMAKE_CURRENT_BINARY_DIR}/.venv) and install packages" - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Pipfile + OUTPUT .uv_installed + COMMAND ${RUNUV} sync + COMMAND ${CMAKE_COMMAND} -E touch .uv_installed + COMMENT "Setup uv virtual environment in build directory (${CMAKE_CURRENT_BINARY_DIR}/.venv) and install packages" + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} VERBATIM ) add_custom_target( generate_docs - COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV} - run python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version - COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV} - run python -m sphinx -W -c ${CMAKE_CURRENT_BINARY_DIR} -b html ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNUV} run + python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version + COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNUV} run + python -m sphinx -W -c ${CMAKE_CURRENT_BINARY_DIR} -b html ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/docbuild/html - DEPENDS generate_docs_setup .pipenv_installed + DEPENDS generate_docs_setup .uv_installed WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Sphinx PlantUML extension now generates diagrams automatically in build tree" ) # Generate PDF documentation (Letter paper size - US standard) add_custom_target( generate_pdf - COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV} - run python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version - COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV} - run python -m sphinx -c ${CMAKE_CURRENT_BINARY_DIR} -b latex -t latex_paper ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNUV} run + python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version + COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNUV} run + python -m sphinx -c ${CMAKE_CURRENT_BINARY_DIR} -b latex -t latex_paper ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex latexmk -pdf -interaction=nonstopmode -f ApacheTrafficServer.tex - DEPENDS generate_docs_setup .pipenv_installed + DEPENDS generate_docs_setup .uv_installed WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Building PDF documentation with Sphinx and LaTeX (Letter paper, PlantUML generates PNG automatically)" ) @@ -122,21 +119,14 @@ add_custom_target( # Generate PDF documentation (A4 paper size - International standard) add_custom_target( generate_pdf_a4 - COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV} - run python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version - COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNPIPENV} - run python -m sphinx -c ${CMAKE_CURRENT_BINARY_DIR} -b latex -t latex_a4 ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNUV} run + python ${CMAKE_CURRENT_SOURCE_DIR}/checkvers.py --check-version + COMMAND ${CMAKE_COMMAND} -E env "JAVA_TOOL_OPTIONS=-Djava.awt.headless=true -Dapple.awt.UIElement=true" ${RUNUV} run + python -m sphinx -c ${CMAKE_CURRENT_BINARY_DIR} -b latex -t latex_a4 ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex-a4 COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/docbuild/latex-a4 latexmk -pdf -interaction=nonstopmode -f ApacheTrafficServer.tex - DEPENDS generate_docs_setup .pipenv_installed + DEPENDS generate_docs_setup .uv_installed WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Building PDF documentation with Sphinx and LaTeX (A4 paper, PlantUML generates PNG automatically)" ) - -#add_custom_command( -# TARGET generate_docs -# POST_BUILD -# COMMAND ${RUNPIPENV} --rm -# COMMENT "Cleaning up pipenv" -#) diff --git a/doc/README.md b/doc/README.md index 1d927e18d0c..62cabb9394f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -6,19 +6,19 @@ This directory contains the source code for Traffic Server documentation. ## Package Requirements Traffic Server documentation is built using the Sphinx documentation generator. -The Sphinx build tool is distributed as a Python package. A Pipfile is provided -to conveniently configure a Python virtual environment with the needed Sphinx -packages. +The Sphinx build tool is distributed as a Python package. A `pyproject.toml` is +provided to conveniently configure a Python virtual environment with the needed +Sphinx packages via uv. In addition to the Sphinx Python package requirements, building the -documentation will also require Java and graphviz system packages to be +documentation will also require Java, graphviz, and uv system packages to be installed. ## Build Steps Building the docs requires passing `-DENABLE_DOCS=ON` to cmake (docs generation is off by default), and then using the appropriate build target. The build steps -will automatically install a Pipenv virtual environment using `docs/Pipfile` and -do what is necessary to build the docs. +will automatically install a virtual environment via uv using `doc/pyproject.toml` +and do what is necessary to build the docs. ### Building HTML Documentation ```sh diff --git a/doc/developer-guide/documentation/building.en.rst b/doc/developer-guide/documentation/building.en.rst index 29135728404..50ebfb6f160 100644 --- a/doc/developer-guide/documentation/building.en.rst +++ b/doc/developer-guide/documentation/building.en.rst @@ -41,16 +41,18 @@ System installs python3 Python 3 is required. The build system will use this to create a virtual environment. - pipenv - Used to manage Python dependencies. Install with ``pip3 install pipenv`` or your system package manager. + uv + Used to manage Python dependencies. See from https://docs.astral.sh/uv/getting-started/installation/ + for installation instructions. ``curl -LsSf https://astral.sh/uv/install.sh | sh``, or + or ``pip3 install uv``, or your system package manager. Python packages - Python dependencies are managed automatically via `pipenv `__ - and the :ts:git:`doc/Pipfile`. The build system will automatically create a virtual environment + Python dependencies are managed automatically via `uv `__ + and the :ts:git:`doc/pyproject.toml`. The build system will automatically create a virtual environment and install all required packages (including Sphinx, sphinx-rtd-theme, sphinxcontrib-plantuml, sphinx-intl for internationalization, and other dependencies) when you build the documentation. - You do not need to manually install these packages or set up pipenv yourself - the CMake build + You do not need to manually install these packages or set up uv yourself - the CMake build targets handle this automatically. Building the documentation @@ -72,8 +74,8 @@ With CMake configured with ``-DENABLE_DOCS=ON``, building the documentation is s The build system will automatically: -1. Create a pipenv virtual environment in the build directory -2. Install all required Python packages from :ts:git:`doc/Pipfile` +1. Create a virtual environment in the build directory via uv +2. Install all required Python packages from :ts:git:`doc/pyproject.toml` 3. Generate the documentation For repeated builds while working on the documentation, simply run the build command again. diff --git a/doc/developer-guide/testing/autests.en.rst b/doc/developer-guide/testing/autests.en.rst index 8c83cff3266..29b0de15576 100644 --- a/doc/developer-guide/testing/autests.en.rst +++ b/doc/developer-guide/testing/autests.en.rst @@ -65,7 +65,6 @@ If |TS| cmake build is configured via ``-DENABLE_AUTEST=ON``, tests can be run w cmake --build build cmake --install build cd build/tests - pipenv install ./autest.sh --sandbox /tmp/sbcursor --clean=none -f For example, to run ``cache-auth.test.py``: diff --git a/doc/developer-guide/testing/blackbox-testing.en.rst b/doc/developer-guide/testing/blackbox-testing.en.rst deleted file mode 100644 index e7827d7692c..00000000000 --- a/doc/developer-guide/testing/blackbox-testing.en.rst +++ /dev/null @@ -1,272 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -.. include:: ../../common.defs - -.. _blackbox-testing: - -|TS| Blackbox Testing -******************************** -|TS| uses the Reusable Gold Testing System (`AuTest `_) -for functional testing. The current layout is: - -:: - - gold_tests/ - contains all the tests that run on the Reusable Gold Testing System (AuTest) - tools/ - contains programs used to help with testing. - -Scripts -======== -To ease the process of running Autest, there is *autest.sh* and *bootstrap.py*. - -**autest.sh** - This file is a simple wrapper that will call the Reusable Gold Testing System (Autest) program in a pipenv. -If the pipenv is not setup, the script will prompt user the missing components. -That will set up the Autest on most systems in a Python virtual environment. -The wrapper adds some basic options to the command to point to the location of the tests. -Run the script from the ``tests/`` directory followed by ``--ats-bin`` and the bin directory where ATS is located (e.g. ``~/ats/bin``) -Use ``--help`` for more details on options for running Autest. - -**bootstrap.py** - This script will check for the necessary packages needed to create a pipenv that can run Autest. -If any package is missing, the script will alert the user. -If all packages are available, it will install a virtual environment using the provided Pipfile. - - -Manual Setup -============= -To run autest manually, the recommended way is to follow these steps: - -1. ``pipenv install``: create the virtual environment(only needed once). -2. ``pipenv shell``: enter a shell in the virtual environment(type ``exit`` to leave the shell). -3. ``cd gold_tests``: enter the directory containing the test files. -4. ``autest --ats-bin user_ats_bin``: run autest where user_ats_bin is the bin directory in the user's ats directory. - - -Advanced Setup -=============== -AuTest and the relevant tools can be install manually instead of using the wrapper script. -By doing this, it is often easier to debug issues with the testing system, or the tests. -There are two ways this can be done. - -1. Run the bootstrap script then source the path with a ``source ./env-test/bin/activate`` command. At this point autest command should run without the wrapper script. -2. Make sure you install python 3.5 or better on your system. From there install these python packages (e.g. ``pip install``): - - hyper - - git+https://bitbucket.org/autestsuite/reusable-gold-testing-system.git - - `traffic-replay `_ (This will automatically install `MicroDNS `_, `MicroServer `_, `TrafficReplayLibrary `_, and dnslib as part of the dependencies.) - -Writing Tests -============== -When writing tests, please refer to the current `documentation `_ for general use of the Autest system. - - -Testing Environment --------------------- -The environment of the testing process will have a number of added environment variables to control |TS| running the in the sandbox location correctly. -This can be used to easily setup other commands that should run under same environment. - -Autest Extensions ------------------- -Autest allows the user to create extensions to help specialize and simplify test writing for a given application domain. - -TrafficServer -~~~~~~~~~~~~~~ - -For TrafficServer, we have defined the following functions and objects in ``tests/gold_tests/autest-site/trafficserver.test.ext``: - -- ``MakeATSProcess(obj, name, command='traffic_server', enable_tls=False)`` - - - name - A name for this instance of ATS - - command - optional argument defining what process to use. Defaults to ``traffic_server`` - - select_ports - have Autest automatically select a nonSSL port to use - - enable_tls - have Autest automatically select SSL port (``select_ports`` must be **True**) - -- ``CopyConfig(file, targetname=None, process=None)`` - - - file - name of the file to copy. Relative paths are relative from the test file location - - targetname - the name of the file when copied to the correct configuration location - - process - optional process object to use for getting path location to copy to. Only needed if the Setup object call is not in the scope of the process object created with the MakeATSProcess(...) API. - -This function copies a given configuration file to the location of the |TS| sandbox used in a test. Given a test might have more than on |TS| instance, it can be difficult to understand the correct location to copy to. This function will deal with the details correctly. - -When automatically selected, the following ports will be allocated for TS: - -- port -- portv6 -- ssl_port -- admin_port - this is set even if select_port is **False** - -A number of file objects are also defined to help test TrafficServer. Files that are currently defined are: - -- squid.log -- error.log -- diags.log -- records.yaml -- cache.config -- hosting.config -- ip_allow.yaml -- logging.yaml -- parent.config -- plugin.config -- remap.config -- sni.yaml -- socks.config -- splitdns.config -- ssl_multicert.config -- storage.config -- volume.config - -Example -++++++++ -.. code-block:: python - - ts1 = Test.MakeATSProcess("ts1",select_ports=False) - # uses the setup object in the scope of the process object - ts1.Setup.ts.CopyConfig('config/records_8090.yaml','records.yaml') - - -Origin Server -~~~~~~~~~~~~~~ - -- ``Test.MakeOriginServer(name, port, s_port, ip, delay, ssl, lookup_key, clientcert, clientkey)`` - - - name - A name for this instance of origin server. - - port - option to specify the nonSSL port. If left unspecified, the port will be autoselected. - - s_port - option to specify the SSL port. If left unspecified, the port will be autoselected (SSL has to be True). - - ip - option to specify IP address. Defaults to ``127.0.0.1``. - - delay - option to have MicroServer delay for set amount of seconds before returning response. Defaults to ``0``. - - ssl - option to enable SSL - - lookup_key - option to change the unique identifier that MicroServer uses to identify each transaction. Defaults to ``PATH``. - - clientcert - path to cert used for SSL. Defaults to the included cert in ``tests/tools/microserver/ssl``. - - clientkey - path to key used for SSL. Same default as above. - -This function returns a AuTest process object that launches the python-based Microserver. -Microserver is a mock server which responds to client http requests. -Microserver needs to be setup for the tests that require an origin server behind ATS. -The server reads a JSON-formatted data file that contains request headers and the corresponding response headers. -Microserver responds with payload if the response header contains Content-Length or Transfer-Encoding specified. - -- ``Test.addResponse(filename, request_header, response_header)`` - - - filename - name of the file where the request header and response header will be written to in JSON format - - request_header - dictionary of request header - - response_header - dictionary of response header corresponding to the request header. - -This function adds the request header and response header to a file which is then read by the microserver to populate request-response map. -The key-fields required for the header dictionary are 'headers', 'timestamp' and 'body'. - -Example -++++++++ -.. code-block:: python - - # create the origin server process - server=Test.MakeOriginServer("server") - # define the request header and the desired response header - request_header={"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} - # desired response form the origin server - response_header={"headers": "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n", "timestamp": "1469733493.993", "body": ""} - # addResponse adds the transaction to a file which is used by the server - server.addResponse("sessionlog.json", request_header, response_header) - - -DNS -~~~~ - -- ``Test.MakeDNServer(name, filename, port, ip, rr, default)`` - - - name - A name for this instance of MicroDNS. - - filename - file containing zone information for MicroDNS to read from. Defaults to ``dns_file.json`` - - port - option for the DNS port. Autoselected if left unspecified. - - ip - option for IP address. Defaults to ``127.0.0.1`` - - rr - option to enable round robin IP. Defaults to ``False`` - - default - option to specify a default IP response when MicroDNS can't find a domain:IP pair. - -- ``dns.addRecords(records, jsonFile)`` - - - records - a dictionary of domain:IP mappings in the form of ``{"domain A": [IP1, IP2], "domain B": [IP3, IP4]}`` - - jsonFile - a JSON file containing domain:IP mappings - -The JSON file must take the form of - -.. code-block:: python - - { - "mappings: [ - {"domain A": [IP1, IP2]}, - {"domain B": [IP3, IP4]} - ] - } - - -Example -++++++++ -.. code-block:: python - - # If everything is mapped to 127.0.0.1 - dns = Test.MakeDNServer("dns", default=['127.0.0.1']) - #------------------------------------------------------ - # Using addRecords method - dns = Test.MakeDNServer("dns") - - dns.addRecords(records={"foo.com.":["127.0.0.1", "127.0.1.1"]}) - # AND/OR - dns.addRecords(jsonFile="zone.json") # where zone.json is in the format described above - - -Condition Testing -+++++++++++++++++ - - ``Condition.HasCurlFeature(feature)`` - This function tests Curl for possible features it has been compiled with. Consult Curl documentation for possible features. - - ``Condition.PluginExists(pluginname)`` - This function tests for the existence of a certain plugin in ATS. - - ``Condition.HasATSFeature(feature)`` - This function tests TrafficServer for possible features it has been compiled with. Current features you can test for are: - - - TS_HAS_LZMA - - TS_HAS_PIPE_BUFFER_SIZE_CONFIG - - TS_HAS_JEMALLOC - - TS_HAS_MIMALLOC - - TS_HAS_IN6_IS_ADDR_UNSPECIFIED - - TS_HAS_BACKTRACE - - TS_HAS_PROFILER - - TS_USE_FAST_SDK - - TS_USE_DIAGS - - TS_USE_EPOLL - - TS_USE_KQUEUE - - TS_USE_POSIX_CAP - - TS_USE_TPROXY - - TS_HAS_SO_MARK - - TS_HAS_IP_TOS - - TS_USE_HWLOC - - TS_USE_TLS13 - - TS_USE_QUIC - - TS_HAS_QUICHE - - TS_HAS_SO_PEERCRED - - TS_USE_REMOTE_UNWINDING - - TS_HAS_128BIT_CAS - - TS_HAS_TESTS - - -Examples: -+++++++++ -.. code-block:: python - - Test.SkipUnless( - Condition.HasCurlFeature('http2'), - ) - - Test.SkipUnless( - Condition.PluginExists('a-plugin.so'), - ) diff --git a/doc/developer-guide/testing/index.en.rst b/doc/developer-guide/testing/index.en.rst index 3810a5f00b2..363dabc4e55 100644 --- a/doc/developer-guide/testing/index.en.rst +++ b/doc/developer-guide/testing/index.en.rst @@ -26,4 +26,3 @@ Testing Traffic Server :maxdepth: 2 autests.en - blackbox-testing.en diff --git a/doc/Pipfile b/doc/pyproject.toml similarity index 52% rename from doc/Pipfile rename to doc/pyproject.toml index e00ab96deea..c357edb9074 100644 --- a/doc/Pipfile +++ b/doc/pyproject.toml @@ -14,34 +14,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true +[project] +name = "ats-docs" +version = "0.1.0" +description = "Documentation dependencies for Apache Traffic Server" +requires-python = ">=3.11" -[dev-packages] +dependencies = [ + # We pin Sphinx because the sphinx-rtd-theme suggests this as a best practice. + # If not, we will often face issues when Sphinx updates their version before + # sphinx-rtd-theme has had time to update their component for the new sphinx + # version. + "sphinx==8.2.3", -[packages] + "sphinx-rtd-theme>=3.0.0,<4.0.0", + "sphinxcontrib-jquery", + "sphinxcontrib-plantuml", + # i18n + "sphinx-intl", -# We pin Sphinx because the sphinx-rtd-theme suggests this as a best practice. -# If not, we will often face issues when Sphinx updates their version before -# sphinx-rtd-theme has had time to update their component for the new sphinx -# version. -sphinx = "==8.2.3" + "pyyaml", -sphinx-rtd-theme = "==3.*" -sphinxcontrib-jquery = "*" -sphinxcontrib-plantuml = "*" -# i18n -sphinx-intl = "*" + # For parsing Doxygen XML output, to add links from an API description + # to the source code for that object. + "lxml", -pyyaml = "*" + "polib>=1.0.3", +] -# For parsing Doxygen XML output, to add links from an API description -# to the source code for that object -lxml = "*" -polib = ">=1.0.3" -[requires] -python_version = "3" diff --git a/plugins/experimental/txn_box/Pipfile b/plugins/experimental/txn_box/pyproject.toml similarity index 79% rename from plugins/experimental/txn_box/Pipfile rename to plugins/experimental/txn_box/pyproject.toml index e9e7bb48d71..103c0e939c0 100644 --- a/plugins/experimental/txn_box/Pipfile +++ b/plugins/experimental/txn_box/pyproject.toml @@ -13,17 +13,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" -[dev-packages] +[project] +name = "txn-box-tests" +version = "0.1.0" +description = "Test dependencies for txn_box plugin" +requires-python = ">=3.6" + +dependencies = [ + "scons", + "scons-parts", + "autest==1.7.4", +] + -[packages] -scons = "*" -scons-parts = "*" -autest = "==1.7.4" -[requires] -python_version = "3" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 593856e4f56..1718ab49113 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -45,7 +45,7 @@ add_subdirectory(gold_tests/pluginTest/TSVConnFd) add_subdirectory(gold_tests/timeout) add_subdirectory(gold_tests/tls) -set(RUNPIPENV PIPENV_VENV_IN_PROJECT=True ${PipEnv}) +set(RUNUV ${UV} --project ${CMAKE_CURRENT_BINARY_DIR}) set(CMAKE_GOLD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/gold_tests") set(CMAKE_SKIP_GOLD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/_skipped_uds_gold_tests") @@ -54,28 +54,26 @@ if(ENABLE_AUTEST_UDS) set(CURL_UDS_FLAG "--curl-uds") endif() -configure_file(Pipfile Pipfile COPYONLY) +configure_file(pyproject.toml pyproject.toml COPYONLY) configure_file(autest.sh.in autest.sh) add_custom_target( autest COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install - COMMAND ${RUNPIPENV} install - COMMAND - ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_GOLD_DIR}/remap:$ENV{PYTHONPATH} ${RUNPIPENV} run env autest --directory - ${CMAKE_GOLD_DIR} --ats-bin=${CMAKE_INSTALL_PREFIX}/bin --proxy-verifier-bin ${PROXY_VERIFIER_PATH} --build-root - ${CMAKE_BINARY_DIR} --sandbox ${AUTEST_SANDBOX} ${CURL_UDS_FLAG} ${AUTEST_OPTIONS_LIST} + COMMAND ${RUNUV} sync + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_GOLD_DIR}/remap:$ENV{PYTHONPATH} ${RUNUV} run autest --directory + ${CMAKE_GOLD_DIR} --ats-bin=${CMAKE_INSTALL_PREFIX}/bin --proxy-verifier-bin ${PROXY_VERIFIER_PATH} + --build-root ${CMAKE_BINARY_DIR} --sandbox ${AUTEST_SANDBOX} ${CURL_UDS_FLAG} ${AUTEST_OPTIONS_LIST} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} USES_TERMINAL ) add_custom_target( autest_no_install - COMMAND ${RUNPIPENV} install - COMMAND - ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_GOLD_DIR}/remap:$ENV{PYTHONPATH} ${RUNPIPENV} run env autest --directory - ${CMAKE_GOLD_DIR} --ats-bin=${CMAKE_INSTALL_PREFIX}/bin --proxy-verifier-bin ${PROXY_VERIFIER_PATH} --build-root - ${CMAKE_BINARY_DIR} --sandbox ${AUTEST_SANDBOX} ${CURL_UDS_FLAG} ${AUTEST_OPTIONS_LIST} + COMMAND ${RUNUV} sync + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_GOLD_DIR}/remap:$ENV{PYTHONPATH} ${RUNUV} run autest --directory + ${CMAKE_GOLD_DIR} --ats-bin=${CMAKE_INSTALL_PREFIX}/bin --proxy-verifier-bin ${PROXY_VERIFIER_PATH} + --build-root ${CMAKE_BINARY_DIR} --sandbox ${AUTEST_SANDBOX} ${CURL_UDS_FLAG} ${AUTEST_OPTIONS_LIST} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} USES_TERMINAL ) @@ -84,14 +82,13 @@ add_custom_target( add_custom_target( autest-uds COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install - COMMAND ${RUNPIPENV} install + COMMAND ${RUNUV} sync COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_GOLD_DIR}/h2 ${CMAKE_SKIP_GOLD_DIR}/h2 COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_GOLD_DIR}/tls ${CMAKE_SKIP_GOLD_DIR}/tls COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_GOLD_DIR}/tls_hooks ${CMAKE_SKIP_GOLD_DIR}/tls_hooks - COMMAND - ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_GOLD_DIR}/remap:$ENV{PYTHONPATH} ${RUNPIPENV} run env autest --directory - ${CMAKE_GOLD_DIR} --ats-bin=${CMAKE_INSTALL_PREFIX}/bin --proxy-verifier-bin ${PROXY_VERIFIER_PATH} --build-root - ${CMAKE_BINARY_DIR} --sandbox ${AUTEST_SANDBOX} ${CURL_UDS_FLAG} ${AUTEST_OPTIONS_LIST} + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_GOLD_DIR}/remap:$ENV{PYTHONPATH} ${RUNUV} run autest --directory + ${CMAKE_GOLD_DIR} --ats-bin=${CMAKE_INSTALL_PREFIX}/bin --proxy-verifier-bin ${PROXY_VERIFIER_PATH} + --build-root ${CMAKE_BINARY_DIR} --sandbox ${AUTEST_SANDBOX} ${CURL_UDS_FLAG} ${AUTEST_OPTIONS_LIST} COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_SKIP_GOLD_DIR}/h2 ${CMAKE_GOLD_DIR}/h2 COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_SKIP_GOLD_DIR}/tls ${CMAKE_GOLD_DIR}/tls COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_SKIP_GOLD_DIR}/tls_hooks ${CMAKE_GOLD_DIR}/tls_hooks diff --git a/tests/Pipfile b/tests/Pipfile deleted file mode 100644 index 6119e276564..00000000000 --- a/tests/Pipfile +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] -pyflakes = "*" - -[packages] - -# Keep init.cli.ext updated with this required autest version. -autest = "==1.10.4" - -traffic-replay = "*" # this should install TRLib, MicroServer, MicroDNS, Traffic-Replay - -h2 = "*" -hyperframe = "*" -hpack = "*" - -dnslib = "*" -# These are likely to be available via yum/dnf or apt-get -requests = "*" -gunicorn = "*" -psutil = "*" - -charset-normalizer = "*" - -# Keep init.cli.ext updated with this required microserver version. -microserver = ">=1.0.8" - -jsonschema = "*" -python-jose = "*" -pyyaml ="*" - -# For the grpc tests. -grpcio = "*" -grpcio-tools = "*" - -pyOpenSSL = "*" -eventlet = "*" - -# To test stats_over_http prometheus exporter. -prometheus_client = "*" - -[requires] -python_version = "3" diff --git a/tests/README.md b/tests/README.md index 5b98a8786cc..5a8ae9dc4ad 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,7 +27,7 @@ Running the tests can be done using the `autest` target $ cmake --build build -t autest -This will build ATS, install it to a temporary directory, setup the pipenv and run all of the autests. +This will build ATS, install it to a temporary directory, setup the virtual environment via uv, and run all of the autests. To run autest again, or to run individual tests, the cmake build generates a helper script in the build directory at `/tests/autest.sh`. This script can be used to run individual tests and further configure autest. diff --git a/tests/autest.sh b/tests/autest.sh index 55f02825763..30e28d4d95a 100755 --- a/tests/autest.sh +++ b/tests/autest.sh @@ -35,6 +35,6 @@ export PYTHONPATH=$(pwd)/gold_tests/remap:$PYTHONPATH hash nc 2>/dev/null || fail "Netcat is not installed." # this is for rhel or centos systems echo "Environment config finished. Running AuTest..." -exec pipenv run env \ +exec uv run env \ HTTP_PROXY= HTTPS_PROXY= NO_PROXY= http_proxy= https_proxy= no_proxy= \ autest -D gold_tests "$@" diff --git a/tests/autest.sh.in b/tests/autest.sh.in index 813f3ccfa3f..6554f42ee63 100755 --- a/tests/autest.sh.in +++ b/tests/autest.sh.in @@ -21,7 +21,7 @@ if [ -n "${CURL_UDS_FLAG}" ]; then fi fi -${RUNPIPENV} run env autest \ +uv run autest \ --sandbox ${AUTEST_SANDBOX} \ --directory ${CMAKE_GOLD_DIR} \ --ats-bin=${CMAKE_INSTALL_PREFIX}/bin \ diff --git a/tests/gold_tests/autest-site/init.cli.ext b/tests/gold_tests/autest-site/init.cli.ext index 318154c8049..dcae951e74e 100644 --- a/tests/gold_tests/autest-site/init.cli.ext +++ b/tests/gold_tests/autest-site/init.cli.ext @@ -27,7 +27,7 @@ found_autest_version = AuTestVersion() if AuTestVersion() < needed_autest_version: host.WriteError( f"Tests need AuTest version {needed_autest_version} or better, found version {found_autest_version}\n" - "Please update AuTest:\n pipenv --rm && pipenv install\n", + "Please update AuTest:\n rm -rf .venv && uv sync\n", show_stack=False) needed_microserver_version = "1.0.8" @@ -35,7 +35,7 @@ found_microserver_version = microserver.__version__ if found_microserver_version < needed_microserver_version: host.WriteError( f"Tests need a MicroServer version {needed_microserver_version} or better, found version {found_microserver_version}\n" - "Please update MicroServer:\n pipenv --rm && pipenv install\n", + "Please update MicroServer:\n rm -rf .venv && uv sync\n", show_stack=False) Settings.path_argument(["--ats-bin"], required=True, help="A user provided directory to ATS bin") diff --git a/tests/pyproject.toml b/tests/pyproject.toml new file mode 100644 index 00000000000..4937dcd0d35 --- /dev/null +++ b/tests/pyproject.toml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[project] +name = "ats-autest" +version = "0.1.0" +description = "AuTest dependencies for Apache Traffic Server testing" +requires-python = ">=3.6" + +dependencies = [ + # Keep init.cli.ext updated with this required autest version. + "autest==1.10.4", + + # This should install TRLib, MicroServer, MicroDNS, Traffic-Replay. + "traffic-replay", + + "h2", + "hyperframe", + "hpack", + + "dnslib", + # These are likely to be available via yum/dnf or apt-get. + "requests", + "gunicorn", + "psutil", + + "charset-normalizer", + + # Keep init.cli.ext updated with this required microserver version. + "microserver>=1.0.8", + + "jsonschema", + "python-jose", + "pyyaml", + + # For the grpc tests. + "grpcio", + "grpcio-tools", + + "pyOpenSSL", + "eventlet", + + # To test stats_over_http prometheus exporter. + "prometheus_client", +] + +[project.optional-dependencies] +dev = [ + "pyflakes", +] + + + diff --git a/tests/test-env-check.sh b/tests/test-env-check.sh index ad59aa46854..dc7bbcc9baf 100755 --- a/tests/test-env-check.sh +++ b/tests/test-env-check.sh @@ -40,18 +40,17 @@ else exit 1 fi -# check for pipenv -pipenv --version &> /dev/null +# check for uv +uv --version &> /dev/null if [ $? -eq 0 ]; then - echo "pipenv detected!" - pipenv --venv &> /dev/null - if [ $? -ne 0 ]; then - echo "Installing a new virtual environment via pipenv" - pipenv install + echo "uv detected!" + if [ ! -d .venv ]; then + echo "Installing a new virtual environment via uv" + uv sync else echo "Using the pre-existing virtual environment." fi else - echo "pipenv is not installed/enabled. " + echo "uv is not installed/enabled. " exit 1 fi diff --git a/tools/cmake-format.sh b/tools/cmake-format.sh index 152ee85b61a..07a1a32e12a 100755 --- a/tools/cmake-format.sh +++ b/tools/cmake-format.sh @@ -22,62 +22,19 @@ # a new cmakelang version is desired. # See: # https://github.com/cheshirekow/cmake_format/tags -CMAKE_FORMAT_VERSION="v0.6.13" +CMAKE_FORMAT_VERSION="0.6.13" VERSION="0.6.13" function main() { - # check for python3 - python3 - << _END_ -import sys - -if sys.version_info.major < 3 or sys.version_info.minor < 8: - exit(1) -_END_ - - if [ $? = 1 ]; then - echo "Python 3.8 or newer is not installed/enabled." - exit 1 - fi - set -e # exit on error - if command -v pip3 &> /dev/null; then - PIP_CMD="pip3" - elif command -v pip &> /dev/null; then - PIP_CMD="pip" - else - echo "pip is not installed." - exit 1 - fi - - if ! type virtualenv >/dev/null 2>/dev/null - then - ${PIP_CMD} install -q virtualenv - fi - - if python3 -m venv --help &> /dev/null; then - VENV_LIB="venv" - elif python3 -m virtualenv --help &> /dev/null; then - VENV_LIB="virtualenv" - else - echo "Neither venv nor virtualenv is available." + # Check for uv. + if ! command -v uv &> /dev/null; then + echo "uv is not installed. Please install it: https://docs.astral.sh/uv/getting-started/installation/" exit 1 fi - - REPO_ROOT=$(cd $(dirname $0) && git rev-parse --show-toplevel) - GIT_DIR=$(git rev-parse --absolute-git-dir) - CMAKE_FORMAT_VENV=${CMAKE_FORMAT_VENV:-${GIT_DIR}/fmt/cmake_format_${CMAKE_FORMAT_VERSION}_venv} - if [ ! -e ${CMAKE_FORMAT_VENV} ] - then - python3 -m ${VENV_LIB} ${CMAKE_FORMAT_VENV} - fi - source ${CMAKE_FORMAT_VENV}/bin/activate - - ${PIP_CMD} install -q --upgrade pip - ${PIP_CMD} install -q "cmakelang==${CMAKE_FORMAT_VERSION}" pyaml - - ver=$(cmake-format --version 2>&1) + ver=$(uv tool run --quiet --from cmakelang@${CMAKE_FORMAT_VERSION} --with pyaml cmake-format --version 2>&1) if [ "$ver" != "$VERSION" ] then echo "Wrong version of cmake-format!" @@ -109,16 +66,13 @@ _END_ # after this we assume was modified by cmake-format. start_time_file=${tmp_dir}/format_start.$$ touch ${start_time_file} - cmake-format -i $(cat ${files_filtered}) + uv tool run --quiet --from cmakelang@${CMAKE_FORMAT_VERSION} --with pyaml cmake-format -i $(cat ${files_filtered}) find $(cat ${files_filtered}) -newer ${start_time_file} rm -rf ${tmp_dir} - deactivate } if [[ "$(basename -- "$0")" == 'cmake-format.sh' ]]; then main "$@" -else - GIT_DIR=$(git rev-parse --absolute-git-dir) - CMAKE_FORMAT_VENV=${CMAKE_FORMAT_VENV:-${GIT_DIR}/fmt/cmake_format_${CMAKE_FORMAT_VERSION}_venv} fi +# When sourced, CMAKE_FORMAT_VERSION is already set at the top of this file. diff --git a/tools/git/pre-commit b/tools/git/pre-commit index e5a710a6a5d..d200824217f 100755 --- a/tools/git/pre-commit +++ b/tools/git/pre-commit @@ -43,17 +43,15 @@ if [ ! -x "$FORMAT" ]; then exit 1 fi -source "$GIT_TOP/tools/yapf.sh" -if [ ! -d ${YAPF_VENV} ]; then - echo "Run \"cmake --build --target yapf\"" +# Check for uv which is required for yapf and cmake-format. +if ! command -v uv &> /dev/null; then + echo "uv is not installed. Please install it: https://docs.astral.sh/uv/getting-started/installation/" exit 1 fi +# Get version variables from the formatting scripts. +source "$GIT_TOP/tools/yapf.sh" source "$GIT_TOP/tools/cmake-format.sh" -if [ ! -d ${CMAKE_FORMAT_VENV} ]; then - echo "Run \"cmake --build --target cmake-format\"" - exit 1 -fi # Where to store the patch clang_patch_file=$(mktemp -t clang-format.XXXXXXXXXX) @@ -62,7 +60,6 @@ cmake_format_patch_file=$(mktemp -t cmake-format.XXXXXXXXXX) trap "rm -f $clang_patch_file $yapf_patch_file $cmake_format_patch_file" 0 1 2 3 5 15 # Loop over all files that are changed, and produce a diff file -source ${YAPF_VENV}/bin/activate REPO_ROOT=$(cd $(dirname $0)/../.. && git rev-parse --show-toplevel) YAPF_CONFIG=${REPO_ROOT}/.style.yapf git diff-index --cached --diff-filter=ACMR --name-only HEAD | grep -vE "lib/(Catch2|fastlz|ls-hpack|swoc|yamlcpp)" | while read file; do @@ -71,9 +68,9 @@ git diff-index --cached --diff-filter=ACMR --name-only HEAD | grep -vE "lib/(Cat ${FORMAT} "$file" | diff -u "$file" - >>"$clang_patch_file" ;; # Keep this list of Python extensions the same with the list of - # extensions searched for in the toosl/yapf.sh script. + # extensions searched for in the tools/yapf.sh script. *.py | *.cli.ext | *.test.ext) - yapf \ + uv tool run --quiet yapf@${YAPF_VERSION} \ --style ${YAPF_CONFIG} \ --parallel \ --diff \ @@ -81,14 +78,11 @@ git diff-index --cached --diff-filter=ACMR --name-only HEAD | grep -vE "lib/(Cat ;; esac done -deactivate # Now repeat the above for CMakeLists.txt files. -source ${CMAKE_FORMAT_VENV}/bin/activate git diff-index --cached --diff-filter=ACMR --name-only HEAD | grep -E 'CMakeLists.txt|\.cmake$' | grep -vE "lib/(Catch2|fastlz|ls-hpack|swoc|yamlcpp)" | while read file; do - cmake-format "$file" | diff -u "$file" - >>"$cmake_format_patch_file" + uv tool run --quiet --from cmakelang@${CMAKE_FORMAT_VERSION} --with pyaml cmake-format "$file" | diff -u "$file" - >>"$cmake_format_patch_file" done -deactivate if [ -s "$clang_patch_file" ]; then echo "The commit is not accepted, because clang-format does not match current" diff --git a/tools/hrw4u/Makefile b/tools/hrw4u/Makefile index 361264dc62f..33ab62873c2 100644 --- a/tools/hrw4u/Makefile +++ b/tools/hrw4u/Makefile @@ -24,7 +24,6 @@ PKG_DIR_HRW4U:=$(BUILD_DIR)/hrw4u PKG_DIR_U4WRH:=$(BUILD_DIR)/u4wrh PKG_DIR_LSP:=$(BUILD_DIR)/hrw4u_lsp -VENV_NAME:=hrw4u ANTLR=antlr # Scripts (each becomes the package __main__.py) @@ -166,26 +165,26 @@ $(PKG_DIR_LSP)/%: src/% cp $< $@ test: - pytest --tb=short tests + uv run pytest --tb=short tests # Build standalone binaries (optional) build: gen - pyinstaller --onefile --name hrw4u --strip $(SCRIPT_HRW4U) - pyinstaller --onefile --name u4wrh --strip $(SCRIPT_U4WRH) - pyinstaller --onefile --name hrw4u-lsp --strip $(SCRIPT_LSP) - pyinstaller --onefile --name hrw4u-kg --strip $(SCRIPT_KG) + uv run pyinstaller --onefile --name hrw4u --strip $(SCRIPT_HRW4U) + uv run pyinstaller --onefile --name u4wrh --strip $(SCRIPT_U4WRH) + uv run pyinstaller --onefile --name hrw4u-lsp --strip $(SCRIPT_LSP) + uv run pyinstaller --onefile --name hrw4u-kg --strip $(SCRIPT_KG) # Wheel packaging (adjust pyproject to include both packages if desired) package: gen @echo "==> Building pip package(s)..." - python3 -m build --wheel --outdir $(DIST_DIR) + uv run python -m build --wheel --outdir $(DIST_DIR) clean: - rm -rf build dist __pycache__ *.spec *.egg-info + rm -rf build dist __pycache__ *.spec *.egg-info .venv find tests -name '__pycache__' -type d -exec rm -r {} + -setup-deps: env - $(PYTHON) -m pip install -r requirements.txt +setup-deps: + uv sync --all-extras activate: - @echo "Run: pyenv activate $(VENV_NAME)" + @echo "Run: source .venv/bin/activate" diff --git a/tools/hrw4u/README.md b/tools/hrw4u/README.md index d01ed3316d2..cd2dfdbe2dd 100644 --- a/tools/hrw4u/README.md +++ b/tools/hrw4u/README.md @@ -23,7 +23,7 @@ Decompiles existing `header_rewrite` rules back into HRW4U source code. - **Python 3.11+** (uses modern type annotations and performance features) - **ANTLR4** for grammar parsing -- **pyenv** (recommended for development) +- **uv** (for Python environment management) ## Development Setup @@ -32,10 +32,11 @@ Decompiles existing `header_rewrite` rules back into HRW4U source code. git clone https://github.com/apache/trafficserver.git cd trafficserver/tools/hrw4u -# Create pyenv virtual environment -pyenv virtualenv 3.11 hrw4u -pyenv activate hrw4u -pip install -r requirements.txt +# Install uv if not already installed +# See: https://docs.astral.sh/uv/getting-started/installation/ + +# Create virtual environment and install dependencies +uv sync --all-extras ``` ### 2. Build the Package @@ -50,10 +51,10 @@ make package ### 3. Install for Development ```bash # Install in development mode -pip install -e . +uv pip install -e . # Or install the built wheel -pip install dist/hrw4u-*.whl +uv pip install dist/hrw4u-*.whl ``` ## Testing @@ -63,10 +64,10 @@ pip install dist/hrw4u-*.whl make test # Run specific test categories -pytest -m examples # Documentation examples -pytest -m conds # Condition tests -pytest -m ops # Operator tests -pytest -m reverse # Reverse compilation tests +uv run pytest -m examples # Documentation examples +uv run pytest -m conds # Condition tests +uv run pytest -m ops # Operator tests +uv run pytest -m reverse # Reverse compilation tests ``` ## Usage diff --git a/tools/hrw4u/bootstrap.sh b/tools/hrw4u/bootstrap.sh index b3f12b10c76..dda781f655c 100755 --- a/tools/hrw4u/bootstrap.sh +++ b/tools/hrw4u/bootstrap.sh @@ -18,34 +18,22 @@ set -e -VENV_NAME="hrw4u" - if ! which antlr; then echo "Make sure antlr is installed, e.g. brew install antlr" echo "Once its in your path, re-run this script." - exit + exit 1 fi -eval "$(pyenv init --path)" -eval "$(pyenv init -)" -# eval "$(pyenv virtualenv-init -)" - -if pyenv virtualenvs | grep hrw4u; then - pyenv uninstall -f "$VENV_NAME" -else - echo "==> Creating virtualenv $VENV_NAME..." - pyenv virtualenv "$VENV_NAME" +if ! command -v uv &> /dev/null; then + echo "uv is not installed. Please install it: https://docs.astral.sh/uv/getting-started/installation/" + exit 1 fi -echo "==> Activating virtualenv..." -pyenv activate "$VENV_NAME" - - -echo "==> Installing dependencies..." -pip install --upgrade pip -pip install -r requirements.txt +echo "==> Creating virtual environment and installing dependencies..." +uv sync --all-extras -echo "==> Done. To activate manually: pyenv activate $VENV_NAME" +echo "==> Done. To run commands, use: uv run " +echo " Or activate manually: source .venv/bin/activate" -# Probably need for running in the local build tree +# Set PYTHONPATH for running in the local build tree. export PYTHONPATH=./build:${PYTHONPATH} diff --git a/tools/hrw4u/requirements.txt b/tools/hrw4u/requirements.txt deleted file mode 100644 index 6b59a19d9e1..00000000000 --- a/tools/hrw4u/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -antlr4-python3-runtime>=4.9,<5.0 -pytest>=7.0,<8.0 -pyinstaller>=5.0,<7.0 -build>=0.8,<2.0 -rapidfuzz>=3.0,<4.0 diff --git a/tools/yapf.sh b/tools/yapf.sh index d3a2539ff53..04af15678cd 100755 --- a/tools/yapf.sh +++ b/tools/yapf.sh @@ -22,62 +22,19 @@ # yapf version is desired. # See: # https://github.com/google/yapf/tags -YAPF_VERSION="v0.43.0" +YAPF_VERSION="0.43.0" VERSION="yapf 0.43.0" function main() { - # check for python3 - python3 - << _END_ -import sys - -if sys.version_info.major < 3 or sys.version_info.minor < 8: - exit(1) -_END_ - - if [ $? = 1 ]; then - echo "Python 3.8 or newer is not installed/enabled." - exit 1 - fi - set -e # exit on error - if command -v pip3 &> /dev/null; then - PIP_CMD="pip3" - elif command -v pip &> /dev/null; then - PIP_CMD="pip" - else - echo "pip is not installed." + # Check for uv. + if ! command -v uv &> /dev/null; then + echo "uv is not installed. Please install it: https://docs.astral.sh/uv/getting-started/installation/" exit 1 fi - if ! type virtualenv >/dev/null 2>/dev/null - then - ${PIP_CMD} install -q virtualenv - fi - - if python3 -m venv --help &> /dev/null; then - VENV_LIB="venv" - elif python3 -m virtualenv --help &> /dev/null; then - VENV_LIB="virtualenv" - else - echo "Neither venv nor virtualenv is available." - exit 1 - fi - - - REPO_ROOT=$(cd $(dirname $0) && git rev-parse --show-toplevel) - GIT_DIR=$(git rev-parse --absolute-git-dir) - YAPF_VENV=${YAPF_VENV:-${GIT_DIR}/fmt/yapf_${YAPF_VERSION}_venv} - if [ ! -e ${YAPF_VENV} ] - then - python3 -m ${VENV_LIB} ${YAPF_VENV} - fi - source ${YAPF_VENV}/bin/activate - - ${PIP_CMD} install -q --upgrade pip - ${PIP_CMD} install -q "yapf==${YAPF_VERSION}" - - ver=$(yapf --version 2>&1) + ver=$(uv tool run --quiet yapf@${YAPF_VERSION} --version 2>&1) if [ "$ver" != "$VERSION" ] then echo "Wrong version of yapf!" @@ -85,6 +42,8 @@ _END_ exit 1 fi + REPO_ROOT=$(cd $(dirname $0) && git rev-parse --show-toplevel) + DIR=${@:-.} # Only run yapf on tracked files. This saves time and possibly avoids @@ -113,7 +72,7 @@ _END_ start_time_file=${tmp_dir}/format_start.$$ touch ${start_time_file} YAPF_CONFIG=${REPO_ROOT}/.style.yapf - yapf \ + uv tool run --quiet yapf@${YAPF_VERSION} \ --style ${YAPF_CONFIG} \ --parallel \ --in-place \ @@ -121,12 +80,8 @@ _END_ find $(cat ${files_filtered}) -newer ${start_time_file} rm -rf ${tmp_dir} - deactivate } if [[ "$(basename -- "$0")" == 'yapf.sh' ]]; then main "$@" -else - GIT_DIR=$(git rev-parse --absolute-git-dir) - YAPF_VENV=${YAPF_VENV:-${GIT_DIR}/fmt/yapf_${YAPF_VERSION}_venv} fi