From 56ff6d7ce93fb8a7138ee607f2eabc08dd2226e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ke=C5=BEl=C3=ADnek?= <103315902+Dakevid@users.noreply.github.com> Date: Fri, 26 Sep 2025 12:18:48 +0200 Subject: [PATCH 1/3] pytrap: add pyproject.toml for modern Python packaging --- .gitignore | 1 + Makefile.am | 2 +- pytrap/Makefile.am | 14 ++++---- pytrap/README | 70 +++++++++++++++++++++++++++++++++----- pytrap/docs/Makefile | 2 +- pytrap/docs/conf.py | 5 ++- pytrap/nemea-pytrap.spec | 14 +++++--- pytrap/pyproject.toml | 72 ++++++++++++++++++++++++++++++++++++++++ pytrap/setup.py | 67 ------------------------------------- 9 files changed, 157 insertions(+), 90 deletions(-) create mode 100644 pytrap/pyproject.toml delete mode 100644 pytrap/setup.py diff --git a/.gitignore b/.gitignore index 14cfbaaa..1ba53bdc 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,4 @@ unirec/tests/test_speed_ur unirec/tests/test_speed_uro unirec/tests/test_template_cmp unirec/tests/test_time +__pycache__ \ No newline at end of file diff --git a/Makefile.am b/Makefile.am index e8b80d71..f9d1376f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,7 +32,7 @@ install-exec-hook: @echo -e "\n\n" @echo "For installation of Python components run the following commands as root:" @for i in ${PYTHON_SUBDIRS}; do \ - echo -e "(cd $${i}; ${PYTHON} setup.py install --record=installed-files.txt;)\n"; \ + echo -e "(pip install ./$${i})\n"; \ done @echo -e "\n\n" diff --git a/pytrap/Makefile.am b/pytrap/Makefile.am index abcde7e3..d8a046be 100644 --- a/pytrap/Makefile.am +++ b/pytrap/Makefile.am @@ -1,6 +1,6 @@ .PHONY: rpm -EXTRA_DIST=MANIFEST.in README README.md test.sh nemea-pytrap.spec setup.py \ +EXTRA_DIST=MANIFEST.in README README.md test.sh nemea-pytrap.spec pyproject.toml \ src/pytrapmodule.c \ src/unirecmodule.c \ src/unirecipaddr.c \ @@ -15,7 +15,7 @@ EXTRA_DIST=MANIFEST.in README README.md test.sh nemea-pytrap.spec setup.py \ rpm: mkdir -p RPMBUILD/SOURCES - python3 setup.py sdist + python3 -m build --sdist cp dist/*.tar.gz RPMBUILD/SOURCES rpmbuild -ba nemea-pytrap.spec --define "_topdir `pwd`/RPMBUILD" mv RPMBUILD/RPMS/*/*.rpm RPMBUILD/SRPMS/*.rpm dist/ @@ -27,13 +27,13 @@ TESTS = test.sh .PHONY: coverage coverage: - CFLAGS=-coverage python setup.py build_ext --inplace - CFLAGS=-coverage python3 setup.py build_ext --inplace - python3 setup.py test || echo "Skipped python3 tests" + CFLAGS=-coverage python3 -m pip install .[test] --no-deps --disable-pip-version-check --no-cache-dir + CFLAGS=-coverage python3 -m pytest --cov=pycommon || echo "Skipped python3 tests" @lcov --capture --directory . --output-file coverage.info 2>/dev/null && \ genhtml coverage.info --output-directory out 2>/dev/null || echo "Skipped coverage analysis" .PHONY: doc doc: - python3 setup.py build_sphinx - + python3 -m pip install .[doc] --no-cache-dir + cd docs + make html diff --git a/pytrap/README b/pytrap/README index 602ab674..aad3a981 100644 --- a/pytrap/README +++ b/pytrap/README @@ -1,29 +1,44 @@ # About pytrap +The pytrap module is a native Python extension that allows for writing NEMEA modules in Python. +TODO This directory contains an implementation of Python extension. The aim is to allow native calls of base TRAP functionality (used by NEMEA modules). # Installation +## Prerequisites + Development package of Python is required (python3-devel etc. according to your OS distribution). It contains needed header files. -Since this module uses libtrap and libunirec, https:/github.com/CESNET/Nemea-framework +Since this module uses libtrap and libunirec, https://github.com/CESNET/Nemea-framework must be installed in the system. -When all requirements are met, run as root: +### Installing Python development headers + +**For Debian/Ubuntu:** +```bash +sudo apt install python3-dev +``` +**For Red Hat/Fedora/CentOS:** +```bash +sudo dnf install python3-devel ``` -python3 setup.py install +## Installing pytrap + +When all requirements are met, run as root: + +```bash +pip install . ``` -Note: for different versions of python, it is needed to perform this command -separately for each of them. -# Help +## Viewing help Help is contained in the python module. After successful installation run in -a python interactive interpret: +a python interactive interpreter: ```python import pytrap @@ -31,7 +46,44 @@ help(pytrap) help(pytrap.pytrap) ``` -# Examples -See https://github.com/CESNET/Nemea-Framework/tree/master/examples/python +# Documentation + +## Building documentation + +The project uses Sphinx for documentation generation. First install documentation dependencies: + +```bash +pip install .[docs] +``` + +Then build the documentation: + +```bash +cd docs + +make html +``` + +The generated documentation will be available in the `dist/doc/` directory. + +# Testing + +The project uses pytest for testing. First install documentation dependencies: + +```bash +pip install .[test] +``` + +```bash +# Run all tests +pytest + +# Run specific test files (following the project's naming convention) +pytest test/*_unittest.py +``` + + +# Examples +See https://github.com/CESNET/Nemea-Framework/tree/master/examples/python \ No newline at end of file diff --git a/pytrap/docs/Makefile b/pytrap/docs/Makefile index d4bb2cbb..79297d82 100644 --- a/pytrap/docs/Makefile +++ b/pytrap/docs/Makefile @@ -6,7 +6,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . -BUILDDIR = _build +BUILDDIR = ../dist/doc # Put it first so that "make" without argument is like "make help". help: diff --git a/pytrap/docs/conf.py b/pytrap/docs/conf.py index e21429a7..a54096cb 100644 --- a/pytrap/docs/conf.py +++ b/pytrap/docs/conf.py @@ -12,8 +12,11 @@ # import os import sys -sys.path.insert(0, os.path.abspath('..')) +from sphinx_pyproject import SphinxConfig +config = SphinxConfig("../pyproject.toml", globalns=globals()) +release = version +project = name # -- Project information ----------------------------------------------------- diff --git a/pytrap/nemea-pytrap.spec b/pytrap/nemea-pytrap.spec index 51ac5c03..fa7f177a 100644 --- a/pytrap/nemea-pytrap.spec +++ b/pytrap/nemea-pytrap.spec @@ -22,6 +22,9 @@ NEMEA modules in Python. Summary: Python extension of the NEMEA project %{?python_provide:%python_provide python%{python3_pkgversion}-%{pypi_name}} Requires: libtrap +BuildRequires: python%{python3_pkgversion}-build +BuildRequires: python%{python3_pkgversion}-pip +BuildRequires: python%{python3_pkgversion}-wheel BuildRequires: python%{python3_pkgversion}-setuptools BuildRequires: python%{python3_pkgversion}-devel BuildRequires: gcc @@ -44,12 +47,15 @@ rm -rf %{pypi_name}.egg-info %install # Must do the subpackages' install first because the scripts in /usr/bin are -# overwritten with every setup.py install. -%{__python3} setup.py install --skip-build --single-version-externally-managed --root %{buildroot} +# Install into the build root using pip (PEP 517 compatible) +%{__python3} -m pip install . \ + --root %{buildroot} \ + --no-deps --disable-pip-version-check --no-cache-dir --verbose %check -TRAP_SOCKET_DIR=/tmp PAGER="" %{__python3} setup.py test - +# Install test dependencies and run tests +%{__python3} -m pip install .[test] --no-deps --disable-pip-version-check --no-cache-dir +TRAP_SOCKET_DIR=/tmp PAGER="" %{__python3} -m pytest -v %files -n python%{python3_pkgversion}-%{pypi_name} %doc README %{python3_sitearch}/* diff --git a/pytrap/pyproject.toml b/pytrap/pyproject.toml new file mode 100644 index 00000000..bb58afe5 --- /dev/null +++ b/pytrap/pyproject.toml @@ -0,0 +1,72 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "nemea-pytrap" +version = "0.17.0" +description = "Python extension of the NEMEA project." +readme = {file = "README.md", content-type = "text/markdown"} +authors = [ + { name = "Tomas Cejka", email = "cejkat@cesnet.cz" }, +] +maintainers = [ + { name = "Tomas Cejka", email = "cejkat@cesnet.cz" }, +] +license = { text = "BSD" } +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 4 - Beta", + "Operating System :: POSIX :: Linux", + "Programming Language :: C", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Networking :: Monitoring" +] +dependencies = [] + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-cov", + "pytest-forked", +] +docs = [ + "sphinx", + "sphinx-rtd-theme", + "sphinx-pyproject" +] + +[project.urls] +Homepage = "https://github.com/CESNET/Nemea-Framework" + +# Configure setuptools to find packages and handle C extensions +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] + +[[tool.setuptools.ext-modules]] +name = "pytrap.pytrap" +sources = [ + "src/pytrapmodule.c", + "src/unirecmodule.c", + "src/unirecipaddr.c", + "src/unirecmacaddr.c", + "src/iplist.c", + "src/fields.c" +] +libraries = ["trap", "unirec"] + + +[tool.pytest.ini_options] +testpaths = ["test"] +python_files = ["*_unittest.py"] +addopts = "--forked" \ No newline at end of file diff --git a/pytrap/setup.py b/pytrap/setup.py deleted file mode 100644 index 8a654829..00000000 --- a/pytrap/setup.py +++ /dev/null @@ -1,67 +0,0 @@ -from setuptools import setup, Extension -import os - -SRC_PATH = os.path.relpath(os.path.join(os.path.dirname(__file__), ".", "src")) - -name = 'nemea-pytrap' -version = '0.17.0' -release = version -description = 'Python extension of the NEMEA project.' -long_description = 'The pytrap module is a native Python extension that allows for writing NEMEA modules in Python.' -author = 'Tomas Cejka' -author_email = 'cejkat@cesnet.cz' -maintainer = 'Tomas Cejka' -maintainer_email = 'cejkat@cesnet.cz' - -pytrapmodule = Extension('pytrap.pytrap', - sources = ['src/pytrapmodule.c', 'src/unirecmodule.c', - 'src/unirecipaddr.c', 'src/unirecmacaddr.c', - 'src/iplist.c', 'src/fields.c'], - libraries = ['trap', 'unirec']) - -try: - from sphinx.setup_command import BuildDoc - cmdclass = {'build_sphinx': BuildDoc} - command_options = {'build_sphinx': { - 'project': ('setup.py', name), - 'version': ('setup.py', version), - 'release': ('setup.py', release), - 'source_dir': ('setup.py', 'docs'), - 'build_dir': ('setup.py', 'dist/doc/')} - } -except: - cmdclass = {} - command_options = {} - -setup(name = name, - version = version, - description = description, - long_description = long_description, - author = author, - author_email = author_email, - maintainer = maintainer, - maintainer_email = maintainer_email, - url = 'https://github.com/CESNET/Nemea-Framework', - license = 'BSD', - test_suite = "test", - platforms = ["Linux"], - classifiers = [ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: C', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython', - 'Topic :: Software Development :: Libraries', - 'Topic :: System :: Networking :: Monitoring' - ], - ext_modules = [pytrapmodule], - packages = ["pytrap"], - package_dir={ "": SRC_PATH, }, - # sphinx: - cmdclass=cmdclass, - # these are optional and override conf.py settings - command_options=command_options, - ) - From f3de622e24d3320b92b6c16c7280d4d00479b92e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ke=C5=BEl=C3=ADnek?= <103315902+Dakevid@users.noreply.github.com> Date: Fri, 26 Sep 2025 12:19:52 +0200 Subject: [PATCH 2/3] pycommon: add pyproject.toml for modern Python packaging --- pycommon/Makefile.am | 4 +-- pycommon/README | 21 ++++++++----- pycommon/README-RPM-release.md | 18 ++++++----- pycommon/nemea-pycommon.spec | 9 ++++-- pycommon/pyproject.toml | 57 ++++++++++++++++++++++++++++++++++ pycommon/setup.py | 31 ------------------ 6 files changed, 89 insertions(+), 51 deletions(-) create mode 100644 pycommon/pyproject.toml delete mode 100644 pycommon/setup.py diff --git a/pycommon/Makefile.am b/pycommon/Makefile.am index cbb59aa8..08f1f324 100644 --- a/pycommon/Makefile.am +++ b/pycommon/Makefile.am @@ -1,11 +1,11 @@ -EXTRA_DIST=report2idea.py ip_prefix_search.py nemea-pycommon.spec setup.py README-RPM-release.md README reporter_config +EXTRA_DIST=report2idea.py ip_prefix_search.py nemea-pycommon.spec pyproject.toml README-RPM-release.md README reporter_config if MAKE_RPMS RPMFILENAME=nemea-pycommon .PHONY: rpm rpm: mkdir -p RPMBUILD/SOURCES - python3 setup.py sdist + python3 -m build --sdist cp dist/*.tar.gz RPMBUILD/SOURCES rpmbuild -ba nemea-pycommon.spec --define "_topdir `pwd`/RPMBUILD" mv RPMBUILD/RPMS/*/*.rpm RPMBUILD/SRPMS/*.rpm dist/ diff --git a/pycommon/README b/pycommon/README index fcc0b7a6..e70a3457 100644 --- a/pycommon/README +++ b/pycommon/README @@ -1,16 +1,21 @@ -# About pycommon +# pycommon -The pycommon module contains functionality for NEMEA modules written in python. -Currently, it is used by reporter modules. +**pycommon** provides shared functionality for [NEMEA](https://github.com/CESNET/Nemea) modules written in Python. +It is primarily used by reporter modules, but can be reused in other NEMEA-related Python projects. -# Installation +--- -Run as root: +## Installation +It is recommended to install using `pip`: + +```bash +pip install . ``` -python3 setup.py install +If you need to install for a specific Python interpreter (e.g., Python 3.9 vs. 3.11), run the above command with the corresponding interpreter: +```bash +python3.11 -m pip install . ``` -Note: for different versions of python, it is needed to perform this command -separately for each of them. +--- diff --git a/pycommon/README-RPM-release.md b/pycommon/README-RPM-release.md index a32aa145..cef92b7e 100644 --- a/pycommon/README-RPM-release.md +++ b/pycommon/README-RPM-release.md @@ -1,22 +1,26 @@ -# How to release new version of RPM? +# How to release a new version of the RPM? +1. Bump the version in `pyproject.toml` and `*.spec`. -1. bump version in `setup.py` and `*.spec` -2. don't forget to commit and push with commit message +2. Don't forget to commit and push with a commit message: ``` pycommon: increased version, released package ``` -3. create python src package using `python setup.py sdist` +3. Create a Python source distribution: + ```bash + python3 -m pip install build + python3 -m build --sdist + ``` -4. upload files in `dist/` using `twine`: +4. Upload files in `dist/` using `twine`: ``` twine upload ./* ``` -5. run `make rpm` -6. your packages are in `RPMBUILD/`, build for other RPM-based systems can be done using: +5. Run `make rpm` +6. Your packages are in `RPMBUILD/`, build for other RPM-based systems can be done using: ``` copr build @CESNET/NEMEA RPMBUILD/SRPMS/ diff --git a/pycommon/nemea-pycommon.spec b/pycommon/nemea-pycommon.spec index fc00b1b1..18f3731e 100644 --- a/pycommon/nemea-pycommon.spec +++ b/pycommon/nemea-pycommon.spec @@ -60,13 +60,16 @@ rm -rf %{pypi_name}.egg-info %install # Must do the subpackages' install first because the scripts in /usr/bin are -# overwritten with every setup.py install. -%{__python3} setup.py install --skip-build --single-version-externally-managed --root %{buildroot} +# overwritten with every install. TODO check +%{__python3} -m pip install . --root %{buildroot} --no-deps --disable-pip-version-check --no-cache-dir --verbose + mkdir -p %{buildroot}/%{_sysconfdir}/nemea/email-templates/; cp reporter_config/default.html %{buildroot}/%{_sysconfdir}/nemea/email-templates/default.html %check -%{__python3} setup.py test +%{__python3} -m pip install .[test] +%{__python3} -m pytest + %files -n python%{python3_pkgversion}-%{pypi_name} %doc README diff --git a/pycommon/pyproject.toml b/pycommon/pyproject.toml new file mode 100644 index 00000000..3c2b9663 --- /dev/null +++ b/pycommon/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "nemea-pycommon" +version = "1.7.0" +description = "Common Python modules and methods of the NEMEA system." +readme = {text = "The module contains methods for creation and submission of incident reports in IDEA format.", content-type = "text/plain"} +license = {text = "BSD"} +authors = [ + {name = "Vaclav Bartos, CESNET", email = "bartos@cesnet.cz"}, +] +maintainers = [ + {name = "Tomas Cejka", email = "cejkat@cesnet.cz"}, +] +requires-python = ">=3.6" +dependencies = [ + "pynspect", + "idea-format", + "PyYAML", + "ply", + "jinja2", + "redis", +] +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Operating System :: POSIX", + "Programming Language :: C", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Networking :: Monitoring", +] + + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-cov", +] + + + +[project.urls] +Homepage = "https://github.com/CESNET/Nemea-Framework" + +[tool.setuptools] +py-modules = ["report2idea", "ip_prefix_search"] +packages = ["reporter_config", "reporter_config.actions"] + + +[tool.pytest.ini_options] +testpaths = ["test"] +python_files = ["*_unittest.py","rc_*.py"] +addopts = "" \ No newline at end of file diff --git a/pycommon/setup.py b/pycommon/setup.py deleted file mode 100644 index 5f26a893..00000000 --- a/pycommon/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -from setuptools import setup - -DESCRIPTION="Common Python modules and methods of the NEMEA system." -LONG_DESCRIPTION="""The module contains methods for creation and submission of incident reports in IDEA format.""" -setup(name='nemea-pycommon', - version='1.7.0', - py_modules=['report2idea', 'ip_prefix_search'], - packages=['reporter_config', 'reporter_config.actions'], - author='Vaclav Bartos, CESNET', - author_email='bartos@cesnet.cz', - maintainer = 'Tomas Cejka', - maintainer_email = 'cejkat@cesnet.cz', - url = 'https://github.com/CESNET/Nemea-Framework', - license="BSD", - test_suite="test", - platforms = ["any"], - install_requires = [ 'pynspect', 'idea-format', 'PyYAML', 'ply', 'jinja2', 'redis' ], - classifiers = [ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Operating System :: POSIX', - 'Programming Language :: C', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Topic :: Software Development :: Libraries', - 'Topic :: System :: Networking :: Monitoring' - ], - description=DESCRIPTION, - long_description=LONG_DESCRIPTION -) - From dc8a4ca504703a10be3d3d02f7bd372bf59f2521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ke=C5=BEl=C3=ADnek?= <103315902+Dakevid@users.noreply.github.com> Date: Fri, 26 Sep 2025 12:59:16 +0200 Subject: [PATCH 3/3] update README --- Makefile.am | 2 +- README.md | 4 ++-- pycommon/nemea-pycommon.spec | 2 +- pytrap/Makefile.am | 2 +- pytrap/README | 5 ++--- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Makefile.am b/Makefile.am index f9d1376f..bdaa9b2c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -30,7 +30,7 @@ ACLOCAL_AMFLAGS=-I m4 install-exec-hook: @echo -e "\n\n" - @echo "For installation of Python components run the following commands as root:" + @echo "For installation of Python components run the following commands:" @for i in ${PYTHON_SUBDIRS}; do \ echo -e "(pip install ./$${i})\n"; \ done diff --git a/README.md b/README.md index 75e398dc..4e643b32 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,11 @@ Python parts must be installed separately when needed. It can be done using: ``` -cd pytrap; sudo python3 setup.py install +pip install ./pytrap ``` and ``` -cd pycommon; sudo python3 setup.py install +pip install ./pycommon ``` Project status: diff --git a/pycommon/nemea-pycommon.spec b/pycommon/nemea-pycommon.spec index 18f3731e..2d332bec 100644 --- a/pycommon/nemea-pycommon.spec +++ b/pycommon/nemea-pycommon.spec @@ -60,7 +60,7 @@ rm -rf %{pypi_name}.egg-info %install # Must do the subpackages' install first because the scripts in /usr/bin are -# overwritten with every install. TODO check +# overwritten with every install. %{__python3} -m pip install . --root %{buildroot} --no-deps --disable-pip-version-check --no-cache-dir --verbose mkdir -p %{buildroot}/%{_sysconfdir}/nemea/email-templates/; cp reporter_config/default.html %{buildroot}/%{_sysconfdir}/nemea/email-templates/default.html diff --git a/pytrap/Makefile.am b/pytrap/Makefile.am index d8a046be..297e9af3 100644 --- a/pytrap/Makefile.am +++ b/pytrap/Makefile.am @@ -34,6 +34,6 @@ coverage: .PHONY: doc doc: - python3 -m pip install .[doc] --no-cache-dir + python3 -m pip install .[docs] --no-cache-dir cd docs make html diff --git a/pytrap/README b/pytrap/README index aad3a981..2a234a16 100644 --- a/pytrap/README +++ b/pytrap/README @@ -1,8 +1,7 @@ # About pytrap +The `pytrap` module is a native Python extension that allows writing NEMEA modules in Python. -The pytrap module is a native Python extension that allows for writing NEMEA modules in Python. -TODO -This directory contains an implementation of Python extension. The aim +This directory contains an implementation of a Python extension. The aim is to allow native calls of base TRAP functionality (used by NEMEA modules). # Installation