diff --git a/.codespellrc b/.codespellrc index 4ee856e..477508c 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,2 +1,2 @@ [codespell] -skip = .git/* +skip = .git/*,*.pdf diff --git a/.copier-answers.yml b/.copier-answers.yml index 835da5b..c10d6ad 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,6 +1,6 @@ # Changes here will be overwritten by Copier -_commit: 0.1.53 +_commit: 0.1.66 _src_path: git@bbpgitlab.epfl.ch:neuromath/python-template.git author_email: '' author_name: Blue Brain Project, EPFL diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index ff4859f..14a08fc 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -8,12 +8,14 @@ jobs: check-pr-title: name: Check PR title runs-on: ubuntu-latest + env: + PR_TITLE: ${{ github.event.pull_request.title }} steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '20' - run: npm install -g --force commitlint @commitlint/cli commitlint-plugin-cleanfeet - run: npm install conventional-changelog-conventionalcommits - run: touch .git/COMMIT_EDITMSG - - run: echo "${{ github.event.pull_request.title }}" | commitlint + - run: echo "$PR_TITLE" | commitlint diff --git a/.github/workflows/publish-sdist.yml b/.github/workflows/publish-sdist.yml index 85abdf9..362a947 100644 --- a/.github/workflows/publish-sdist.yml +++ b/.github/workflows/publish-sdist.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.10" - name: Build a wheel and a source tarball run: | pip install setuptools>=42 build setuptools_scm[toml]>=3.4 diff --git a/.github/workflows/run-tox.yml b/.github/workflows/run-tox.yml index 817d12a..d1fe1df 100644 --- a/.github/workflows/run-tox.yml +++ b/.github/workflows/run-tox.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] min_versions: ["min_versions", "latest_versions"] exclude: - min_versions: "min_versions" include: - - python-version: "3.8" + - python-version: "3.9" min_versions: "min_versions" steps: @@ -57,7 +57,7 @@ jobs: pip install tox tox run -e min_versions - name: JUnit Report Action - uses: mikepenz/action-junit-report@v4 + uses: mikepenz/action-junit-report@v5 if: always() # always run even if the previous step fails with: report_paths: 'reports/pytest-*.xml' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6fa9e94..6e8187f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,6 @@ -default_language_version: - python: python3.8 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -13,22 +11,22 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook - rev: v9.5.0 + rev: v9.18.0 hooks: - id: commitlint stages: - commit-msg additional_dependencies: ['conventional-changelog-conventionalcommits'] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.3.0 hooks: - id: codespell args: ["-x", ".codespellignorelines"] @@ -38,6 +36,6 @@ repos: - id: pydocstyle additional_dependencies: ["tomli"] - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.1.1 hooks: - id: flake8 diff --git a/.pylintrc b/.pylintrc index 9b10b78..4800d68 100644 --- a/.pylintrc +++ b/.pylintrc @@ -2,6 +2,7 @@ disable= arguments-differ, invalid-name, + too-many-positional-arguments [FORMAT] # Regexp for a line that is allowed to be longer than the limit. diff --git a/.readthedocs.yml b/.readthedocs.yml index c51d15e..91cce21 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -6,9 +6,9 @@ version: 2 build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/source/conf.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aed9db3..2a85dfb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,7 +59,7 @@ When you wish to contribute to the code base, please consider the following guid or ```shell - tox -e py38 -e lint -e docs -e check-packaging + tox run -e py39,lint,docs,check-packaging ``` * Commit your changes using a descriptive commit message. diff --git a/data_validation_framework/__init__.py b/data_validation_framework/__init__.py index a7c3263..327b602 100644 --- a/data_validation_framework/__init__.py +++ b/data_validation_framework/__init__.py @@ -2,6 +2,7 @@ Simple framework to create data validation workflows. """ + import importlib.metadata from data_validation_framework import result # noqa diff --git a/data_validation_framework/report.py b/data_validation_framework/report.py index 9ace5c9..1b8b6b7 100644 --- a/data_validation_framework/report.py +++ b/data_validation_framework/report.py @@ -1,4 +1,5 @@ """Specific tasks.""" + import copy import json import logging diff --git a/data_validation_framework/result.py b/data_validation_framework/result.py index abb60b1..1f85b35 100644 --- a/data_validation_framework/result.py +++ b/data_validation_framework/result.py @@ -1,4 +1,5 @@ """Util functions.""" + import logging import warnings from copy import deepcopy diff --git a/data_validation_framework/rst_tools.py b/data_validation_framework/rst_tools.py index db9f519..b943f29 100644 --- a/data_validation_framework/rst_tools.py +++ b/data_validation_framework/rst_tools.py @@ -1,4 +1,5 @@ """Some tools to generate RST files.""" + import re import textwrap diff --git a/data_validation_framework/target.py b/data_validation_framework/target.py index 5032034..011fd8b 100644 --- a/data_validation_framework/target.py +++ b/data_validation_framework/target.py @@ -1,4 +1,5 @@ """Specific targets.""" + import warnings from luigi_tools.target import OutputLocalTarget diff --git a/data_validation_framework/task.py b/data_validation_framework/task.py index 2d767ad..d30c1c4 100644 --- a/data_validation_framework/task.py +++ b/data_validation_framework/task.py @@ -1,4 +1,5 @@ """Specific tasks.""" + import logging import sys import time @@ -19,7 +20,11 @@ from luigi_tools.parameter import OptionalBoolParameter from luigi_tools.task import LogTargetMixin from luigi_tools.task import RerunMixin -from numpy import VisibleDeprecationWarning + +try: + from numpy import VisibleDeprecationWarning +except ImportError: # pragma: no cover + from numpy.exceptions import VisibleDeprecationWarning from data_validation_framework.report import make_report from data_validation_framework.result import ValidationResult @@ -39,7 +44,7 @@ class ValidationError(Exception): class TagResultOutputMixin: - """Initialize target prefixes and optionally add a tag to the resut directory. + """Initialize target prefixes and optionally add a tag to the result directory. .. warning:: @@ -163,7 +168,7 @@ def inputs(self): data_dir = OptionalStrParameter( default="data", description=( - ":str: name of folder to store addittional files created by a task (the provided " + ":str: name of folder to store additional files created by a task (the provided " "validation function must take this as argument)." ), ) diff --git a/data_validation_framework/util.py b/data_validation_framework/util.py index 09dd7ed..984b8e7 100644 --- a/data_validation_framework/util.py +++ b/data_validation_framework/util.py @@ -1,4 +1,5 @@ """Util functions.""" + import contextlib import logging import multiprocessing @@ -128,7 +129,7 @@ def apply_to_df(df, func, *args, nb_processes=None, redirect_stdout=None, **kwar # Start the computation results_list = pool.imap( _apply_to_df_internal, - zip(chunks, repeat([func] + list(args)), repeat(kwargs)), + zip(chunks, repeat([func] + list(args)), repeat(kwargs)), # pylint: disable=E0606 ) pool.close() diff --git a/docs/source/conf.py b/docs/source/conf.py index 787dbf1..2b31832 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,10 +14,11 @@ # -- Project information ----------------------------------------------------- -project = "data-validation-framework" +project_name = "Data Validation Framework" +package_name = "data-validation-framework" # The short X.Y version -version = metadata.version("data-validation-framework") +version = metadata.version(package_name) # The full version, including alpha/beta/rc tags release = version @@ -61,10 +62,10 @@ # html_static_path = ['_static'] html_theme_options = { - "metadata_distribution": "data-validation-framework", + "metadata_distribution": package_name, } -html_title = "Data Validation Framework" +html_title = project_name # If true, links to the reST sources are added to the pages. html_show_sourcelink = False diff --git a/pyproject.toml b/pyproject.toml index 38acbda..c5f5726 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,10 @@ build-backend = "setuptools.build_meta" [tool.black] line-length = 100 target-version = [ - "py38", "py39", "py310", "py311", + "py312", ] [tool.pydocstyle] diff --git a/setup.py b/setup.py index 550f360..2ae3c36 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ """Setup for the data-validation-framework package.""" + from pathlib import Path from setuptools import find_namespace_packages @@ -10,7 +11,7 @@ "numpy>=1.21", "pandas>=1.3", "rst2pdf>=0.99", - "sphinx>=4,<8", + "sphinx>=7.4", "tqdm>=4.40", ] doc_reqs = [ @@ -23,8 +24,8 @@ "diff_pdf_visually>=1.6.2", "pause>=0.2", "pytest>=7", - "pytest-cov>=3", - "pytest-html>=3.1", + "pytest-cov>=4.1", + "pytest-html>=3.2", ] setup( @@ -40,7 +41,11 @@ }, license="Apache License 2.0", packages=find_namespace_packages(include=["data_validation_framework*"]), - python_requires=">=3.8", + python_requires=">=3.9", + use_scm_version=True, + setup_requires=[ + "setuptools_scm", + ], install_requires=reqs, extras_require={ "docs": doc_reqs, @@ -54,10 +59,10 @@ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], diff --git a/tests/__init__.py b/tests/__init__.py index 6a49c9d..fdd6a02 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,5 @@ """Tests suite for the data-validation-framework package.""" + import re from pathlib import Path diff --git a/tests/conftest.py b/tests/conftest.py index 4d32209..2739ae5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """Configuration for the pytest test suite.""" + # pylint: disable=missing-function-docstring import os from pathlib import Path diff --git a/tests/data/test_report/report_latexpdf.pdf b/tests/data/test_report/report_latexpdf.pdf index 82a2dd0..260f5d7 100644 Binary files a/tests/data/test_report/report_latexpdf.pdf and b/tests/data/test_report/report_latexpdf.pdf differ diff --git a/tests/data/test_report/report_rst2pdf.pdf b/tests/data/test_report/report_rst2pdf.pdf index afee63a..4d07867 100644 Binary files a/tests/data/test_report/report_rst2pdf.pdf and b/tests/data/test_report/report_rst2pdf.pdf differ diff --git a/tests/data/test_report_before_run/report_latexpdf_with_config.pdf b/tests/data/test_report_before_run/report_latexpdf_with_config.pdf index 2aba29a..dde9830 100644 Binary files a/tests/data/test_report_before_run/report_latexpdf_with_config.pdf and b/tests/data/test_report_before_run/report_latexpdf_with_config.pdf differ diff --git a/tests/test_result.py b/tests/test_result.py index 3bba7ae..f8d1961 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -1,4 +1,5 @@ """Test the data_validation_framework.result module.""" + import pandas as pd import pytest @@ -122,7 +123,6 @@ def test_constructor_ValidationResultSet(self): new_df = result.ValidationResultSet(df) assert new_df.equals(df) - @pytest.mark.filterwarnings("ignore::numpy.VisibleDeprecationWarning") def test_column_order(self): """Check column order.""" df = result.ValidationResultSet( diff --git a/tests/test_rst_tools.py b/tests/test_rst_tools.py index 44c3a28..388a606 100644 --- a/tests/test_rst_tools.py +++ b/tests/test_rst_tools.py @@ -1,4 +1,5 @@ """Test the data_validation_framework.rst_tools module.""" + # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring # pylint: disable=redefined-outer-name diff --git a/tests/test_target.py b/tests/test_target.py index 1dc5167..72fecd3 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -1,4 +1,5 @@ """Test the data_validation_framework.target module.""" + # pylint: disable=missing-function-docstring # pylint: disable=redefined-outer-name # pylint: disable=unused-argument diff --git a/tests/test_task.py b/tests/test_task.py index 3374987..429d672 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -1,4 +1,5 @@ """Test the data_validation_framework.task module.""" + # pylint: disable=missing-class-docstring # pylint: disable=missing-function-docstring # pylint: disable=redefined-outer-name diff --git a/tests/test_util.py b/tests/test_util.py index 317027e..2c92d1c 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,4 +1,5 @@ """Test the data_validation_framework.util module.""" + # pylint: disable=missing-function-docstring import os import re diff --git a/tox.ini b/tox.ini index bc685af..52c0f15 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,8 @@ envlist = lint docs min_versions - py{38,39,310,311} + py{39,310,311,312} + coverage minversion = 3.18 @@ -16,12 +17,7 @@ minversion = 3.18 extras = test setenv = COVERAGE_FILE = {env:COVERAGE_FILE:.coverage-{envname}} -deps = - min_versions: Requirements-Builder commands = - min_versions: requirements-builder --level=min --extras=test -o {envtmpdir}/requirements_min.txt setup.py - min_versions: pip install -r {envtmpdir}/requirements_min.txt - min_versions: pip freeze pytest \ --basetemp={envtmpdir} \ --cov={[base]name} \ @@ -34,8 +30,19 @@ commands = --html reports/pytest-{envname}.html \ --junit-xml=reports/pytest-{envname}.xml \ --self-contained-html \ + --durations 10 \ + --durations-min=2.0 \ {posargs} +[testenv:coverage] +skip_install = true +deps = coverage +allowlist_externals = /bin/bash +commands = + /bin/bash -c 'coverage combine .coverage-py*' + coverage xml + coverage report + [testenv:check-packaging] skip_install = true deps = @@ -46,10 +53,16 @@ commands = twine check {envtmpdir}/dist/* [testenv:min_versions] -basepython = python3.8 +basepython = python3.9 +deps = + Requirements-Builder +commands_pre = + requirements-builder --level=min --extras=test -o {envtmpdir}/requirements_min.txt setup.py + pip install -r {envtmpdir}/requirements_min.txt + pip freeze [testenv:lint] -basepython = python3.8 +basepython = python3.9 deps = pre-commit pylint @@ -58,7 +71,7 @@ commands = pylint -j {env:PYLINT_NPROCS:1} {[base]files} [testenv:format] -basepython = python3.8 +basepython = python3.9 skip_install = true deps = codespell @@ -78,7 +91,7 @@ commands = make html SPHINXOPTS=-W [gh-actions] python = - 3.8: py38, lint - 3.9: py39, docs + 3.9: py39, lint 3.10: py310, check-packaging - 3.11: py311 + 3.11: py311, docs + 3.12: py312