diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index 82a6896b6..c9d2d3152 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -36,9 +36,12 @@ jobs: with: python-version: "3.10" - - name: setup poetry - run: | - curl -sSL https://install.python-poetry.org | python - + - name: Cache Hugging Face + id: cache-hf + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: docs-cache-hf - name: Install pandoc run: | @@ -46,23 +49,23 @@ jobs: - name: Install dependencies run: | - poetry install --with docs + pip install .[docs] - name: Run tests if: github.event_name != 'workflow_dispatch' run: | echo "Testing documentation build..." - make test-docs + python -m sphinx build -b doctest docs/source docs/build/html - name: Build documentation if: ${{ github.ref == 'refs/heads/dev' }} && github.event_name != 'workflow_dispatch' run: | - make docs + python -m sphinx build -b html docs/source docs/build/html - name: build multiversion documentation if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' run: | - make multi-version-docs + sphinx-multiversion docs/source docs/build/html - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 diff --git a/.github/workflows/reusable-test.yaml b/.github/workflows/reusable-test.yaml new file mode 100644 index 000000000..ac8f0b169 --- /dev/null +++ b/.github/workflows/reusable-test.yaml @@ -0,0 +1,46 @@ +name: Reusable Test Workflow + +on: + workflow_call: + inputs: + test_command: + required: true + type: string + description: 'Command to run tests' + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.10", "3.11", "3.12" ] + include: + - os: windows-latest + python-version: "3.10" + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache Hugging Face + id: cache-hf + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: ${{ runner.os }}-hf + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install dependencies + run: | + pip install .[test] + + - name: Run tests + run: | + ${{ inputs.test_command }} \ No newline at end of file diff --git a/.github/workflows/test-inference.yaml b/.github/workflows/test-inference.yaml index a68ef07e8..d89e4503f 100644 --- a/.github/workflows/test-inference.yaml +++ b/.github/workflows/test-inference.yaml @@ -8,31 +8,6 @@ on: jobs: test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.10", "3.11", "3.12" ] - include: - - os: windows-latest - python-version: "3.10" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - pip install . - pip install pytest pytest-asyncio - - - name: Run tests - run: | - pytest tests/pipeline/test_inference.py + uses: ./.github/workflows/reusable-test.yaml + with: + test_command: pytest -n auto tests/pipeline/test_inference.py diff --git a/.github/workflows/test-nodes.yaml b/.github/workflows/test-nodes.yaml index b10161724..c1914913c 100644 --- a/.github/workflows/test-nodes.yaml +++ b/.github/workflows/test-nodes.yaml @@ -8,31 +8,6 @@ on: jobs: test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.10", "3.11", "3.12" ] - include: - - os: windows-latest - python-version: "3.10" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - pip install . - pip install pytest pytest-asyncio - - - name: Run tests - run: | - pytest tests/nodes + uses: ./.github/workflows/reusable-test.yaml + with: + test_command: pytest -n auto tests/nodes diff --git a/.github/workflows/test-optimization.yaml b/.github/workflows/test-optimization.yaml index 4625f39d7..ad3168dd1 100644 --- a/.github/workflows/test-optimization.yaml +++ b/.github/workflows/test-optimization.yaml @@ -8,31 +8,6 @@ on: jobs: test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.10", "3.11", "3.12" ] - include: - - os: windows-latest - python-version: "3.10" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - pip install . - pip install pytest pytest-asyncio - - - name: Run tests - run: | - pytest tests/pipeline/test_optimization.py + uses: ./.github/workflows/reusable-test.yaml + with: + test_command: pytest -n auto tests/pipeline/test_optimization.py diff --git a/.github/workflows/test-presets.yaml b/.github/workflows/test-presets.yaml index ab4a6723d..836c58fa9 100644 --- a/.github/workflows/test-presets.yaml +++ b/.github/workflows/test-presets.yaml @@ -8,31 +8,6 @@ on: jobs: test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.10", "3.11", "3.12" ] - include: - - os: windows-latest - python-version: "3.10" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - pip install . - pip install pytest pytest-asyncio - - - name: Run tests - run: | - pytest tests/pipeline/test_presets.py + uses: ./.github/workflows/reusable-test.yaml + with: + test_command: pytest -n auto tests/pipeline/test_presets.py diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml index eb0c374ff..dfe873e68 100644 --- a/.github/workflows/typing.yml +++ b/.github/workflows/typing.yml @@ -11,14 +11,9 @@ jobs: python-version: "3.10" cache: "pip" - - name: Install Poetry - run: | - curl -sSL https://install.python-poetry.org | python3 - - echo "$HOME/.poetry/bin" >> $GITHUB_PATH - - name: Install dependencies run: | - poetry install --with typing + pip install .[typing] - name: Run mypy - run: make typing + run: mypy autointent diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 5883080eb..4d8164f26 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -8,31 +8,6 @@ on: jobs: test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.10", "3.11", "3.12" ] - include: - - os: windows-latest - python-version: "3.10" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - pip install . - pip install pytest pytest-asyncio - - - name: Run tests - run: | - pytest --ignore=tests/nodes --ignore=tests/pipeline + uses: ./.github/workflows/reusable-test.yaml + with: + test_command: pytest -n auto --ignore=tests/nodes --ignore=tests/pipeline diff --git a/pyproject.toml b/pyproject.toml index 884fab15b..1aaa71e34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,44 @@ dependencies = [ "codecarbon (==2.6)", ] +[project.optional-dependencies] +dev = [ + "tach (>=0.11.3,<1.0.0)", + "ipykernel (>=6.29.5,<7.0.0)", + "ipywidgets (>=8.1.5,<9.0.0)", + "ruff (==0.8.4)", +] +test = [ + "pytest (>=8.3.2,<9.0.0)", + "pytest-cov (>=5.0.0,<6.0.0)", + "coverage (>=7.6.1,<8.0.0)", + "pytest-asyncio (>=0.24.0,<1.0.0)", + "pytest-rerunfailures (>=15.0,<16.0)", + "pytest-xdist (>=3.6.1,<4.0.0)", +] +typing = [ + "mypy (>=1,<2)", + "types-pyyaml (>=6.0.12.20240917,<7.0.0)", + "types-pygments (>=2.18.0.20240506,<3.0.0)", + "types-setuptools (>=75.2.0.20241019,<76.0.0)", + "joblib-stubs (>=1.4.2.5.20240918,<2.0.0)", +] +docs = [ + "sphinx (>=8.1.3,<9.0.0)", + "pydata-sphinx-theme (>=0.16.0,<1.0.0)", + "jupytext (>=1.16.4,<2.0.0)", + "nbsphinx (>=0.9.5,<1.0.0)", + "sphinx-autodoc-typehints (>=2.5.0,<3.0.0)", + "sphinx-copybutton (>=0.5.2,<1.0.0)", + "sphinx-autoapi (>=3.3.3,<4.0.0)", + "ipykernel (>=6.29.5,<7.0.0)", + "tensorboardx (>=2.6.2.2,<3.0.0)", + "sphinx-multiversion (>=0.2.4,<1.0.0)", +] +dspy = [ + "dspy (>=2.6.5,<3.0.0)", +] + [project.urls] Homepage = "https://deeppavlov.github.io/AutoIntent/" Repository = "https://github.com/deeppavlov/AutoIntent" @@ -57,57 +95,6 @@ Documentation = "https://deeppavlov.github.io/AutoIntent/" "basic-aug" = "autointent.generation.utterances.basic.cli:main" "evolution-aug" = "autointent.generation.utterances.evolution.cli:main" -[tool.poetry.group.dev] -optional = true - -[tool.poetry.group.dev.dependencies] -tach = "^0.11.3" -ipykernel = "^6.29.5" -ipywidgets = "^8.1.5" -ruff = "==0.8.4" - -[tool.poetry.group.test] -optional = true - -[tool.poetry.group.test.dependencies] -pytest = "8.3.2" -pytest-cov = "^5.0.0" -coverage = "^7.6.1" -pytest-asyncio = "^0.24.0" - -[tool.poetry.group.typing] -optional = true - -[tool.poetry.group.typing.dependencies] -mypy = "^1" -types-pyyaml = "^6.0.12.20240917" -types-pygments = "^2.18.0.20240506" -types-setuptools = "^75.2.0.20241019" -joblib-stubs = "^1.4.2.5.20240918" - -[tool.poetry.group.docs] -optional = true - -[tool.poetry.group.docs.dependencies] -sphinx = "^8.1.3" -pydata-sphinx-theme = "^0.16.0" -jupytext = "^1.16.4" -nbsphinx = "^0.9.5" -sphinx-autodoc-typehints = "^2.5.0" -sphinx-copybutton = "^0.5.2" -sphinx-autoapi = "^3.3.3" -ipykernel = "^6.29.5" -tensorboardx = "^2.6.2.2" -sphinx-multiversion = "^0.2.4" - -[tool.poetry.group.dspy] -optional = true - - -[tool.poetry.group.dspy.dependencies] -dspy = "^2.6.5" - - [tool.ruff] line-length = 120 indent-width = 4 @@ -149,11 +136,28 @@ build-backend = "poetry.core.masonry.api" [tool.pytest.ini_options] minversion = "8.0" -addopts = "-ra" # `--cov` option breaks pycharm's test debugger testpaths = [ "tests", ] pythonpath = "autointent" +# `--cov` option breaks pycharm's test debugger +addopts = """ + -ra + --reruns 3 + --only-rerun requests.exceptions.ReadTimeout + --only-rerun huggingface_hub.errors.HfHubHTTPError + --only-rerun huggingface_hub.errors.LocalEntryNotFoundError + --only-rerun FileNotFoundError + --only-rerun OSError + --durations 5 + --reruns-delay 10 +""" +# --reruns 3 -> # Retry failed tests 3 times +# requests.exceptions.ReadTimeout -> # HF Read timed out +# huggingface_hub.errors.HfHubHTTPError -> # HF is unavailable +# huggingface_hub.errors.LocalEntryNotFoundError -> # Gateway Time-out from HF +# FileNotFoundError -> HF Cache is broken +# --reruns-delay 10 -> Delay between reruns in seconds to avoid running into the same issue again [tool.coverage.run] branch = true @@ -213,3 +217,4 @@ module = [ "autointent.modules.abc.*", ] warn_unreachable = false + diff --git a/user_guides/advanced/01_data.py b/user_guides/advanced/01_data.py index 27314a8d4..51778775c 100644 --- a/user_guides/advanced/01_data.py +++ b/user_guides/advanced/01_data.py @@ -6,9 +6,8 @@ """ # %% -import importlib.resources as ires - import datasets +import huggingface_hub from autointent import Dataset @@ -180,7 +179,11 @@ """ # %% -path_to_dataset = ires.files("tests.assets.data").joinpath("clinc_subset.json") +path_to_dataset = huggingface_hub.hf_hub_download( + repo_id="DeepPavlov/clinc150_subset", + filename="clinc_subset.json", + repo_type="dataset", +) dataset = Dataset.from_json(path_to_dataset) # %% [markdown] diff --git a/user_guides/basic_usage/01_data.py b/user_guides/basic_usage/01_data.py index a03f39bb3..f56294a76 100644 --- a/user_guides/basic_usage/01_data.py +++ b/user_guides/basic_usage/01_data.py @@ -6,9 +6,8 @@ """ # %% -import importlib.resources as ires - import datasets +import huggingface_hub from autointent import Dataset @@ -53,7 +52,11 @@ """ # %% -path_to_dataset = ires.files("tests.assets.data").joinpath("clinc_subset_unsplitted.json") +path_to_dataset = huggingface_hub.hf_hub_download( + repo_id="DeepPavlov/clinc150_subset", + filename="clinc_subset_unsplitted.json", + repo_type="dataset", +) dataset = Dataset.from_json(path_to_dataset) # %% [markdown]