Fix packaging errors including cython module build / import issues (#1936)

* Avoid cychon compile on Windows

* Fix packaging errors

* Fix pre-commit

* Remove use_develop option from tox tests

* Fix pre-commit

* Fix tox -e -> -re for clean build

* Fix import error

* Add missing __init__.py

* Fix relative -> absolute import

* Revert to editable package in tox tests

---------

Signed-off-by: Songki Choi <[email protected]>

Author: Songki Choi

.github/workflows/pre_merge.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Unit-testing
-        run: tox -re pre-merge -- tests/unit
+        run: tox -e pre-merge -- tests/unit
       - name: Upload coverage reports to Codecov
         run: |
           # If the workflow is triggered from PR then it gets the commit id from the PR.
@@ -72,7 +72,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Integration-testing
-        run: tox -re pre-merge -- tests/integration/cli/test_cli.py
+        run: tox -e pre-merge -- tests/integration/cli/test_cli.py
   Pre-Merge-Integration-Cls-Test:
     runs-on: [self-hosted, linux, x64, dev]
     needs: Pre-Merge-Unit-Test
@@ -83,7 +83,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Integration-testing
-        run: tox -re pre-merge-cls
+        run: tox -e pre-merge-cls
   Pre-Merge-Integration-Det-Test:
     runs-on: [self-hosted, linux, x64, dev]
     needs: Pre-Merge-Unit-Test
@@ -94,7 +94,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Integration-testing
-        run: tox -re pre-merge-det
+        run: tox -e pre-merge-det
   Pre-Merge-Integration-Seg-Test:
     runs-on: [self-hosted, linux, x64, dev]
     needs: Pre-Merge-Unit-Test
@@ -105,7 +105,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Integration-testing
-        run: tox -re pre-merge-seg
+        run: tox -e pre-merge-seg
   Pre-Merge-Integration-Action-Test:
     runs-on: [self-hosted, linux, x64, dev]
     needs: Pre-Merge-Unit-Test
@@ -116,7 +116,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Integration-testing
-        run: tox -re pre-merge-action
+        run: tox -e pre-merge-action
   Pre-Merge-Integration-Anomaly-Test:
     runs-on: [self-hosted, linux, x64, dev]
     needs: Pre-Merge-Unit-Test
@@ -127,4 +127,4 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Integration-testing
-        run: tox -re pre-merge-anomaly
+        run: tox -e pre-merge-anomaly

MANIFEST.in

@@ -2,3 +2,6 @@ recursive-include requirements *
 recursive-include otx *.pyx
 recursive-include otx *.yaml
 recursive-include otx *.json
+recursive-exclude otx *.c
+graft tests
+global-exclude *.py[cod]

otx/algorithms/common/adapters/mmcv/__init__.py

@@ -34,7 +34,6 @@
 )
 from .nncf.hooks import CompressionHook
 from .nncf.runners import AccuracyAwareRunner
-from .pipelines.transforms import pil_augment
 from .runner import EpochRunnerWithCancel, IterBasedRunnerWithCancel
 
 __all__ = [
@@ -56,7 +55,6 @@
     "StopLossNanTrainingHook",
     "EMAMomentumUpdateHook",
     "CompressionHook",
-    "pil_augment",
     "AccuracyAwareRunner",
     "TwoCropTransformHook",
 ]

otx/algorithms/common/adapters/mmcv/pipelines/__init__.py

@@ -0,0 +1,4 @@
+"""Pipelines for mmcv."""
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#

otx/algorithms/common/adapters/mmcv/pipelines/transforms/__init__.py

@@ -2,9 +2,3 @@
 # Copyright (C) 2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 #
-
-# flake8: noqa
-
-from .cython_augments import pil_augment
-
-__all__ = ["pil_augment"]

otx/algorithms/common/adapters/mmcv/pipelines/transforms/augments.py

@@ -11,9 +11,9 @@
 from PIL.Image import Image as PILImage
 from PIL.Image import Resampling
 
-from otx.algorithms.common.adapters.mmcv.pipelines.transforms.cython_augments import (
-    pil_augment as pil_aug,
-)
+# type: ignore[attr-defined]
+# pylint: disable = no-name-in-module
+import otx.algorithms.common.adapters.mmcv.pipelines.transforms.cython_augments.pil_augment as pil_aug
 
 ImgTypes = Union[PILImage, CvImage]
 

otx/algorithms/common/adapters/mmcv/pipelines/transforms/cython_augments/__init__.py

@@ -1,9 +1,3 @@
 """Module to init cython augments."""
 # Copyright (C) 2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-# ignore mypy attr-defined error by cython modules
-# pylint: disable=import-self
-
-from . import pil_augment  # type: ignore[attr-defined]
-
-__all__ = ["pil_augment"]

otx/algorithms/common/adapters/torch/__init__.py

@@ -0,0 +1,4 @@
+"""Adapters for OTX Common Algorithms."""
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#

setup.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
+import platform
 import subprocess
 import sys
 import warnings
@@ -100,6 +101,9 @@ def get_requirements(requirement_files: Union[str, List[str]]) -> List[str]:
 
 
 def get_extensions():
+    if platform.system() == "Windows":
+        return []
+
     def _cython_modules():
         package_root = os.path.dirname(__file__)
 
@@ -111,14 +115,14 @@ def _cython_modules():
         ext_modules = [
             Extension(
                 cython_file.rstrip(".pyx").replace("/", "."),
-                [os.path.join(package_root, cython_file)],
+                [cython_file],
                 include_dirs=[numpy.get_include()],
                 extra_compile_args=["-O3"],
             )
             for cython_file in cython_files
         ]
 
-        return cythonize(ext_modules, annotate=True)
+        return cythonize(ext_modules)
 
     extensions = []
     extensions.extend(_cython_modules())

tox.ini

@@ -173,11 +173,11 @@ deps =
 skip_install = true
 allowlist_externals =
     rm
+    find
 commands =
     rm -rf ./dist
     python -m build --sdist
-    python -m pip install dist/otx-1.1.0rc1.tar.gz[full]
-    # python -m pip install otx[full]==1.1.0rc1
+    find ./dist -type f -name *.tar.gz -exec pip install {}[full] \;
     pytest {posargs:tests/unit tests/integration/cli}