Make pretrained models optional to install (#590)

* separate pretrained model tests

* only test on python 3.10 and 3.11

* fix indentation

* fix lint

* remove pretrained pipelines from tutorial tests

* add installation options

Author: sarahmish

.github/workflows/tests.yml

@@ -124,3 +124,21 @@ jobs:
       run: pip install invoke jupyter matplotlib .
     - name: invoke tutorials
       run: invoke tutorials
+
+
+  pretrained:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11']
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install package and dependencies
+      run: pip install invoke jupyter .[pretrained]
+    - name: invoke pretrained
+      run: invoke pretrained

Makefile

@@ -81,6 +81,14 @@ install-test: clean-build clean-pyc ## install the package and test dependencies
 install-develop: clean-build clean-pyc ## install the package in editable mode and dependencies for development
 	pip install -e .[dev]
 
+.PHONY: install-pretrained
+install-pretrained: clean-build clean-pyc ## install the package and pretrained pipelines
+	pip install -e .[pretrained]
+
+.PHONY: install-pretrained-develop
+install-pretrained-develop: clean-build clean-pyc ## install the package with pretrained in editable mode and dependencies for development
+	pip install -e .[pretrained-dev]
+
 MINIMUM := $(shell sed -n '/install_requires = \[/,/]/p' setup.py | grep -v -e '[][]' | sed 's/ *\(.*\),$?$$/\1/g' | tr '>' '=')
 
 .PHONY: install-minimum
@@ -127,6 +135,10 @@ test-readme: ## run the readme snippets
 test-tutorials: ## run the tutorial notebooks
 	invoke tutorials
 
+.PHONY: test-pretrained-tutorials
+test-pretrained-tutorials: ## run the tutorial notebooks
+	invoke pretrained
+
 .PHONY: test
 test: test-unit test-readme test-tutorials ## test everything that needs test dependencies
 

orion/primitives/units.py

@@ -14,14 +14,14 @@
 import numpy as np
 import torch
 import torch.nn.functional as F
-from smart_open import open as smart_open
-from timm.layers import DropPath, Mlp
-from timm.layers.helpers import to_2tuple
 from torch import nn
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 
 from orion.data import BUCKET, S3_URL
+from smart_open import open as smart_open
+from timm.layers import DropPath, Mlp
+from timm.layers.helpers import to_2tuple
 
 warnings.filterwarnings('ignore')
 

setup.py

@@ -37,14 +37,12 @@
 
     # fix conflict
     'protobuf<4',
-    
-    # fails on python 3.6
-    'opencv-python<4.7',
+]
 
-    #UniTS
-    'timm', 
+pretrained_requires = [
+    #units
+    'timm',
     'smart_open',
-
 ]
 
 setup_requires = [
@@ -128,6 +126,8 @@
     extras_require={
         'test': tests_require,
         'dev': development_requires + tests_require,
+        'pretrained': pretrained_requires,
+        'pretrained-dev': pretrained_requires + development_requires + tests_require,
     },
     include_package_data=True,
     install_requires=install_requires,

tasks.py

@@ -92,15 +92,29 @@ def readme(c):
 
 @task
 def tutorials(c):
+    pipelines = os.listdir(os.path.join('orion', 'pipelines', 'pretrained'))
     for ipynb_file in glob.glob('tutorials/*.ipynb') + glob.glob('tutorials/**/*.ipynb'):
-        if 'OrionDBExplorer' in ipynb_file: # skip db testing
-            continue
-        if '.ipynb_checkpoints' not in ipynb_file:
-            c.run((
-                'jupyter nbconvert --execute --ExecutePreprocessor.timeout=4400 '
-                f'--to=html --stdout {ipynb_file}'
-            ), hide='out')
+        for pipeline in pipelines:
+            if pipeline in ipynb_file: # skip pretrained pipelines
+                continue
+            if '.ipynb_checkpoints' not in ipynb_file:
+                c.run((
+                    'jupyter nbconvert --execute --ExecutePreprocessor.timeout=4400 '
+                    f'--to=html --stdout {ipynb_file}'
+                ), hide='out')
 
+@task
+def pretrained(c):
+    pipelines = os.listdir(os.path.join('orion', 'pipelines', 'pretrained'))
+    for ipynb_file in glob.glob('tutorials/pipelines/*.ipynb'):
+        for pipeline in pipelines: 
+            if pipeline not in ipynb_file: # skip non-pretrained pipelines
+                continue
+            if '.ipynb_checkpoints' not in ipynb_file:
+                c.run((
+                    'jupyter nbconvert --execute --ExecutePreprocessor.timeout=4400 '
+                    f'--to=html --stdout {ipynb_file}'
+                ), hide='out')
 
 @task
 def lint(c):