Refactor unit and integration tests and corresponding github actions (#686)

* refactor integration tests

* move legacy to integration

* move legacy to integration

* temporarily disable notebooks

* update unit tests

* update coverage action

* fix job name

* fix coverage action

* reinstate notebooks

* use pip

* tidy up linting

* wip fix or ignore mypy errors

* fix or ignore mypy errors

* fix plasmodium tests

* fix typing

Author: alimanfoo

.github/workflows/coverage.yml

@@ -15,20 +15,17 @@ jobs:
             - name: Checkout source
               uses: actions/checkout@v4
 
-            - name: Install poetry
-              run: pipx install poetry==1.8.3
-
             - name: Setup python
               uses: actions/setup-python@v5
               with:
                   python-version: '3.12'
-                  cache: 'poetry'
+                  cache: 'pip'
 
-            - name: Install dependencies
-              run: poetry install
+            - name: Install package
+              run: pip install .[dev]
 
-            - name: Run tests with coverage
-              run: poetry run pytest --durations=20 -v --cov malariagen_data/anoph --cov-report=xml tests/anoph
+            - name: Run unit tests with coverage
+              run: pytest -v tests --ignore tests/integration --cov malariagen_data/anoph --cov-report=xml
 
             - name: Upload coverage report
               uses: codecov/codecov-action@v3

.github/workflows/legacy_tests.yml renamed to .github/workflows/integration_tests.yml

@@ -1,4 +1,4 @@
-name: legacy_tests
+name: integration_tests
 on:
   push:
     branches:
@@ -7,7 +7,7 @@ on:
     branches:
       - master
 jobs:
-  legacy_tests:
+  integration_tests:
     strategy:
       fail-fast: true
       matrix:
@@ -23,17 +23,14 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@v4
 
-      - name: Install poetry
-        run: pipx install poetry==1.8.3
-
       - name: Setup python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+          cache: "pip"
 
-      - name: Install dependencies
-        run: poetry install
+      - name: Install package
+        run: pip install .[dev]
 
       - id: 'auth'
         name: 'Set up Google Cloud authentication'
@@ -53,14 +50,14 @@ jobs:
         uses: actions/cache/restore@v3
         with:
           path: gcs_cache
-          key: gcs_cache_tests_20240922
+          key: gcs_cache_integration_tests_20240922
 
-      - name: Run full test suite
-        run: poetry run pytest --durations=20 --ignore=tests/anoph -v tests
+      - name: Run integration tests
+        run: pytest --durations=20 -v tests/integration
 
       - name: Save GCS cache
         uses: actions/cache/save@v3
         if: always()
         with:
           path: gcs_cache
-          key: gcs_cache_tests_20240922
+          key: gcs_cache_integration_tests_20240922

.github/workflows/linting.yml

@@ -12,13 +12,21 @@ jobs:
             fail-fast: true
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v4
-            - uses: actions/setup-python@v5
+            - name: Checkout source
+              uses: actions/checkout@v4
+
+            - name: Setup python
+              uses: actions/setup-python@v5
               with:
                   python-version: '3.12'
-            - uses: pre-commit/[email protected]
-            - name: Install mypy
-              run: pip install mypy
+                  cache: 'pip'
+
+            - name: Run pre-commit checks
+              uses: pre-commit/[email protected]
+
+            - name: Install package
+              run: pip install .[dev]
+
             - name: Run mypy
               run: |
                   mypy malariagen_data tests --ignore-missing-imports

.github/workflows/notebooks.yml

@@ -19,17 +19,14 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@v4
 
-      - name: Install poetry
-        run: pipx install poetry==1.8.3
-
       - name: Setup python
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-          cache: "poetry"
+          cache: "pip"
 
-      - name: Install dependencies
-        run: poetry install
+      - name: Install package
+        run: pip install .[dev]
 
       - id: 'auth'
         name: 'Set up Google Cloud authentication'
@@ -52,7 +49,7 @@ jobs:
           key: gcs_cache_notebooks_20240922
 
       - name: Run notebooks
-        run: poetry run jupyter nbconvert --execute notebooks/*.ipynb --inplace
+        run: jupyter nbconvert --execute notebooks/*.ipynb --inplace
 
       - name: Save GCS cache
         uses: actions/cache/save@v3

.github/workflows/tests.yml

@@ -32,5 +32,5 @@ jobs:
       - name: Install package
         run: pip install "${{ matrix.numpy-version }}" .[dev]
 
-      - name: Run fast unit tests
-        run: pytest -v tests/anoph --typeguard-packages=malariagen_data,malariagen_data.anoph
+      - name: Run unit tests
+        run: pytest -v tests --ignore tests/integration --typeguard-packages=malariagen_data,malariagen_data.anoph

malariagen_data/anoph/genome_features.py

@@ -415,11 +415,12 @@ def plot_genes(
 
             # Increase the figure height by a certain factor, to accommodate labels.
             height_increase_factor = 1.3
+            assert fig.height is not None
             fig.height = int(fig.height * height_increase_factor)
 
             # Get the original y_range.
             # Note: fig.y_range is not subscriptable.
-            orig_y_range = fig.y_range.start, fig.y_range.end
+            orig_y_range = fig.y_range.start, fig.y_range.end  # type: ignore
 
             # Determine the midpoint of the original range, to rescale outward from there.
             orig_mid_y_range = (orig_y_range[0] + orig_y_range[1]) / 2
@@ -432,7 +433,7 @@ def plot_genes(
             new_y_end = orig_mid_y_range + new_y_range_extent_half
 
             # Set the new y_range.
-            fig.y_range = bokeh.models.Range1d(new_y_start, new_y_end)
+            fig.y_range = bokeh.models.Range1d(new_y_start, new_y_end)  # type: ignore
 
             debug("determine midpoint of each gene rectangle")
             data["mid_x"] = (data["start"] + data["end"]) / 2
@@ -474,7 +475,7 @@ def plot_genes(
             data_as_cds = bokeh.models.ColumnDataSource(data)
 
             # Create a LabelSet for the gene pointers.
-            gene_pointers_ls = bokeh.models.LabelSet(
+            gene_pointers_ls = bokeh.models.LabelSet(  # type: ignore
                 source=data_as_cds,
                 x="mid_x",
                 y="pointer_y",
@@ -486,7 +487,7 @@ def plot_genes(
             )
 
             # Create a LabelSet for the gene labels.
-            gene_labels_ls = bokeh.models.LabelSet(
+            gene_labels_ls = bokeh.models.LabelSet(  # type: ignore
                 source=data_as_cds,
                 x="mid_x",
                 y="label_y",

malariagen_data/anoph/gplt_params.py

@@ -127,6 +127,6 @@
 ]
 
 gene_labelset: TypeAlias = Annotated[
-    bokeh.models.LabelSet,
+    bokeh.models.LabelSet,  # type: ignore
     "A LabelSet to use in the plot.",
 ]

malariagen_data/anoph/h12.py

@@ -823,7 +823,7 @@ def plot_h12_gwss_multi_panel(
         figs.append(fig2)
 
         # Combine plots into a single figure.
-        fig = bokeh.layouts.gridplot(
+        fig = bokeh.layouts.gridplot(  # type: ignore
             figs,
             ncols=1,
             toolbar_location="above",

malariagen_data/anoph/hap_frq.py

@@ -107,7 +107,7 @@ def haplotypes_frequencies(
             hap_dict.update(f)
             freq_cols["frq_" + coh] = list(hap_dict.values())
 
-        df_freqs = pd.DataFrame(freq_cols, index=hap_dict.keys())
+        df_freqs = pd.DataFrame(freq_cols, index=list(hap_dict.keys()))
 
         # Compute max_af.
         df_max_af = pd.DataFrame({"max_af": df_freqs.max(axis=1)})
@@ -230,14 +230,14 @@ def haplotypes_frequencies_advanced(
             freq_cols["frq_" + cohort_key_str] = list(hap_freq.values())
             nobs_cols["nobs_" + cohort_key_str] = list(hap_nob.values())
 
-        df_freqs = pd.DataFrame(freq_cols, index=hap_freq.keys())
+        df_freqs = pd.DataFrame(freq_cols, index=list(hap_freq.keys()))
 
         # Compute max_af.
         df_max_af = pd.DataFrame({"max_af": df_freqs.max(axis=1)})
 
-        df_counts = pd.DataFrame(count_cols, index=hap_count.keys())
+        df_counts = pd.DataFrame(count_cols, index=list(hap_count.keys()))
 
-        df_nobs = pd.DataFrame(nobs_cols, index=hap_nob.keys())
+        df_nobs = pd.DataFrame(nobs_cols, index=list(hap_nob.keys()))
 
         # Build the final dataframe.
         df_haps = pd.concat([df_freqs, df_counts, df_nobs, df_max_af], axis=1)

malariagen_data/anoph/sample_metadata.py

@@ -1270,7 +1270,7 @@ def cohorts(
             df_cohorts = pd.read_csv(f, sep=",", na_values="")
 
         # Ensure all column names are lower case.
-        df_cohorts.columns = [c.lower() for c in df_cohorts.columns]
+        df_cohorts.columns = [c.lower() for c in df_cohorts.columns]  # type: ignore
 
         return df_cohorts