ci: Import numpy before pyarrow in tests to resolve import warning (#2187)

* Fix: Import numpy before pyarrow in tests to resolve import warning

A `PytestDeprecationWarning` was occurring in several test files
because `pyarrow`, when imported by `pytest.importorskip`, would fail
to import `numpy.core.multiarray`.

This change addresses the warning by explicitly importing `numpy`
before `pytest.importorskip("pyarrow", ...)` in the affected
test files. This ensures that numpy is fully initialized before
pyarrow attempts to use it, resolving the underlying import error.

I also updated the test execution to use `nox -s unit`, which
correctly sets up the test environment and dependencies, allowing
the tests to pass and confirm the warning is resolved. Pre-existing
failures in `tests/unit/test_magics.py` are unrelated to this change.

* Update tests/unit/test__pyarrow_helpers.py

* revisions to numpy handling

* adds import or skip commands to accompany pyarrow import or skips

* Update tests/unit/test__pandas_helpers.py

* updates an import step and restores gc import

* Updates magics.context and removes unneeded? reference to numpy/pyarrow

---------

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>

Author: chalmerlowe

noxfile.py

@@ -163,8 +163,7 @@ def unit_noextras(session):
     # so that it continues to be an optional dependency.
     # https://github.com/googleapis/python-bigquery/issues/1877
     if session.python == UNIT_TEST_PYTHON_VERSIONS[0]:
-        session.install("pyarrow==4.0.0")
-
+        session.install("pyarrow==4.0.0", "numpy==1.20.2")
     default(session, install_extras=False)
 
 

testing/constraints-3.9.txt

@@ -20,6 +20,7 @@ ipykernel==6.2.0
 opentelemetry-api==1.1.0
 opentelemetry-instrumentation==0.20b0
 opentelemetry-sdk==1.1.0
+numpy==1.20.2
 packaging==24.2.0
 pandas==1.3.0
 pandas-gbq==0.26.1

tests/unit/test__pandas_helpers.py

@@ -1856,6 +1856,7 @@ def test__download_table_bqstorage_shuts_down_workers(
     Make sure that when the top-level iterator goes out of scope (is deleted),
     the child threads are also stopped.
     """
+    pytest.importorskip("google.cloud.bigquery_storage_v1")
     from google.cloud.bigquery import dataset
     from google.cloud.bigquery import table
     import google.cloud.bigquery_storage_v1.reader

tests/unit/test__pyarrow_helpers.py

@@ -14,7 +14,7 @@
 
 import pytest
 
-
+numpy = pytest.importorskip("numpy")
 pyarrow = pytest.importorskip("pyarrow", minversion="3.0.0")
 
 

tests/unit/test_dbapi__helpers.py

@@ -210,6 +210,7 @@ def test_empty_iterable(self):
         self.assertEqual(list(result), [])
 
     def test_non_empty_iterable(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("pyarrow")
         from tests.unit.helpers import _to_pyarrow
 

tests/unit/test_magics.py

@@ -1276,6 +1276,11 @@ def test_bigquery_magic_with_no_query_cache(monkeypatch):
     bigquery.load_ipython_extension(ip)
     conn = make_connection()
     monkeypatch.setattr(magics.context, "_connection", conn)
+    monkeypatch.setattr(
+        magics.context,
+        "credentials",
+        mock.create_autospec(google.auth.credentials.Credentials, instance=True),
+    )
     monkeypatch.setattr(magics.context, "project", "project-from-context")
 
     # --no_query_cache option should override context.

tests/unit/test_table.py

@@ -2416,13 +2416,15 @@ def test_to_arrow_error_if_pyarrow_is_none(self):
             row_iterator.to_arrow()
 
     def test_to_arrow(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip("pyarrow")
         row_iterator = self._make_one()
         tbl = row_iterator.to_arrow()
         self.assertIsInstance(tbl, pyarrow.Table)
         self.assertEqual(tbl.num_rows, 0)
 
     def test_to_arrow_iterable(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip(
             "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION
         )
@@ -3089,6 +3091,7 @@ def test_to_arrow_iterable_w_bqstorage(self):
         bqstorage_client._transport.grpc_channel.close.assert_not_called()
 
     def test_to_arrow(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip(
             "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION
         )
@@ -3173,6 +3176,7 @@ def test_to_arrow(self):
         )
 
     def test_to_arrow_w_nulls(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip(
             "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION
         )
@@ -3209,6 +3213,7 @@ def test_to_arrow_w_nulls(self):
         self.assertEqual(ages, [32, 29, None, 111])
 
     def test_to_arrow_w_unknown_type(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip(
             "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION
         )
@@ -3254,6 +3259,7 @@ def test_to_arrow_w_unknown_type(self):
         self.assertTrue(all("sport" in str(warning) for warning in warned))
 
     def test_to_arrow_w_empty_table(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip(
             "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION
         )
@@ -3295,6 +3301,7 @@ def test_to_arrow_w_empty_table(self):
         self.assertEqual(child_field.type.value_type[1].name, "age")
 
     def test_to_arrow_max_results_w_explicit_bqstorage_client_warning(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("pyarrow")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery.schema import SchemaField
@@ -3337,6 +3344,7 @@ def test_to_arrow_max_results_w_explicit_bqstorage_client_warning(self):
         mock_client._ensure_bqstorage_client.assert_not_called()
 
     def test_to_arrow_max_results_w_create_bqstorage_client_no_warning(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("pyarrow")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery.schema import SchemaField
@@ -3375,6 +3383,7 @@ def test_to_arrow_max_results_w_create_bqstorage_client_no_warning(self):
         mock_client._ensure_bqstorage_client.assert_not_called()
 
     def test_to_arrow_w_bqstorage(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip("pyarrow")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery import schema
@@ -3458,6 +3467,7 @@ def test_to_arrow_w_bqstorage(self):
         bqstorage_client._transport.grpc_channel.close.assert_not_called()
 
     def test_to_arrow_w_bqstorage_creates_client(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("pyarrow")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery import schema
@@ -3491,6 +3501,7 @@ def test_to_arrow_w_bqstorage_creates_client(self):
         bqstorage_client._transport.grpc_channel.close.assert_called_once()
 
     def test_to_arrow_ensure_bqstorage_client_wo_bqstorage(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip(
             "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION
         )
@@ -3524,6 +3535,7 @@ def mock_verify_version(raise_if_error: bool = False):
         self.assertEqual(tbl.num_rows, 2)
 
     def test_to_arrow_w_bqstorage_no_streams(self):
+        pytest.importorskip("numpy")
         pyarrow = pytest.importorskip("pyarrow")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery import schema
@@ -3563,6 +3575,7 @@ def test_to_arrow_w_bqstorage_no_streams(self):
         self.assertEqual(actual_table.schema[2].name, "colB")
 
     def test_to_arrow_progress_bar(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("pyarrow")
         pytest.importorskip("tqdm")
         pytest.importorskip("tqdm.notebook")
@@ -3696,6 +3709,7 @@ def test_to_dataframe_iterable_with_dtypes(self):
         self.assertEqual(df_2["age"][0], 33)
 
     def test_to_dataframe_iterable_w_bqstorage(self):
+        pytest.importorskip("numpy")
         pandas = pytest.importorskip("pandas")
         pyarrow = pytest.importorskip("pyarrow")
         pytest.importorskip("google.cloud.bigquery_storage")
@@ -3770,6 +3784,7 @@ def test_to_dataframe_iterable_w_bqstorage(self):
         bqstorage_client._transport.grpc_channel.close.assert_not_called()
 
     def test_to_dataframe_iterable_w_bqstorage_max_results_warning(self):
+        pytest.importorskip("numpy")
         pandas = pytest.importorskip("pandas")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery import schema
@@ -4513,7 +4528,7 @@ def test_to_dataframe_w_none_dtypes_mapper(self):
 
     def test_to_dataframe_w_unsupported_dtypes_mapper(self):
         pytest.importorskip("pandas")
-        import numpy
+        numpy = pytest.importorskip("numpy")
         from google.cloud.bigquery.schema import SchemaField
 
         schema = [
@@ -4797,6 +4812,7 @@ def test_to_dataframe_max_results_w_create_bqstorage_client_no_warning(self):
         mock_client._ensure_bqstorage_client.assert_not_called()
 
     def test_to_dataframe_w_bqstorage_creates_client(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("pandas")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery import schema
@@ -4830,6 +4846,7 @@ def test_to_dataframe_w_bqstorage_creates_client(self):
         bqstorage_client._transport.grpc_channel.close.assert_called_once()
 
     def test_to_dataframe_w_bqstorage_no_streams(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("pandas")
         pytest.importorskip("google.cloud.bigquery_storage")
         from google.cloud.bigquery import schema
@@ -4858,6 +4875,7 @@ def test_to_dataframe_w_bqstorage_no_streams(self):
         self.assertTrue(got.empty)
 
     def test_to_dataframe_w_bqstorage_logs_session(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("google.cloud.bigquery_storage")
         pytest.importorskip("pandas")
         pytest.importorskip("pyarrow")
@@ -4882,6 +4900,7 @@ def test_to_dataframe_w_bqstorage_logs_session(self):
         )
 
     def test_to_dataframe_w_bqstorage_empty_streams(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("google.cloud.bigquery_storage")
         pytest.importorskip("pandas")
         pyarrow = pytest.importorskip("pyarrow")
@@ -4936,6 +4955,7 @@ def test_to_dataframe_w_bqstorage_empty_streams(self):
         self.assertTrue(got.empty)
 
     def test_to_dataframe_w_bqstorage_nonempty(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("google.cloud.bigquery_storage")
         pytest.importorskip("pandas")
         pyarrow = pytest.importorskip("pyarrow")
@@ -5018,6 +5038,7 @@ def test_to_dataframe_w_bqstorage_nonempty(self):
         bqstorage_client._transport.grpc_channel.close.assert_not_called()
 
     def test_to_dataframe_w_bqstorage_multiple_streams_return_unique_index(self):
+        pytest.importorskip("numpy")
         bigquery_storage = pytest.importorskip("google.cloud.bigquery_storage")
         pytest.importorskip("pandas")
         pyarrow = pytest.importorskip("pyarrow")
@@ -5070,6 +5091,7 @@ def test_to_dataframe_w_bqstorage_multiple_streams_return_unique_index(self):
         self.assertTrue(got.index.is_unique)
 
     def test_to_dataframe_w_bqstorage_updates_progress_bar(self):
+        pytest.importorskip("numpy")
         bigquery_storage = pytest.importorskip("google.cloud.bigquery_storage")
         pytest.importorskip("pandas")
         pyarrow = pytest.importorskip("pyarrow")
@@ -5147,6 +5169,7 @@ def blocking_to_arrow(*args, **kwargs):
             tqdm_mock().close.assert_called_once()
 
     def test_to_dataframe_w_bqstorage_exits_on_keyboardinterrupt(self):
+        pytest.importorskip("numpy")
         bigquery_storage = pytest.importorskip("google.cloud.bigquery_storage")
         pytest.importorskip("pandas")
         pyarrow = pytest.importorskip("pyarrow")
@@ -5322,6 +5345,7 @@ def test_to_dataframe_w_bqstorage_snapshot(self):
             row_iterator.to_dataframe(bqstorage_client)
 
     def test_to_dataframe_concat_categorical_dtype_w_pyarrow(self):
+        pytest.importorskip("numpy")
         pytest.importorskip("google.cloud.bigquery_storage")
         pandas = pytest.importorskip("pandas")
         pyarrow = pytest.importorskip("pyarrow")
@@ -5604,7 +5628,7 @@ def test_rowiterator_to_geodataframe_delegation(self, to_dataframe):
         """
         pandas = pytest.importorskip("pandas")
         geopandas = pytest.importorskip("geopandas")
-        import numpy
+        numpy = pytest.importorskip("numpy")
         from shapely import wkt
 
         row_iterator = self._make_one_from_data(

tests/unit/test_table_arrow.py

@@ -18,7 +18,8 @@
 import google.cloud.bigquery.table
 
 
-pyarrow = pytest.importorskip("pyarrow", minversion="3.0.0")
+pytest.importorskip("numpy")
+pytest.importorskip("pyarrow", minversion="3.0.0")
 
 
 def test_to_arrow_with_jobs_query_response():