diff --git a/pymsis/__init__.py b/pymsis/__init__.py index 05d45fa..1b92db7 100644 --- a/pymsis/__init__.py +++ b/pymsis/__init__.py @@ -7,4 +7,4 @@ __version__ = importlib.metadata.version("pymsis") -__all__ = ["__version__", "Variable", "calculate"] +__all__ = ["Variable", "__version__", "calculate"] diff --git a/pymsis/utils.py b/pymsis/utils.py index 725c07c..c2e78f6 100644 --- a/pymsis/utils.py +++ b/pymsis/utils.py @@ -43,8 +43,7 @@ def download_f107_ap() -> None: Space Weather, https://doi.org/10.1029/2020SW002641 """ warnings.warn(f"Downloading ap and F10.7 data from {_F107_AP_URL}") - req = urllib.request.urlopen(_F107_AP_URL) - with _F107_AP_PATH.open("wb") as f: + with _F107_AP_PATH.open("wb") as f, urllib.request.urlopen(_F107_AP_URL) as req: f.write(req.read()) @@ -90,27 +89,21 @@ def _load_f107_ap_data() -> dict[str, npt.NDArray]: # Use a buffer to read in and load so we can quickly get rid of # the extra "PRD" lines at the end of the file (unknown length # so we can't just go back in line lengths) - with _F107_AP_PATH.open() as fin: - with BytesIO() as fout: - for line in fin: - if "PRM" in line: - # We don't want the monthly predicted values - continue - if ",,,,,,,," in line: - # We don't want lines with missing values - continue - fout.write(line.encode("utf-8")) - fout.seek(0) - arr = np.loadtxt( - fout, delimiter=",", dtype=dtype, usecols=usecols, skiprows=1 - ) # type: ignore + with _F107_AP_PATH.open() as fin, BytesIO() as fout: + for line in fin: + if "PRM" in line or ",,,,,,,," in line: + # We don't want the monthly predicted values or missing values + continue + fout.write(line.encode("utf-8")) + fout.seek(0) + arr = np.loadtxt(fout, delimiter=",", dtype=dtype, usecols=usecols, skiprows=1) # type: ignore # transform each day's 8 3-hourly ap values into a single column ap = np.empty(len(arr) * 8, dtype=float) daily_ap = arr["Ap"].astype(float) dates = np.repeat(arr["date"], 8).astype("datetime64[m]") for i in range(8): - ap[i::8] = arr[f"ap{i+1}"] + ap[i::8] = arr[f"ap{i + 1}"] dates[i::8] += i * np.timedelta64(3, "h") # data file has missing values as negatives @@ -208,6 +201,12 @@ def get_f107_ap(dates: npt.ArrayLike) -> tuple[npt.NDArray, npt.NDArray, npt.NDA """ dates = np.asarray(dates, dtype=np.datetime64) data = _DATA or _load_f107_ap_data() + # If our requested data time is after the cached values we have, + # go and download a new file to refresh the local file cache + last_time_in_file = data["dates"][7::8][~data["warn_data"]].max() + if np.any((dates > last_time_in_file) & (dates < np.datetime64("now"))): + download_f107_ap() + data = _load_f107_ap_data() data_start = data["dates"][0] data_end = data["dates"][-1] diff --git a/pyproject.toml b/pyproject.toml index 213f762..81464fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,10 @@ testpaths = [ addopts = [ "--import-mode=importlib", ] +filterwarnings = [ +# Ignore warnings loading from file specifically + 'ignore:Downloading ap and F10.7 data from file:UserWarning', +] [tool.cibuildwheel] # skip Python <3.10 diff --git a/tests/conftest.py b/tests/conftest.py index bbe64d9..9048616 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,21 +6,15 @@ @pytest.fixture(autouse=True) -def local_path(monkeypatch): - # Update the data location to our test data - test_file = Path(__file__).parent / "f107_ap_test_data.txt" +def _path_setup(monkeypatch, tmp_path): # Monkeypatch the url and expected download location, so we aren't # dependent on an internet connection. - monkeypatch.setattr(utils, "_F107_AP_PATH", test_file) - return test_file - + monkeypatch.setattr(utils, "_F107_AP_PATH", tmp_path / "f107_ap_test_data.txt") -@pytest.fixture(autouse=True) -def remote_path(monkeypatch, local_path): # Update the remote URL to point to a local file system test path # by prepending file:// so that it can be opened by urlopen() - test_url = local_path.absolute().as_uri() + test_file = Path(__file__).parent / "f107_ap_test_data.txt" + test_url = test_file.absolute().as_uri() # Monkeypatch the url and expected download location, so we aren't # dependent on an internet connection. monkeypatch.setattr(utils, "_F107_AP_URL", test_url) - return test_url diff --git a/tests/test_utils.py b/tests/test_utils.py index 7c13cae..76cf2fc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,6 @@ +from pathlib import Path +from unittest.mock import patch + import numpy as np import pytest from numpy.testing import assert_allclose, assert_array_equal @@ -130,3 +133,41 @@ def test_get_f107_ap_interpolated_warns(dates): UserWarning, match="There is data that was either interpolated or" ): utils.get_f107_ap(dates) + + +@patch("pymsis.utils.download_f107_ap") +def test_auto_refresh(download_data_mock, monkeypatch): + test_file = Path(__file__).parent / "f107_ap_test_data.txt" + # Monkeypatch the url and expected download location, so we aren't + # dependent on an internet connection. + monkeypatch.setattr(utils, "_F107_AP_PATH", test_file) + + def call_with_time(time): + try: + utils.get_f107_ap(time) + except ValueError: + # There is no data in our test file for this, so we will error later + # But this is enough to trigger an attempt at a refresh + pass + + # Should not trigger a refresh, data before the time in the file + call_with_time(np.datetime64("1990-12-31T23:00")) + assert download_data_mock.call_count == 0 + + # Final observed time in the file + call_with_time(np.datetime64("2000-12-29T21:00")) + assert download_data_mock.call_count == 0 + + # One hour beyond our current time shouldn't trigger a refresh + # there would be no data to get for that time period + call_with_time(np.datetime64("now") + np.timedelta64(1, "h")) + assert download_data_mock.call_count == 0 + + # Within the predicted data in the file should try to get a refresh + with pytest.warns(UserWarning, match="There is data that was either"): + call_with_time(np.datetime64("2000-12-30T00:00")) + assert download_data_mock.call_count == 1 + + # Should trigger a refresh, after the data in the file but before current time + call_with_time(np.datetime64("2005-01-01T00:00")) + assert download_data_mock.call_count == 2 # noqa: PLR2004 diff --git a/tools/download_source.py b/tools/download_source.py index 5304d51..7faeadf 100644 --- a/tools/download_source.py +++ b/tools/download_source.py @@ -24,7 +24,7 @@ def get_source(): if not Path("src/msis2.0/msis_init.F90").exists(): # No source code yet, so go download and extract it try: - warnings.warn("Downloading the MSIS2.0 source code from " f"{MSIS20_FILE}") + warnings.warn(f"Downloading the MSIS2.0 source code from {MSIS20_FILE}") with urllib.request.urlopen(MSIS20_FILE) as stream: tf = tarfile.open(fileobj=stream, mode="r|gz") tf.extractall(path=Path("src/msis2.0")) @@ -49,7 +49,7 @@ def get_source(): if not Path("src/msis2.1/msis_init.F90").exists(): # No source code yet, so go download and extract it try: - warnings.warn("Downloading the MSIS2.1 source code from " f"{MSIS21_FILE}") + warnings.warn(f"Downloading the MSIS2.1 source code from {MSIS21_FILE}") with urllib.request.urlopen(MSIS21_FILE) as stream: tf = tarfile.open(fileobj=stream, mode="r|gz") tf.extractall(path=Path("src/msis2.1")) @@ -76,7 +76,7 @@ def get_source(): local_msis00_path.parent.mkdir(parents=True, exist_ok=True) # No source code yet, so go download and extract it try: - warnings.warn("Downloading the MSIS-00 source code from " f"{MSIS00_FILE}") + warnings.warn(f"Downloading the MSIS-00 source code from {MSIS00_FILE}") with urllib.request.urlopen(MSIS00_FILE) as response: with open(local_msis00_path, "wb") as f: