release(1.2.7) update MODFLOW 6 version to 6.4.1 (#128)

Update download._request_get() and dowload._request_header() to improve
ability to retry url requests. Improved flakyness of autotest/test_requests.py.
Update to usgsprograms.export_json() to handle directory paths appended to
fpth by separating it into a appdir and file_name. Also writing sorted code.json
and code.md.

Author: jdhughes-dev

.github/workflows/pymake-requests.yml

@@ -32,7 +32,7 @@ jobs:
 
       - name: Run pytest
         run: |
-          pytest -v -m requests --durations=0 --cov=pymake --cov-report=xml autotest/
+          pytest -v -n=auto -m requests --durations=0 --cov=pymake --cov-report=xml autotest/
 
       - name: Run scheduled tests
         if: ${{ github.event_name == 'schedule' }}

.gitignore

@@ -62,18 +62,14 @@ target/
 
 # files in the autotest directory
 autotest/temp*/
+autotest/*.json
+autotest/code.md
 mod_temp/
 obj_temp/
 src_temp/
 *.exe
 Dependencies
 
-# files in the examples directory
-examples/win32/
-examples/win64/
-examples/mac/
-examples/linux/
-examples/temp/
-
+# files in the doc directory
 docs/source/
 

autotest/test_requests.py

@@ -42,12 +42,12 @@ def initialize_working_dir():
     os.makedirs(dstpth, exist_ok=True)
 
 
-def export_code_json():
+def export_code_json(file_name="code.json"):
     # make sure the test directory exists
     initialize_working_dir()
 
     # make the json file
-    fpth = os.path.join(dstpth, "code.test.json")
+    fpth = os.path.join(dstpth, file_name)
     pymake.usgs_program_data.export_json(
         fpth=fpth,
         current=True,
@@ -302,7 +302,7 @@ def test_target_keys():
 @pytest.mark.requests
 def test_usgsprograms_export_json():
     # export code.json and return json file path
-    fpth = export_code_json()
+    fpth = export_code_json(file_name="code.export.json")
 
     # test the json export
     with open(fpth, "r") as f:
@@ -353,7 +353,7 @@ def test_usgsprograms_load_json():
     print("test_usgsprograms_load_json()")
 
     # export code.json and return json file path
-    fpth = export_code_json()
+    fpth = export_code_json(file_name="code.load.json")
 
     json_dict = pymake.usgs_program_data.load_json(fpth)
 
@@ -383,7 +383,7 @@ def test_usgsprograms_list_json():
     print("test_usgsprograms_list_json()")
 
     # export code.json and return json file path
-    fpth = export_code_json()
+    fpth = export_code_json(file_name="code.list.json")
 
     # list the contents of the json file
     pymake.usgs_program_data.list_json(fpth=fpth)

pymake/cmds/createjson.py

@@ -91,7 +91,7 @@ def main() -> None:
             "tag": ("--write_markdown",),
             "help": "If True, write markdown file that includes the "
             + "target name, version, and the last-modified date of "
-            + "the download asset (url). Default is False.",
+            + "the download asset (url). Default is True.",
             "default": True,
             "choices": None,
             "action": "store_true",

pymake/utils/download.py

@@ -15,8 +15,8 @@
 import shutil
 import sys
 import tarfile
-import time
 import timeit
+from http.client import responses
 from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
 
 import requests
@@ -202,32 +202,38 @@ def _request_get(url, verify=True, timeout=1, max_requests=10, verbose=False):
         request object for url
 
     """
+    if verbose:
+        print(f"request url '{url}'")
+
     for idx in range(max_requests):
         if verbose:
-            msg = f"open request attempt {idx + 1} of {max_requests}"
-            print(msg)
+            print(f"  request attempt {idx + 1} of {max_requests}")
         req = None
         try:
             req = requests.get(
-                url, stream=True, verify=verify, timeout=timeout
+                url,
+                stream=True,
+                verify=verify,
+                timeout=timeout,
             )
+            if verbose:
+                print(f"    status: {responses[req.status_code]}")
         except:
-            if idx < max_requests - 1:
-                time.sleep(13)
-                continue
-            else:
-                msg = "Cannot open request from:\n" + f"    {url}\n\n"
-                print(msg)
-                if req is not None:
-                    req.raise_for_status()
+            continue
 
-        # successful request
-        break
+        if req.status_code == 200:
+            break
+
+    # final test for success
+    if req is None:
+        raise ConnectionError(f"Could not get data from: {url}")
+    else:
+        req.raise_for_status()
 
     return req
 
 
-def _request_header(url, max_requests=10, verbose=False):
+def _request_header(url, max_requests=10, timeout=1, verbose=False):
     """Get the headers from a url
 
     Parameters
@@ -236,6 +242,8 @@ def _request_header(url, max_requests=10, verbose=False):
         url address for the zip file
     max_requests : int
         number of url download request attempts (default is 10)
+    timeout : int
+        url request time out length (default is 1 seconds)
     verbose : bool
         boolean indicating if output will be printed to the terminal
         (default is False)
@@ -246,23 +254,32 @@ def _request_header(url, max_requests=10, verbose=False):
         request header object for url
 
     """
+    if verbose:
+        print(f"request url: '{url}'")
+
     for idx in range(max_requests):
         if verbose:
-            msg = f"open request attempt {idx + 1} of {max_requests}"
-            print(msg)
+            print(f"  request attempt {idx + 1} of {max_requests}")
+        header = None
+        try:
+            header = requests.head(
+                url,
+                allow_redirects=True,
+                timeout=timeout,
+            )
+            if verbose:
+                print(f"    status: {responses[header.status_code]}")
+        except:
+            continue
 
-        header = requests.head(url, allow_redirects=True)
-        if header.status_code != 200:
-            if idx < max_requests - 1:
-                time.sleep(13)
-                continue
-            else:
-                msg = "Cannot open request from:\n" + f"    {url}\n\n"
-                print(msg)
-                header.raise_for_status()
+        if header.status_code == 200:
+            break
 
-        # successful header request
-        break
+    # final test for success
+    if header is None:
+        raise ConnectionError(f"Could not get header from: {url}")
+    else:
+        header.raise_for_status()
 
     return header
 

pymake/utils/usgsprograms.py

@@ -21,6 +21,7 @@
 import datetime
 import json
 import os
+import pathlib as pl
 import sys
 
 from .download import _request_header
@@ -341,12 +342,25 @@ def export_json(
         print(
             f'writing a json file ("{fpth}") '
             + f"of {sel} USGS programs\n"
-            + f'in the "{program_data_file}" database.'
+            + f'in the "{program_data_file}" database.\n'
         )
         if prog_data is not None:
             for idx, key in enumerate(prog_data.keys()):
                 print(f"    {idx + 1:>2d}: {key}")
-        print("\n")
+
+        # process the passed file path into appdir and file_name
+        appdir = pl.Path(".")
+        file_name = pl.Path(fpth)
+        if file_name.parent != str(appdir):
+            appdir = file_name.parent
+            file_name = file_name.name
+        else:
+            for idx, argv in enumerate(sys.argv):
+                if argv in ("--appdir", "-ad"):
+                    appdir = pl.Path(sys.argv[idx + 1])
+
+        if str(appdir) != ".":
+            appdir.mkdir(parents=True, exist_ok=True)
 
         # get usgs program data
         udata = usgs_program_data.get_program_dict()
@@ -392,48 +406,38 @@ def export_json(
 
         # export file
         try:
-            with open(fpth, "w") as f:
-                json.dump(prog_data, f, indent=4)
+            with open(file_name, "w") as file_obj:
+                json.dump(prog_data, file_obj, indent=4, sort_keys=True)
         except:
-            msg = f'could not export json file "{fpth}"'
+            msg = f'could not export json file "{file_name}"'
             raise IOError(msg)
 
-        # export code.json to --appdir directory, if the
-        # command line argument was specified. Only done if not CI
-        # command line argument was specified. Only done if not CI
-        appdir = "."
-        for idx, argv in enumerate(sys.argv):
-            if argv in ("--appdir", "-ad"):
-                appdir = sys.argv[idx + 1]
-
-        # make appdir if it does not already exist
-        if not os.path.isdir(appdir):
-            os.makedirs(appdir)
-
         # write code.json
-        if appdir != ".":
-            dst = os.path.join(appdir, fpth)
-            with open(dst, "w") as f:
-                json.dump(prog_data, f, indent=4)
+        if str(appdir) != ".":
+            dst = appdir / file_name
+            with open(dst, "w") as file_obj:
+                json.dump(prog_data, file_obj, indent=4, sort_keys=True)
 
         # write code.md
         if prog_data is not None and write_markdown:
-            file_obj = open("code.md", "w")
-            line = "| Program | Version | UTC Date |"
-            file_obj.write(line + "\n")
-            line = "| ------- | ------- | ---- |"
-            file_obj.write(line + "\n")
-            for target, target_dict in prog_data.items():
-                keys = list(target_dict.keys())
-                line = f"| {target} | {target_dict['version']} |"
-                date_key = "url_download_asset_date"
-                if date_key in keys:
-                    line += f" {target_dict[date_key]} |"
-                else:
-                    line += " |"
-                line += "\n"
-                file_obj.write(line)
-            file_obj.close()
+            sorted_prog_data = {
+                key: prog_data[key] for key in sorted(list(prog_data.keys()))
+            }
+            with open("code.md", "w") as file_obj:
+                line = "| Program | Version | UTC Date |"
+                file_obj.write(line + "\n")
+                line = "| ------- | ------- | ---- |"
+                file_obj.write(line + "\n")
+                for target, target_dict in sorted_prog_data.items():
+                    keys = list(target_dict.keys())
+                    line = f"| {target} | {target_dict['version']} |"
+                    date_key = "url_download_asset_date"
+                    if date_key in keys:
+                        line += f" {target_dict[date_key]} |"
+                    else:
+                        line += " |"
+                    line += "\n"
+                    file_obj.write(line)
 
         return
 

pymake/utils/usgsprograms.txt

@@ -1,7 +1,7 @@
 target     ,  version,  current, url                                                                                            , dirname             , srcdir              ,    standard_switch, double_switch, shared_object
-mf6        ,  6.4.0  ,  True   , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.0/mf6.4.0_linux.zip             , mf6.4.0             , src                 ,    True           , False        , False
-zbud6      ,  6.4.0  ,  True   , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.0/mf6.4.0_linux.zip             , mf6.4.0            , utils/zonebudget/src,    True           , False        , False
-libmf6     ,  6.4.0  ,  True   , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.0/mf6.4.0_linux.zip             , mf6.4.0            , srcbmi              ,    True           , False        , True
+mf6        ,  6.4.1  ,  True   , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.1/mf6.4.1_linux.zip             , mf6.4.1_linux       , src                 ,    True           , False        , False
+zbud6      ,  6.4.1  ,  True   , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.1/mf6.4.1_linux.zip             , mf6.4.1_linux       , utils/zonebudget/src,    True           , False        , False
+libmf6     ,  6.4.1  ,  True   , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.1/mf6.4.1_linux.zip             , mf6.4.1_linux       , srcbmi              ,    True           , False        , True
 mp7        ,  7.2.001,  True   , https://water.usgs.gov/water-resources/software/MODPATH/modpath_7_2_001.zip                    , modpath_7_2_001     , source              ,    True           , False        , False
 mt3dms     ,  5.3.0  ,  True   , https://hydro.geo.ua.edu/mt3d/mt3dms_530.exe                                                   , mt3dms5.3.0         , src/true-binary     ,    True           , False        , False
 mt3dusgs   ,  1.1.0  ,  True   , https://water.usgs.gov/water-resources/software/MT3D-USGS/mt3dusgs1.1.0.zip                    , mt3dusgs1.1.0       , src                 ,    True           , False        , False

pytest.ini

@@ -1,5 +1,6 @@
 [pytest]
 markers =
     base: base tests
-    regression: base and regression tests
+    regression: regression tests
     requests: usgsprograms requests tests
+    schedule: tests to run if a scheduled job