util: net: cached_download_unpack_archive

Signed-off-by: John Andersen <[email protected]>

Author: pdxjohnny

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Tensorflow hub NLP models.
 - Notes on development dependencies in `setup.py` files to codebase notes.
 - Test for `cached_download`
+- `dffml.util.net.cached_download_unpack_archive` to run a cached download and
+  unpack the archive, very useful for testing. Documented on the Networking
+  Helpers API docs page.
 - Directions on how to read the CI under the Git and GitHub page of the
   contributing documentation.
 - HTTP API

dffml/util/net.py

@@ -1,9 +1,12 @@
+import shutil
 import hashlib
 import pathlib
 import functools
 import urllib.request
 from typing import List
 
+from .os import chdir
+
 
 class HashValidationError(Exception):
     """
@@ -34,10 +37,16 @@ def __str__(self):
         return f"Protocol of URL {self.url!r} is not in allowlist: {self.allowlist!r}"
 
 
+# Default list of URL protocols allowed
 DEFAULT_PROTOCOL_ALLOWLIST: List[str] = ["https://"]
 
 
 def sync_urlopen(url, protocol_allowlist=DEFAULT_PROTOCOL_ALLOWLIST):
+    """
+    Check that ``url`` has a protocol defined in ``protocol_allowlist``, then
+    return the result of calling :py:func:`urllib.request.urlopen` passing it
+    ``url``.
+    """
     allowed_protocol = False
     for protocol in protocol_allowlist:
         if url.startswith(protocol):
@@ -54,6 +63,24 @@ def cached_download(
     expected_hash,
     protocol_allowlist=DEFAULT_PROTOCOL_ALLOWLIST,
 ):
+    """
+    Download a file and verify the hash of the downloaded file. If the file
+    already exists and the hash matches, do not re-download the file.
+
+    Examples
+    --------
+
+    >>> @cached_download(
+    ...     "https://github.com/intel/dffml/raw/152c2b92535fac6beec419236f8639b0d75d707d/MANIFEST.in",
+    ...     "MANIFEST.in",
+    ...     "f7aadf5cdcf39f161a779b4fa77ec56a49630cf7680e21fb3dc6c36ce2d8c6fae0d03d5d3094a6aec4fea1561393c14c",
+    ... )
+    ... async def first_line_in_manifest_152c2b(manifest):
+    ...     return manifest.read_text().split()[:2]
+    >>>
+    >>> asyncio.run(first_line_in_manifest_152c2b())
+    ['include', 'README.md']
+    """
     target_path = pathlib.Path(target_path)
 
     def validate_hash(error: bool = True):
@@ -69,7 +96,7 @@ def validate_hash(error: bool = True):
     def mkwrapper(func):
         @functools.wraps(func)
         async def wrapper(*args, **kwds):
-            args = list(args) + [str(target_path)]
+            args = list(args) + [target_path]
             if not target_path.is_file() or not validate_hash(error=False):
                 # TODO(p5) Blocking request in coroutine
                 with sync_urlopen(
@@ -82,3 +109,60 @@ async def wrapper(*args, **kwds):
         return wrapper
 
     return mkwrapper
+
+
+def cached_download_unpack_archive(
+    url,
+    file_path,
+    directory_path,
+    expected_hash,
+    protocol_allowlist=DEFAULT_PROTOCOL_ALLOWLIST,
+):
+    """
+    Download an archive and extract it to a directory on disk.
+
+    Verify the hash of the downloaded file. If the hash matches the file is not
+    re-downloaded.
+
+    .. warning::
+
+        This function does not verify the integrity of the unpacked archive on
+        disk. Only the downloaded file.
+
+    Examples
+    --------
+
+    >>> @cached_download_unpack_archive(
+    ...     "https://github.com/intel/dffml/archive/152c2b92535fac6beec419236f8639b0d75d707d.tar.gz",
+    ...     "dffml.tar.gz",
+    ...     "dffml",
+    ...     "32ba082cd8056ff4ddcb68691a590c3cb8fea2ff75c0265b8d844c5edc7eaef54136160c6090750e562059b957355b15",
+    ... )
+    ... async def files_in_dffml_commit_152c2b(dffml_dir):
+    ...     return len(list(dffml_dir.rglob("**/*")))
+    >>>
+    >>> asyncio.run(files_in_dffml_commit_152c2b())
+    594
+    """
+    directory_path = pathlib.Path(directory_path)
+
+    async def extractor(download_path):
+        download_path = download_path.absolute()
+        with chdir(directory_path):
+            shutil.unpack_archive(str(download_path), ".")
+
+    extract = cached_download(
+        url, file_path, expected_hash, protocol_allowlist=protocol_allowlist,
+    )(extractor)
+
+    def mkwrapper(func):
+        @functools.wraps(func)
+        async def wrapper(*args, **kwds):
+            if not directory_path.is_dir():
+                directory_path.mkdir(parents=True)
+                await extract()
+            return await func(*(list(args) + [directory_path]), **kwds)
+
+        return wrapper
+
+    return mkwrapper

docs/api/util/index.rst

@@ -1,5 +1,5 @@
-API Helper Utilities Reference
-==============================
+Utilities
+=========
 
 :py:mod:`asyncio` testing, command line, and configuration helpers live here.
 
@@ -9,3 +9,4 @@ API Helper Utilities Reference
     :caption: Contents:
 
     asynchelper
+    net

docs/api/util/net.rst

@@ -0,0 +1,5 @@
+Networking Helpers
+==================
+
+.. automodule:: dffml.util.net
+   :members:

docs/conf.py

@@ -88,10 +88,23 @@
 napoleon_numpy_docstring = True
 
 doctest_global_setup = """
+import os
 import sys
+import shutil
+import atexit
 import inspect
 import asyncio
+import tempfile
+import functools
+
+# Create a temporary directory for test to run in
+DOCTEST_TEMPDIR = tempfile.mkdtemp()
+# Remove it when the test exits
+atexit.register(functools.partial(shutil.rmtree, DOCTEST_TEMPDIR))
+# Change the current working directory to the temporary directory
+os.chdir(DOCTEST_TEMPDIR)
 
 from dffml.base import *
 from dffml.df.base import *
+from dffml.util.net import *
 """

scripts/docs.sh

@@ -10,9 +10,5 @@ ln -s "${PWD}/service/http/docs/" docs/plugins/service/http
 python3.7 scripts/docs.py
 python3.7 -c 'import os, pkg_resources; [e.load() for e in pkg_resources.iter_entry_points("console_scripts") if e.name.startswith("sphinx-build")][0]()' -b html docs pages \
   || (echo "[ERROR] Failed run sphinx, is it installed (pip install -U .[dev])?" 1>&2 ; exit 1)
-find pages/ -name \*.html -exec \
-  sed -i 's/<span class="gp">\&gt;\&gt;\&gt; <\/span>//g' {} \;
-find pages/ -name \*.html -exec \
-  sed -i 's/<span class="go">\&gt;\&gt;\&gt;<\/span>//g' {} \;
 cp -r docs/images pages/
 touch pages/.nojekyll

tests/source/test_idx.py

@@ -31,7 +31,7 @@ class TestIDXSources(AsyncTestCase):
     async def test_idx1(self, filename):
         feature_name = "label"
         async with IDX1Source(
-            IDX1SourceConfig(filename=filename, feature=feature_name)
+            IDX1SourceConfig(filename=str(filename), feature=feature_name)
         ) as source:
             async with source() as sctx:
                 records = [record async for record in sctx.records()]
@@ -43,7 +43,7 @@ async def test_idx1(self, filename):
     async def test_idx3(self, filename):
         feature_name = "image"
         async with IDX3Source(
-            IDX3SourceConfig(filename=filename, feature=feature_name)
+            IDX3SourceConfig(filename=str(filename), feature=feature_name)
         ) as source:
             async with source() as sctx:
                 records = [record async for record in sctx.records()]

tests/util/test_net.py

@@ -10,7 +10,7 @@
 import httptest
 
 from dffml.util.os import chdir
-from dffml.util.net import cached_download
+from dffml.util.net import cached_download, cached_download_unpack_archive
 from dffml.util.asynctestcase import AsyncTestCase
 
 
@@ -44,8 +44,17 @@ def do_GET(self):
 
 
 class TestNet(AsyncTestCase):
+    def verify_extracted_contents(self, extracted):
+        self.assertTrue((extracted / "somedir").is_dir())
+        self.assertEqual(
+            (extracted / "somedir" / "hello.txt").read_text(), "world"
+        )
+        self.assertEqual(
+            (extracted / "somedir" / "dead.bin").read_bytes(), b"\xBE\xEF"
+        )
+
     @httptest.Server(TestCachedDownloadServer)
-    async def test_call_response(self, ts=httptest.NoServer()):
+    async def test_cached_download(self, ts=httptest.NoServer()):
         with tempfile.TemporaryDirectory() as tempdir:
 
             @cached_download(
@@ -63,11 +72,22 @@ async def func(filename):
             # Unpack the archive
             shutil.unpack_archive(await func(), extracted)
 
-            # Verify contents are correct
-            self.assertTrue((extracted / "somedir").is_dir())
-            self.assertEqual(
-                (extracted / "somedir" / "hello.txt").read_text(), "world"
-            )
-            self.assertEqual(
-                (extracted / "somedir" / "dead.bin").read_bytes(), b"\xBE\xEF"
+            self.verify_extracted_contents(extracted)
+
+    @httptest.Server(TestCachedDownloadServer)
+    async def test_cached_download_unpack_archive(
+        self, ts=httptest.NoServer()
+    ):
+        with tempfile.TemporaryDirectory() as tempdir:
+
+            @cached_download_unpack_archive(
+                ts.url() + "/archive.tar.gz",
+                pathlib.Path(tempdir) / "archive.tar.gz",
+                pathlib.Path(tempdir) / "archive",
+                "cd538a17ce51458e3315639eba0650e96740d3d6abadbf174209ee7c5cae000ac739e99d9f32c9c2ba417b0cf67e69b8",
+                protocol_allowlist=["http://"],
             )
+            async def func(extracted):
+                return extracted
+
+            self.verify_extracted_contents(await func())