Tests: Improve test cases to verify universal source access

Author: amotl

hubspot_tech_writing/core.py

@@ -14,7 +14,7 @@
 from hubspot_tech_writing.hubspot_api import HubSpotAdapter, HubSpotBlogPost, HubSpotFile
 from hubspot_tech_writing.util.common import ContentTypeResolver
 from hubspot_tech_writing.util.html import HTMLImageTranslator
-from hubspot_tech_writing.util.io import path_from_url, to_io
+from hubspot_tech_writing.util.io import open_url, to_io
 
 logger = logging.getLogger(__name__)
 
@@ -77,7 +77,7 @@ def upload(
 ):
     source_path: Path
     if isinstance(source, str):
-        source_path = path_from_url(source)
+        source_path = open_url(source)
     else:
         source_path = source
     logger.info(f"Source: {source_path}")

hubspot_tech_writing/util/common.py

@@ -4,7 +4,8 @@
 import colorlog
 from colorlog.escape_codes import escape_codes
 from pathlibfs import Path
-from yarl import URL
+
+from hubspot_tech_writing.util.io import path_without_scheme
 
 
 def setup_logging(level=logging.INFO, verbose: bool = False):
@@ -25,10 +26,7 @@ class ContentTypeResolver:
     TEXT_SUFFIXES = MARKUP_SUFFIXES + HTML_SUFFIXES + [".txt"]
 
     def __init__(self, filepath: t.Union[str, Path]):
-        self.url = URL(str(filepath))
-        if self.url.is_absolute():
-            self.url = self.url.with_scheme("")
-        self.path = Path(str(self.url))
+        self.path = path_without_scheme(filepath)
         self.suffix = self.path.suffix
 
     def is_markup(self):
@@ -42,8 +40,3 @@ def is_text(self):
 
     def is_file(self):
         return not self.is_text()
-
-
-def url_to_path(filepath: str):
-    url = URL(str(filepath)).with_scheme("")
-    return Path(str(url))

hubspot_tech_writing/util/html.py

@@ -88,9 +88,8 @@ def upload(self) -> "HTMLImageTranslator":
             return self
         for image_local in self.images_local:
             hs_file = self.uploader(source=image_local.src, name=image_local.src.name)
-            image_url = hs_file.url
             image_remote: HTMLImage = deepcopy(image_local)
-            image_remote.src = image_url
+            image_remote.src = hs_file.url
             self.images_remote.append(image_remote)
         return self
 

hubspot_tech_writing/util/io.py

@@ -9,29 +9,29 @@
 
 @contextlib.contextmanager
 def to_io(source: t.Union[str, Path, t.IO]) -> t.Generator[t.IO, None, None]:
+    """
+    Main context manager for accessing resources.
+    Before accessing / opening, it converges a path string, object, or IO handle, to an IO handle.
+    """
     fp: t.IO
     if isinstance(source, io.TextIOWrapper):
         fp = source
     elif isinstance(source, (str, Path, PathPlus)):
         source = str(source)
-        path = path_from_url(source)
+        path = open_url(source)
         fp = path.open(mode="rt")
-        """
-        if source.startswith("http://") or source.startswith("https://"):
-            response = requests.get(source, timeout=10.0)
-            fp = io.StringIO(response.text)
-        else:
-            fp = open(source, "r")
-        """
     else:
         raise TypeError(f"Unable to converge to IO handle. type={type(source)}, value={source}")
     yield fp
     fp.close()
 
 
-def path_from_url(url: str) -> PathPlus:
+def open_url(url: str) -> PathPlus:
     """
-    Convert GitHub HTTP URL to pathlibfs / fsspec URL.
+    Access URL, with specific handling for GitHub URLs.
+
+    When approached using a GitHub HTTP URL, converge it to a pathlibfs / fsspec URL,
+    and open it.
 
     Input URLs
     ----------
@@ -54,7 +54,7 @@ def path_from_url(url: str) -> PathPlus:
         }
 
         real_path_fragments = path_fragments[2:]
-        if path_fragments[2] == "blob":
+        if path_fragments[2] in ["blob", "raw"]:
             real_path_fragments = path_fragments[4:]
 
         downstream_url = "github://" + "/".join(real_path_fragments)
@@ -63,3 +63,13 @@ def path_from_url(url: str) -> PathPlus:
     else:
         path = PathPlus(url)
     return path
+
+
+def path_without_scheme(url_like: str) -> PathPlus:
+    """
+    Return a pathlibfs Path, without the scheme.
+    """
+    url = URL(str(url_like))
+    if url.is_absolute():
+        url = url.with_scheme("")
+    return PathPlus(str(url))

tests/conftest.py

@@ -15,6 +15,30 @@ def markdownfile() -> Path:
     return Path(__file__).parent / "data" / "hubspot-blog-post-original.md"
 
 
+def get_markdownurl(infix: str = "", scheme: str = "https:") -> str:
+    return f"{scheme}//github.com/crate-workbench/hubspot-tech-writing/{infix}tests/data/hubspot-blog-post-original.md"
+
+
+@pytest.fixture
+def markdownurl_https_raw() -> str:
+    return get_markdownurl(infix="raw/main/")
+
+
+@pytest.fixture
+def markdownurl_github_https_bare() -> str:
+    return get_markdownurl(scheme="github+https:")
+
+
+@pytest.fixture
+def markdownurl_github_https_raw() -> str:
+    return get_markdownurl(infix="raw/main/", scheme="github+https:")
+
+
+@pytest.fixture
+def markdownurl_github_https_blob() -> str:
+    return get_markdownurl(infix="blob/main/", scheme="github+https:")
+
+
 @pytest.fixture
 def markdownfile_minimal_broken_links() -> Path:
     return Path(__file__).parent / "data" / "minimal-broken-links.md"

tests/test_core.py

@@ -13,9 +13,8 @@ def test_convert_file(markdownfile):
     check_content(html)
 
 
-def test_convert_url():
-    url = "https://github.com/crate-workbench/hubspot-tech-writing/raw/main/tests/data/hubspot-blog-post-original.md"
-    html = convert(url)
+def test_convert_url(markdownurl_https_raw):
+    html = convert(markdownurl_https_raw)
     check_content(html)
 
 

tests/test_hubspot_blogpost.py

@@ -1,19 +1,25 @@
-import json
 import os
 
 import pytest
-from hubspot.cms.blogs.blog_posts.rest import RESTResponse
-from urllib3 import HTTPResponse
 
 from hubspot_tech_writing.core import delete_blogpost, upload
+from tests.test_hubspot_file import response_simulator_upload
+from tests.util import mkresponse
 
 
-def mkresponse(data, status=200, reason="OK"):
-    body = json.dumps(data).encode("utf-8")
-    return RESTResponse(HTTPResponse(body=body, status=status, reason=reason))
+def response_simulator_create(self, method, url, **kwargs):
+    if method == "GET" and url == "https://api.hubapi.com/cms/v3/blogs/posts":
+        response = mkresponse({"total": 0, "results": []})
+    elif method == "POST" and url == "https://api.hubapi.com/cms/v3/blogs/posts":
+        response = mkresponse({"id": "12345"}, status=201, reason="Created")
+    elif method == "PATCH" and url == "https://api.hubapi.com/cms/v3/blogs/posts/12345":
+        response = mkresponse({"id": "12345"})
+    else:
+        raise ValueError(f"No HTTP conversation mock for: method={method}, url={url}")
+    return response
 
 
-def response_simulator_create(self, method, url, **kwargs):
+def response_simulator_create_with_image(self, method, url, **kwargs):
     if method == "GET" and url == "https://api.hubapi.com/cms/v3/blogs/posts":
         response = mkresponse({"total": 0, "results": []})
     elif method == "POST" and url == "https://api.hubapi.com/cms/v3/blogs/posts":
@@ -93,7 +99,7 @@ def test_upload_blogpost_create_from_markdown(hubspot_access_token, mocker, capl
 
 
 def test_upload_blogpost_update(hubspot_access_token, mocker, caplog, tmp_path):
-    tmpfile = tmp_path / "foo.html"
+    tmpfile = tmp_path / "foo.md"
     tmpfile.write_text("# Foobar\nFranz jagt im komplett verwahrlosten Taxi quer durch Bayern.")
 
     mocker.patch("hubspot.cms.blogs.blog_posts.rest.RESTClientObject.request", response_simulator_update)
@@ -109,6 +115,35 @@ def test_upload_blogpost_update(hubspot_access_token, mocker, caplog, tmp_path):
     assert "Saving blog post: HubSpotBlogPost identifier=12345, name=hstw-test" in caplog.text
 
 
+def test_upload_blogpost_with_image(hubspot_access_token, mocker, caplog, tmp_path):
+    mdfile = tmp_path / "foo.md"
+    pngfile = tmp_path / "images" / "bar.png"
+    pngfile.parent.mkdir()
+    mdfile.write_text("![bar](images/bar.png)")
+    pngfile.write_bytes(b"")
+
+    mocker.patch("hubspot.cms.blogs.blog_posts.rest.RESTClientObject.request", response_simulator_create_with_image)
+    mocker.patch("hubspot.files.files.rest.RESTClientObject.request", response_simulator_upload)
+    upload(
+        source=mdfile,
+        name="hstw-test",
+        content_group_id="55844199082",
+        folder_path="/path/to/assets",
+        access_token=hubspot_access_token,
+    )
+
+    assert "Uploading file:" in caplog.text
+
+    assert "Loading file: HubSpotFile identifier=None, name=bar.png, folder=/path/to/assets" in caplog.text
+    assert "Searching for 'bar.png' in folder path '/path/to/assets'" in caplog.text
+    assert "File does not exist: bar.png" in caplog.text
+    assert "Creating: HubSpotFile identifier=None, name=bar.png, folder=/path/to/assets" in caplog.text
+    assert "Saving file: HubSpotFile identifier=12345, name=bar.png, folder=/path/to/assets" in caplog.text
+
+    assert "Loading blog post: HubSpotBlogPost identifier=None, name=hstw-test" in caplog.text
+    assert "Saving blog post: HubSpotBlogPost identifier=12345, name=hstw-test" in caplog.text
+
+
 def test_delete_by_identifier(hubspot_access_token, mocker, caplog):
     mocker.patch.dict(os.environ, {"CONFIRM": "yes"})
     mocker.patch("hubspot.cms.blogs.blog_posts.rest.RESTClientObject.request", response_simulator_delete_id)

tests/test_hubspot_file.py

@@ -5,16 +5,18 @@
 
 from hubspot_tech_writing.core import delete_file, upload
 
-from .test_hubspot_blogpost import mkresponse
+from .util import mkresponse
 
 
 def response_simulator_upload(self, method, url, **kwargs):
     if method == "GET" and url == "https://api.hubapi.com/files/v3/files/search":
         response = mkresponse({"total": 0, "results": []})
     elif method == "POST" and url == "https://api.hubapi.com/files/v3/files":
-        response = mkresponse({"id": "12345"}, status=201, reason="Created")
+        response = mkresponse(
+            {"id": "12345", "url": "https://site.example/hubfs/any.png"}, status=201, reason="Created"
+        )
     elif method == "PUT" and url == "https://api.hubapi.com/files/v3/files/12345":
-        response = mkresponse({"id": "12345"})
+        response = mkresponse({"id": "12345", "url": "https://site.example/hubfs/any.png"})
     else:
         raise ValueError(f"No HTTP conversation mock for: method={method}, url={url}")
     return response

tests/test_util.py

@@ -0,0 +1,80 @@
+from hubspot_tech_writing.util.common import ContentTypeResolver
+from hubspot_tech_writing.util.io import open_url, path_without_scheme
+
+
+def test_content_type_resolver_local_html():
+    ctr = ContentTypeResolver("/path/to/document.html")
+    assert ctr.is_file() is False
+    assert ctr.is_text() is True
+    assert ctr.is_markup() is False
+    assert ctr.is_html() is True
+    assert ctr.suffix == ".html"
+
+
+def test_content_type_resolver_local_markdown():
+    ctr = ContentTypeResolver("/path/to/document.md")
+    assert ctr.is_file() is False
+    assert ctr.is_text() is True
+    assert ctr.is_markup() is True
+    assert ctr.is_html() is False
+    assert ctr.suffix == ".md"
+
+
+def test_content_type_resolver_local_text():
+    ctr = ContentTypeResolver("/path/to/document.txt")
+    assert ctr.is_file() is False
+    assert ctr.is_text() is True
+    assert ctr.is_markup() is False
+    assert ctr.is_html() is False
+    assert ctr.suffix == ".txt"
+
+
+def test_content_type_resolver_local_image():
+    ctr = ContentTypeResolver("/path/to/document.png")
+    assert ctr.is_file() is True
+    assert ctr.is_text() is False
+    assert ctr.is_markup() is False
+    assert ctr.is_html() is False
+    assert ctr.suffix == ".png"
+
+
+def test_content_type_resolver_remote_https():
+    ctr = ContentTypeResolver("https://site.example/path/to/document.md")
+    assert ctr.is_file() is False
+    assert ctr.is_text() is True
+    assert ctr.is_markup() is True
+    assert ctr.is_html() is False
+    assert ctr.suffix == ".md"
+
+
+def test_content_type_resolver_remote_github():
+    ctr = ContentTypeResolver("github://site.example/path/to/document.md")
+    assert ctr.is_file() is False
+    assert ctr.is_text() is True
+    assert ctr.is_markup() is True
+    assert ctr.is_html() is False
+    assert ctr.suffix == ".md"
+
+
+def test_content_type_resolver_remote_github_https():
+    ctr = ContentTypeResolver("github+https://site.example/path/to/document.md")
+    assert ctr.is_file() is False
+    assert ctr.is_text() is True
+    assert ctr.is_markup() is True
+    assert ctr.is_html() is False
+    assert ctr.suffix == ".md"
+
+
+def test_path_without_scheme_local():
+    assert str(path_without_scheme("/path/to/document.md")) == "/path/to/document.md"
+
+
+def test_path_without_scheme_url():
+    assert str(path_without_scheme("https://site.example/path/to/document.md")) == "//site.example/path/to/document.md"
+
+
+def test_path_from_url(markdownurl_github_https_bare, markdownurl_github_https_raw, markdownurl_github_https_blob):
+    reference = "github://tests/data/hubspot-blog-post-original.md"
+    assert str(open_url(markdownurl_github_https_bare)) == reference
+    assert str(open_url(markdownurl_github_https_raw)) == reference
+    assert str(open_url(markdownurl_github_https_blob)) == reference

tests/util.py

@@ -0,0 +1,9 @@
+import json
+
+from hubspot.cms.blogs.blog_posts.rest import RESTResponse
+from urllib3 import HTTPResponse
+
+
+def mkresponse(data, status=200, reason="OK"):
+    body = json.dumps(data).encode("utf-8")
+    return RESTResponse(HTTPResponse(body=body, status=status, reason=reason))