fix: Support for gitlab doc URLs

Author: Alan Christie

README.rst

@@ -34,15 +34,28 @@ there::
     pip install im-data-manager-job-decoder
 
 Once installed you can validate the definition (expected to be a dictionary
-formed from the definition YAML file) with:
+formed from the definition YAML file) with::
 
->>> from decoder import decoder
->>> error = decoder.validate_manifest_schema(manifest)
->>> error = decoder.validate_job_schema(job_definition)
+    >>> from decoder import decoder
+    >>> error: Optional[str] = decoder.validate_manifest_schema(manifest)
+    >>> error: Optional[str] = decoder.validate_job_schema(job_definition)
 
 And run the decoder with:
 
->>> decoded, success = decoder.decode(text, variables, 'command', decoder.TextEncoding.JINJA2_3_0)
+    >>> decoded, success = decoder.decode(text, variables, 'command', decoder.TextEncoding.JINJA2_3_0)
+
+A method exists to generate a Data Manger Job documentation URL
+for the supported repository types. For example, to get the
+auto-generated documentation URL for a GitHub repository-based
+job definition you can do this::
+
+    >>> doc_url: str = decoder.get_job_doc_url("github",
+                                               collection
+                                               job_id,
+                                               job_definition,
+                                               manifest_url)
+
+We support ``github`` and ``gitlab`` as repository types.
 
 .. _PyPI: https://pypi.org/project/im-data-manager-job-decoder
 

decoder/decoder.py

@@ -6,7 +6,7 @@
 """
 import enum
 import os
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 import jsonschema
 import yaml
@@ -40,6 +40,10 @@
     _MANIFEST_SCHEMA: Dict[str, Any] = yaml.load(schema_file, Loader=yaml.FullLoader)
 assert _MANIFEST_SCHEMA
 
+REPO_TYPE_GITHUB: str = "github"
+REPO_TYPE_GITLAB: str = "gitlab"
+_REPO_TYPES: List[str] = [REPO_TYPE_GITHUB, REPO_TYPE_GITLAB]
+
 
 class TextEncoding(enum.Enum):
     """A general text encoding format, used initially for Job text fields."""
@@ -81,14 +85,73 @@ def validate_job_schema(job_definition: Dict[str, Any]) -> Optional[str]:
     return None
 
 
+def get_supported_repository_types() -> List[str]:
+    """Returns a copy of the supported Git repository types."""
+    return _REPO_TYPES[:]
+
+
+def _get_github_job_doc_url(
+    manifest_url: str, collection: str, job_id: str, doc_url: Optional[str]
+) -> str:
+    """Returns the path to the doc for a GitHub public reference,
+    based on the manifest URL, collection and Job ID.
+    """
+    manifest_directory_url, _ = os.path.split(manifest_url)
+    if doc_url and not doc_url.startswith("https://"):
+        # doc-url defined but does not start 'https://'
+        # the doc_url is a path within the docs directory
+        doc_url = f"{manifest_directory_url}/docs/{doc_url}"
+    elif doc_url is None:
+        # No doc-url.
+        # The doc is expected to be in 'docs',
+        # as '{job}.md' in the {collection} directory.
+        doc_url = f"{manifest_directory_url}/docs/{collection}/{job_id}.md"
+    else:
+        # How did we get here?
+        assert False
+
+    return doc_url
+
+
+def _get_gitlab_job_doc_url(
+    manifest_url: str, collection: str, job_id: str, doc_url: Optional[str]
+) -> str:
+    """Returns the path to the doc for a GitLab reference,
+    based on the manifest URL, collection and Job ID.
+    """
+    manifest_directory_url, _ = manifest_url.split("%2f")
+    if doc_url and not doc_url.startswith("https://"):
+        # doc-url defined but does not start 'https://'
+        # the doc_url is a path within the docs directory
+        doc_url = doc_url.replace("/", "%2f")
+        doc_url = f"{manifest_directory_url}%2fdocs%2f{doc_url}/raw"
+    elif doc_url is None:
+        # No doc-url.
+        # The doc is expected to be in 'docs',
+        # as '{job}.md' in the {collection} directory.
+        doc_url = f"{manifest_directory_url}%2fdocs%2f{collection}%2f{job_id}.md/raw"
+    else:
+        # How did we get here?
+        assert False
+
+    return doc_url
+
+
 def get_job_doc_url(
-    collection: str, job: str, job_definition: Dict[str, Any], manifest_url: str
+    repo_type: str,
+    collection: str,
+    job: str,
+    job_definition: Dict[str, Any],
+    manifest_url: str,
 ) -> str:
     """Returns the Job's documentation URL for a specific Job using the JOb's
     'doc-url' and manifest URL. The job manifest is expected to
     have been validated, and we expect to have been given one job structure
     from the job's definition, not the whole job definition file.
+
+    repo_type must be one of the supported types.
     """
+    assert repo_type in _REPO_TYPES
     assert collection
     assert job
     assert isinstance(job_definition, dict)
@@ -101,12 +164,12 @@ def get_job_doc_url(
     # 3. If the doc-url does not begin https:// then is
     #    assumed to be relative to the manifest directory URL
 
-    # A typical manifest URl will look like this...
+    # A typical GitHub manifest URL will look like this...
     #
     #   https://raw.githubusercontent.com/InformaticsMatters/
     #       virtual-screening/main/data-manager/manifest-virtual-screening.yaml
     #
-    # The base for an auto-generated doc-url (if we need it)
+    # And, in this case the base for an auto-generated doc-url (if we need it)
     # will be: -
     #
     #   https://raw.githubusercontent.com/InformaticsMatters/
@@ -117,21 +180,14 @@ def get_job_doc_url(
     # If doc-url starts 'https://' just return it
     if doc_url and doc_url.startswith("https://"):
         return doc_url
-
-    # If the doc-url is set (it's not https://)
-    # so we assume is simply relative to the 'docs' directory
-    # where the manifest is found.
-    manifest_directory_url, _ = os.path.split(manifest_url)
-    if doc_url and not doc_url.startswith("https://"):
-        # doc-url defined but does not start 'https://'
-        doc_url = f"{manifest_directory_url}/docs/{doc_url}"
-    elif doc_url is None:
-        # No doc-url.
-        # The
-        doc_url = f"{manifest_directory_url}/docs/{collection}/{job}.md"
-    else:
-        # How did we get here?
-        assert False
+    if doc_url is not None:
+        assert not doc_url.startswith("/")
+        assert not doc_url.endswith("/")
+
+    if repo_type == REPO_TYPE_GITHUB:
+        doc_url = _get_github_job_doc_url(manifest_url, collection, job, doc_url)
+    elif repo_type == REPO_TYPE_GITLAB:
+        doc_url = _get_gitlab_job_doc_url(manifest_url, collection, job, doc_url)
 
     assert doc_url
     return doc_url

tests/test_get_job_doc_url.py

@@ -8,7 +8,7 @@
 from decoder import decoder
 
 
-def test_get_job_doc_url_with_fq_doc_url():
+def test_get_github_job_doc_url_with_fq_doc_url():
     # Arrange
     collection: str = "collection-x"
     job: str = "job-x"
@@ -23,14 +23,16 @@ def test_get_job_doc_url_with_fq_doc_url():
     expected_doc_url: str = "https://example.com/docs/doc.md"
 
     # Act
-    doc_url = decoder.get_job_doc_url(collection, job, job_definition, manifest_url)
+    doc_url = decoder.get_job_doc_url(
+        decoder.REPO_TYPE_GITHUB, collection, job, job_definition, manifest_url
+    )
 
     # Assert
     assert doc_url
     assert doc_url == expected_doc_url
 
 
-def test_get_job_doc_url_with_partial_doc_url():
+def test_get_github_job_doc_url_with_partial_doc_url():
     # Arrange
     collection: str = "collection-x"
     job: str = "job-x"
@@ -47,14 +49,16 @@ def test_get_job_doc_url_with_partial_doc_url():
     )
 
     # Act
-    doc_url = decoder.get_job_doc_url(collection, job, job_definition, manifest_url)
+    doc_url = decoder.get_job_doc_url(
+        decoder.REPO_TYPE_GITHUB, collection, job, job_definition, manifest_url
+    )
 
     # Assert
     assert doc_url
     assert doc_url == expected_doc_url
 
 
-def test_get_job_doc_url_with_no_doc_url():
+def test_get_github_job_doc_url_with_no_doc_url():
     # Arrange
     collection: str = "collection-x"
     job: str = "job-x"
@@ -71,7 +75,57 @@ def test_get_job_doc_url_with_no_doc_url():
     )
 
     # Act
-    doc_url = decoder.get_job_doc_url(collection, job, job_definition, manifest_url)
+    doc_url = decoder.get_job_doc_url(
+        decoder.REPO_TYPE_GITHUB, collection, job, job_definition, manifest_url
+    )
+
+    # Assert
+    assert doc_url
+    assert doc_url == expected_doc_url
+
+
+def test_get_gitlab_job_doc_url_with_partial_doc_url():
+    # Arrange
+    collection: str = "collection-x"
+    job: str = "job-x"
+    job_definition: Dict = {"doc-url": "special/doc.md"}
+    manifest_url: str = (
+        "https://gitlab.com/api/v4/projects/00000000/repository/files/"
+        "data-manager%2fmanifest.yaml/raw"
+    )
+    expected_doc_url: str = (
+        "https://gitlab.com/api/v4/projects/00000000/repository/files/"
+        "data-manager%2fdocs%2fspecial%2fdoc.md/raw"
+    )
+
+    # Act
+    doc_url = decoder.get_job_doc_url(
+        decoder.REPO_TYPE_GITLAB, collection, job, job_definition, manifest_url
+    )
+
+    # Assert
+    assert doc_url
+    assert doc_url == expected_doc_url
+
+
+def test_get_gitlab_job_doc_url_with_no_doc_url():
+    # Arrange
+    collection: str = "collection-x"
+    job: str = "job-x"
+    job_definition: Dict = {}
+    manifest_url: str = (
+        "https://gitlab.com/api/v4/projects/00000000/repository/files/"
+        "data-manager%2fmanifest.yaml/raw"
+    )
+    expected_doc_url: str = (
+        "https://gitlab.com/api/v4/projects/00000000/repository/files/"
+        "data-manager%2fdocs%2fcollection-x%2fjob-x.md/raw"
+    )
+
+    # Act
+    doc_url = decoder.get_job_doc_url(
+        decoder.REPO_TYPE_GITLAB, collection, job, job_definition, manifest_url
+    )
 
     # Assert
     assert doc_url