diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7429365..8bd7e07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,6 @@ jobs: run: | python -m build - name: Install package - run: python -m pip install --find-links=dist --no-index --ignore-installed docstring_to_markdown + run: python -m pip install --find-links=dist --ignore-installed docstring_to_markdown - name: Pip check run: python -m pip check diff --git a/README.md b/README.md index 94231e4..97a3703 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ On the fly conversion of Python docstrings to markdown -- Python 3.6+ (tested on 3.8 up to 3.13) +- Python 3.7+ (tested on 3.8 up to 3.13) - can recognise reStructuredText and convert multiple of its features to Markdown - since v0.13 includes initial support for Google-formatted docstrings @@ -35,6 +35,11 @@ Traceback (most recent call last): docstring_to_markdown.UnknownFormatError ``` +### Extensibility + +`docstring_to_markdown` entry point group allows to add custom converters which follow the `Converter` protocol. +The built-in converters can be customized by providing entry point with matching name. + ### Development ```bash diff --git a/docstring_to_markdown/__init__.py b/docstring_to_markdown/__init__.py index 62fa804..a9367f0 100644 --- a/docstring_to_markdown/__init__.py +++ b/docstring_to_markdown/__init__.py @@ -1,27 +1,59 @@ -from .cpython import cpython_to_markdown -from .google import google_to_markdown, looks_like_google -from .plain import looks_like_plain_text, plain_text_to_markdown -from .rst import looks_like_rst, rst_to_markdown +from importlib_metadata import entry_points +from typing import List, TYPE_CHECKING -__version__ = "0.15" +from .types import Converter + +if TYPE_CHECKING: + from importlib_metadata import EntryPoint + +__version__ = "0.16" class UnknownFormatError(Exception): pass -def convert(docstring: str) -> str: - if looks_like_rst(docstring): - return rst_to_markdown(docstring) +def _entry_points_sort_key(entry_point: 'EntryPoint'): + if entry_point.dist is None: + return 1 + if entry_point.dist.name == "docstring-to-markdown": + return 0 + return 1 + + +def _load_converters() -> List[Converter]: + converter_entry_points = entry_points( + group="docstring_to_markdown" + ) + # sort so that the default ones can be overridden + sorted_entry_points = sorted( + converter_entry_points, + key=_entry_points_sort_key + ) + # de-duplicate + unique_entry_points = {} + for entry_point in sorted_entry_points: + unique_entry_points[entry_point.name] = entry_point - if looks_like_google(docstring): - return google_to_markdown(docstring) + converters = [] + for entry_point in unique_entry_points.values(): + converter_class = entry_point.load() + converters.append(converter_class()) - if looks_like_plain_text(docstring): - return plain_text_to_markdown(docstring) + converters.sort(key=lambda converter: -converter.priority) - cpython = cpython_to_markdown(docstring) - if cpython: - return cpython + return converters + + +_CONVERTERS = None + + +def convert(docstring: str) -> str: + global _CONVERTERS + if _CONVERTERS is None: + _CONVERTERS = _load_converters() + for converter in _CONVERTERS: + if converter.can_convert(docstring): + return converter.convert(docstring) raise UnknownFormatError() diff --git a/docstring_to_markdown/cpython.py b/docstring_to_markdown/cpython.py index e2cae78..974ea60 100644 --- a/docstring_to_markdown/cpython.py +++ b/docstring_to_markdown/cpython.py @@ -1,8 +1,10 @@ from typing import Union, List from re import fullmatch +from .types import Converter from ._utils import escape_markdown + def _is_cpython_signature_line(line: str) -> bool: """CPython uses signature lines in the following format: @@ -30,8 +32,29 @@ def cpython_to_markdown(text: str) -> Union[str, None]: escape_markdown('\n'.join(other_lines)) ]) + def looks_like_cpython(text: str) -> bool: return cpython_to_markdown(text) is not None -__all__ = ['looks_like_cpython', 'cpython_to_markdown'] +class CPythonConverter(Converter): + + priority = 10 + + def __init__(self) -> None: + self._last_docstring: Union[str, None] = None + self._converted: Union[str, None] = None + + def can_convert(self, docstring): + self._last_docstring = docstring + self._converted = cpython_to_markdown(docstring) + return self._converted is not None + + def convert(self, docstring): + if docstring != self._last_docstring: + self._last_docstring = docstring + self._converted = cpython_to_markdown(docstring) + return self._converted + + +__all__ = ['looks_like_cpython', 'cpython_to_markdown', 'CPythonConverter'] diff --git a/docstring_to_markdown/google.py b/docstring_to_markdown/google.py index e3ca5a8..156f3da 100644 --- a/docstring_to_markdown/google.py +++ b/docstring_to_markdown/google.py @@ -2,6 +2,9 @@ from textwrap import dedent from typing import List +from .types import Converter + + # All possible sections in Google style docstrings SECTION_HEADERS: List[str] = [ "Args", @@ -169,3 +172,14 @@ def google_to_markdown(text: str, extract_signature: bool = True) -> str: docstring = GoogleDocstring(text) return docstring.as_markdown() + + +class GoogleConverter(Converter): + + priority = 75 + + def can_convert(self, docstring): + return looks_like_google(docstring) + + def convert(self, docstring): + return google_to_markdown(docstring) diff --git a/docstring_to_markdown/plain.py b/docstring_to_markdown/plain.py index d3bf7fb..3c42253 100644 --- a/docstring_to_markdown/plain.py +++ b/docstring_to_markdown/plain.py @@ -1,4 +1,5 @@ from re import fullmatch +from .types import Converter from ._utils import escape_markdown @@ -24,4 +25,16 @@ def looks_like_plain_text(value: str) -> bool: def plain_text_to_markdown(text: str) -> str: return escape_markdown(text) -__all__ = ['looks_like_plain_text', 'plain_text_to_markdown'] + +class PlainTextConverter(Converter): + + priority = 50 + + def can_convert(self, docstring): + return looks_like_plain_text(docstring) + + def convert(self, docstring): + return plain_text_to_markdown(docstring) + + +__all__ = ['looks_like_plain_text', 'plain_text_to_markdown', 'PlainTextConverter'] diff --git a/docstring_to_markdown/rst.py b/docstring_to_markdown/rst.py index ba50d91..4149773 100644 --- a/docstring_to_markdown/rst.py +++ b/docstring_to_markdown/rst.py @@ -4,6 +4,8 @@ from typing import Callable, Match, Union, List, Dict import re +from .types import Converter + class Directive: def __init__( @@ -532,7 +534,7 @@ def _consume_row(self, line: str): self._rows.append(self._split(line)) self._expecting_row_content = not self._expecting_row_content else: - self._state += 1 + self._state += 1 # pragma: no cover class BlockParser(IParser): @@ -651,11 +653,13 @@ def initiate_parsing(self, line: str, current_language: str): if line.strip() == '.. autosummary::': language = '' line = '' + suffix = '' else: line = re.sub(r'::$', '', line) + suffix = '\n\n' self._start_block(language) - return IBlockBeginning(remainder=line.rstrip() + '\n\n') + return IBlockBeginning(remainder=line.rstrip() + suffix) class MathBlockParser(IndentedBlockParser): @@ -825,3 +829,14 @@ def flush_buffer(): if active_parser: markdown += active_parser.finish_consumption(True) return markdown + + +class ReStructuredTextConverter(Converter): + + priority = 100 + + def can_convert(self, docstring): + return looks_like_rst(docstring) + + def convert(self, docstring): + return rst_to_markdown(docstring) diff --git a/docstring_to_markdown/types.py b/docstring_to_markdown/types.py new file mode 100644 index 0000000..ef7fb18 --- /dev/null +++ b/docstring_to_markdown/types.py @@ -0,0 +1,14 @@ +from typing_extensions import Protocol + + +class Converter(Protocol): + + def convert(self, docstring: str) -> str: + """Convert given docstring to markdown.""" + + def can_convert(self, docstring: str) -> bool: + """Check if conversion to markdown can be performed.""" + + # The higher the priority, the sooner the conversion + # with this converter will be attempted. + priority: int diff --git a/setup.cfg b/setup.cfg index 0bb013f..817ea9e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,21 +28,36 @@ warn_unused_configs = True [options] packages = find: -python_requires = >=3.6 +python_requires = >=3.7 +install_requires = + importlib-metadata>=3.6 + typing_extensions>=4.6 [options.package_data] docstring-to-markdown = py.typed +[options.entry_points] +docstring_to_markdown = + rst = docstring_to_markdown.rst:ReStructuredTextConverter + google = docstring_to_markdown.google:GoogleConverter + plain = docstring_to_markdown.plain:PlainTextConverter + cpython = docstring_to_markdown.cpython:CPythonConverter + [tool:pytest] addopts = --pyargs tests --cov docstring_to_markdown - --cov-fail-under=99 + --cov-fail-under=100 --cov-report term-missing:skip-covered -p no:warnings --flake8 -vv +[coverage:report] +exclude_lines = + pragma: no cover + if TYPE_CHECKING: + [flake8] max-line-length = 120 max-complexity = 15 diff --git a/tests/test_convert.py b/tests/test_convert.py index 7642aeb..5b030da 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,4 +1,10 @@ +from contextlib import contextmanager from docstring_to_markdown import convert, UnknownFormatError +from docstring_to_markdown.types import Converter +from docstring_to_markdown.cpython import CPythonConverter +from importlib_metadata import EntryPoint, entry_points, distribution +from unittest.mock import patch +import docstring_to_markdown import pytest CPYTHON = """\ @@ -55,3 +61,72 @@ def test_convert_rst(): def test_unknown_format(): with pytest.raises(UnknownFormatError): convert('ARGS [arg1, arg2] RETURNS: str OR None') + + +class HighPriorityConverter(Converter): + priority = 120 + + def convert(self, docstring): + return "HighPriority" + + def can_convert(self, docstring): + return True + + +class MockEntryPoint(EntryPoint): + def load(self): + return self.value + + dist = None + + +class DistMockEntryPoint(MockEntryPoint): + # Pretend it is contributed by `pytest`. + # It could be anything else, but `pytest` + # is guaranteed to be installed during tests. + dist = distribution('pytest') + + +class CustomCPythonConverter(CPythonConverter): + priority = 10 + + def convert(self, docstring): + return 'CustomCPython' + + def can_convert(self, docstring): + return True + + +@contextmanager +def custom_entry_points(entry_points): + old = docstring_to_markdown._CONVERTERS + docstring_to_markdown._CONVERTERS = None + with patch.object(docstring_to_markdown, 'entry_points', return_value=entry_points): + yield + docstring_to_markdown._CONVERTERS = old + + +def test_adding_entry_point(): + original_entry_points = entry_points(group="docstring_to_markdown") + mock_entry_point = MockEntryPoint( + name='high-priority-converter', + group='docstring_to_markdown', + value=HighPriorityConverter, + ) + with custom_entry_points([*original_entry_points, mock_entry_point]): + assert convert('test') == 'HighPriority' + + +def test_replacing_entry_point(): + assert convert(CPYTHON) == CPYTHON_MD + original_entry_points = entry_points(group="docstring_to_markdown") + mock_entry_point = DistMockEntryPoint( + name='cpython', + group='docstring_to_markdown', + value=CustomCPythonConverter + ) + with custom_entry_points([*original_entry_points, mock_entry_point]): + assert convert('test') == 'test' + assert convert(GOOGLE) == GOOGLE_MD + assert convert(RST) == RST_MD + assert convert(CPYTHON) == 'CustomCPython' diff --git a/tests/test_cpython.py b/tests/test_cpython.py index 2b7245d..b292c66 100644 --- a/tests/test_cpython.py +++ b/tests/test_cpython.py @@ -1,5 +1,5 @@ import pytest -from docstring_to_markdown.cpython import looks_like_cpython, cpython_to_markdown +from docstring_to_markdown.cpython import looks_like_cpython, cpython_to_markdown, CPythonConverter BOOL = """\ bool(x) -> bool @@ -101,3 +101,10 @@ def test_conversion_bool(): def test_conversion_str(): assert cpython_to_markdown(STR) == STR_MD + + +def test_convert(): + converter = CPythonConverter() + assert converter.can_convert(BOOL) + assert not converter.can_convert('this is plain text') + assert converter.convert(BOOL) == BOOL_MD diff --git a/tests/test_google.py b/tests/test_google.py index 78486bc..ef0d785 100644 --- a/tests/test_google.py +++ b/tests/test_google.py @@ -1,6 +1,6 @@ import pytest -from docstring_to_markdown.google import google_to_markdown, looks_like_google +from docstring_to_markdown.google import google_to_markdown, looks_like_google, GoogleConverter BASIC_EXAMPLE = """Do **something**. @@ -133,3 +133,10 @@ def test_looks_like_google_ignores_plain_text(): ) def test_google_to_markdown(google, markdown): assert google_to_markdown(google) == markdown + + +def test_converter(): + converter = GoogleConverter() + assert converter.can_convert(BASIC_EXAMPLE) + assert not converter.can_convert("This is plain text") + assert converter.convert(BASIC_EXAMPLE) == BASIC_EXAMPLE_MD diff --git a/tests/test_plain.py b/tests/test_plain.py index 61ef4b4..0e30eef 100644 --- a/tests/test_plain.py +++ b/tests/test_plain.py @@ -1,5 +1,5 @@ import pytest -from docstring_to_markdown.plain import looks_like_plain_text, plain_text_to_markdown +from docstring_to_markdown.plain import looks_like_plain_text, plain_text_to_markdown, PlainTextConverter @pytest.mark.parametrize("text", [ @@ -40,3 +40,10 @@ def test_rejects_code(text): def test_conversion(): assert plain_text_to_markdown("test") == "test" + + +def test_convert(): + converter = PlainTextConverter() + assert not converter.can_convert('def test():') + assert converter.can_convert('this is plain text') + assert converter.convert('test') == 'test' diff --git a/tests/test_rst.py b/tests/test_rst.py index 3830563..c645def 100644 --- a/tests/test_rst.py +++ b/tests/test_rst.py @@ -1,6 +1,6 @@ import pytest -from docstring_to_markdown.rst import looks_like_rst, rst_to_markdown +from docstring_to_markdown.rst import looks_like_rst, rst_to_markdown, ReStructuredTextConverter SEE_ALSO = """ @@ -119,6 +119,26 @@ A function definition is an executable statement. """ +RST_AUTOSUMMARY_BLOCK = """ +Summary + +.. autosummary:: + + environment.BuildEnvironment + util.relative_uri +""" + + +RST_AUTOSUMMARY_BLOCK_MARKDOWN = """ +Summary + +``` +environment.BuildEnvironment +util.relative_uri +``` +""" + + RST_COLON_CODE_BLOCK = """ For example, the following code :: @@ -598,6 +618,26 @@ def func(): pass """ +BROKEN_GRID_TABLE = """ ++------------+-----------+------------+-----------------+---+---------+ +|param_kernel|param_gamma|param_degree|split0_test_score|...|rank_t...| ++============+===========+============+=================+===+=========+ +| 'poly' | -- | 2 | 0.80 |...| 2 | ++------------+-----------+------------+-----------------+---+---------+ +| 'poly' | -- | 3 | 0.70 |...| 4 | +someone forgot to close the row above. +""" + + +BROKEN_GRID_TABLE_MARKDOWN = """ +| param_kernel | param_gamma | param_degree | split0_test_score | ... | rank_t... | +| ------------ | ----------- | ------------ | ----------------- | --- | --------- | +| 'poly' | -- | 2 | 0.80 | ... | 2 | +| 'poly' | -- | 3 | 0.70 | ... | 4 | +someone forgot to close the row above. +""" + + NESTED_PARAMETERS = """ Parameters ---------- @@ -733,6 +773,10 @@ def foo(): 'rst': NUMPY_EXAMPLE, 'md': NUMPY_EXAMPLE_MARKDOWN }, + 'converts autosummary block': { + 'rst': RST_AUTOSUMMARY_BLOCK, + 'md': RST_AUTOSUMMARY_BLOCK_MARKDOWN + }, 'converts version changed': { 'rst': '.. versionchanged:: 0.23.0', 'md': '*Changed in 0.23.0*' @@ -835,6 +879,10 @@ def foo(): 'rst': GRID_TABLE_IN_SKLEARN, 'md': GRID_TABLE_IN_SKLEARN_MARKDOWN }, + 'converts broken grid table': { + 'rst': BROKEN_GRID_TABLE, + 'md': BROKEN_GRID_TABLE_MARKDOWN + }, 'converts nested parameter lists': { 'rst': NESTED_PARAMETERS, 'md': NESTED_PARAMETERS_MARKDOWN @@ -888,3 +936,10 @@ def test_rst_to_markdown(rst, markdown): converted = rst_to_markdown(rst) print(converted) assert converted == markdown + + +def test_converter(): + converter = ReStructuredTextConverter() + assert converter.can_convert('.. versionadded:: 0.1') + assert not converter.can_convert('this is plain text') + assert converter.convert(PEP_287_CODE_BLOCK) == PEP_287_CODE_BLOCK_MARKDOWN