Replacement for #3125 (#3146)

* add tests to check SPARQLStore against public endpoints. By default it is disabled (slow, and public endpoints loves to go down whenever they can, so we don't want to have them run as part of the default CI/CD). We need to add the flag --public-endpoints to the pytest command to enable them. The list of tested endpoints is the same as the one used in SPARQLWrapper tests, with a new entry for a Qlever endpoint. Documentation about this have been added to the developers docs page

* Improve type annotations for the SPARQLStore, uses Literal, so that people using it will get proper autocomplete suggestion for the returnFormats and methods available. Add better docs on how to use SPARQLStore in its docstring

* Fix mypy type checking errors:  Unused "type: ignore" comment and  Unsupported operand types for + ("str" and "int")

* fix doctests

* improve mkdocs page linking

* add additions to developers.rst to developers.md

---------

Co-authored-by: Vincent Emonet <[email protected]>

Author: nicholascar

docs/developers.md

@@ -128,6 +128,18 @@ poetry install --all-extras
 poetry run pytest
 ```
 
+By default, tests of the `SPARQLStore` against remote public endpoints are skipped, to enable them add the flag:
+
+```bash
+poetry run pytest --public-endpoints
+```
+
+Or exclusively run the SPARQLStore tests:
+
+```bash
+poetry run pytest test/test_store/test_store_sparqlstore_public.py --public-endpoints
+```
+
 ### Writing tests
 
 New tests should be written for [pytest](https://docs.pytest.org/en/latest/) instead of for python's built-in `unittest` module as pytest provides advanced features such as parameterization and more flexibility in writing expected failure tests than `unittest`.

examples/custom_eval.py

@@ -6,7 +6,7 @@
 asking for `rdf:type` triples.
 
 Here the custom eval function is added manually, normally you would use
-entry points to do it. See the [Plugins Usage Documentation](/plugins/).
+entry points to do it. See the [Plugins Usage Documentation](../plugins.md).
 """
 
 from pathlib import Path

rdflib/compare.py

@@ -466,14 +466,14 @@ def _traces(
                 best_experimental_score = experimental_score
             elif best_score > color_score:
                 # prune this branch.
-                if stats is not None:
-                    stats["prunings"] = int(stats.get("prunings", 0)) + 1
+                if stats is not None and isinstance(stats["prunings"], int):
+                    stats["prunings"] += 1
             elif experimental_score != best_experimental_score:
                 best.append(refined_coloring)
             else:
                 # prune this branch.
-                if stats is not None:
-                    stats["prunings"] = int(stats.get("prunings", 0)) + 1
+                if stats is not None and isinstance(stats["prunings"], int):
+                    stats["prunings"] += 1
         discrete: list[list[Color]] = [x for x in best if self._discrete(x)]
         if len(discrete) == 0:
             best_score = None

rdflib/plugin.py

@@ -3,7 +3,7 @@
 either automatically through entry points or by calling
 rdf.plugin.register directly.
 
-For more details, see the [Plugins Usage Documentation](/plugins/).
+For more details, see the [Plugins Usage Documentation](../plugins.md).
 """
 
 from __future__ import annotations

rdflib/plugins/stores/sparqlconnector.py

@@ -17,6 +17,9 @@
 if TYPE_CHECKING:
     import typing_extensions as te
 
+    SUPPORTED_METHODS = te.Literal["GET", "POST", "POST_FORM"]
+    SUPPORTED_FORMATS = te.Literal["xml", "json", "csv", "tsv", "application/rdf+xml"]
+
 
 class SPARQLConnectorException(Exception):  # noqa: N818
     pass
@@ -41,8 +44,8 @@ def __init__(
         self,
         query_endpoint: str | None = None,
         update_endpoint: str | None = None,
-        returnFormat: str = "xml",  # noqa: N803
-        method: te.Literal["GET", "POST", "POST_FORM"] = "GET",
+        returnFormat: SUPPORTED_FORMATS = "xml",  # noqa: N803
+        method: SUPPORTED_METHODS = "GET",
         auth: tuple[str, str] | None = None,
         **kwargs,
     ):

rdflib/plugins/stores/sparqlstore.py

@@ -44,6 +44,7 @@
     )
     from rdflib.plugins.sparql.sparql import Query, Update
     from rdflib.query import Result, ResultRow
+    from .sparqlconnector import SUPPORTED_FORMATS, SUPPORTED_METHODS
 
 from .sparqlconnector import SPARQLConnector
 
@@ -67,11 +68,37 @@ def _node_to_sparql(node: Node) -> str:
 
 
 class SPARQLStore(SPARQLConnector, Store):
-    """An RDFLib store around a SPARQL endpoint
+    """An RDFLib store around a SPARQL endpoint.
 
     This is context-aware and should work as expected
     when a context is specified.
 
+    ### Usage example
+
+    ```python
+    >>> from rdflib import Dataset
+    >>> from rdflib.plugins.stores.sparqlstore import SPARQLStore
+    >>>
+    >>> g = Dataset(
+    ...    SPARQLStore("https://query.wikidata.org/sparql", returnFormat="xml"),
+    ...    default_union=True
+    ... )
+    >>>
+    >>> res = g.query("SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 5")
+    >>>
+    >>> # Iterate the results
+    >>> for row in res:
+    ...     pass  # but really you'd do something like: print(row)
+    >>>
+    >>> # Or serialize the results
+    >>> # something like: print(res.serialize(format="json").decode())
+    ```
+
+    !!! warning "Not all SPARQL endpoints support the same features"
+
+        Checkout the `test suite on public endpoints <https://github.com/RDFLib/rdflib/blob/main/test/test_store/test_store_sparqlstore_public.py>`_
+        for more details on how to successfully query different types of endpoints.
+
     For ConjunctiveGraphs, reading is done from the "default graph". Exactly
     what this means depends on your endpoint, because SPARQL does not offer a
     simple way to query the union of all graphs as it would be expected for a
@@ -83,7 +110,7 @@ class SPARQLStore(SPARQLConnector, Store):
 
     !!! warning "Blank nodes
 
-        By default the SPARQL Store does not support blank-nodes!
+        By default, the SPARQL Store does not support blank-nodes!
 
         As blank-nodes act as variables in SPARQL queries,
         there is no way to query for a particular blank node without
@@ -96,13 +123,15 @@ class SPARQLStore(SPARQLConnector, Store):
     "<bnode:b0001>", you can use a function like this:
 
     ```python
-    >>> def my_bnode_ext(node):
-    ...    if isinstance(node, BNode):
-    ...        return '<bnode:b%s>' % node
-    ...    return _node_to_sparql(node)
-    >>> store = SPARQLStore('http://dbpedia.org/sparql',
-    ...                     node_to_sparql=my_bnode_ext)
-
+    >> def my_bnode_ext(node):
+    ...     if isinstance(node, BNode):
+    ...         return f"<bnode:b{node}>"
+    ...     return _node_to_sparql(node)
+    ...
+    >> store = SPARQLStore(
+    ...     "http://dbpedia.org/sparql",
+    ...     node_to_sparql=my_bnode_ext
+    ... )
     ```
 
     You can request a particular result serialization with the
@@ -115,14 +144,11 @@ class SPARQLStore(SPARQLConnector, Store):
     urllib when doing HTTP calls. I.e. you have full control of
     cookies/auth/headers.
 
-    Form example:
+    HTTP basic auth is available with:
 
     ```python
-    >>> store = SPARQLStore('...my endpoint ...', auth=('user','pass'))
-
+    >> store = SPARQLStore('...my endpoint ...', auth=('user','pass'))
     ```
-
-    will use HTTP basic auth.
     """
 
     formula_aware = False
@@ -136,13 +162,15 @@ def __init__(
         sparql11: bool = True,
         context_aware: bool = True,
         node_to_sparql: _NodeToSparql = _node_to_sparql,
-        returnFormat: str = "xml",  # noqa: N803
+        returnFormat: SUPPORTED_FORMATS = "xml",  # noqa: N803
+        method: SUPPORTED_METHODS = "GET",
         auth: tuple[str, str] | None = None,
         **sparqlconnector_kwargs,
     ):
         super(SPARQLStore, self).__init__(
             query_endpoint=query_endpoint,
             returnFormat=returnFormat,
+            method=method,
             auth=auth,
             **sparqlconnector_kwargs,
         )

test/conftest.py

@@ -104,8 +104,15 @@ def exit_stack() -> Generator[ExitStack, None, None]:
 
 
 @pytest.hookimpl(tryfirst=True)
-def pytest_collection_modifyitems(items: Iterable[pytest.Item]):
+def pytest_collection_modifyitems(config: pytest.Config, items: Iterable[pytest.Item]):
     for item in items:
+        if config and not config.getoption("--public-endpoints", False):
+            # Skip tests marked with public_endpoints if the option is not provided
+            if "public_endpoints" in item.keywords:
+                item.add_marker(
+                    pytest.mark.skip(reason="need --public-endpoints option to run")
+                )
+
         parent_name = (
             str(Path(item.parent.module.__file__).relative_to(PROJECT_ROOT))
             if item.parent is not None
@@ -117,3 +124,19 @@ def pytest_collection_modifyitems(items: Iterable[pytest.Item]):
             extra_markers = EXTRA_MARKERS[(parent_name, item.name)]
             for extra_marker in extra_markers:
                 item.add_marker(extra_marker)
+
+
+def pytest_addoption(parser):
+    """Add optional pytest markers to run tests on public endpoints"""
+    parser.addoption(
+        "--public-endpoints",
+        action="store_true",
+        default=False,
+        help="run tests that require public SPARQL endpoints",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "public_endpoints: mark tests that require public SPARQL endpoints"
+    )