Merge branch 'master' into develop

Author: rememberYou

CONTRIBUTING.rst

@@ -219,19 +219,19 @@ To achieve this, there are 5 points there are 5 points to follow:
         """Defines Foo embedding technique."""
 
         def fit(self,  walks: List[List[SWalk]], is_updated: bool = False) -> Embedder:
-           """Fits the model based on provided walks.
+            """Fits the model based on provided walks.
 
-           Args:
-               walks: The walks to create the corpus to to fit the model.
-               is_update: True if the new walks should be added to old model's
-                   walks, False otherwise.
-                   Defaults to False.
+            Args:
+                walks: The walks to create the corpus to to fit the model.
+                is_update: True if the new walks should be added to old model's
+                    walks, False otherwise.
+                    Defaults to False.
 
-           Returns:
-               The fitted Foo model.
+            Returns:
+                The fitted Foo model.
 
-        """
-        # TODO: to be implemented
+            """
+            # TODO: to be implemented
 
         def transform(self, entities: Entities) -> Embeddings:
             """The features vector of the provided entities.
@@ -244,14 +244,14 @@ To achieve this, there are 5 points there are 5 points to follow:
             Returns:
                 The features vector of the provided entities.
 
-        """
-        # TODO: to be implemented
+            """
+            # TODO: to be implemented
 
 4. **Create unit tests of your embedding technique:**
 
 Create a ``tests/embedders/foo.py`` file and see `how the tests are done for
 Word2Vec
-<https://github.com/IBCNServices/pyRDF2Vec/blob/master/tests/embedders/word2vec.py>`__
+<https://github.com/IBCNServices/pyRDF2Vec/blob/master/tests/embedders/test_word2vec.py>`__
 as an example.
 
 Once this is done, run your tests:

README.rst

@@ -100,7 +100,7 @@ embeddings and get literals from a given Knowledge Graph (KG) and entities:
    # ]
 
    transformer = RDF2VecTransformer(
-       Word2Vec(iter=10),
+       Word2Vec(epochs=10),
        walkers=[RandomWalker(4, 10, with_reverse=False, n_jobs=2)],
        # verbose=1
    )

pyrdf2vec/connectors.py

@@ -11,8 +11,6 @@
 import requests
 from cachetools import Cache, TTLCache, cachedmethod
 from cachetools.keys import hashkey
-from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util import Retry
 
 from pyrdf2vec.typings import Literal, Response
 
@@ -24,8 +22,6 @@ class Connector(ABC):
     Attributes:
         _asession: The aiohttp session to use for asynchrone requests.
             Defaults to None.
-        _session: The requests session to use for synchrone requests.
-            Defaults to requests.Session.
         _headers: The HTTP headers to use.
             Defaults to {"Accept": "application/sparql-results+json"}.
         cache: The policy and size cache to use.
@@ -50,16 +46,10 @@ class Connector(ABC):
         init=False,
         type=Dict[str, str],
         repr=False,
-        default={
-            "Accept": "application/sparql-results+json",
-        },
+        default={"Accept": "application/sparql-results+json"},
     )
 
     _asession = attr.ib(init=False, default=None)
-    _session = attr.ib(
-        init=False,
-        factory=lambda: requests.Session(),
-    )
 
     async def close(self) -> None:
         """Closes the aiohttp session."""
@@ -73,7 +63,7 @@ def fetch(self, query: str):
             query: The query to fetch the result
 
         Returns:
-            The generated dictionary from the ['results']['bindings'] json.
+            The JSON response.
 
         Raises:
             NotImplementedError: If this method is called, without having
@@ -90,8 +80,6 @@ class SPARQLConnector(Connector):
     Attributes:
         _asession: The aiohttp session to use for asynchrone requests.
             Defaults to None.
-        _session: The requests session to use for synchrone requests.
-            Defaults to requests.Session.
         _headers: The HTTP headers to use.
             Defaults to {"Accept": "application/sparql-results+json"}.
         cache: The policy and size cache to use.
@@ -100,17 +88,6 @@ class SPARQLConnector(Connector):
 
     """
 
-    def __attrs_post_init__(self):
-        adapter = HTTPAdapter(
-            Retry(
-                total=3,
-                status_forcelist=[429, 500, 502, 503, 504],
-                method_whitelist=["HEAD", "GET", "OPTIONS"],
-            )
-        )
-        self._session.mount("http", adapter)
-        self._session.mount("https", adapter)
-
     async def afetch(self, queries: List[str]) -> List[List[Response]]:
         """Fetchs the result of SPARQL queries asynchronously.
 
@@ -141,8 +118,7 @@ async def _fetch(self, query) -> Response:
         """
         url = f"{self.endpoint}/query?query={parse.quote(query)}"
         async with self._asession.get(url, headers=self._headers) as res:
-            res = await res.json()
-            return res["results"]["bindings"]
+            return await res.json()
 
     @cachedmethod(operator.attrgetter("cache"), key=partial(hashkey, "fetch"))
     def fetch(self, query: str) -> Response:
@@ -156,8 +132,8 @@ def fetch(self, query: str) -> Response:
 
         """
         url = f"{self.endpoint}/query?query={parse.quote(query)}"
-        res = self._session.get(url, headers=self._headers).json()
-        return res["results"]["bindings"]
+        with requests.get(url, headers=self._headers) as res:
+            return res.json()
 
     def get_query(self, entity: str, preds: Optional[List[str]] = None) -> str:
         """Gets the SPARQL query for an entity.

pyrdf2vec/graphs/kg.py

@@ -53,6 +53,10 @@ class KG:
             Defaults to False.
         skip_predicates: The label predicates to skip from the KG.
             Defaults to set.
+        skip_verify: To skip or not the verification of existing entities in a
+            Knowledge Graph. Its deactivation can improve HTTP latency for KG
+            remotes.
+            Defaults to False.
 
     """
 
@@ -95,6 +99,13 @@ class KG:
         validator=attr.validators.instance_of(bool),
     )
 
+    skip_verify = attr.ib(
+        kw_only=True,
+        type=bool,
+        default=False,
+        validator=attr.validators.instance_of(bool),
+    )
+
     cache = attr.ib(
         kw_only=True,
         type=Cache,
@@ -206,9 +217,6 @@ def add_walk(self, subj: Vertex, pred: Vertex, obj: Vertex) -> bool:
             return True
         return False
 
-    @cachedmethod(
-        operator.attrgetter("cache"), key=partial(hashkey, "fetch_hops")
-    )
     def fetch_hops(self, vertex: Vertex) -> List[Hop]:
         """Fetchs the hops of the vertex from a SPARQL endpoint server and
         add the hops for this vertex in a cache dictionary.
@@ -229,7 +237,7 @@ def fetch_hops(self, vertex: Vertex) -> List[Hop]:
             "https://"
         ):
             res = self.connector.fetch(self.connector.get_query(vertex.name))
-            hops = self._res2hops(vertex, res)
+            hops = self._res2hops(vertex, res["results"]["bindings"])
         return hops
 
     def get_hops(self, vertex: Vertex, is_reverse: bool = False) -> List[Hop]:
@@ -283,7 +291,10 @@ def get_literals(self, entities: Entities, verbose: int = 0) -> Literals:
                 responses = [self.connector.fetch(query) for query in queries]
 
             literals_responses = [
-                self.connector.res2literals(res) for res in responses
+                self.connector.res2literals(
+                    res["results"]["bindings"]  # type: ignore
+                )
+                for res in responses
             ]
             return [
                 literals_responses[
@@ -340,6 +351,31 @@ def get_pliterals(self, entity: str, preds: List[str]) -> List[str]:
             frontier = new_frontier
         return list(frontier)
 
+    def is_exist(self, entities: Entities) -> bool:
+        """Checks that all provided entities exists in the Knowledge Graph.
+
+        Args:
+            entities: The entities to check the existence
+
+        Returns:
+            True if all the entities exists, False otherwise.
+
+        """
+        if self._is_remote:
+            queries = [
+                f"ASK WHERE {{ <{entity}> ?p ?o . }}" for entity in entities
+            ]
+            if self.mul_req:
+                responses = [
+                    res["boolean"]  # type: ignore
+                    for res in asyncio.run(self.connector.afetch(queries))
+                ]
+            else:
+                responses = [self.connector.fetch(query) for query in queries]
+                responses = [res["boolean"] for res in responses]
+            return False not in responses
+        return all([Vertex(entity) in self._vertices for entity in entities])
+
     def remove_edge(self, v1: Vertex, v2: Vertex) -> bool:
         """Removes the edge (v1 -> v2) if present.
 
@@ -403,7 +439,9 @@ def _fill_hops(self, entities: Entities) -> None:
             entities,
             asyncio.run(self.connector.afetch(queries)),
         ):
-            hops = self._res2hops(Vertex(entity), res)
+            hops = self._res2hops(
+                Vertex(entity), res["results"]["bindings"]  # type: ignore
+            )
             self._entity_hops.update({entity: hops})
 
     @cachedmethod(
@@ -418,8 +456,8 @@ def _get_hops(self, vertex: Vertex, is_reverse: bool = False) -> List[Hop]:
                 vertex. Otherwise, get the child nodes for this vertex.
                 Defaults to False.
 
-        Returns:
-            The hops of a vertex in a (predicate, object) form.
+         Returns:
+             The hops of a vertex in a (predicate, object) form.
 
         """
         matrix = self._transition_matrix
@@ -452,6 +490,6 @@ def _res2hops(self, vertex: Vertex, res) -> List[Hop]:
                 vprev=vertex,
                 vnext=obj,
             )
-            if self.add_walk(vertex, pred, obj):
+            if pred.name not in self.skip_predicates:
                 hops.append((pred, obj))
         return hops

pyrdf2vec/rdf2vec.py

@@ -8,7 +8,7 @@
 import attr
 
 from pyrdf2vec.embedders import Embedder, Word2Vec
-from pyrdf2vec.graphs import KG, Vertex
+from pyrdf2vec.graphs import KG
 from pyrdf2vec.typings import Embeddings, Entities, Literals, SWalk
 from pyrdf2vec.walkers import RandomWalker, Walker
 
@@ -160,15 +160,15 @@ def get_walks(self, kg: KG, entities: Entities) -> List[List[SWalk]]:
             ValueError: If the provided entities aren't in the Knowledge Graph.
 
         """
-        if not kg._is_remote and not all(
-            [Vertex(entity) in kg._vertices for entity in entities]
-        ):
-            raise ValueError(
-                "The provided entities must be in the Knowledge Graph."
-            )
-
         # Avoids duplicate entities for unnecessary walk extractions.
         entities = list(set(entities))
+        if kg.skip_verify is False and not kg.is_exist(entities):
+            if kg.mul_req:
+                asyncio.run(kg.connector.close())
+            raise ValueError(
+                "At least one provided entity does not exist in the "
+                + "Knowledge Graph."
+            )
 
         if self.verbose == 2:
             print(kg)

pyrdf2vec/utils/validation.py

@@ -91,6 +91,9 @@ def is_valid_url(url: str) -> bool:
 
     """
     try:
-        return requests.get(url).status_code == requests.codes.ok
+        return (
+            requests.head(url, headers={"Accept": "text/html"}).status_code
+            == requests.codes.ok
+        )
     except Exception:
         return False

tests/test_graph.py

@@ -219,6 +219,20 @@ def test_invalid_url(self):
         with pytest.raises(ValueError):
             KG("http://foo")
 
+    def test_is_exist(self, setup):
+        assert LOCAL_KG.is_exist([f"{URL}#Alice", "foo"]) is False
+        assert (
+            LOCAL_KG.is_exist(
+                [
+                    f"{URL}#Alice",
+                    f"{URL}#Bob",
+                    f"{URL}#Casper",
+                    f"{URL}#Dean",
+                ]
+            )
+            is True
+        )
+
     def test_remove_edge(self, setup):
         vtx_alice = Vertex(f"{URL}#Alice")