Add telemetry

Author: YoanSallami

README.md

@@ -399,6 +399,12 @@ For more information, or help for implementing your ideas (or ones from a paper)
 
 Beware that every additional metric/module/optimizer should be approved by the core team, we want to keep the library minimal and clean as possible to avoid an uncontrolled growth leading to bad software practices like in most current leading LM frameworks.
 
+If you have specific feedbacks or features request we invite you to open an [issue](https://github.com/SynaLinks/synalinks/issues).
+
+### Telemetry
+
+We collect usage and software performance information to enhance and catch errors as fast as possible in order to make the best experience for you. No private, personal or confidential information is recorded. To get the complete list of what we collect, and how to disable it, please read the following [document](TELEMETRY.md).
+
 ### Community
 
 Join our community to learn more about neuro-symbolic systems and the future of AI. We welcome the participation of people from very different backgrounds or education levels.

TELEMETRY.md

@@ -0,0 +1,24 @@
+# Usage telemetry
+
+In order to enhance Synalinks and provide you with the best software possible we collect usage information with [Sentry](https://sentry.io/welcome/).
+
+Here is the list of what we collect, and the reason behind:
+
+- *Modules exceptions*: To solve errors before they can go into production
+- *Language & embedding models exceptions*: The monitor integrations stability
+- *Knowledge bases exceptions*: To monitor integrations stability
+- *Uncatched exceptions*: To enhance your experience by providing you with better error messages
+
+You can disable the telemetry by setting the following evironment variable `export SYNALINKS_TELEMETRY=false` in your `~/.bashrc` (or equivalent).
+
+OR you can use `synalinks.disable_telemetry()` at the beginning of your scripts
+
+```python
+import synalinks
+
+synalinks.disable_telemetry()
+```
+
+**Note**: We might collect more information in the future, but we will **NEVER** collect private, personal or confidential (like your prompts/data models) informations.
+
+If you have specific feedbacks or feature request we invite you to open an [issue](https://github.com/SynaLinks/synalinks/issues).

coverage-badge.svg

@@ -37,7 +37,8 @@ dependencies = [
     "graphviz",
     "click",
     "inquirer",
-    "neo4j>=5.28.1",
+    "neo4j",
+    "sentry-sdk",
 ]
 
 [project.urls]

pyproject.toml

@@ -9,12 +9,13 @@
 from synalinks.cli.utils.setup_utils import secrets_setup_config
 from synalinks.cli.utils.setup_utils import setup_project
 from synalinks.src.version import version as get_version
+from synalinks.src.backend.config import maybe_initialize_telemetry
 
 
 @click.group()
 def magic_cli():
     """Synalinks AI powered command-line interface."""
-    pass
+    maybe_initialize_telemetry()
 
 
 @magic_cli.command()

synalinks/cli/magic_cli.py

@@ -4,8 +4,8 @@
 
 import json
 import os
-
 import nest_asyncio
+import sentry_sdk
 
 from synalinks.src.api_export import synalinks_export
 
@@ -28,6 +28,78 @@
 # Available backends
 _AVAILABLE_BACKEND = ["pydantic"]
 
+# Enable/Disable telemetry
+_synalinks_telemetry_enabled = False
+
+# Check if sentry have been initialized
+_telemetry_initialized = False
+
+# Attemp to know if the telemetry is enabled from env variables
+if "SYNALINKS_TELEMETRY" in os.environ:
+    try:
+        _synalinks_telemetry_enabled = bool(os.environ.get("SYNALINKS_TELEMETRY"))
+    except Exception:
+        _synalinks_telemetry_enabled = True
+else:
+    _synalinks_telemetry_enabled = True
+
+
+@synalinks_export(
+    [
+        "synalinks.config.is_telemetry_enabled",
+        "synalinks.backend.is_telemetry_enabled",
+    ]
+)
+def is_telemetry_enabled():
+    """Return wether of not the telemetry is enabled
+    
+    Returns:
+        (bool): True if the telemetry is enabled False otherwise.
+    """
+    return _synalinks_telemetry_enabled
+
+
+@synalinks_export(
+    [
+        "synalinks.config.disable_telemetry",
+        "synalinks.disable_telemetry",
+    ]
+)
+def disable_telemetry():
+    """Disable the telemetry
+    
+    Returns:
+        (bool): True if the telemetry is enabled False otherwise.
+    """
+    global _synalinks_telemetry_enabled
+    _synalinks_telemetry_enabled = False
+
+
+def maybe_initialize_telemetry():
+    global _telemetry_initialized
+    if not _telemetry_initialized and _synalinks_telemetry_enabled:
+        sentry_sdk.init(
+            dsn="https://b53fc10adc81c3c865734c9b5115082a@o4509456897933312.ingest.de.sentry.io/4509456899637328", # noqa 501
+            max_breadcrumbs=50,
+            debug=False,
+            # we do NOT collect private/personal information
+            send_default_pii=False,
+            # Set traces_sample_rate to 1.0 to capture 100%
+            # of transactions for tracing.
+            traces_sample_rate=1.0,
+        )
+        _telemetry_initialized = True
+
+
+def capture_exception(error):
+    if is_telemetry_enabled():
+        sentry_sdk.capture_exception(error)
+
+
+def capture_message(msg):
+    if is_telemetry_enabled():
+        sentry_sdk.capture_message(msg)
+
 
 @synalinks_export(["synalinks.config.floatx", "synalinks.backend.floatx"])
 def floatx():
@@ -239,6 +311,7 @@ def set_backend(backend):
     _BACKEND = backend
 
 
+
 # Attemp to get the API key from env variables
 _api_key = os.getenv("SYNALINKS_API_KEY", None)
 if _api_key:
@@ -284,4 +357,4 @@ def set_backend(backend):
             f.write(json.dumps(_config, indent=4))
     except IOError:
         # Except permission denied.
-        pass
+        pass

synalinks/src/backend/config.py

@@ -7,6 +7,9 @@
 from synalinks.src.api_export import synalinks_export
 from synalinks.src.saving.synalinks_saveable import SynalinksSaveable
 
+from synalinks.src.backend.config import maybe_initialize_telemetry
+from synalinks.src.backend.config import capture_exception
+
 
 @synalinks_export(
     ["synalinks.EmbeddingModel", "synalinks.embedding_models.EmbeddingModel"]
@@ -82,9 +85,17 @@ class EmbeddingModel(SynalinksSaveable):
         model (str): The model to use.
         api_base (str): Optional. The endpoint to use.
         retry (int): Optional. The number of retry.
+        fallback (EmbeddingModel): Optional. The embedding model to fallback
+            if anything is wrong.
     """
 
-    def __init__(self, model=None, api_base=None, retry=5):
+    def __init__(
+        self,
+        model=None,
+        api_base=None,
+        retry=5,
+        fallback=None,
+    ):
         if model is None:
             raise ValueError(
                 "You need to set the `model` argument for any EmbeddingModel"
@@ -95,6 +106,7 @@ def __init__(self, model=None, api_base=None, retry=5):
         else:
             self.api_base = api_base
         self.retry = retry
+        self.fallback = fallback
 
     async def __call__(self, texts, **kwargs):
         """
@@ -106,6 +118,8 @@ async def __call__(self, texts, **kwargs):
         Returns:
             (list): The list of corresponding vectors.
         """
+        maybe_initialize_telemetry()
+        
         for i in range(self.retry):
             try:
                 if self.api_base:
@@ -127,24 +141,44 @@ async def __call__(self, texts, **kwargs):
                 return {"embeddings": vectors}
             except Exception as e:
                 warnings.warn(f"Error occured while trying to call {self}: " + str(e))
-        raise RuntimeError(
-            f"Failed to retrieve embeddings with {self} after {self.retry} attempts."
-        )
+                capture_exception(e)
+        if self.fallback:
+            return self.fallback(
+                texts,
+                **kwargs,
+            )
+        else:
+            return None
 
     def _obj_type(self):
         return "EmbeddingModel"
 
     def get_config(self):
-        return {
+        config = {
             "model": self.model,
             "api_base": self.api_base,
             "retry": self.retry,
         }
+        if self.fallback:
+            fallback_config = {
+                "fallback": serialization_lib.serialize_synalinks_object(
+                    self.fallback,
+                )
+            }
+            return {**fallback_config, **config}
+        else:
+            return config
 
     @classmethod
     def from_config(cls, config):
-        return cls(**config)
-
+        if "fallback" in config:
+            fallback = serialization_lib.deserialize_synalinks_object(
+                config.pop("fallback")
+            )
+            return cls(fallback=fallback, **config)
+        else:
+            return cls(**config)
+        
     def __repr__(self):
         api_base = f" api_base={self.api_base}" if self.api_base else ""
         return f"<EmbeddingModel model={self.model}{api_base}>"

synalinks/src/embedding_models/embedding_model.py

@@ -6,6 +6,8 @@
 from synalinks.src.saving import serialization_lib
 from synalinks.src.saving.synalinks_saveable import SynalinksSaveable
 
+from synalinks.src.backend.config import maybe_initialize_telemetry
+
 
 @synalinks_export("synalinks.KnowledgeBase")
 class KnowledgeBase(SynalinksSaveable):
@@ -45,7 +47,7 @@ class IsPartOf(synalinks.Relation):
     )
     ```
     
-    Note: Obviously, use an `.env` file and `.gitignore` to avoid putting your username and password
+    **Note**: Obviously, use an `.env` file and `.gitignore` to avoid putting your username and password
     in the code or a config file that can lead to leackage when pushing it into repositories.
 
     Args:
@@ -102,6 +104,7 @@ async def update(
                 Entities with similarity above this threshold will be merged.
                 Should be between 0.0 and 1.0 (Defaults to 0.8).
         """
+        maybe_initialize_telemetry()
         return await self.adapter.update(data_model)
 
     async def query(self, query: str):
@@ -114,6 +117,7 @@ async def query(self, query: str):
         Returns:
             (GenericResult): the query results
         """
+        maybe_initialize_telemetry()
         return await self.adapter.query(query)
 
     async def similarity_search(
@@ -135,6 +139,7 @@ async def similarity_search(
                 Entities with similarity below this threshold are excluded.
                 Should be between 0.0 and 1.0 (Defaults to 0.8).
         """
+        maybe_initialize_telemetry()
         return await self.adapter.similarity_search(
             similarity_search,
             k=k,
@@ -160,6 +165,7 @@ async def triplet_search(
                 Triplets with similarity below this threshold are excluded.
                 Should be between 0.0 and 1.0. (Defaults to 0.8).
         """
+        maybe_initialize_telemetry()
         return await self.adapter.triplet_search(
             triplet_search,
             k=k,

synalinks/src/knowledge_bases/knowledge_base.py

@@ -11,6 +11,9 @@
 from synalinks.src.saving import serialization_lib
 from synalinks.src.saving.synalinks_saveable import SynalinksSaveable
 
+from synalinks.src.backend.config import maybe_initialize_telemetry
+from synalinks.src.backend.config import capture_exception
+
 
 @synalinks_export(
     [
@@ -145,7 +148,6 @@ def __init__(
         model=None,
         api_base=None,
         retry=5,
-        timeout=30,
         fallback=None,
     ):
         if model is None:
@@ -162,7 +164,6 @@ def __init__(
         else:
             self.api_base = api_base
         self.retry = retry
-        self.timeout = timeout
 
     async def __call__(self, messages, schema=None, streaming=False, **kwargs):
         """
@@ -180,6 +181,8 @@ async def __call__(self, messages, schema=None, streaming=False, **kwargs):
         Returns:
             (dict): The generated structured response.
         """
+        maybe_initialize_telemetry()
+        
         formatted_messages = messages.get_json().get("messages", [])
         json_instance = {}
         input_kwargs = copy.deepcopy(kwargs)
@@ -293,7 +296,8 @@ async def __call__(self, messages, schema=None, streaming=False, **kwargs):
                     json_instance = {"role": ChatRole.ASSISTANT, "content": response_str}
                 return json_instance
             except Exception as e:
-                warnings.warn(str(e))
+                warnings.warn(f"Error occured while trying to call {self}: " + str(e))
+                capture_exception(e)
         if self.fallback:
             return self.fallback(
                 messages,

synalinks/src/language_models/language_model.py

@@ -21,6 +21,9 @@
 from synalinks.src.utils import python_utils
 from synalinks.src.utils import tracking
 
+from synalinks.src.backend.config import maybe_initialize_telemetry
+from synalinks.src.backend.config import capture_exception
+
 if backend.backend() == "pydantic":
     from synalinks.src.backend.pydantic.module import PydanticModule as BackendModule
 else:
@@ -494,6 +497,8 @@ def get_variable(self, name=None, index=None):
         )
 
     async def __call__(self, *args, **kwargs):
+        maybe_initialize_telemetry()
+        
         self._check_super_called()
         self._called = True
 
@@ -572,6 +577,9 @@ async def __call__(self, *args, **kwargs):
 
             if not self.built:
                 self.built = True
+        except Exception as e:
+            capture_exception(e)
+            raise e
         finally:
             # Destroy call context if we created it
             self._maybe_reset_call_context()