Bugfixes to sending user feedback and consolidating env var (#261)

* ENV var bugfixes, send user feedback fixes

* Slack template

* Bump version

Author: karthikscale3

src/examples/inspect_ai_example/basic_eval.py

@@ -1,10 +1,12 @@
 import fsspec
+from dotenv import find_dotenv, load_dotenv
 from inspect_ai import Task, task
 from inspect_ai.dataset import csv_dataset, Sample
 from inspect_ai.scorer import model_graded_qa
 from inspect_ai.solver import chain_of_thought, self_critique
 from langtrace_python_sdk.extensions.langtrace_filesystem import LangTraceFileSystem
 
+_ = load_dotenv(find_dotenv())
 
 # Manually register the filesystem with fsspec
 # Note: This is only necessary because the filesystem is not registered.
@@ -24,9 +26,9 @@ def hydrate_with_question(record):
 
 
 @task
-def pricing_question():
+def basic_eval():
     return Task(
-        dataset=csv_dataset("langtracefs://clyythmcs0001145cuvi426zi", hydrate_with_question),
+        dataset=csv_dataset("langtracefs://clz0p4i1t000fwv0xjtlvkxyx"),
         plan=[chain_of_thought(), self_critique()],
         scorer=model_graded_qa(),
     )

src/examples/openai_example/send_user_feedback.py

@@ -0,0 +1,40 @@
+from dotenv import find_dotenv, load_dotenv
+from openai import OpenAI
+from langtrace_python_sdk import langtrace, with_langtrace_root_span, SendUserFeedback
+
+_ = load_dotenv(find_dotenv())
+
+# Initialize Langtrace SDK
+langtrace.init()
+client = OpenAI()
+
+
+def api(span_id, trace_id):
+    response = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "user", "content": "What is the best place to live in the US?"},
+        ],
+        stream=False,
+    )
+
+    # Collect user feedback and send it to Langtrace
+    user_score = 1  # Example user score
+    user_id = 'user_1234'  # Example user ID
+    data = {
+        "userScore": user_score,
+        "userId": user_id,
+        "spanId": span_id,
+        "traceId": trace_id
+    }
+    SendUserFeedback().evaluate(data=data)
+
+    # Return the response
+    return response.choices[0].message.content
+
+
+# wrap the API call with the Langtrace root span
+wrapped_api = with_langtrace_root_span()(api)
+
+# Call the wrapped API
+wrapped_api()

src/examples/routellm_example/basic.py

@@ -0,0 +1,41 @@
+import sys
+
+sys.path.insert(0, "/Users/karthikkalyanaraman/work/langtrace/langtrace-python-sdk/src")
+
+from langtrace_python_sdk import langtrace
+from langtrace_python_sdk.utils.with_root_span import with_langtrace_root_span
+from routellm.controller import Controller
+from dotenv import load_dotenv
+
+load_dotenv()
+
+langtrace.init()
+
+# litellm.set_verbose=True
+client = Controller(
+    routers=["mf"],
+    strong_model="claude-3-opus-20240229",
+    weak_model="claude-3-opus-20240229",
+)
+
+
+@with_langtrace_root_span("Routellm")
+def Routellm(prompt):
+    try:
+
+        response = client.chat.completions.create(
+            model="router-mf-0.11593", messages=[{"role": "user", "content": prompt}]
+        )
+
+        for chunk in response:
+            if hasattr(chunk, "choices"):
+                print(chunk.choices[0].delta.content or "", end="")
+            else:
+                print(chunk)
+
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+
+Routellm("what is the square root of 12182382932.99")
+Routellm("Write me a short story")

src/langtrace_python_sdk/extensions/langtrace_filesystem.py

@@ -27,13 +27,25 @@ def __new__(cls, value):
 
 
 class LangTraceFile(io.BytesIO):
-    _host: str = os.environ.get("LANGTRACE_API_HOST", None) or LANGTRACE_REMOTE_URL
 
     def __init__(self, fs: "LangTraceFileSystem", path: str, mode: OpenMode):
         super().__init__()
         self.fs = fs
         self.path = path
         self.mode = mode
+        self._host: str = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL)
+        self._api_key: str = os.environ.get("LANGTRACE_API_KEY", None)
+        if self._host.endswith("/api/trace"):
+            self._host = self._host.replace("/api/trace", "")
+
+        if self._api_key is None:
+            print(Fore.RED)
+            print(
+                f"Missing Langtrace API key, proceed to {self._host} to create one"
+            )
+            print("Set the API key as an environment variable LANGTRACE_API_KEY")
+            print(Fore.RESET)
+            return
 
     def close(self) -> None:
         if not self.closed:
@@ -71,7 +83,7 @@ def upload_to_server(self, file_data: bytes) -> None:
                 data=json.dumps(data),
                 headers={
                     "Content-Type": "application/json",
-                    "x-api-key": os.environ.get("LANGTRACE_API_KEY"),
+                    "x-api-key": self._api_key,
                 },
                 timeout=20,
             )
@@ -82,14 +94,26 @@ def upload_to_server(self, file_data: bytes) -> None:
 
 
 class LangTraceFileSystem(AbstractFileSystem):
-    _host: str = os.environ.get("LANGTRACE_API_HOST", None) or LANGTRACE_REMOTE_URL
     protocol = "langtracefs"
     sep = "/"
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.files = {}
         self.dirs = set()
+        self._host: str = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL)
+        self._api_key: str = os.environ.get("LANGTRACE_API_KEY", None)
+        if self._host.endswith("/api/trace"):
+            self._host = self._host.replace("/api/trace", "")
+
+        if self._api_key is None:
+            print(Fore.RED)
+            print(
+                f"Missing Langtrace API key, proceed to {self._host} to create one"
+            )
+            print("Set the API key as an environment variable LANGTRACE_API_KEY")
+            print(Fore.RESET)
+            return
 
     def open(
         self,
@@ -118,7 +142,7 @@ def fetch_file_from_api(self, dataset_id: str) -> bytes:
                 url=f"{self._host}/api/dataset/download?id={dataset_id}",
                 headers={
                     "Content-Type": "application/json",
-                    "x-api-key": os.environ.get("LANGTRACE_API_KEY"),
+                    "x-api-key": self._api_key,
                 },
                 timeout=20,
             )

src/langtrace_python_sdk/utils/with_root_span.py

@@ -25,6 +25,9 @@
 from opentelemetry.trace import SpanKind
 from opentelemetry.trace.propagation import set_span_in_context
 
+from langtrace_python_sdk.constants.exporter.langtrace_exporter import (
+    LANGTRACE_REMOTE_URL,
+)
 from langtrace_python_sdk.constants.instrumentation.common import (
     LANGTRACE_ADDITIONAL_SPAN_ATTRIBUTES_KEY,
 )
@@ -142,7 +145,10 @@ class SendUserFeedback:
     _langtrace_api_key: str
 
     def __init__(self):
-        self._langtrace_host = os.environ["LANGTRACE_API_HOST"]
+        self._langtrace_host = os.environ.get("LANGTRACE_API_HOST", LANGTRACE_REMOTE_URL)
+        # When the host is set to /api/trace, remove the /api/trace
+        if self._langtrace_host.endswith("/api/trace"):
+            self._langtrace_host = self._langtrace_host.replace("/api/trace", "")
         self._langtrace_api_key = os.environ.get("LANGTRACE_API_KEY", None)
 
     def evaluate(self, data: EvaluationAPIData) -> None:
@@ -155,6 +161,16 @@ def evaluate(self, data: EvaluationAPIData) -> None:
                 print("Set the API key as an environment variable LANGTRACE_API_KEY")
                 print(Fore.RESET)
                 return
+
+            # convert spanId and traceId to hexadecimals
+            span_hex_number = hex(int(data["spanId"], 10))[2:]  # Convert to hex and remove the '0x' prefix
+            formatted_span_hex_number = span_hex_number.zfill(16)  # Pad with zeros to 16 characters
+            data["spanId"] = f"0x{formatted_span_hex_number}"
+
+            trace_hex_number = hex(int(data["traceId"], 10))[2:]  # Convert to hex and remove the '0x' prefix
+            formatted_trace_hex_number = trace_hex_number.zfill(32)  # Pad with zeros to 32 characters
+            data["traceId"] = f"0x{formatted_trace_hex_number}"
+
             evaluation = self.get_evaluation(data["spanId"])
             headers = {"x-api-key": self._langtrace_api_key}
             if evaluation is not None:

src/langtrace_python_sdk/version.py

@@ -1 +1 @@
-__version__ = "2.2.7"
+__version__ = "2.2.8"