fix: total tokens and docs

Author: PeriniM

docs/source/scrapers/telemetry.rst

@@ -29,6 +29,7 @@ Additionally, the following properties are collected:
        "source_type": source_type,
        "execution_time": execution_time,
        "error_node": error_node_name,
+       "total_tokens": total_tokens,
    }
 
 For more details, refer to the `telemetry.py <https://github.com/VinciGit00/Scrapegraph-ai/blob/main/scrapegraphai/telemetry/telemetry.py>`_ module.

scrapegraphai/graphs/base_graph.py

@@ -140,6 +140,16 @@ def _execute_standard(self, initial_state: dict) -> Tuple[dict, list]:
                     result = current_node.execute(state)
                 except Exception as e:
                     error_node = current_node.node_name
+                    
+                    graph_execution_time = time.time() - start_time
+                    log_graph_execution(
+                        graph_name=self.graph_name,
+                        llm_model=llm_model,
+                        embedder_model=embedder_model,
+                        source_type=source_type,
+                        execution_time=graph_execution_time,
+                        error_node=error_node
+                    )
                     raise e
                 node_exec_time = time.time() - curr_time
                 total_exec_time += node_exec_time
@@ -187,7 +197,7 @@ def _execute_standard(self, initial_state: dict) -> Tuple[dict, list]:
             embedder_model=embedder_model,
             source_type=source_type,
             execution_time=graph_execution_time,
-            error_node=error_node
+            total_tokens=cb_total["total_tokens"] if cb_total["total_tokens"] > 0 else None,
         )
 
         return state, exec_info

scrapegraphai/telemetry/telemetry.py

@@ -156,14 +156,15 @@ def log_event(event: str, properties: Dict[str, any]):
         send_event_json(event_json)
 
 
-def log_graph_execution(graph_name: str, llm_model: str, embedder_model: str, source_type: str, execution_time: float, error_node: str = None):
+def log_graph_execution(graph_name: str, llm_model: str, embedder_model: str, source_type: str, execution_time: float, error_node: str = None, total_tokens: int = None):
     properties = {
         "graph_name": graph_name,
         "llm_model": llm_model,
         "embedder_model": embedder_model,
         "source_type": source_type,
         "execution_time": execution_time,
         "error_node": error_node,
+        "total_tokens": total_tokens,
     }
     log_event("graph_execution", properties)