ai agents

Signed-off-by: Saurabh Misra <[email protected]>

Author: misrasaurabh1

codeflash/after_aiagents.py

@@ -0,0 +1,15 @@
+def _estimate_string_tokens(content: str | Sequence[UserContent]) -> int:
+    if not content:
+        return 0
+    if isinstance(content, str):
+        return len(_TOKEN_SPLIT_RE.split(content.strip()))
+    tokens = 0
+    for part in content:
+        if isinstance(part, str):
+            tokens += len(_TOKEN_SPLIT_RE.split(part.strip()))
+        elif isinstance(part, BinaryContent):
+            tokens += len(part.data)
+    return tokens
+
+
+_TOKEN_SPLIT_RE = re.compile(r'[\s",.:]+')

codeflash/before_aiagents.py

@@ -0,0 +1,16 @@
+def _estimate_string_tokens(content: str | Sequence[UserContent]) -> int:
+    if not content:
+        return 0
+    if isinstance(content, str):
+        return len(re.split(r'[\s",.:]+', content.strip()))
+    tokens = 0
+    for part in content:
+        if isinstance(part, str):
+            tokens += len(re.split(r'[\s",.:]+', part.strip()))
+        if isinstance(part, (AudioUrl, ImageUrl)):
+            tokens += 0
+        elif isinstance(part, BinaryContent):
+            tokens += len(part.data)
+        else:
+            tokens += 0
+    return tokens