New hspylib-askai revision: 'v1.2.18' >> No commits

Author: yorevs

README.md

@@ -6,7 +6,7 @@
 
 [![Terminal](https://badgen.net/badge/icon/terminal?icon=terminal&label)](https://github.com/yorevs/homesetup)
 [![License](https://badgen.net/badge/license/MIT/gray)](LICENSE.md)
-[![Release](https://badgen.net/badge/release/v1.2.17/gray)](docs/CHANGELOG.md#unreleased)
+[![Release](https://badgen.net/badge/release/v1.2.19/gray)](docs/CHANGELOG.md#unreleased)
 [![Donate](https://badgen.net/badge/paypal/donate/yellow)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=J5CDEFLF6M3H4)
 [![build-and-test](https://github.com/yorevs/askai/actions/workflows/build-and-test.yml/badge.svg)](https://github.com/yorevs/askai/actions/workflows/build-and-test.yml)
 

bumpver.toml

@@ -1,5 +1,5 @@
 [bumpver]
-current_version = "1.2.17"
+current_version = "1.2.19"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit_message = "++version {old_version} -> {new_version}"
 commit = false

dependencies.hspd

@@ -69,11 +69,12 @@ package: opentelemetry-proto, version: 1.36.0, mode: compat
 package: opentelemetry-semantic-conventions, version: 0.48b0, mode: compat
 package: opentelemetry-instrumentation-fastapi, version: 0.48b0, mode: compat
 package: opentelemetry-instrumentation-asgi, version: 0.48b0, mode: compat
-package: transformers, version: 4.55.2, mode: compat
-package: unstructured, version: 0.18.13, mode: compat
-package: unstructured[md], version: 0.18.13, mode: compat
-package: tiktoken, version: 0.11.0, mode: compat
-package: nltk, version: 3.9.1, mode: compat
+package: transformers, version: 4.57.3, mode: compat
+package: unstructured, version: 0.18.21, mode: compat
+package: unstructured[md], version: 0.18.21, mode: compat
+package: tiktoken, version: 0.12.0, mode: compat
+package: nltk, version: 3.9.2, mode: compat
+package: tokenizers, version: 0.22.1, mode: compat
 
 /* Vector Store and Databases */
 package: faiss-cpu, version: 1.13.0, mode: compat

gradle.properties

@@ -2,7 +2,7 @@ author        = <B>H</B>ugo <B>S</B>aporetti <B>J</B>unior
 mailTo        = yorevs@hotmail.com
 siteUrl       = https://github.com/yorevs/askai
 app_name      = hspylib-askai
-app_version   = 1.2.17
+app_version   = 1.2.19
 
 # Build Tools
 buildTools    = setuptools, wheel, build, twine, bumpver, pipenv, isort, black, mypy, pylint, pdoc, pytest

src/demo/__init__.py

@@ -1,6 +1,6 @@
 # _*_ coding: utf-8 _*_
 #
-# hspylib-askai v1.2.17
+# hspylib-askai v1.2.19
 #
 # Package: demo
 """Package initialization."""
@@ -14,4 +14,4 @@
     'sports_api', 
     'utils'
 ]
-__version__ = '1.2.17'
+__version__ = '1.2.19'

src/demo/components/__init__.py

@@ -1,6 +1,6 @@
 # _*_ coding: utf-8 _*_
 #
-# hspylib-askai v1.2.17
+# hspylib-askai v1.2.19
 #
 # Package: demo.components
 """Package initialization."""
@@ -15,4 +15,4 @@
     'vision_demo', 
     'webcam_demo'
 ]
-__version__ = '1.2.17'
+__version__ = '1.2.19'

src/demo/devel/__init__.py

@@ -1,6 +1,6 @@
 # _*_ coding: utf-8 _*_
 #
-# hspylib-askai v1.2.17
+# hspylib-askai v1.2.19
 #
 # Package: demo.devel
 """Package initialization."""
@@ -21,4 +21,4 @@
     'vision-tests', 
     'whisper_tests'
 ]
-__version__ = '1.2.17'
+__version__ = '1.2.19'

src/demo/features/__init__.py

@@ -1,6 +1,6 @@
 # _*_ coding: utf-8 _*_
 #
-# hspylib-askai v1.2.17
+# hspylib-askai v1.2.19
 #
 # Package: demo.features
 """Package initialization."""
@@ -10,4 +10,4 @@
     'router_demo', 
     'tools'
 ]
-__version__ = '1.2.17'
+__version__ = '1.2.19'

src/demo/features/rag/__init__.py

@@ -1,11 +1,11 @@
 # _*_ coding: utf-8 _*_
 #
-# hspylib-askai v1.2.17
+# hspylib-askai v1.2.19
 #
 # Package: demo.features.rag
 """Package initialization."""
 
 __all__ = [
     'x_refs_demo'
 ]
-__version__ = '1.2.17'
+__version__ = '1.2.19'

src/demo/features/rag/x_refs_demo.py

@@ -3,26 +3,39 @@
 from askai.core.support.utilities import display_text
 from utils import get_resource, init_context
 
-if __name__ == "__main__":
-    queries_files: list[str] = [
-        "Open 1",
-        "Open second",
-        "Open 3",
-        "Open <AC/DC_Song>",
-        "Open it",
-        "Open any",
-        "Open the most recent",
-        "Open the most oldest",
-        "open metalica",
-    ]
-    init_context("x_refs_demo")
-    ctx: str = get_resource("songs")
-    display_text(f"\n```bash\n{ctx}\n```\n")
-    # Provide the context
-    shared.context.push("HISTORY", ctx)
-    shared.context.push("HISTORY", "Is there any song about cars?")
-    shared.context.push("HISTORY", 'Yes, there is. The song is "Highway to Hell"', "assistant")
-    for query in queries_files:
-        print(f"({shared.context.size('HISTORY')}/{shared.context.max_context_size})", "QUESTION: ", query)
-        resp: str = resolve_x_refs(query)
-        display_text(f"Response: {resp}")
+# if __name__ == "__main__":
+#     queries_files: list[str] = [
+#         "Open 1",
+#         "Open second",
+#         "Open 3",
+#         "Open <AC/DC_Song>",
+#         "Open it",
+#         "Open any",
+#         "Open the most recent",
+#         "Open the most oldest",
+#         "open metalica",
+#     ]
+#     init_context("x_refs_demo")
+#     ctx: str = get_resource("songs")
+#     display_text(f"\n```bash\n{ctx}\n```\n")
+#     # Provide the context
+#     shared.context.push("HISTORY", ctx)
+#     shared.context.push("HISTORY", "Is there any song about cars?")
+#     shared.context.push("HISTORY", 'Yes, there is. The song is "Highway to Hell"', "assistant")
+#     for query in queries_files:
+#         print(f"({shared.context.size('HISTORY')}/{shared.context.max_context_size})", "QUESTION: ", query)
+#         resp: str = resolve_x_refs(query)
+#         display_text(f"Response: {resp}")
+
+from tokenizers.decoders import DecodeStream
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct')
+stream = DecodeStream(skip_special_tokens=False)
+
+prompt = "To modify below `HTML` to `{{ '{KEY}' | transalte }}` obeys below Keys.\nExample:\n{{ 'Text.Action\\_Edit' | transalte }}\nKeys:\nText.Action\\_Edit\nText.Action\\_Log\nText.Action\\_View\nText.Action\\_Clone\nText.Action\\_Credit\nText.Action\\_CancelReward\nHTML:\n \n [View](#)\n[Edit](17.reward-template-setting.html)\n[Clone](#)\n[Credit](#)\n[Cancel Rewards](#)\n[Log](#)"
+
+token_ids = tok(prompt).input_ids
+# [1271, 5719, 3770, 1595, 5959, 63, 311, 1595, 3052, 11834, 4889, 11923, 765, 1380, 93420, 3954, 63, 98502, 1065, 3770, 25104, 627, 13617, 512, 3052, 364, 1199, 11614, 76838, 4126, 6, 765, 1380, 93420, 8256, 9026, 512, 1199, 11614, 76838, 4126, 198, 1199, 11614, 76838, 2250, 198, 1199, 11614, 76838, 860, 198, 1199, 11614, 76838, 38777, 198, 1199, 11614, 76838, 34593, 198, 1199, 11614, 76838, 9453, 60722, 198, 5959, 512, 720, 510, 860, 9725, 2, 340, 58, 4126, 9725, 1114, 83480, 34509, 61556, 2628, 340, 58, 38777, 9725, 2, 340, 58, 34593, 9725, 2, 340, 58, 9453, 50868, 9725, 2, 340, 58, 2250, 9725, 2, 8]
+
+for token_id in token_ids:
+   stream.step(tok._tokenizer, token_id)