diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7dd32ff..feb9d46 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.3.0 +current_version = 3.4.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? diff --git a/README.md b/README.md index 37157ca..8e4ed69 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Red Teaming python-framework for testing chatbots and GenAI systems ## Install 🚀 ```bash -pip install llamator==3.3.0 +pip install llamator==3.4.0 ``` ## Documentation 📚 diff --git a/docs/howtos.md b/docs/howtos.md index b0eac24..607e215 100644 --- a/docs/howtos.md +++ b/docs/howtos.md @@ -13,7 +13,7 @@ ## Installation Guide ```bash -pip install llamator==3.3.0 +pip install llamator==3.4.0 ``` ## Usage Guide (using LM Studio) diff --git a/examples/llamator-api.ipynb b/examples/llamator-api.ipynb index 7abccdd..a18da79 100644 --- a/examples/llamator-api.ipynb +++ b/examples/llamator-api.ipynb @@ -11,20 +11,24 @@ }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-06T13:58:57.453098Z", - "start_time": "2025-07-06T13:58:57.059519Z" + "end_time": "2025-09-23T23:15:49.914432Z", + "start_time": "2025-09-23T23:15:48.231631Z" } }, + "source": [ + "%pip install llamator python-dotenv requests --upgrade --quiet\n", + "%pip show llamator" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Note: you may need to restart the kernel to use updated packages.\n", "Name: llamator\r\n", - "Version: 3.3.0\r\n", + "Version: 3.4.0\r\n", "Summary: Framework for testing vulnerabilities of GenAI systems.\r\n", "Home-page: https://github.com/LLAMATOR-Core/llamator\r\n", "Author: Roman Neronov, Timur Nizamov, Nikita Ivanov\r\n", @@ -37,34 +41,36 @@ ] } ], - "source": [ - "%pip install llamator python-dotenv requests --upgrade --quiet\n", - "%pip show llamator" - ] + "execution_count": 1 }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:52:14.326798Z", - "start_time": "2025-07-15T10:52:08.534205Z" + "end_time": "2025-09-23T23:16:02.192727Z", + "start_time": "2025-09-23T23:15:53.249841Z" } }, - "outputs": [], "source": [ "import llamator" - ] + ], + "outputs": [], + "execution_count": 2 }, { "cell_type": "code", - "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2025-04-05T21:56:28.196836Z", - "start_time": "2025-04-05T21:56:28.195091Z" + "end_time": "2025-09-23T23:16:02.202719Z", + "start_time": "2025-09-23T23:16:02.198275Z" } }, + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv(\".env\") # example of environment variables in the .env.example file" + ], "outputs": [ { "data": { @@ -77,12 +83,7 @@ "output_type": "execute_result" } ], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "\n", - "load_dotenv(\".env\") # example of environment variables in the .env.example file" - ] + "execution_count": 3 }, { "cell_type": "markdown", @@ -210,13 +211,15 @@ }, { "cell_type": "code", - "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:52:22.481186Z", - "start_time": "2025-07-15T10:52:22.474697Z" + "end_time": "2025-09-23T23:16:02.720737Z", + "start_time": "2025-09-23T23:16:02.716536Z" } }, + "source": [ + "llamator.print_test_preset(\"all\")" + ], "outputs": [ { "name": "stdout", @@ -240,6 +243,17 @@ " \"num_transformations\": 5,\n", " \"sigma\": 0.4\n", " }),\n", + " (\"cop\", {\n", + " \"custom_dataset\": None,\n", + " \"initial_principles\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 12,\n", + " \"num_attempts\": 3,\n", + " \"principle_library_size\": 12,\n", + " \"similarity_penalty_threshold\": 0.8,\n", + " \"similarity_penalty_weight\": 0.2,\n", + " \"use_similarity_judge\": True\n", + " }),\n", " (\"crescendo\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -261,6 +275,7 @@ " \"multistage_depth\": 20,\n", " \"num_attempts\": 3\n", " }),\n", + " (\"repetition_token\", { \"num_attempts\": 3, \"repeat_count\": 10 }),\n", " (\"shuffle\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -296,9 +311,7 @@ ] } ], - "source": [ - "llamator.print_test_preset(\"all\")" - ] + "execution_count": 4 }, { "cell_type": "code", diff --git a/examples/llamator-langchain-custom-attack.ipynb b/examples/llamator-langchain-custom-attack.ipynb index 1842723..7df60b7 100644 --- a/examples/llamator-langchain-custom-attack.ipynb +++ b/examples/llamator-langchain-custom-attack.ipynb @@ -11,20 +11,24 @@ }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-06T13:59:29.416433Z", - "start_time": "2025-07-06T13:59:29.025054Z" + "end_time": "2025-09-23T23:16:15.245732Z", + "start_time": "2025-09-23T23:16:13.703407Z" } }, + "source": [ + "%pip install llamator python-dotenv --upgrade --quiet\n", + "%pip show llamator" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Note: you may need to restart the kernel to use updated packages.\n", "Name: llamator\r\n", - "Version: 3.3.0\r\n", + "Version: 3.4.0\r\n", "Summary: Framework for testing vulnerabilities of GenAI systems.\r\n", "Home-page: https://github.com/LLAMATOR-Core/llamator\r\n", "Author: Roman Neronov, Timur Nizamov, Nikita Ivanov\r\n", @@ -37,34 +41,36 @@ ] } ], - "source": [ - "%pip install llamator python-dotenv --upgrade --quiet\n", - "%pip show llamator" - ] + "execution_count": 1 }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:51:52.329146Z", - "start_time": "2025-07-15T10:51:46.534968Z" + "end_time": "2025-09-23T23:16:21.088402Z", + "start_time": "2025-09-23T23:16:15.249583Z" } }, - "outputs": [], "source": [ "import llamator" - ] + ], + "outputs": [], + "execution_count": 2 }, { "cell_type": "code", - "execution_count": 9, "metadata": { "ExecuteTime": { - "end_time": "2025-05-29T09:49:52.436622Z", - "start_time": "2025-05-29T09:49:52.431517Z" + "end_time": "2025-09-23T23:16:21.131645Z", + "start_time": "2025-09-23T23:16:21.127145Z" } }, + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv(\".env\") # example of environment variables in the .env.example file" + ], "outputs": [ { "data": { @@ -72,17 +78,12 @@ "True" ] }, - "execution_count": 9, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "\n", - "load_dotenv(\".env\") # example of environment variables in the .env.example file" - ] + "execution_count": 3 }, { "cell_type": "markdown", @@ -100,13 +101,15 @@ }, { "cell_type": "code", - "execution_count": 10, "metadata": { "ExecuteTime": { - "end_time": "2025-05-29T09:49:55.067102Z", - "start_time": "2025-05-29T09:49:52.450219Z" + "end_time": "2025-09-23T23:16:24.212362Z", + "start_time": "2025-09-23T23:16:21.142375Z" } }, + "source": [ + "llamator.print_chat_models_info()" + ], "outputs": [ { "name": "stdout", @@ -896,9 +899,7 @@ ] } ], - "source": [ - "llamator.print_chat_models_info()" - ] + "execution_count": 4 }, { "cell_type": "markdown", @@ -1192,14 +1193,104 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:52:02.010099Z", - "start_time": "2025-07-15T10:52:02.002716Z" + "end_time": "2025-09-23T23:16:30.347240Z", + "start_time": "2025-09-23T23:16:30.341119Z" } }, "cell_type": "code", - "outputs": [], - "execution_count": null, - "source": "llamator.print_test_preset(\"all\")" + "source": "llamator.print_test_preset(\"all\")", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Example configuration for preset 'all':\n", + "basic_tests = [\n", + " (\"aim_jailbreak\", { \"num_attempts\": 3 }),\n", + " (\"autodan_turbo\", {\n", + " \"custom_dataset\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 10,\n", + " \"num_attempts\": 3,\n", + " \"strategy_library_size\": 10\n", + " }),\n", + " (\"base64_injection\", { \"custom_dataset\": None, \"num_attempts\": 3 }),\n", + " (\"bon\", {\n", + " \"custom_dataset\": None,\n", + " \"language\": \"any\",\n", + " \"num_attempts\": 3,\n", + " \"num_transformations\": 5,\n", + " \"sigma\": 0.4\n", + " }),\n", + " (\"cop\", {\n", + " \"custom_dataset\": None,\n", + " \"initial_principles\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 12,\n", + " \"num_attempts\": 3,\n", + " \"principle_library_size\": 12,\n", + " \"similarity_penalty_threshold\": 0.8,\n", + " \"similarity_penalty_weight\": 0.2,\n", + " \"use_similarity_judge\": True\n", + " }),\n", + " (\"crescendo\", {\n", + " \"custom_dataset\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 5,\n", + " \"num_attempts\": 3\n", + " }),\n", + " (\"dan\", { \"language\": \"any\", \"num_attempts\": 3 }),\n", + " (\"deceptive_delight\", { \"custom_dataset\": None, \"num_attempts\": 3 }),\n", + " (\"dialogue_injection_devmode\", { \"custom_dataset\": None, \"num_attempts\": 3 }),\n", + " (\"dialogue_injection_continuation\", { \"custom_dataset\": None, \"language\": \"any\", \"num_attempts\": 3 }),\n", + " (\"ethical_compliance\", { \"custom_dataset\": None, \"num_attempts\": 3 }),\n", + " (\"harmbench\", { \"custom_dataset\": None, \"language\": \"any\", \"num_attempts\": 3 }),\n", + " (\"linguistic_evasion\", { \"num_attempts\": 3 }),\n", + " (\"linguistic_sandwich\", { \"custom_dataset\": None, \"num_attempts\": 3, \"num_translations\": 5 }),\n", + " (\"logical_inconsistencies\", { \"multistage_depth\": 20, \"num_attempts\": 3 }),\n", + " (\"pair\", {\n", + " \"custom_dataset\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 20,\n", + " \"num_attempts\": 3\n", + " }),\n", + " (\"repetition_token\", { \"num_attempts\": 3, \"repeat_count\": 10 }),\n", + " (\"shuffle\", {\n", + " \"custom_dataset\": None,\n", + " \"language\": \"any\",\n", + " \"num_attempts\": 3,\n", + " \"num_transformations\": 5\n", + " }),\n", + " (\"suffix\", { \"custom_dataset\": None, \"num_attempts\": 3 }),\n", + " (\"sycophancy\", { \"multistage_depth\": 20, \"num_attempts\": 3 }),\n", + " (\"system_prompt_leakage\", { \"custom_dataset\": None, \"multistage_depth\": 20, \"num_attempts\": 3 }),\n", + " (\"time_machine\", {\n", + " \"custom_dataset\": None,\n", + " \"language\": \"any\",\n", + " \"num_attempts\": 3,\n", + " \"time_context\": \"any\"\n", + " }),\n", + " (\"ucar\", { \"language\": \"any\", \"num_attempts\": 3 }),\n", + " (\"vlm_lowres_docs\", {\n", + " \"custom_pdf_dir\": None,\n", + " \"is_long_pdf\": False,\n", + " \"num_attempts\": 3,\n", + " \"overwrite_existing_pdfs\": False,\n", + " \"rescale\": 0.25\n", + " }),\n", + " (\"vlm_m_attack\", {\n", + " \"attack_data_base\": None,\n", + " \"attack_source\": \"parquet\",\n", + " \"dataset\": \"bigscale_100\",\n", + " \"dataset_variations\": None,\n", + " \"num_attempts\": 3\n", + " }),\n", + " (\"vlm_text_hallucination\", { \"attack_types\": None, \"num_attempts\": 3 }),\n", + "]\n" + ] + } + ], + "execution_count": 5 }, { "cell_type": "code", diff --git a/examples/llamator-selenium.ipynb b/examples/llamator-selenium.ipynb index 0dcf7dd..1826494 100644 --- a/examples/llamator-selenium.ipynb +++ b/examples/llamator-selenium.ipynb @@ -11,20 +11,24 @@ }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-06T14:00:19.793925Z", - "start_time": "2025-07-06T14:00:19.395177Z" + "end_time": "2025-09-23T23:17:13.137703Z", + "start_time": "2025-09-23T23:17:11.445903Z" } }, + "source": [ + "%pip install llamator python-dotenv selenium --upgrade --quiet\n", + "%pip show llamator" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Note: you may need to restart the kernel to use updated packages.\n", "Name: llamator\r\n", - "Version: 3.3.0\r\n", + "Version: 3.4.0\r\n", "Summary: Framework for testing vulnerabilities of GenAI systems.\r\n", "Home-page: https://github.com/LLAMATOR-Core/llamator\r\n", "Author: Roman Neronov, Timur Nizamov, Nikita Ivanov\r\n", @@ -37,43 +41,68 @@ ] } ], - "source": [ - "%pip install llamator python-dotenv selenium --upgrade --quiet\n", - "%pip show llamator" - ] + "execution_count": 1 }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2025-09-23T23:17:13.626926Z", + "start_time": "2025-09-23T23:17:13.146108Z" + } + }, "source": [ "%pip show selenium" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: selenium\r\n", + "Version: 4.35.0\r\n", + "Summary: Official Python bindings for Selenium WebDriver\r\n", + "Home-page: https://www.selenium.dev\r\n", + "Author: \r\n", + "Author-email: \r\n", + "License: Apache-2.0\r\n", + "Location: /Users/roman/Projects/PycharmProjects/LLAMATORS/llamator/venv/lib/python3.10/site-packages\r\n", + "Requires: certifi, trio, trio-websocket, typing_extensions, urllib3, websocket-client\r\n", + "Required-by: \r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 2 }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:51:35.104571Z", - "start_time": "2025-07-15T10:51:29.231950Z" + "end_time": "2025-09-23T23:17:19.411792Z", + "start_time": "2025-09-23T23:17:13.630838Z" } }, - "outputs": [], "source": [ "import llamator" - ] + ], + "outputs": [], + "execution_count": 3 }, { "cell_type": "code", - "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2025-04-05T21:56:28.196836Z", - "start_time": "2025-04-05T21:56:28.195091Z" + "end_time": "2025-09-23T23:17:03.514694Z", + "start_time": "2025-09-23T23:17:03.510729Z" } }, + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv(\".env\") # example of environment variables in the .env.example file" + ], "outputs": [ { "data": { @@ -81,17 +110,12 @@ "True" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "\n", - "load_dotenv(\".env\") # example of environment variables in the .env.example file" - ] + "execution_count": 4 }, { "cell_type": "markdown", @@ -254,13 +278,15 @@ }, { "cell_type": "code", - "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:51:40.094148Z", - "start_time": "2025-07-15T10:51:40.087442Z" + "end_time": "2025-09-23T23:17:32.203486Z", + "start_time": "2025-09-23T23:17:32.198844Z" } }, + "source": [ + "llamator.print_test_preset(\"all\")" + ], "outputs": [ { "name": "stdout", @@ -284,6 +310,17 @@ " \"num_transformations\": 5,\n", " \"sigma\": 0.4\n", " }),\n", + " (\"cop\", {\n", + " \"custom_dataset\": None,\n", + " \"initial_principles\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 12,\n", + " \"num_attempts\": 3,\n", + " \"principle_library_size\": 12,\n", + " \"similarity_penalty_threshold\": 0.8,\n", + " \"similarity_penalty_weight\": 0.2,\n", + " \"use_similarity_judge\": True\n", + " }),\n", " (\"crescendo\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -305,6 +342,7 @@ " \"multistage_depth\": 20,\n", " \"num_attempts\": 3\n", " }),\n", + " (\"repetition_token\", { \"num_attempts\": 3, \"repeat_count\": 10 }),\n", " (\"shuffle\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -340,9 +378,7 @@ ] } ], - "source": [ - "llamator.print_test_preset(\"all\")" - ] + "execution_count": 4 }, { "cell_type": "code", diff --git a/examples/llamator-telegram.ipynb b/examples/llamator-telegram.ipynb index b1904f2..a6ca776 100644 --- a/examples/llamator-telegram.ipynb +++ b/examples/llamator-telegram.ipynb @@ -11,20 +11,24 @@ }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-06T14:00:30.433445Z", - "start_time": "2025-07-06T14:00:30.050566Z" + "end_time": "2025-09-23T23:19:41.455820Z", + "start_time": "2025-09-23T23:19:39.804836Z" } }, + "source": [ + "%pip install llamator python-dotenv telethon --upgrade --quiet\n", + "%pip show llamator" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Note: you may need to restart the kernel to use updated packages.\n", "Name: llamator\r\n", - "Version: 3.3.0\r\n", + "Version: 3.4.0\r\n", "Summary: Framework for testing vulnerabilities of GenAI systems.\r\n", "Home-page: https://github.com/LLAMATOR-Core/llamator\r\n", "Author: Roman Neronov, Timur Nizamov, Nikita Ivanov\r\n", @@ -37,43 +41,68 @@ ] } ], - "source": [ - "%pip install llamator python-dotenv telethon --upgrade --quiet\n", - "%pip show llamator" - ] + "execution_count": 1 }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2025-09-23T23:19:41.952191Z", + "start_time": "2025-09-23T23:19:41.467157Z" + } + }, "source": [ "%pip show telethon" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: Telethon\r\n", + "Version: 1.41.2\r\n", + "Summary: Full-featured Telegram client library for Python 3\r\n", + "Home-page: https://github.com/LonamiWebs/Telethon\r\n", + "Author: Lonami Exo\r\n", + "Author-email: totufals@hotmail.com\r\n", + "License: MIT\r\n", + "Location: /Users/roman/Projects/PycharmProjects/LLAMATORS/llamator/venv/lib/python3.10/site-packages\r\n", + "Requires: pyaes, rsa\r\n", + "Required-by: \r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 2 }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:51:16.297322Z", - "start_time": "2025-07-15T10:51:10.568105Z" + "end_time": "2025-09-23T23:19:47.794911Z", + "start_time": "2025-09-23T23:19:41.956187Z" } }, - "outputs": [], "source": [ "import llamator" - ] + ], + "outputs": [], + "execution_count": 3 }, { "cell_type": "code", - "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2025-04-05T21:56:28.196836Z", - "start_time": "2025-04-05T21:56:28.195091Z" + "end_time": "2025-09-23T23:19:47.835573Z", + "start_time": "2025-09-23T23:19:47.831303Z" } }, + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv(\".env\") # example of environment variables in the .env.example file" + ], "outputs": [ { "data": { @@ -86,12 +115,7 @@ "output_type": "execute_result" } ], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "\n", - "load_dotenv(\".env\") # example of environment variables in the .env.example file" - ] + "execution_count": 4 }, { "cell_type": "markdown", @@ -122,9 +146,7 @@ }, { "cell_type": "code", - "execution_count": 6, "metadata": {}, - "outputs": [], "source": [ "from telethon import TelegramClient\n", "\n", @@ -136,7 +158,9 @@ ")\n", "await telegram_client.start()\n", "await telegram_client.connect()" - ] + ], + "outputs": [], + "execution_count": null }, { "cell_type": "code", @@ -281,13 +305,15 @@ }, { "cell_type": "code", - "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:51:20.321653Z", - "start_time": "2025-07-15T10:51:20.314202Z" + "end_time": "2025-09-23T23:19:49.178659Z", + "start_time": "2025-09-23T23:19:49.174895Z" } }, + "source": [ + "llamator.print_test_preset(\"all\")" + ], "outputs": [ { "name": "stdout", @@ -311,6 +337,17 @@ " \"num_transformations\": 5,\n", " \"sigma\": 0.4\n", " }),\n", + " (\"cop\", {\n", + " \"custom_dataset\": None,\n", + " \"initial_principles\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 12,\n", + " \"num_attempts\": 3,\n", + " \"principle_library_size\": 12,\n", + " \"similarity_penalty_threshold\": 0.8,\n", + " \"similarity_penalty_weight\": 0.2,\n", + " \"use_similarity_judge\": True\n", + " }),\n", " (\"crescendo\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -332,6 +369,7 @@ " \"multistage_depth\": 20,\n", " \"num_attempts\": 3\n", " }),\n", + " (\"repetition_token\", { \"num_attempts\": 3, \"repeat_count\": 10 }),\n", " (\"shuffle\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -367,9 +405,7 @@ ] } ], - "source": [ - "llamator.print_test_preset(\"all\")" - ] + "execution_count": 5 }, { "cell_type": "code", diff --git a/examples/llamator-vlm-attacks.ipynb b/examples/llamator-vlm-attacks.ipynb index 7621b42..67c3886 100644 --- a/examples/llamator-vlm-attacks.ipynb +++ b/examples/llamator-vlm-attacks.ipynb @@ -11,20 +11,24 @@ }, { "cell_type": "code", - "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2025-07-06T14:00:38.774461Z", - "start_time": "2025-07-06T14:00:38.365374Z" + "end_time": "2025-09-23T23:20:38.845227Z", + "start_time": "2025-09-23T23:20:37.478171Z" } }, + "source": [ + "%pip install llamator python-dotenv --upgrade --quiet\n", + "%pip show llamator" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Note: you may need to restart the kernel to use updated packages.\n", "Name: llamator\r\n", - "Version: 3.3.0\r\n", + "Version: 3.4.0\r\n", "Summary: Framework for testing vulnerabilities of GenAI systems.\r\n", "Home-page: https://github.com/LLAMATOR-Core/llamator\r\n", "Author: Roman Neronov, Timur Nizamov, Nikita Ivanov\r\n", @@ -37,34 +41,36 @@ ] } ], - "source": [ - "%pip install llamator python-dotenv --upgrade --quiet\n", - "%pip show llamator" - ] + "execution_count": 1 }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:49:58.325707Z", - "start_time": "2025-07-15T10:49:52.565014Z" + "end_time": "2025-09-23T23:20:44.733611Z", + "start_time": "2025-09-23T23:20:38.849352Z" } }, - "outputs": [], "source": [ "import llamator" - ] + ], + "outputs": [], + "execution_count": 2 }, { "cell_type": "code", - "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2025-04-12T16:55:00.087159Z", - "start_time": "2025-04-12T16:55:00.083547Z" + "end_time": "2025-09-23T23:20:44.788451Z", + "start_time": "2025-09-23T23:20:44.783939Z" } }, + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv(\".env\") # example of environment variables in the .env.example file" + ], "outputs": [ { "data": { @@ -77,12 +83,7 @@ "output_type": "execute_result" } ], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "\n", - "load_dotenv(\".env\") # example of environment variables in the .env.example file" - ] + "execution_count": 3 }, { "cell_type": "markdown", @@ -214,13 +215,15 @@ }, { "cell_type": "code", - "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:50:09.724355Z", - "start_time": "2025-07-15T10:50:09.719536Z" + "end_time": "2025-09-23T23:21:00.395376Z", + "start_time": "2025-09-23T23:21:00.391455Z" } }, + "source": [ + "llamator.print_test_preset(\"vlm\")" + ], "outputs": [ { "name": "stdout", @@ -247,9 +250,7 @@ ] } ], - "source": [ - "llamator.print_test_preset(\"vlm\")" - ] + "execution_count": 4 }, { "cell_type": "code", diff --git a/examples/llamator-whatsapp.ipynb b/examples/llamator-whatsapp.ipynb index ab56590..8029288 100644 --- a/examples/llamator-whatsapp.ipynb +++ b/examples/llamator-whatsapp.ipynb @@ -11,20 +11,24 @@ }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-06T14:00:58.891888Z", - "start_time": "2025-07-06T14:00:58.499676Z" + "end_time": "2025-09-23T23:21:13.729346Z", + "start_time": "2025-09-23T23:21:11.888267Z" } }, + "source": [ + "%pip install llamator python-dotenv selenium --upgrade --quiet\n", + "%pip show llamator" + ], "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Note: you may need to restart the kernel to use updated packages.\n", "Name: llamator\r\n", - "Version: 3.3.0\r\n", + "Version: 3.4.0\r\n", "Summary: Framework for testing vulnerabilities of GenAI systems.\r\n", "Home-page: https://github.com/LLAMATOR-Core/llamator\r\n", "Author: Roman Neronov, Timur Nizamov, Nikita Ivanov\r\n", @@ -37,50 +41,81 @@ ] } ], - "source": [ - "%pip install llamator python-dotenv selenium --upgrade --quiet\n", - "%pip show llamator" - ] + "execution_count": 1 }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "metadata": { + "ExecuteTime": { + "end_time": "2025-09-23T23:21:14.220168Z", + "start_time": "2025-09-23T23:21:13.733463Z" + } + }, "source": [ "%pip show selenium" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: selenium\r\n", + "Version: 4.35.0\r\n", + "Summary: Official Python bindings for Selenium WebDriver\r\n", + "Home-page: https://www.selenium.dev\r\n", + "Author: \r\n", + "Author-email: \r\n", + "License: Apache-2.0\r\n", + "Location: /Users/roman/Projects/PycharmProjects/LLAMATORS/llamator/venv/lib/python3.10/site-packages\r\n", + "Requires: certifi, trio, trio-websocket, typing_extensions, urllib3, websocket-client\r\n", + "Required-by: \r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "execution_count": 2 }, { "cell_type": "code", - "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:49:04.034637Z", - "start_time": "2025-07-15T10:48:58.497279Z" + "end_time": "2025-09-23T23:21:20.102034Z", + "start_time": "2025-09-23T23:21:14.225345Z" } }, - "outputs": [], "source": [ "import llamator" - ] + ], + "outputs": [], + "execution_count": 3 }, { "cell_type": "code", - "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2025-04-05T21:56:28.196836Z", - "start_time": "2025-04-05T21:56:28.195091Z" + "end_time": "2025-09-23T23:21:20.143864Z", + "start_time": "2025-09-23T23:21:20.139837Z" } }, - "outputs": [], "source": [ "import os\n", "from dotenv import load_dotenv\n", "\n", "load_dotenv(\".env\") # example of environment variables in the .env.example file" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 4 }, { "cell_type": "markdown", @@ -391,13 +426,15 @@ }, { "cell_type": "code", - "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2025-07-15T10:49:45.346723Z", - "start_time": "2025-07-15T10:49:45.339107Z" + "end_time": "2025-09-23T23:21:35.859944Z", + "start_time": "2025-09-23T23:21:35.854052Z" } }, + "source": [ + "llamator.print_test_preset(\"all\")" + ], "outputs": [ { "name": "stdout", @@ -421,6 +458,17 @@ " \"num_transformations\": 5,\n", " \"sigma\": 0.4\n", " }),\n", + " (\"cop\", {\n", + " \"custom_dataset\": None,\n", + " \"initial_principles\": None,\n", + " \"language\": \"any\",\n", + " \"multistage_depth\": 12,\n", + " \"num_attempts\": 3,\n", + " \"principle_library_size\": 12,\n", + " \"similarity_penalty_threshold\": 0.8,\n", + " \"similarity_penalty_weight\": 0.2,\n", + " \"use_similarity_judge\": True\n", + " }),\n", " (\"crescendo\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -442,6 +490,7 @@ " \"multistage_depth\": 20,\n", " \"num_attempts\": 3\n", " }),\n", + " (\"repetition_token\", { \"num_attempts\": 3, \"repeat_count\": 10 }),\n", " (\"shuffle\", {\n", " \"custom_dataset\": None,\n", " \"language\": \"any\",\n", @@ -477,9 +526,7 @@ ] } ], - "source": [ - "llamator.print_test_preset(\"all\")" - ] + "execution_count": 5 }, { "cell_type": "code", diff --git a/src/llamator/__version__.py b/src/llamator/__version__.py index d7dea7f..71a1a73 100644 --- a/src/llamator/__version__.py +++ b/src/llamator/__version__.py @@ -1,5 +1,5 @@ """Version information.""" # fmt: off -__version__ = '3.3.0' # noqa +__version__ = '3.4.0' # noqa # fmt: on