diff --git a/backend/core/poetry.lock b/backend/core/poetry.lock index 6994fba464de..577b400a67cf 100644 --- a/backend/core/poetry.lock +++ b/backend/core/poetry.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +[[package]] +name = "aiofiles" +version = "24.1.0" +description = "File support for asyncio." +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5"}, + {file = "aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c"}, +] + [[package]] name = "aiohttp" version = "3.9.5" @@ -120,6 +131,94 @@ files = [ {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] +[[package]] +name = "anyio" +version = "4.4.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, + {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "asyncpg" +version = "0.29.0" +description = "An asyncio PostgreSQL driver" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "asyncpg-0.29.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72fd0ef9f00aeed37179c62282a3d14262dbbafb74ec0ba16e1b1864d8a12169"}, + {file = "asyncpg-0.29.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:52e8f8f9ff6e21f9b39ca9f8e3e33a5fcdceaf5667a8c5c32bee158e313be385"}, + {file = "asyncpg-0.29.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9e6823a7012be8b68301342ba33b4740e5a166f6bbda0aee32bc01638491a22"}, + {file = "asyncpg-0.29.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:746e80d83ad5d5464cfbf94315eb6744222ab00aa4e522b704322fb182b83610"}, + {file = "asyncpg-0.29.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ff8e8109cd6a46ff852a5e6bab8b0a047d7ea42fcb7ca5ae6eaae97d8eacf397"}, + {file = "asyncpg-0.29.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97eb024685b1d7e72b1972863de527c11ff87960837919dac6e34754768098eb"}, + {file = "asyncpg-0.29.0-cp310-cp310-win32.whl", hash = "sha256:5bbb7f2cafd8d1fa3e65431833de2642f4b2124be61a449fa064e1a08d27e449"}, + {file = "asyncpg-0.29.0-cp310-cp310-win_amd64.whl", hash = "sha256:76c3ac6530904838a4b650b2880f8e7af938ee049e769ec2fba7cd66469d7772"}, + {file = "asyncpg-0.29.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4900ee08e85af01adb207519bb4e14b1cae8fd21e0ccf80fac6aa60b6da37b4"}, + {file = "asyncpg-0.29.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a65c1dcd820d5aea7c7d82a3fdcb70e096f8f70d1a8bf93eb458e49bfad036ac"}, + {file = "asyncpg-0.29.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b52e46f165585fd6af4863f268566668407c76b2c72d366bb8b522fa66f1870"}, + {file = "asyncpg-0.29.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc600ee8ef3dd38b8d67421359779f8ccec30b463e7aec7ed481c8346decf99f"}, + {file = "asyncpg-0.29.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:039a261af4f38f949095e1e780bae84a25ffe3e370175193174eb08d3cecab23"}, + {file = "asyncpg-0.29.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6feaf2d8f9138d190e5ec4390c1715c3e87b37715cd69b2c3dfca616134efd2b"}, + {file = "asyncpg-0.29.0-cp311-cp311-win32.whl", hash = "sha256:1e186427c88225ef730555f5fdda6c1812daa884064bfe6bc462fd3a71c4b675"}, + {file = "asyncpg-0.29.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfe73ffae35f518cfd6e4e5f5abb2618ceb5ef02a2365ce64f132601000587d3"}, + {file = "asyncpg-0.29.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6011b0dc29886ab424dc042bf9eeb507670a3b40aece3439944006aafe023178"}, + {file = "asyncpg-0.29.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b544ffc66b039d5ec5a7454667f855f7fec08e0dfaf5a5490dfafbb7abbd2cfb"}, + {file = "asyncpg-0.29.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d84156d5fb530b06c493f9e7635aa18f518fa1d1395ef240d211cb563c4e2364"}, + {file = "asyncpg-0.29.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54858bc25b49d1114178d65a88e48ad50cb2b6f3e475caa0f0c092d5f527c106"}, + {file = "asyncpg-0.29.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bde17a1861cf10d5afce80a36fca736a86769ab3579532c03e45f83ba8a09c59"}, + {file = "asyncpg-0.29.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:37a2ec1b9ff88d8773d3eb6d3784dc7e3fee7756a5317b67f923172a4748a175"}, + {file = "asyncpg-0.29.0-cp312-cp312-win32.whl", hash = "sha256:bb1292d9fad43112a85e98ecdc2e051602bce97c199920586be83254d9dafc02"}, + {file = "asyncpg-0.29.0-cp312-cp312-win_amd64.whl", hash = "sha256:2245be8ec5047a605e0b454c894e54bf2ec787ac04b1cb7e0d3c67aa1e32f0fe"}, + {file = "asyncpg-0.29.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0009a300cae37b8c525e5b449233d59cd9868fd35431abc470a3e364d2b85cb9"}, + {file = "asyncpg-0.29.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cad1324dbb33f3ca0cd2074d5114354ed3be2b94d48ddfd88af75ebda7c43cc"}, + {file = "asyncpg-0.29.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:012d01df61e009015944ac7543d6ee30c2dc1eb2f6b10b62a3f598beb6531548"}, + {file = "asyncpg-0.29.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000c996c53c04770798053e1730d34e30cb645ad95a63265aec82da9093d88e7"}, + {file = "asyncpg-0.29.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0bfe9c4d3429706cf70d3249089de14d6a01192d617e9093a8e941fea8ee775"}, + {file = "asyncpg-0.29.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:642a36eb41b6313ffa328e8a5c5c2b5bea6ee138546c9c3cf1bffaad8ee36dd9"}, + {file = "asyncpg-0.29.0-cp38-cp38-win32.whl", hash = "sha256:a921372bbd0aa3a5822dd0409da61b4cd50df89ae85150149f8c119f23e8c408"}, + {file = "asyncpg-0.29.0-cp38-cp38-win_amd64.whl", hash = "sha256:103aad2b92d1506700cbf51cd8bb5441e7e72e87a7b3a2ca4e32c840f051a6a3"}, + {file = "asyncpg-0.29.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5340dd515d7e52f4c11ada32171d87c05570479dc01dc66d03ee3e150fb695da"}, + {file = "asyncpg-0.29.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e17b52c6cf83e170d3d865571ba574577ab8e533e7361a2b8ce6157d02c665d3"}, + {file = "asyncpg-0.29.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f100d23f273555f4b19b74a96840aa27b85e99ba4b1f18d4ebff0734e78dc090"}, + {file = "asyncpg-0.29.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48e7c58b516057126b363cec8ca02b804644fd012ef8e6c7e23386b7d5e6ce83"}, + {file = "asyncpg-0.29.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f9ea3f24eb4c49a615573724d88a48bd1b7821c890c2effe04f05382ed9e8810"}, + {file = "asyncpg-0.29.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8d36c7f14a22ec9e928f15f92a48207546ffe68bc412f3be718eedccdf10dc5c"}, + {file = "asyncpg-0.29.0-cp39-cp39-win32.whl", hash = "sha256:797ab8123ebaed304a1fad4d7576d5376c3a006a4100380fb9d517f0b59c1ab2"}, + {file = "asyncpg-0.29.0-cp39-cp39-win_amd64.whl", hash = "sha256:cce08a178858b426ae1aa8409b5cc171def45d4293626e7aa6510696d46decd8"}, + {file = "asyncpg-0.29.0.tar.gz", hash = "sha256:d1c49e1f44fffafd9a55e1a9b101590859d881d639ea2922516f5d9c512d354e"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.3", markers = "python_version < \"3.12.0\""} + +[package.extras] +docs = ["Sphinx (>=5.3.0,<5.4.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["flake8 (>=6.1,<7.0)", "uvloop (>=0.15.3)"] + [[package]] name = "attrs" version = "23.2.0" @@ -249,6 +348,124 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecation" +version = "2.1.0" +description = "A library to handle automated deprecations" +optional = true +python-versions = "*" +files = [ + {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, + {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, +] + +[package.dependencies] +packaging = "*" + +[[package]] +name = "dnspython" +version = "2.6.1" +description = "DNS toolkit" +optional = false +python-versions = ">=3.8" +files = [ + {file = "dnspython-2.6.1-py3-none-any.whl", hash = "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50"}, + {file = "dnspython-2.6.1.tar.gz", hash = "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc"}, +] + +[package.extras] +dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "sphinx (>=7.2.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] +dnssec = ["cryptography (>=41)"] +doh = ["h2 (>=4.1.0)", "httpcore (>=1.0.0)", "httpx (>=0.26.0)"] +doq = ["aioquic (>=0.9.25)"] +idna = ["idna (>=3.6)"] +trio = ["trio (>=0.23)"] +wmi = ["wmi (>=1.5.1)"] + +[[package]] +name = "email-validator" +version = "2.2.0" +description = "A robust email address syntax and deliverability validation library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631"}, + {file = "email_validator-2.2.0.tar.gz", hash = "sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7"}, +] + +[package.dependencies] +dnspython = ">=2.0.0" +idna = ">=2.0.0" + +[[package]] +name = "fastapi" +version = "0.111.0" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastapi-0.111.0-py3-none-any.whl", hash = "sha256:97ecbf994be0bcbdadedf88c3150252bed7b2087075ac99735403b1b76cc8fc0"}, + {file = "fastapi-0.111.0.tar.gz", hash = "sha256:b9db9dd147c91cb8b769f7183535773d8741dd46f9dc6676cd82eab510228cd7"}, +] + +[package.dependencies] +email_validator = ">=2.0.0" +fastapi-cli = ">=0.0.2" +httpx = ">=0.23.0" +jinja2 = ">=2.11.2" +orjson = ">=3.2.1" +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" +python-multipart = ">=0.0.7" +starlette = ">=0.37.2,<0.38.0" +typing-extensions = ">=4.8.0" +ujson = ">=4.0.1,<4.0.2 || >4.0.2,<4.1.0 || >4.1.0,<4.2.0 || >4.2.0,<4.3.0 || >4.3.0,<5.0.0 || >5.0.0,<5.1.0 || >5.1.0" +uvicorn = {version = ">=0.12.0", extras = ["standard"]} + +[package.extras] +all = ["email_validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.7)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] + +[[package]] +name = "fastapi-cli" +version = "0.0.4" +description = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastapi_cli-0.0.4-py3-none-any.whl", hash = "sha256:a2552f3a7ae64058cdbb530be6fa6dbfc975dc165e4fa66d224c3d396e25e809"}, + {file = "fastapi_cli-0.0.4.tar.gz", hash = "sha256:e2e9ffaffc1f7767f488d6da34b6f5a377751c996f397902eb6abb99a67bde32"}, +] + +[package.dependencies] +typer = ">=0.12.3" + +[package.extras] +standard = ["fastapi", "uvicorn[standard] (>=0.15.0)"] + [[package]] name = "frozenlist" version = "1.4.1" @@ -335,6 +552,21 @@ files = [ {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, ] +[[package]] +name = "gotrue" +version = "2.5.4" +description = "Python Client Library for Supabase Auth" +optional = true +python-versions = "<4.0,>=3.8" +files = [ + {file = "gotrue-2.5.4-py3-none-any.whl", hash = "sha256:6f45003bc73cdee612a2d0be79cffed39c91cc8ad43a7440c02c320c7ad03a8e"}, + {file = "gotrue-2.5.4.tar.gz", hash = "sha256:acf0644a2e5d1bd70f66452361bfea4ba9621a0354a13154a333671a4c751c53"}, +] + +[package.dependencies] +httpx = ">=0.23,<0.28" +pydantic = ">=1.10,<3" + [[package]] name = "greenlet" version = "3.0.3" @@ -406,6 +638,110 @@ files = [ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.5" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, + {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.26.0)"] + +[[package]] +name = "httptools" +version = "0.6.1" +description = "A collection of framework independent HTTP protocol utils." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d2f6c3c4cb1948d912538217838f6e9960bc4a521d7f9b323b3da579cd14532f"}, + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:00d5d4b68a717765b1fabfd9ca755bd12bf44105eeb806c03d1962acd9b8e563"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:639dc4f381a870c9ec860ce5c45921db50205a37cc3334e756269736ff0aac58"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e57997ac7fb7ee43140cc03664de5f268813a481dff6245e0075925adc6aa185"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0ac5a0ae3d9f4fe004318d64b8a854edd85ab76cffbf7ef5e32920faef62f142"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3f30d3ce413088a98b9db71c60a6ada2001a08945cb42dd65a9a9fe228627658"}, + {file = "httptools-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:1ed99a373e327f0107cb513b61820102ee4f3675656a37a50083eda05dc9541b"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7a7ea483c1a4485c71cb5f38be9db078f8b0e8b4c4dc0210f531cdd2ddac1ef1"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85ed077c995e942b6f1b07583e4eb0a8d324d418954fc6af913d36db7c05a5a0"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b0bb634338334385351a1600a73e558ce619af390c2b38386206ac6a27fecfc"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9ceb2c957320def533671fc9c715a80c47025139c8d1f3797477decbc6edd2"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4f0f8271c0a4db459f9dc807acd0eadd4839934a4b9b892f6f160e94da309837"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6a4f5ccead6d18ec072ac0b84420e95d27c1cdf5c9f1bc8fbd8daf86bd94f43d"}, + {file = "httptools-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:5cceac09f164bcba55c0500a18fe3c47df29b62353198e4f37bbcc5d591172c3"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75c8022dca7935cba14741a42744eee13ba05db00b27a4b940f0d646bd4d56d0"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:48ed8129cd9a0d62cf4d1575fcf90fb37e3ff7d5654d3a5814eb3d55f36478c2"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f58e335a1402fb5a650e271e8c2d03cfa7cea46ae124649346d17bd30d59c90"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ad80d7176aa5788902f207a4e79885f0576134695dfb0fefc15b7a4648d503"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9bb68d3a085c2174c2477eb3ffe84ae9fb4fde8792edb7bcd09a1d8467e30a84"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b512aa728bc02354e5ac086ce76c3ce635b62f5fbc32ab7082b5e582d27867bb"}, + {file = "httptools-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:97662ce7fb196c785344d00d638fc9ad69e18ee4bfb4000b35a52efe5adcc949"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8e216a038d2d52ea13fdd9b9c9c7459fb80d78302b257828285eca1c773b99b3"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3e802e0b2378ade99cd666b5bffb8b2a7cc8f3d28988685dc300469ea8dd86cb"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd3e488b447046e386a30f07af05f9b38d3d368d1f7b4d8f7e10af85393db97"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe467eb086d80217b7584e61313ebadc8d187a4d95bb62031b7bab4b205c3ba3"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3c3b214ce057c54675b00108ac42bacf2ab8f85c58e3f324a4e963bbc46424f4"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8ae5b97f690badd2ca27cbf668494ee1b6d34cf1c464271ef7bfa9ca6b83ffaf"}, + {file = "httptools-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:405784577ba6540fa7d6ff49e37daf104e04f4b4ff2d1ac0469eaa6a20fde084"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:95fb92dd3649f9cb139e9c56604cc2d7c7bf0fc2e7c8d7fbd58f96e35eddd2a3"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dcbab042cc3ef272adc11220517278519adf8f53fd3056d0e68f0a6f891ba94e"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cf2372e98406efb42e93bfe10f2948e467edfd792b015f1b4ecd897903d3e8d"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:678fcbae74477a17d103b7cae78b74800d795d702083867ce160fc202104d0da"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e0b281cf5a125c35f7f6722b65d8542d2e57331be573e9e88bc8b0115c4a7a81"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:95658c342529bba4e1d3d2b1a874db16c7cca435e8827422154c9da76ac4e13a"}, + {file = "httptools-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ebaec1bf683e4bf5e9fbb49b8cc36da482033596a415b3e4ebab5a4c0d7ec5e"}, + {file = "httptools-0.6.1.tar.gz", hash = "sha256:c6e26c30455600b95d94b1b836085138e82f177351454ee841c148f93a9bad5a"}, +] + +[package.extras] +test = ["Cython (>=0.29.24,<0.30.0)"] + +[[package]] +name = "httpx" +version = "0.27.0" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"}, + {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + [[package]] name = "idna" version = "3.7" @@ -417,6 +753,23 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "jinja2" +version = "3.1.4" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "jsonpatch" version = "1.33" @@ -523,6 +876,110 @@ pydantic = [ ] requests = ">=2,<3" +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "multidict" version = "6.0.5" @@ -733,6 +1190,23 @@ files = [ {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] +[[package]] +name = "postgrest" +version = "0.16.8" +description = "PostgREST client for Python. This library provides an ORM interface to PostgREST." +optional = true +python-versions = "<4.0,>=3.8" +files = [ + {file = "postgrest-0.16.8-py3-none-any.whl", hash = "sha256:c353a24452f51ab9760cf2b884c4b7457a2653ff36444e66b12615bc4cc8e23e"}, + {file = "postgrest-0.16.8.tar.gz", hash = "sha256:7b3802a514dc1e0fc8b5bbdeb2c99af35a0bd910e4ddb17855ca4e3422350c84"}, +] + +[package.dependencies] +deprecation = ">=2.1.0,<3.0.0" +httpx = ">=0.24,<0.28" +pydantic = ">=1.9,<3.0" +strenum = ">=0.4.9,<0.5.0" + [[package]] name = "pydantic" version = "2.7.4" @@ -843,6 +1317,62 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "pygments" +version = "2.18.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-multipart" +version = "0.0.9" +description = "A streaming multipart parser for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python_multipart-0.0.9-py3-none-any.whl", hash = "sha256:97ca7b8ea7b05f977dc3849c3ba99d51689822fab725c3703af7c866a0c2b215"}, + {file = "python_multipart-0.0.9.tar.gz", hash = "sha256:03f54688c663f1b7977105f021043b0793151e4cb1c1a9d4a11fc13d622c4026"}, +] + +[package.extras] +dev = ["atomicwrites (==1.4.1)", "attrs (==23.2.0)", "coverage (==7.4.1)", "hatch", "invoke (==2.2.0)", "more-itertools (==10.2.0)", "pbr (==6.0.0)", "pluggy (==1.4.0)", "py (==1.11.0)", "pytest (==8.0.0)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.2.0)", "pyyaml (==6.0.1)", "ruff (==0.2.1)"] + [[package]] name = "pyyaml" version = "6.0.1" @@ -903,6 +1433,22 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] +[[package]] +name = "realtime" +version = "1.0.6" +description = "" +optional = true +python-versions = "<4.0,>=3.8" +files = [ + {file = "realtime-1.0.6-py3-none-any.whl", hash = "sha256:c66918a106d8ef348d1821f2dbf6683d8833825580d95b2fdea9995406b42838"}, + {file = "realtime-1.0.6.tar.gz", hash = "sha256:2be0d8a6305513d423604ee319216108fc20105cb7438922d5c8958c48f40a47"}, +] + +[package.dependencies] +python-dateutil = ">=2.8.1,<3.0.0" +typing-extensions = ">=4.12.2,<5.0.0" +websockets = ">=11,<13" + [[package]] name = "requests" version = "2.32.3" @@ -924,6 +1470,57 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "rich" +version = "13.7.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, + {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + [[package]] name = "sqlalchemy" version = "2.0.31" @@ -1011,6 +1608,103 @@ postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] pymysql = ["pymysql"] sqlcipher = ["sqlcipher3_binary"] +[[package]] +name = "sqlmodel" +version = "0.0.19" +description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." +optional = false +python-versions = ">=3.7" +files = [ + {file = "sqlmodel-0.0.19-py3-none-any.whl", hash = "sha256:6c8125d4101970d031e9aae970b20cbeaf44149989f8366d939f4ab21aab8763"}, + {file = "sqlmodel-0.0.19.tar.gz", hash = "sha256:95449b0b48a40a3eecf0a629fa5735b9dfc8a5574a91090d24ca17f02246ad96"}, +] + +[package.dependencies] +pydantic = ">=1.10.13,<3.0.0" +SQLAlchemy = ">=2.0.14,<2.1.0" + +[[package]] +name = "starlette" +version = "0.37.2" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.8" +files = [ + {file = "starlette-0.37.2-py3-none-any.whl", hash = "sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee"}, + {file = "starlette-0.37.2.tar.gz", hash = "sha256:9af890290133b79fc3db55474ade20f6220a364a0402e0b556e7cd5e1e093823"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<5" + +[package.extras] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7)", "pyyaml"] + +[[package]] +name = "storage3" +version = "0.7.6" +description = "Supabase Storage client for Python." +optional = true +python-versions = "<4.0,>=3.8" +files = [ + {file = "storage3-0.7.6-py3-none-any.whl", hash = "sha256:d8c23bf87b3a88cafb03761b7f936e4e49daca67741d571513edf746e0f8ba72"}, + {file = "storage3-0.7.6.tar.gz", hash = "sha256:0b7781cea7fe6382e6b9349b84395808c5f4203dfcac31478304eedc2f81acf6"}, +] + +[package.dependencies] +httpx = ">=0.24,<0.28" +python-dateutil = ">=2.8.2,<3.0.0" +typing-extensions = ">=4.2.0,<5.0.0" + +[[package]] +name = "strenum" +version = "0.4.15" +description = "An Enum that inherits from str." +optional = true +python-versions = "*" +files = [ + {file = "StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659"}, + {file = "StrEnum-0.4.15.tar.gz", hash = "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff"}, +] + +[package.extras] +docs = ["myst-parser[linkify]", "sphinx", "sphinx-rtd-theme"] +release = ["twine"] +test = ["pylint", "pytest", "pytest-black", "pytest-cov", "pytest-pylint"] + +[[package]] +name = "supabase" +version = "2.5.1" +description = "Supabase client for Python." +optional = true +python-versions = "<4.0,>=3.8" +files = [ + {file = "supabase-2.5.1-py3-none-any.whl", hash = "sha256:74a1f24f04fede1967ef084b50dea688228f7b10eb2f9d73350fe2251a865188"}, + {file = "supabase-2.5.1.tar.gz", hash = "sha256:c50e0eba5b03de3abd5ac0f887957ca43558ba44c4d17bb44e73ec454b41734c"}, +] + +[package.dependencies] +gotrue = ">=1.3,<3.0" +httpx = ">=0.24,<0.28" +postgrest = ">=0.14,<0.17.0" +realtime = ">=1.0.0,<2.0.0" +storage3 = ">=0.5.3,<0.8.0" +supafunc = ">=0.3.1,<0.5.0" + +[[package]] +name = "supafunc" +version = "0.4.6" +description = "Library for Supabase Functions" +optional = true +python-versions = "<4.0,>=3.8" +files = [ + {file = "supafunc-0.4.6-py3-none-any.whl", hash = "sha256:f7ca7b244365e171da7055a64edb462c2ec449cdaa210fc418cfccd132f4cf98"}, + {file = "supafunc-0.4.6.tar.gz", hash = "sha256:92db51f8f8568d1430285219c9c0072e44207409c416622d7387f609e31928a6"}, +] + +[package.dependencies] +httpx = ">=0.24,<0.28" + [[package]] name = "tenacity" version = "8.4.2" @@ -1026,6 +1720,23 @@ files = [ doc = ["reno", "sphinx"] test = ["pytest", "tornado (>=4.5)", "typeguard"] +[[package]] +name = "typer" +version = "0.12.3" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.7" +files = [ + {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, + {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, +] + +[package.dependencies] +click = ">=8.0.0" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + [[package]] name = "typing-extensions" version = "4.12.2" @@ -1037,6 +1748,93 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "ujson" +version = "5.10.0" +description = "Ultra fast JSON encoder and decoder for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "ujson-5.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2601aa9ecdbee1118a1c2065323bda35e2c5a2cf0797ef4522d485f9d3ef65bd"}, + {file = "ujson-5.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:348898dd702fc1c4f1051bc3aacbf894caa0927fe2c53e68679c073375f732cf"}, + {file = "ujson-5.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22cffecf73391e8abd65ef5f4e4dd523162a3399d5e84faa6aebbf9583df86d6"}, + {file = "ujson-5.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26b0e2d2366543c1bb4fbd457446f00b0187a2bddf93148ac2da07a53fe51569"}, + {file = "ujson-5.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:caf270c6dba1be7a41125cd1e4fc7ba384bf564650beef0df2dd21a00b7f5770"}, + {file = "ujson-5.10.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a245d59f2ffe750446292b0094244df163c3dc96b3ce152a2c837a44e7cda9d1"}, + {file = "ujson-5.10.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:94a87f6e151c5f483d7d54ceef83b45d3a9cca7a9cb453dbdbb3f5a6f64033f5"}, + {file = "ujson-5.10.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:29b443c4c0a113bcbb792c88bea67b675c7ca3ca80c3474784e08bba01c18d51"}, + {file = "ujson-5.10.0-cp310-cp310-win32.whl", hash = "sha256:c18610b9ccd2874950faf474692deee4223a994251bc0a083c114671b64e6518"}, + {file = "ujson-5.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:924f7318c31874d6bb44d9ee1900167ca32aa9b69389b98ecbde34c1698a250f"}, + {file = "ujson-5.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a5b366812c90e69d0f379a53648be10a5db38f9d4ad212b60af00bd4048d0f00"}, + {file = "ujson-5.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:502bf475781e8167f0f9d0e41cd32879d120a524b22358e7f205294224c71126"}, + {file = "ujson-5.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b91b5d0d9d283e085e821651184a647699430705b15bf274c7896f23fe9c9d8"}, + {file = "ujson-5.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:129e39af3a6d85b9c26d5577169c21d53821d8cf68e079060602e861c6e5da1b"}, + {file = "ujson-5.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f77b74475c462cb8b88680471193064d3e715c7c6074b1c8c412cb526466efe9"}, + {file = "ujson-5.10.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7ec0ca8c415e81aa4123501fee7f761abf4b7f386aad348501a26940beb1860f"}, + {file = "ujson-5.10.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ab13a2a9e0b2865a6c6db9271f4b46af1c7476bfd51af1f64585e919b7c07fd4"}, + {file = "ujson-5.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:57aaf98b92d72fc70886b5a0e1a1ca52c2320377360341715dd3933a18e827b1"}, + {file = "ujson-5.10.0-cp311-cp311-win32.whl", hash = "sha256:2987713a490ceb27edff77fb184ed09acdc565db700ee852823c3dc3cffe455f"}, + {file = "ujson-5.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:f00ea7e00447918ee0eff2422c4add4c5752b1b60e88fcb3c067d4a21049a720"}, + {file = "ujson-5.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:98ba15d8cbc481ce55695beee9f063189dce91a4b08bc1d03e7f0152cd4bbdd5"}, + {file = "ujson-5.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9d2edbf1556e4f56e50fab7d8ff993dbad7f54bac68eacdd27a8f55f433578e"}, + {file = "ujson-5.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6627029ae4f52d0e1a2451768c2c37c0c814ffc04f796eb36244cf16b8e57043"}, + {file = "ujson-5.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ccb77b3e40b151e20519c6ae6d89bfe3f4c14e8e210d910287f778368bb3d1"}, + {file = "ujson-5.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3caf9cd64abfeb11a3b661329085c5e167abbe15256b3b68cb5d914ba7396f3"}, + {file = "ujson-5.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6e32abdce572e3a8c3d02c886c704a38a1b015a1fb858004e03d20ca7cecbb21"}, + {file = "ujson-5.10.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a65b6af4d903103ee7b6f4f5b85f1bfd0c90ba4eeac6421aae436c9988aa64a2"}, + {file = "ujson-5.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:604a046d966457b6cdcacc5aa2ec5314f0e8c42bae52842c1e6fa02ea4bda42e"}, + {file = "ujson-5.10.0-cp312-cp312-win32.whl", hash = "sha256:6dea1c8b4fc921bf78a8ff00bbd2bfe166345f5536c510671bccececb187c80e"}, + {file = "ujson-5.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:38665e7d8290188b1e0d57d584eb8110951a9591363316dd41cf8686ab1d0abc"}, + {file = "ujson-5.10.0-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:618efd84dc1acbd6bff8eaa736bb6c074bfa8b8a98f55b61c38d4ca2c1f7f287"}, + {file = "ujson-5.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:38d5d36b4aedfe81dfe251f76c0467399d575d1395a1755de391e58985ab1c2e"}, + {file = "ujson-5.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67079b1f9fb29ed9a2914acf4ef6c02844b3153913eb735d4bf287ee1db6e557"}, + {file = "ujson-5.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7d0e0ceeb8fe2468c70ec0c37b439dd554e2aa539a8a56365fd761edb418988"}, + {file = "ujson-5.10.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:59e02cd37bc7c44d587a0ba45347cc815fb7a5fe48de16bf05caa5f7d0d2e816"}, + {file = "ujson-5.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2a890b706b64e0065f02577bf6d8ca3b66c11a5e81fb75d757233a38c07a1f20"}, + {file = "ujson-5.10.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:621e34b4632c740ecb491efc7f1fcb4f74b48ddb55e65221995e74e2d00bbff0"}, + {file = "ujson-5.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b9500e61fce0cfc86168b248104e954fead61f9be213087153d272e817ec7b4f"}, + {file = "ujson-5.10.0-cp313-cp313-win32.whl", hash = "sha256:4c4fc16f11ac1612f05b6f5781b384716719547e142cfd67b65d035bd85af165"}, + {file = "ujson-5.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:4573fd1695932d4f619928fd09d5d03d917274381649ade4328091ceca175539"}, + {file = "ujson-5.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a984a3131da7f07563057db1c3020b1350a3e27a8ec46ccbfbf21e5928a43050"}, + {file = "ujson-5.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:73814cd1b9db6fc3270e9d8fe3b19f9f89e78ee9d71e8bd6c9a626aeaeaf16bd"}, + {file = "ujson-5.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61e1591ed9376e5eddda202ec229eddc56c612b61ac6ad07f96b91460bb6c2fb"}, + {file = "ujson-5.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c75269f8205b2690db4572a4a36fe47cd1338e4368bc73a7a0e48789e2e35a"}, + {file = "ujson-5.10.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7223f41e5bf1f919cd8d073e35b229295aa8e0f7b5de07ed1c8fddac63a6bc5d"}, + {file = "ujson-5.10.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc2fd6b3067c0782e7002ac3b38cf48608ee6366ff176bbd02cf969c9c20fe"}, + {file = "ujson-5.10.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:232cc85f8ee3c454c115455195a205074a56ff42608fd6b942aa4c378ac14dd7"}, + {file = "ujson-5.10.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:cc6139531f13148055d691e442e4bc6601f6dba1e6d521b1585d4788ab0bfad4"}, + {file = "ujson-5.10.0-cp38-cp38-win32.whl", hash = "sha256:e7ce306a42b6b93ca47ac4a3b96683ca554f6d35dd8adc5acfcd55096c8dfcb8"}, + {file = "ujson-5.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:e82d4bb2138ab05e18f089a83b6564fee28048771eb63cdecf4b9b549de8a2cc"}, + {file = "ujson-5.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dfef2814c6b3291c3c5f10065f745a1307d86019dbd7ea50e83504950136ed5b"}, + {file = "ujson-5.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4734ee0745d5928d0ba3a213647f1c4a74a2a28edc6d27b2d6d5bd9fa4319e27"}, + {file = "ujson-5.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d47ebb01bd865fdea43da56254a3930a413f0c5590372a1241514abae8aa7c76"}, + {file = "ujson-5.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dee5e97c2496874acbf1d3e37b521dd1f307349ed955e62d1d2f05382bc36dd5"}, + {file = "ujson-5.10.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7490655a2272a2d0b072ef16b0b58ee462f4973a8f6bbe64917ce5e0a256f9c0"}, + {file = "ujson-5.10.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ba17799fcddaddf5c1f75a4ba3fd6441f6a4f1e9173f8a786b42450851bd74f1"}, + {file = "ujson-5.10.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:2aff2985cef314f21d0fecc56027505804bc78802c0121343874741650a4d3d1"}, + {file = "ujson-5.10.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ad88ac75c432674d05b61184178635d44901eb749786c8eb08c102330e6e8996"}, + {file = "ujson-5.10.0-cp39-cp39-win32.whl", hash = "sha256:2544912a71da4ff8c4f7ab5606f947d7299971bdd25a45e008e467ca638d13c9"}, + {file = "ujson-5.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:3ff201d62b1b177a46f113bb43ad300b424b7847f9c5d38b1b4ad8f75d4a282a"}, + {file = "ujson-5.10.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5b6fee72fa77dc172a28f21693f64d93166534c263adb3f96c413ccc85ef6e64"}, + {file = "ujson-5.10.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:61d0af13a9af01d9f26d2331ce49bb5ac1fb9c814964018ac8df605b5422dcb3"}, + {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecb24f0bdd899d368b715c9e6664166cf694d1e57be73f17759573a6986dd95a"}, + {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbd8fd427f57a03cff3ad6574b5e299131585d9727c8c366da4624a9069ed746"}, + {file = "ujson-5.10.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:beeaf1c48e32f07d8820c705ff8e645f8afa690cca1544adba4ebfa067efdc88"}, + {file = "ujson-5.10.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:baed37ea46d756aca2955e99525cc02d9181de67f25515c468856c38d52b5f3b"}, + {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7663960f08cd5a2bb152f5ee3992e1af7690a64c0e26d31ba7b3ff5b2ee66337"}, + {file = "ujson-5.10.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:d8640fb4072d36b08e95a3a380ba65779d356b2fee8696afeb7794cf0902d0a1"}, + {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78778a3aa7aafb11e7ddca4e29f46bc5139131037ad628cc10936764282d6753"}, + {file = "ujson-5.10.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0111b27f2d5c820e7f2dbad7d48e3338c824e7ac4d2a12da3dc6061cc39c8e6"}, + {file = "ujson-5.10.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:c66962ca7565605b355a9ed478292da628b8f18c0f2793021ca4425abf8b01e5"}, + {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ba43cc34cce49cf2d4bc76401a754a81202d8aa926d0e2b79f0ee258cb15d3a4"}, + {file = "ujson-5.10.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac56eb983edce27e7f51d05bc8dd820586c6e6be1c5216a6809b0c668bb312b8"}, + {file = "ujson-5.10.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44bd4b23a0e723bf8b10628288c2c7c335161d6840013d4d5de20e48551773b"}, + {file = "ujson-5.10.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c10f4654e5326ec14a46bcdeb2b685d4ada6911050aa8baaf3501e57024b804"}, + {file = "ujson-5.10.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0de4971a89a762398006e844ae394bd46991f7c385d7a6a3b93ba229e6dac17e"}, + {file = "ujson-5.10.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e1402f0564a97d2a52310ae10a64d25bcef94f8dd643fcf5d310219d915484f7"}, + {file = "ujson-5.10.0.tar.gz", hash = "sha256:b3cd8f3c5d8c7738257f1018880444f7b7d9b66232c64649f562d7ba86ad4bc1"}, +] + [[package]] name = "urllib3" version = "2.2.2" @@ -1054,6 +1852,243 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uvicorn" +version = "0.30.1" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.8" +files = [ + {file = "uvicorn-0.30.1-py3-none-any.whl", hash = "sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81"}, + {file = "uvicorn-0.30.1.tar.gz", hash = "sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8"}, +] + +[package.dependencies] +click = ">=7.0" +colorama = {version = ">=0.4", optional = true, markers = "sys_platform == \"win32\" and extra == \"standard\""} +h11 = ">=0.8" +httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""} +python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} +uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + +[[package]] +name = "uvloop" +version = "0.19.0" +description = "Fast implementation of asyncio event loop on top of libuv" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de4313d7f575474c8f5a12e163f6d89c0a878bc49219641d49e6f1444369a90e"}, + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5588bd21cf1fcf06bded085f37e43ce0e00424197e7c10e77afd4bbefffef428"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b1fd71c3843327f3bbc3237bedcdb6504fd50368ab3e04d0410e52ec293f5b8"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a05128d315e2912791de6088c34136bfcdd0c7cbc1cf85fd6fd1bb321b7c849"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cd81bdc2b8219cb4b2556eea39d2e36bfa375a2dd021404f90a62e44efaaf957"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f17766fb6da94135526273080f3455a112f82570b2ee5daa64d682387fe0dcd"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ce6b0af8f2729a02a5d1575feacb2a94fc7b2e983868b009d51c9a9d2149bef"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:31e672bb38b45abc4f26e273be83b72a0d28d074d5b370fc4dcf4c4eb15417d2"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:570fc0ed613883d8d30ee40397b79207eedd2624891692471808a95069a007c1"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5138821e40b0c3e6c9478643b4660bd44372ae1e16a322b8fc07478f92684e24"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:91ab01c6cd00e39cde50173ba4ec68a1e578fee9279ba64f5221810a9e786533"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:47bf3e9312f63684efe283f7342afb414eea4d3011542155c7e625cd799c3b12"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:da8435a3bd498419ee8c13c34b89b5005130a476bda1d6ca8cfdde3de35cd650"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:02506dc23a5d90e04d4f65c7791e65cf44bd91b37f24cfc3ef6cf2aff05dc7ec"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693049be9d36fef81741fddb3f441673ba12a34a704e7b4361efb75cf30befc"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7010271303961c6f0fe37731004335401eb9075a12680738731e9c92ddd96ad6"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5daa304d2161d2918fa9a17d5635099a2f78ae5b5960e742b2fcfbb7aefaa593"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7207272c9520203fea9b93843bb775d03e1cf88a80a936ce760f60bb5add92f3"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:78ab247f0b5671cc887c31d33f9b3abfb88d2614b84e4303f1a63b46c046c8bd"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:472d61143059c84947aa8bb74eabbace30d577a03a1805b77933d6bd13ddebbd"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45bf4c24c19fb8a50902ae37c5de50da81de4922af65baf760f7c0c42e1088be"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271718e26b3e17906b28b67314c45d19106112067205119dddbd834c2b7ce797"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:34175c9fd2a4bc3adc1380e1261f60306344e3407c20a4d684fd5f3be010fa3d"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e27f100e1ff17f6feeb1f33968bc185bf8ce41ca557deee9d9bbbffeb72030b7"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13dfdf492af0aa0a0edf66807d2b465607d11c4fa48f4a1fd41cbea5b18e8e8b"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e3d4e85ac060e2342ff85e90d0c04157acb210b9ce508e784a944f852a40e67"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca4956c9ab567d87d59d49fa3704cf29e37109ad348f2d5223c9bf761a332e7"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f467a5fd23b4fc43ed86342641f3936a68ded707f4627622fa3f82a120e18256"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:492e2c32c2af3f971473bc22f086513cedfc66a130756145a931a90c3958cb17"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2df95fca285a9f5bfe730e51945ffe2fa71ccbfdde3b0da5772b4ee4f2e770d5"}, + {file = "uvloop-0.19.0.tar.gz", hash = "sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd"}, +] + +[package.extras] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] + +[[package]] +name = "watchfiles" +version = "0.22.0" +description = "Simple, modern and high performance file watching and code reload in python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "watchfiles-0.22.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da1e0a8caebf17976e2ffd00fa15f258e14749db5e014660f53114b676e68538"}, + {file = "watchfiles-0.22.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:61af9efa0733dc4ca462347becb82e8ef4945aba5135b1638bfc20fad64d4f0e"}, + {file = "watchfiles-0.22.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d9188979a58a096b6f8090e816ccc3f255f137a009dd4bbec628e27696d67c1"}, + {file = "watchfiles-0.22.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2bdadf6b90c099ca079d468f976fd50062905d61fae183f769637cb0f68ba59a"}, + {file = "watchfiles-0.22.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:067dea90c43bf837d41e72e546196e674f68c23702d3ef80e4e816937b0a3ffd"}, + {file = "watchfiles-0.22.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bbf8a20266136507abf88b0df2328e6a9a7c7309e8daff124dda3803306a9fdb"}, + {file = "watchfiles-0.22.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1235c11510ea557fe21be5d0e354bae2c655a8ee6519c94617fe63e05bca4171"}, + {file = "watchfiles-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2444dc7cb9d8cc5ab88ebe792a8d75709d96eeef47f4c8fccb6df7c7bc5be71"}, + {file = "watchfiles-0.22.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c5af2347d17ab0bd59366db8752d9e037982e259cacb2ba06f2c41c08af02c39"}, + {file = "watchfiles-0.22.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9624a68b96c878c10437199d9a8b7d7e542feddda8d5ecff58fdc8e67b460848"}, + {file = "watchfiles-0.22.0-cp310-none-win32.whl", hash = "sha256:4b9f2a128a32a2c273d63eb1fdbf49ad64852fc38d15b34eaa3f7ca2f0d2b797"}, + {file = "watchfiles-0.22.0-cp310-none-win_amd64.whl", hash = "sha256:2627a91e8110b8de2406d8b2474427c86f5a62bf7d9ab3654f541f319ef22bcb"}, + {file = "watchfiles-0.22.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8c39987a1397a877217be1ac0fb1d8b9f662c6077b90ff3de2c05f235e6a8f96"}, + {file = "watchfiles-0.22.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a927b3034d0672f62fb2ef7ea3c9fc76d063c4b15ea852d1db2dc75fe2c09696"}, + {file = "watchfiles-0.22.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:052d668a167e9fc345c24203b104c313c86654dd6c0feb4b8a6dfc2462239249"}, + {file = "watchfiles-0.22.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e45fb0d70dda1623a7045bd00c9e036e6f1f6a85e4ef2c8ae602b1dfadf7550"}, + {file = "watchfiles-0.22.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c49b76a78c156979759d759339fb62eb0549515acfe4fd18bb151cc07366629c"}, + {file = "watchfiles-0.22.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4a65474fd2b4c63e2c18ac67a0c6c66b82f4e73e2e4d940f837ed3d2fd9d4da"}, + {file = "watchfiles-0.22.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc0cba54f47c660d9fa3218158b8963c517ed23bd9f45fe463f08262a4adae1"}, + {file = "watchfiles-0.22.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94ebe84a035993bb7668f58a0ebf998174fb723a39e4ef9fce95baabb42b787f"}, + {file = "watchfiles-0.22.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e0f0a874231e2839abbf473256efffe577d6ee2e3bfa5b540479e892e47c172d"}, + {file = "watchfiles-0.22.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:213792c2cd3150b903e6e7884d40660e0bcec4465e00563a5fc03f30ea9c166c"}, + {file = "watchfiles-0.22.0-cp311-none-win32.whl", hash = "sha256:b44b70850f0073b5fcc0b31ede8b4e736860d70e2dbf55701e05d3227a154a67"}, + {file = "watchfiles-0.22.0-cp311-none-win_amd64.whl", hash = "sha256:00f39592cdd124b4ec5ed0b1edfae091567c72c7da1487ae645426d1b0ffcad1"}, + {file = "watchfiles-0.22.0-cp311-none-win_arm64.whl", hash = "sha256:3218a6f908f6a276941422b035b511b6d0d8328edd89a53ae8c65be139073f84"}, + {file = "watchfiles-0.22.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:c7b978c384e29d6c7372209cbf421d82286a807bbcdeb315427687f8371c340a"}, + {file = "watchfiles-0.22.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bd4c06100bce70a20c4b81e599e5886cf504c9532951df65ad1133e508bf20be"}, + {file = "watchfiles-0.22.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:425440e55cd735386ec7925f64d5dde392e69979d4c8459f6bb4e920210407f2"}, + {file = "watchfiles-0.22.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:68fe0c4d22332d7ce53ad094622b27e67440dacefbaedd29e0794d26e247280c"}, + {file = "watchfiles-0.22.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8a31bfd98f846c3c284ba694c6365620b637debdd36e46e1859c897123aa232"}, + {file = "watchfiles-0.22.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc2e8fe41f3cac0660197d95216c42910c2b7e9c70d48e6d84e22f577d106fc1"}, + {file = "watchfiles-0.22.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b7cc10261c2786c41d9207193a85c1db1b725cf87936df40972aab466179b6"}, + {file = "watchfiles-0.22.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28585744c931576e535860eaf3f2c0ec7deb68e3b9c5a85ca566d69d36d8dd27"}, + {file = "watchfiles-0.22.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:00095dd368f73f8f1c3a7982a9801190cc88a2f3582dd395b289294f8975172b"}, + {file = "watchfiles-0.22.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:52fc9b0dbf54d43301a19b236b4a4614e610605f95e8c3f0f65c3a456ffd7d35"}, + {file = "watchfiles-0.22.0-cp312-none-win32.whl", hash = "sha256:581f0a051ba7bafd03e17127735d92f4d286af941dacf94bcf823b101366249e"}, + {file = "watchfiles-0.22.0-cp312-none-win_amd64.whl", hash = "sha256:aec83c3ba24c723eac14225194b862af176d52292d271c98820199110e31141e"}, + {file = "watchfiles-0.22.0-cp312-none-win_arm64.whl", hash = "sha256:c668228833c5619f6618699a2c12be057711b0ea6396aeaece4ded94184304ea"}, + {file = "watchfiles-0.22.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d47e9ef1a94cc7a536039e46738e17cce058ac1593b2eccdede8bf72e45f372a"}, + {file = "watchfiles-0.22.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28f393c1194b6eaadcdd8f941307fc9bbd7eb567995232c830f6aef38e8a6e88"}, + {file = "watchfiles-0.22.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd64f3a4db121bc161644c9e10a9acdb836853155a108c2446db2f5ae1778c3d"}, + {file = "watchfiles-0.22.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2abeb79209630da981f8ebca30a2c84b4c3516a214451bfc5f106723c5f45843"}, + {file = "watchfiles-0.22.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4cc382083afba7918e32d5ef12321421ef43d685b9a67cc452a6e6e18920890e"}, + {file = "watchfiles-0.22.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d048ad5d25b363ba1d19f92dcf29023988524bee6f9d952130b316c5802069cb"}, + {file = "watchfiles-0.22.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:103622865599f8082f03af4214eaff90e2426edff5e8522c8f9e93dc17caee13"}, + {file = "watchfiles-0.22.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3e1f3cf81f1f823e7874ae563457828e940d75573c8fbf0ee66818c8b6a9099"}, + {file = "watchfiles-0.22.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8597b6f9dc410bdafc8bb362dac1cbc9b4684a8310e16b1ff5eee8725d13dcd6"}, + {file = "watchfiles-0.22.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0b04a2cbc30e110303baa6d3ddce8ca3664bc3403be0f0ad513d1843a41c97d1"}, + {file = "watchfiles-0.22.0-cp38-none-win32.whl", hash = "sha256:b610fb5e27825b570554d01cec427b6620ce9bd21ff8ab775fc3a32f28bba63e"}, + {file = "watchfiles-0.22.0-cp38-none-win_amd64.whl", hash = "sha256:fe82d13461418ca5e5a808a9e40f79c1879351fcaeddbede094028e74d836e86"}, + {file = "watchfiles-0.22.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:3973145235a38f73c61474d56ad6199124e7488822f3a4fc97c72009751ae3b0"}, + {file = "watchfiles-0.22.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:280a4afbc607cdfc9571b9904b03a478fc9f08bbeec382d648181c695648202f"}, + {file = "watchfiles-0.22.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a0d883351a34c01bd53cfa75cd0292e3f7e268bacf2f9e33af4ecede7e21d1d"}, + {file = "watchfiles-0.22.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9165bcab15f2b6d90eedc5c20a7f8a03156b3773e5fb06a790b54ccecdb73385"}, + {file = "watchfiles-0.22.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc1b9b56f051209be458b87edb6856a449ad3f803315d87b2da4c93b43a6fe72"}, + {file = "watchfiles-0.22.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dc1fc25a1dedf2dd952909c8e5cb210791e5f2d9bc5e0e8ebc28dd42fed7562"}, + {file = "watchfiles-0.22.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dc92d2d2706d2b862ce0568b24987eba51e17e14b79a1abcd2edc39e48e743c8"}, + {file = "watchfiles-0.22.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97b94e14b88409c58cdf4a8eaf0e67dfd3ece7e9ce7140ea6ff48b0407a593ec"}, + {file = "watchfiles-0.22.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:96eec15e5ea7c0b6eb5bfffe990fc7c6bd833acf7e26704eb18387fb2f5fd087"}, + {file = "watchfiles-0.22.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:28324d6b28bcb8d7c1041648d7b63be07a16db5510bea923fc80b91a2a6cbed6"}, + {file = "watchfiles-0.22.0-cp39-none-win32.whl", hash = "sha256:8c3e3675e6e39dc59b8fe5c914a19d30029e36e9f99468dddffd432d8a7b1c93"}, + {file = "watchfiles-0.22.0-cp39-none-win_amd64.whl", hash = "sha256:25c817ff2a86bc3de3ed2df1703e3d24ce03479b27bb4527c57e722f8554d971"}, + {file = "watchfiles-0.22.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b810a2c7878cbdecca12feae2c2ae8af59bea016a78bc353c184fa1e09f76b68"}, + {file = "watchfiles-0.22.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f7e1f9c5d1160d03b93fc4b68a0aeb82fe25563e12fbcdc8507f8434ab6f823c"}, + {file = "watchfiles-0.22.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:030bc4e68d14bcad2294ff68c1ed87215fbd9a10d9dea74e7cfe8a17869785ab"}, + {file = "watchfiles-0.22.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ace7d060432acde5532e26863e897ee684780337afb775107c0a90ae8dbccfd2"}, + {file = "watchfiles-0.22.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5834e1f8b71476a26df97d121c0c0ed3549d869124ed2433e02491553cb468c2"}, + {file = "watchfiles-0.22.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:0bc3b2f93a140df6806c8467c7f51ed5e55a931b031b5c2d7ff6132292e803d6"}, + {file = "watchfiles-0.22.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fdebb655bb1ba0122402352b0a4254812717a017d2dc49372a1d47e24073795"}, + {file = "watchfiles-0.22.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c8e0aa0e8cc2a43561e0184c0513e291ca891db13a269d8d47cb9841ced7c71"}, + {file = "watchfiles-0.22.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2f350cbaa4bb812314af5dab0eb8d538481e2e2279472890864547f3fe2281ed"}, + {file = "watchfiles-0.22.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7a74436c415843af2a769b36bf043b6ccbc0f8d784814ba3d42fc961cdb0a9dc"}, + {file = "watchfiles-0.22.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00ad0bcd399503a84cc688590cdffbe7a991691314dde5b57b3ed50a41319a31"}, + {file = "watchfiles-0.22.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72a44e9481afc7a5ee3291b09c419abab93b7e9c306c9ef9108cb76728ca58d2"}, + {file = "watchfiles-0.22.0.tar.gz", hash = "sha256:988e981aaab4f3955209e7e28c7794acdb690be1efa7f16f8ea5aba7ffdadacb"}, +] + +[package.dependencies] +anyio = ">=3.0.0" + +[[package]] +name = "websockets" +version = "12.0" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, + {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, + {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, + {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, + {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, + {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, + {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, + {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, + {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, + {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, + {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, + {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, + {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, + {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, + {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, + {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, + {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, + {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, + {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, + {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, + {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, + {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, + {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, + {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, + {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, +] + [[package]] name = "yarl" version = "1.9.4" @@ -1160,4 +2195,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "dc6781ae577a76ccefee8f8286035abf35e9e5b2486c2c671b2d4bbb9c99bdc5" +content-hash = "982c385f8e526d48c67312f0852b1643703473608cfa4ef54f0a1bdae56c346e" diff --git a/backend/core/pyproject.toml b/backend/core/pyproject.toml index 8dfd73d3107a..a8c4823be462 100644 --- a/backend/core/pyproject.toml +++ b/backend/core/pyproject.toml @@ -12,6 +12,15 @@ pydantic = "^2.7.4" langchain-core = "^0.2.10" langchain = "^0.2.6" +[tool.poetry.group.api.dependencies] +aiofiles = "^24.1.0" +python-dotenv = "^1.0.1" +fastapi = "^0.111.0" +sqlmodel = "^0.0.19" +asyncpg = "^0.29.0" +uvicorn = "^0.30.1" +supabase = { version = "^2.5.1", optional = true } + [tool.ruff] line-length = 88 exclude = [".git", "__pycache__", ".mypy_cache", ".pytest_cache"] diff --git a/backend/core/quivr_core/api/__init__.py b/backend/core/quivr_core/api/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/celery_config.py b/backend/core/quivr_core/api/celery_config.py new file mode 100644 index 000000000000..e4752f8dec35 --- /dev/null +++ b/backend/core/quivr_core/api/celery_config.py @@ -0,0 +1,41 @@ +# celery_config.py +import os + +import dotenv +from celery import Celery + +dotenv.load_dotenv() + +CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "") +CELERY_BROKER_QUEUE_NAME = os.getenv("CELERY_BROKER_QUEUE_NAME", "quivr") + +celery = Celery(__name__) + +if CELERY_BROKER_URL.startswith("sqs"): + broker_transport_options = { + CELERY_BROKER_QUEUE_NAME: { + "my-q": { + "url": CELERY_BROKER_URL, + } + } + } + celery = Celery( + __name__, + broker=CELERY_BROKER_URL, + task_serializer="json", + task_concurrency=4, + worker_prefetch_multiplier=1, + broker_transport_options=broker_transport_options, + ) + celery.conf.task_default_queue = CELERY_BROKER_QUEUE_NAME +elif CELERY_BROKER_URL.startswith("redis"): + celery = Celery( + __name__, + broker=f"{CELERY_BROKER_URL}", + backend=f"{CELERY_BROKER_URL}", + task_concurrency=4, + worker_prefetch_multiplier=2, + task_serializer="json", + ) +else: + raise ValueError(f"Unsupported broker URL: {CELERY_BROKER_URL}") diff --git a/backend/core/quivr_core/api/celery_worker.py b/backend/core/quivr_core/api/celery_worker.py new file mode 100644 index 000000000000..b4f75fdac9d3 --- /dev/null +++ b/backend/core/quivr_core/api/celery_worker.py @@ -0,0 +1,174 @@ +import os +from tempfile import NamedTemporaryFile +from uuid import UUID + +from celery.schedules import crontab + +from quivr_core.api.celery_config import celery +from quivr_core.api.logger import get_logger +from quivr_core.api.models.files import File +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.brain.service.brain_service import BrainService +from quivr_core.api.modules.brain.service.brain_vector_service import BrainVectorService +from quivr_core.api.modules.notification.dto.inputs import ( + NotificationUpdatableProperties, +) +from quivr_core.api.modules.notification.entity.notification import ( + NotificationsStatusEnum, +) +from quivr_core.api.modules.notification.service.notification_service import ( + NotificationService, +) +from quivr_core.api.packages.files.crawl.crawler import CrawlWebsite, slugify +from quivr_core.api.packages.files.parsers.github import process_github +from quivr_core.api.packages.files.processors import filter_file +from quivr_core.api.packages.utils.telemetry import maybe_send_telemetry + +logger = get_logger(__name__) + +notification_service = NotificationService() +brain_service = BrainService() + + +@celery.task(name="process_file_and_notify") +def process_file_and_notify( + file_name: str, + file_original_name: str, + brain_id, + notification_id=None, + integration=None, + delete_file=False, +): + try: + supabase_client = get_supabase_client() + tmp_name = file_name.replace("/", "_") + base_file_name = os.path.basename(file_name) + _, file_extension = os.path.splitext(base_file_name) + + with NamedTemporaryFile( + suffix="_" + tmp_name, # pyright: ignore reportPrivateUsage=none + ) as tmp_file: + res = supabase_client.storage.from_("quivr").download(file_name) + tmp_file.write(res) + tmp_file.flush() + file_instance = File( + file_name=base_file_name, + tmp_file_path=tmp_file.name, + bytes_content=res, + file_size=len(res), + file_extension=file_extension, + ) + brain_vector_service = BrainVectorService(brain_id) + if delete_file: # TODO fix bug + brain_vector_service.delete_file_from_brain( + file_original_name, only_vectors=True + ) + + filter_file( + file=file_instance, + brain_id=brain_id, + original_file_name=file_original_name, + ) + + if notification_id: + notification_service.update_notification_by_id( + notification_id, + NotificationUpdatableProperties( + status=NotificationsStatusEnum.SUCCESS, + description="Your file has been properly uploaded!", + ), + ) + brain_service.update_brain_last_update_time(brain_id) + + return True + + except TimeoutError: + logger.error("TimeoutError") + + except Exception as e: + logger.exception(e) + notification_service.update_notification_by_id( + notification_id, + NotificationUpdatableProperties( + status=NotificationsStatusEnum.ERROR, + description=f"An error occurred while processing the file: {e}", + ), + ) + + +@celery.task(name="process_crawl_and_notify") +def process_crawl_and_notify( + crawl_website_url: str, + brain_id: UUID, + notification_id=None, +): + crawl_website = CrawlWebsite(url=crawl_website_url) + + if not crawl_website.checkGithub(): + # Build file data + extracted_content = crawl_website.process() + extracted_content_bytes = extracted_content.encode("utf-8") + file_name = slugify(crawl_website.url) + ".txt" + + with NamedTemporaryFile( + suffix="_" + file_name, # pyright: ignore reportPrivateUsage=none + ) as tmp_file: + tmp_file.write(extracted_content_bytes) + tmp_file.flush() + file_instance = File( + file_name=file_name, + tmp_file_path=tmp_file.name, + bytes_content=extracted_content_bytes, + file_size=len(extracted_content), + file_extension=".txt", + ) + filter_file( + file=file_instance, + brain_id=brain_id, + original_file_name=crawl_website_url, + ) + notification_service.update_notification_by_id( + notification_id, + NotificationUpdatableProperties( + status=NotificationsStatusEnum.SUCCESS, + description="Your URL has been properly crawled!", + ), + ) + else: + process_github( + repo=crawl_website.url, + brain_id=brain_id, + ) + + if notification_id: + notification_service.update_notification_by_id( + notification_id, + NotificationUpdatableProperties( + status=NotificationsStatusEnum.SUCCESS, + description="Your file has been properly uploaded!", + ), + ) + + brain_service.update_brain_last_update_time(brain_id) + return True + + +@celery.task +def ping_telemetry(): + maybe_send_telemetry("ping", {"ping": "pong"}) + + +celery.conf.beat_schedule = { + "ping_telemetry": { + "task": f"{__name__}.ping_telemetry", + "schedule": crontab(minute="*/30", hour="*"), + }, + "process_sync_active": { + "task": "process_sync_active", + "schedule": crontab(minute="*/1", hour="*"), + }, + "process_premium_users": { + "task": "check_if_is_premium_user", + "schedule": crontab(minute="*/1", hour="*"), + }, +} diff --git a/backend/core/quivr_core/api/logger.py b/backend/core/quivr_core/api/logger.py new file mode 100644 index 000000000000..b839e9aef4f1 --- /dev/null +++ b/backend/core/quivr_core/api/logger.py @@ -0,0 +1,45 @@ +import logging +import os +from logging.handlers import RotatingFileHandler + +from colorlog import ( + ColoredFormatter, +) + + +def get_logger(logger_name, log_file="application.log"): + log_level = os.getenv("LOG_LEVEL", "WARNING").upper() + logger = logging.getLogger(logger_name) + logger.setLevel(log_level) + logger.propagate = False # Prevent log propagation to avoid double logging + + formatter = logging.Formatter( + "[%(levelname)s] %(name)s [%(filename)s:%(lineno)d]: %(message)s" + ) + + color_formatter = ColoredFormatter( + "%(log_color)s[%(levelname)s]%(reset)s %(name)s [%(filename)s:%(lineno)d]: %(message)s", + log_colors={ + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red,bg_white", + }, + reset=True, + style="%", + ) + + console_handler = logging.StreamHandler() + console_handler.setFormatter(color_formatter) + + file_handler = RotatingFileHandler( + log_file, maxBytes=5000000, backupCount=5 + ) # 5MB file + file_handler.setFormatter(formatter) + + if not logger.handlers: + logger.addHandler(console_handler) + logger.addHandler(file_handler) + + return logger diff --git a/backend/core/quivr_core/api/main.py b/backend/core/quivr_core/api/main.py new file mode 100644 index 000000000000..d19efc3b2835 --- /dev/null +++ b/backend/core/quivr_core/api/main.py @@ -0,0 +1,40 @@ +import logging + +from fastapi import FastAPI +from quivr_core.api.modules.brain.controller import brain_router +from quivr_core.api.modules.chat.controller import chat_router +from quivr_core.api.modules.knowledge.controller import knowledge_router +from quivr_core.api.modules.prompt.controller import prompt_router +from quivr_core.api.modules.upload.controller import upload_router +from quivr_core.api.modules.user.controller import user_router + +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("LiteLLM").setLevel(logging.WARNING) +logging.getLogger("litellm").setLevel(logging.WARNING) + +app = FastAPI() + + +app.include_router(brain_router) +app.include_router(chat_router) +app.include_router(upload_router) +app.include_router(user_router) +app.include_router(prompt_router) +app.include_router(knowledge_router) + + +@app.get("/") +async def root(): + return {"status": "OK"} + + +@app.get("/healthz", tags=["Health"]) +async def healthz(): + return {"status": "ok"} + + +if __name__ == "__main__": + # run main.py to debug backend + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=5050, log_level="debug", access_log=False) diff --git a/backend/core/quivr_core/api/models/__init__.py b/backend/core/quivr_core/api/models/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/models/file_repository.py b/backend/core/quivr_core/api/models/file_repository.py new file mode 100644 index 000000000000..3fc263ae0b17 --- /dev/null +++ b/backend/core/quivr_core/api/models/file_repository.py @@ -0,0 +1,32 @@ +from uuid import UUID + +from quivr_core.api.models.settings import get_supabase_client + + +class FileRepository: + def __init__(self): + self.db = get_supabase_client() + + def set_file_vectors_ids(self, file_sha1: bytes): + response = ( + self.db.table("vectors") + .select("id") + .filter("file_sha1", "eq", str(file_sha1)) + .execute() + ) + return response.data + + def get_brain_vectors_by_brain_id_and_file_sha1( + self, brain_id: UUID, file_sha1: bytes + ): + self.set_file_vectors_ids(file_sha1) + # Check if file exists in that brain + response = ( + self.db.table("brains_vectors") + .select("brain_id, vector_id") + .filter("brain_id", "eq", str(brain_id)) + .filter("file_sha1", "eq", str(file_sha1)) + .execute() + ) + + return response diff --git a/backend/core/quivr_core/api/models/files.py b/backend/core/quivr_core/api/models/files.py new file mode 100644 index 000000000000..8f0e6938989f --- /dev/null +++ b/backend/core/quivr_core/api/models/files.py @@ -0,0 +1,105 @@ +from pathlib import Path +from typing import List, Optional + +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain_core.documents import Document +from pydantic import BaseModel + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.file_repository import FileRepository +from quivr_core.api.models.vectors import Vector +from quivr_core.api.modules.brain.service.brain_vector_service import BrainVectorService +from quivr_core.api.packages.files.file import compute_sha1_from_content + +logger = get_logger(__name__) + + +class File(BaseModel): + file_name: str + tmp_file_path: Path + bytes_content: bytes + file_size: int + file_extension: str + chunk_size: int = 400 + chunk_overlap: int = 100 + documents: List[Document] = [] + file_sha1: Optional[str] = None + vectors_ids: Optional[list] = [] + + def __init__(self, **data): + super().__init__(**data) + self.vectors_repository = Vector() + self.file_repository = FileRepository() + data["file_sha1"] = compute_sha1_from_content(data["bytes_content"]) + + def compute_documents(self, loader_class): + """ + Compute the documents from the file + + Args: + loader_class (class): The class of the loader to use to load the file + """ + logger.info(f"Computing documents from file {self.file_name}") + loader = loader_class(self.tmp_file_path) + documents = loader.load() + + text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap + ) + + self.documents = text_splitter.split_documents(documents) + + def set_file_vectors_ids(self): + """ + Set the vectors_ids property with the ids of the vectors + that are associated with the file in the vectors table + """ + self.vectors_ids = self.vectors_repository.get_vectors_by_file_sha1( + self.file_sha1 + ).data + + def file_already_exists(self): + """ + Check if file already exists in vectors table + """ + self.set_file_vectors_ids() + + # if the file does not exist in vectors then no need to go check in brains_vectors + if len(self.vectors_ids) == 0: # pyright: ignore reportPrivateUsage=none + return False + + return True + + def file_already_exists_in_brain(self, brain_id): + """ + Check if file already exists in a brain + + Args: + brain_id (str): Brain id + """ + response = self.file_repository.get_brain_vectors_by_brain_id_and_file_sha1( + brain_id, + self.file_sha1, # type: ignore + ) + + if len(response.data) == 0: + return False + + return True + + def file_is_empty(self): + """ + Check if file is empty by checking if the file pointer is at the beginning of the file + """ + return self.file_size < 1 # pyright: ignore reportPrivateUsage=none + + def link_file_to_brain(self, brain_id): + self.set_file_vectors_ids() + + if self.vectors_ids is None: + return + + brain_vector_service = BrainVectorService(brain_id) + + for vector_id in self.vectors_ids: # pyright: ignore reportPrivateUsage=none + brain_vector_service.create_brain_vector(vector_id["id"], self.file_sha1) diff --git a/backend/core/quivr_core/api/models/settings.py b/backend/core/quivr_core/api/models/settings.py new file mode 100644 index 000000000000..f35385c306b1 --- /dev/null +++ b/backend/core/quivr_core/api/models/settings.py @@ -0,0 +1,171 @@ +from typing import Optional +from uuid import UUID + +from dotenv import load_dotenv +from langchain.embeddings.base import Embeddings +from langchain_community.embeddings.ollama import OllamaEmbeddings +from langchain_community.vectorstores.supabase import SupabaseVectorStore +from langchain_openai import OpenAIEmbeddings +from posthog import Posthog +from pydantic_settings import BaseSettings, SettingsConfigDict +from sqlalchemy import Engine, create_engine +from supabase.client import Client, create_client + +from quivr_core.api.logger import get_logger + +load_dotenv() +logger = get_logger(__name__) + + +# The `PostHogSettings` class is used to initialize and interact with the PostHog analytics service. +class PostHogSettings(BaseSettings): + model_config = SettingsConfigDict(validate_default=False) + posthog_api_key: str | None = None + posthog_api_url: str | None = None + posthog: Posthog | None = None + + def __init__(self, *args, **kwargs): + """ + The function initializes the "posthog" attribute and calls the "initialize_posthog" method. + """ + super().__init__(*args, **kwargs) + self.posthog = None + self.initialize_posthog() + + def initialize_posthog(self): + """ + The function initializes a PostHog client with an API key and URL. + """ + if self.posthog_api_key and self.posthog_api_url: + self.posthog = Posthog( + api_key=self.posthog_api_key, host=self.posthog_api_url + ) + + def log_event(self, user_id: UUID, event_name: str, event_properties: dict): + """ + The function logs an event with a user ID, event name, and event properties using the PostHog + analytics tool. + + :param user_id: The user_id parameter is a UUID (Universally Unique Identifier) that uniquely + identifies a user. It is typically used to track and identify individual users in an application + or system + :type user_id: UUID + :param event_name: The event_name parameter is a string that represents the name or type of the + event that you want to log. It could be something like "user_signed_up", "item_purchased", or + "page_viewed" + :type event_name: str + :param event_properties: The event_properties parameter is a dictionary that contains additional + information or properties related to the event being logged. These properties provide more + context or details about the event and can be used for analysis or filtering purposes + :type event_properties: dict + """ + if self.posthog: + self.posthog.capture(user_id, event_name, event_properties) + + def set_user_properties(self, user_id: UUID, event_name, properties: dict): + """ + The function sets user properties for a given user ID and event name using the PostHog analytics + tool. + + :param user_id: The user_id parameter is a UUID (Universally Unique Identifier) that uniquely + identifies a user. It is used to associate the user with the event and properties being captured + :type user_id: UUID + :param event_name: The `event_name` parameter is a string that represents the name of the event + that you want to capture. It could be something like "user_signed_up" or "item_purchased" + :param properties: The `properties` parameter is a dictionary that contains the user properties + that you want to set. Each key-value pair in the dictionary represents a user property, where + the key is the name of the property and the value is the value you want to set for that property + :type properties: dict + """ + if self.posthog: + self.posthog.capture( + user_id, event=event_name, properties={"$set": properties} + ) + + def set_once_user_properties(self, user_id: UUID, event_name, properties: dict): + """ + The function sets user properties for a specific event, ensuring that the properties are only + set once. + + :param user_id: The user_id parameter is a UUID (Universally Unique Identifier) that uniquely + identifies a user + :type user_id: UUID + :param event_name: The `event_name` parameter is a string that represents the name of the event + that you want to capture. It could be something like "user_signed_up" or "item_purchased" + :param properties: The `properties` parameter is a dictionary that contains the user properties + that you want to set. Each key-value pair in the dictionary represents a user property, where + the key is the property name and the value is the property value + :type properties: dict + """ + if self.posthog: + self.posthog.capture( + user_id, event=event_name, properties={"$set_once": properties} + ) + + +class BrainSettings(BaseSettings): + model_config = SettingsConfigDict(validate_default=False) + pg_database_url: str + pg_database_async_url: str + openai_api_key: str = "" + supabase_url: str = "" + supabase_service_key: str = "" + resend_api_key: str = "null" + resend_email_address: str = "brain@mail.quivr.app" + ollama_api_base_url: str | None = None + langfuse_public_key: str | None = None + langfuse_secret_key: str | None = None + + +# Global variables to store the Supabase client and database instances +_supabase_client: Optional[Client] = None +_db_engine: Optional[Engine] = None +_embedding_service = None + +settings = BrainSettings() # type: ignore + + +def get_pg_database_engine(): + global _db_engine + if _db_engine is None: + logger.info("Creating Postgres DB engine") + _db_engine = create_engine(settings.pg_database_url, pool_pre_ping=True) + return _db_engine + + +def get_pg_database_async_engine(): + global _db_engine + if _db_engine is None: + logger.info("Creating Postgres DB engine") + _db_engine = create_engine(settings.pg_database_async_url, pool_pre_ping=True) + return _db_engine + + +def get_supabase_client() -> Client: + global _supabase_client + if _supabase_client is None: + logger.info("Creating Supabase client") + _supabase_client = create_client( + settings.supabase_url, settings.supabase_service_key + ) + return _supabase_client + + +def get_embedding_client() -> Embeddings: + global _embedding_service + if settings.ollama_api_base_url: + embeddings = OllamaEmbeddings( + base_url=settings.ollama_api_base_url, + ) # pyright: ignore reportPrivateUsage=none + else: + embeddings = OpenAIEmbeddings() # pyright: ignore reportPrivateUsage=none + return embeddings + + +def get_documents_vector_store() -> SupabaseVectorStore: + embeddings = get_embedding_client() + supabase_client: Client = get_supabase_client() + documents_vector_store = SupabaseVectorStore( + supabase_client, embeddings, table_name="vectors" + ) + return documents_vector_store diff --git a/backend/core/quivr_core/api/models/vectors.py b/backend/core/quivr_core/api/models/vectors.py new file mode 100644 index 000000000000..e2d207d4d88b --- /dev/null +++ b/backend/core/quivr_core/api/models/vectors.py @@ -0,0 +1,78 @@ +from postgrest.types import CountMethod + +from quivr_core.api.models.settings import get_supabase_client + + +class Vector: + def __init__(self): + self.db = get_supabase_client() + + def get_vectors_by_file_name(self, file_name): + response = ( + self.db.table("vectors") + .select( + "metadata->>file_name, metadata->>file_size, metadata->>file_extension, metadata->>file_url", + "content", + "brains_vectors(brain_id,vector_id)", + ) + .match({"metadata->>file_name": file_name}) + .execute() + ) + + return response + + def get_vectors_by_file_sha1(self, file_sha1): + response = ( + self.db.table("vectors") + .select("id") + .filter("file_sha1", "eq", file_sha1) + .execute() + ) + + return response + + # TODO: remove duplicate similarity_search in supabase vector store + def similarity_search(self, query_embedding, table, k, threshold): + response = self.db.rpc( + table, + { + "query_embedding": query_embedding, + "match_count": k, + "match_threshold": threshold, + }, + ).execute() + return response + + def update_summary(self, document_id, summary_id): + return ( + self.db.table("summaries") + .update({"document_id": document_id}) + .match({"id": summary_id}) + .execute() + ) + + def get_vectors_by_batch(self, batch_id): + response = ( + self.db.table("vectors") + .select( + "name:metadata->>file_name, size:metadata->>file_size", + count=CountMethod.exact, + ) + .eq("id", batch_id) + .execute() + ) + + return response + + def get_vectors_in_batch(self, batch_ids): + response = ( + self.db.table("vectors") + .select( + "name:metadata->>file_name, size:metadata->>file_size", + count=CountMethod.exact, + ) + .in_("id", batch_ids) + .execute() + ) + + return response diff --git a/backend/core/quivr_core/api/modules/__init__.py b/backend/core/quivr_core/api/modules/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/brain/__init__.py b/backend/core/quivr_core/api/modules/brain/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/brain/controller/__init__.py b/backend/core/quivr_core/api/modules/brain/controller/__init__.py new file mode 100644 index 000000000000..7e54fbb96673 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/controller/__init__.py @@ -0,0 +1 @@ +from .brain_routes import brain_router diff --git a/backend/core/quivr_core/api/modules/brain/controller/brain_routes.py b/backend/core/quivr_core/api/modules/brain/controller/brain_routes.py new file mode 100644 index 000000000000..396b3d7c76f6 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/controller/brain_routes.py @@ -0,0 +1,143 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Request + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.dto.inputs import ( + BrainQuestionRequest, + BrainUpdatableProperties, + CreateBrainProperties, +) +from quivr_core.api.modules.brain.entity.brain_entity import PublicBrain, RoleEnum +from quivr_core.api.modules.brain.entity.integration_brain import ( + IntegrationDescriptionEntity, +) +from quivr_core.api.modules.brain.service.brain_service import BrainService +from quivr_core.api.modules.brain.service.brain_user_service import BrainUserService +from quivr_core.api.modules.brain.service.get_question_context_from_brain import ( + get_question_context_from_brain, +) +from quivr_core.api.modules.brain.service.integration_brain_service import ( + IntegrationBrainDescriptionService, +) +from quivr_core.api.modules.dependencies import get_current_user, get_service +from quivr_core.api.modules.prompt.service.prompt_service import PromptService +from quivr_core.api.modules.user.entity.user_identity import UserIdentity +from quivr_core.api.packages.utils.telemetry import maybe_send_telemetry + +logger = get_logger(__name__) +brain_router = APIRouter() + +brain_service = BrainService() +brain_user_service = BrainUserService() +integration_brain_description_service = IntegrationBrainDescriptionService() + +PromptServiceDep = Annotated[PromptService, Depends(get_service(PromptService))] + + +@brain_router.get( + "/brains/integrations/", +) +async def get_integration_brain_description() -> list[IntegrationDescriptionEntity]: + """Retrieve the integration brain description.""" + return integration_brain_description_service.get_all_integration_descriptions() + + +@brain_router.get("/brains/", tags=["Brain"]) +async def retrieve_all_brains_for_user( + current_user: UserIdentity = Depends(get_current_user), +): + """Retrieve all brains for the current user.""" + brains = brain_user_service.get_user_brains(current_user.id) + return {"brains": brains} + + +@brain_router.get("/brains/public", tags=["Brain"]) +async def retrieve_public_brains() -> list[PublicBrain]: + """Retrieve all Quivr public brains.""" + return brain_service.get_public_brains() + + +@brain_router.get( + "/brains/{brain_id}/", + tags=["Brain"], +) +async def retrieve_brain_by_id( + brain_id: UUID, + current_user: UserIdentity = Depends(get_current_user), +): + """Retrieve details of a specific brain by its ID.""" + brain_details = brain_service.get_brain_details(brain_id, current_user.id) + if brain_details is None: + raise HTTPException(status_code=404, detail="Brain details not found") + return brain_details + + +@brain_router.post("/brains/", tags=["Brain"]) +async def create_new_brain( + brain: CreateBrainProperties, + request: Request, + current_user: UserIdentity = Depends(get_current_user), +): + """Create a new brain for the user.""" + + maybe_send_telemetry("create_brain", {"brain_name": brain.name}, request) + new_brain = brain_service.create_brain( + brain=brain, + user_id=current_user.id, + ) + brain_user_service.create_brain_user( + user_id=current_user.id, + brain_id=new_brain.brain_id, + rights=RoleEnum.Owner, + is_default_brain=True, + ) + + return {"id": new_brain.brain_id, "name": brain.name, "rights": "Owner"} + + +@brain_router.put( + "/brains/{brain_id}/", + dependencies=[], + tags=["Brain"], +) +async def update_existing_brain( + brain_id: UUID, + brain_update_data: BrainUpdatableProperties, + prompt_service: PromptServiceDep, + current_user: UserIdentity = Depends(get_current_user), +): + """Update an existing brain's configuration.""" + existing_brain = brain_service.get_brain_details(brain_id, current_user.id) + if existing_brain is None: + raise HTTPException(status_code=404, detail="Brain not found") + + if brain_update_data.prompt_id is None and existing_brain.prompt_id: + prompt = prompt_service.get_prompt_by_id(existing_brain.prompt_id) + if prompt and prompt.status == "private": + prompt_service.delete_prompt_by_id(existing_brain.prompt_id) + + return {"message": f"Prompt {brain_id} has been updated."} + + elif brain_update_data.status == "private" and existing_brain.status == "public": + brain_user_service.delete_brain_users(brain_id) + return {"message": f"Brain {brain_id} has been deleted."} + + else: + brain_service.update_brain_by_id(brain_id, brain_update_data) + + return {"message": f"Brain {brain_id} has been updated."} + + +@brain_router.post( + "/brains/{brain_id}/documents", + tags=["Brain"], +) +async def get_question_context_for_brain( + brain_id: UUID, question: BrainQuestionRequest +): + # TODO: Move this endpoint to AnswerGenerator service + """Retrieve the question context from a specific brain.""" + context = get_question_context_from_brain(brain_id, question.question) + return {"docs": context} diff --git a/backend/core/quivr_core/api/modules/brain/dto/__init__.py b/backend/core/quivr_core/api/modules/brain/dto/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/brain/dto/inputs.py b/backend/core/quivr_core/api/modules/brain/dto/inputs.py new file mode 100644 index 000000000000..068751ac510f --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/dto/inputs.py @@ -0,0 +1,88 @@ +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.entity.api_brain_definition_entity import ( + ApiBrainAllowedMethods, + ApiBrainDefinitionEntity, + ApiBrainDefinitionSchema, + ApiBrainDefinitionSecret, +) +from quivr_core.api.modules.brain.entity.brain_entity import BrainType +from quivr_core.api.modules.brain.entity.integration_brain import IntegrationType + +logger = get_logger(__name__) + + +class CreateApiBrainDefinition(BaseModel, extra="ignore"): + method: ApiBrainAllowedMethods + url: str + params: Optional[ApiBrainDefinitionSchema] = ApiBrainDefinitionSchema() + search_params: ApiBrainDefinitionSchema = ApiBrainDefinitionSchema() + secrets: Optional[list[ApiBrainDefinitionSecret]] = [] + raw: Optional[bool] = False + jq_instructions: Optional[str] = None + + +class CreateIntegrationBrain(BaseModel, extra="ignore"): + integration_name: str + integration_logo_url: str + connection_settings: dict + integration_type: IntegrationType + description: str + max_files: int + + +class BrainIntegrationSettings(BaseModel, extra="ignore"): + integration_id: str + settings: dict + + +class BrainIntegrationUpdateSettings(BaseModel, extra="ignore"): + settings: dict + + +class CreateBrainProperties(BaseModel, extra="ignore"): + name: Optional[str] = "Default brain" + description: str = "This is a description" + status: Optional[str] = "private" + model: Optional[str] = None + temperature: Optional[float] = 0.0 + max_tokens: Optional[int] = 2000 + prompt_id: Optional[UUID] = None + brain_type: Optional[BrainType] = BrainType.doc + brain_definition: Optional[CreateApiBrainDefinition] = None + brain_secrets_values: Optional[dict] = {} + connected_brains_ids: Optional[list[UUID]] = [] + integration: Optional[BrainIntegrationSettings] = None + + def dict(self, *args, **kwargs): + brain_dict = super().dict(*args, **kwargs) + if brain_dict.get("prompt_id"): + brain_dict["prompt_id"] = str(brain_dict.get("prompt_id")) + return brain_dict + + +class BrainUpdatableProperties(BaseModel, extra="ignore"): + name: Optional[str] = None + description: Optional[str] = None + temperature: Optional[float] = None + model: Optional[str] = None + max_tokens: Optional[int] = None + status: Optional[str] = None + prompt_id: Optional[UUID] = None + brain_definition: Optional[ApiBrainDefinitionEntity] = None + connected_brains_ids: Optional[list[UUID]] = [] + integration: Optional[BrainIntegrationUpdateSettings] = None + + def dict(self, *args, **kwargs): + brain_dict = super().dict(*args, **kwargs) + if brain_dict.get("prompt_id"): + brain_dict["prompt_id"] = str(brain_dict.get("prompt_id")) + return brain_dict + + +class BrainQuestionRequest(BaseModel): + question: str diff --git a/backend/core/quivr_core/api/modules/brain/entity/__init__.py b/backend/core/quivr_core/api/modules/brain/entity/__init__.py new file mode 100644 index 000000000000..ba083445ccf1 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/entity/__init__.py @@ -0,0 +1 @@ +from .api_brain_definition_entity import ApiBrainDefinitionEntity diff --git a/backend/core/quivr_core/api/modules/brain/entity/api_brain_definition_entity.py b/backend/core/quivr_core/api/modules/brain/entity/api_brain_definition_entity.py new file mode 100644 index 000000000000..d327d44ac3ba --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/entity/api_brain_definition_entity.py @@ -0,0 +1,47 @@ +from enum import Enum +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, Extra + + +class ApiBrainDefinitionSchemaProperty(BaseModel, extra=Extra.forbid): + type: str + description: str + enum: Optional[list] = None + name: str + + def dict(self, **kwargs): + result = super().dict(**kwargs) + if "enum" in result and result["enum"] is None: + del result["enum"] + return result + + +class ApiBrainDefinitionSchema(BaseModel, extra=Extra.forbid): + properties: list[ApiBrainDefinitionSchemaProperty] = [] + required: list[str] = [] + + +class ApiBrainDefinitionSecret(BaseModel, extra=Extra.forbid): + name: str + type: str + description: Optional[str] = None + + +class ApiBrainAllowedMethods(str, Enum): + GET = "GET" + POST = "POST" + PUT = "PUT" + DELETE = "DELETE" + + +class ApiBrainDefinitionEntity(BaseModel, extra=Extra.forbid): + brain_id: UUID + method: ApiBrainAllowedMethods + url: str + params: ApiBrainDefinitionSchema + search_params: ApiBrainDefinitionSchema + secrets: list[ApiBrainDefinitionSecret] + raw: bool = False + jq_instructions: Optional[str] = None diff --git a/backend/core/quivr_core/api/modules/brain/entity/brain_entity.py b/backend/core/quivr_core/api/modules/brain/entity/brain_entity.py new file mode 100644 index 000000000000..bf963fcccae1 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/entity/brain_entity.py @@ -0,0 +1,135 @@ +from datetime import datetime +from enum import Enum +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel + +# from sqlmodel import Enum as PGEnum +from sqlalchemy.dialects.postgresql import ENUM as PGEnum +from sqlalchemy.ext.asyncio import AsyncAttrs +from sqlmodel import TIMESTAMP, Column, Field, Relationship, SQLModel, text +from sqlmodel import UUID as PGUUID + +from quivr_core.api.modules.brain.entity.api_brain_definition_entity import ( + ApiBrainDefinitionEntity, +) +from quivr_core.api.modules.brain.entity.integration_brain import ( + IntegrationDescriptionEntity, + IntegrationEntity, +) +from quivr_core.api.modules.prompt.entity.prompt import Prompt + + +class BrainType(str, Enum): + doc = "doc" + api = "api" + composite = "composite" + integration = "integration" + + +class Brain(AsyncAttrs, SQLModel, table=True): + __tablename__ = "brains" # type: ignore + + brain_id: UUID | None = Field( + default=None, + sa_column=Column( + PGUUID, + server_default=text("uuid_generate_v4()"), + primary_key=True, + ), + ) + name: str + description: str + status: str | None = None + model: str | None = None + max_tokens: int | None = None + temperature: float | None = None + last_update: datetime | None = Field( + default=None, + sa_column=Column( + TIMESTAMP(timezone=False), + server_default=text("CURRENT_TIMESTAMP"), + ), + ) + brain_type: BrainType | None = Field( + sa_column=Column( + PGEnum(BrainType, name="brain_type_enum", create_type=False), + default=BrainType.integration, + ), + ) + brain_chat_history: List["ChatHistory"] = Relationship( # noqa: F821 + back_populates="brain", sa_relationship_kwargs={"lazy": "select"} + ) + prompt_id: UUID | None = Field(default=None, foreign_key="prompts.id") + prompt: Prompt | None = Relationship( # noqa: f821 + back_populates="brain", sa_relationship_kwargs={"lazy": "joined"} + ) + + # TODO : add + # "meaning" "public"."vector", + # "tags" "public"."tags"[] + + +class BrainEntity(BaseModel): + brain_id: UUID + name: str + description: Optional[str] = None + temperature: Optional[float] = None + model: Optional[str] = None + max_tokens: Optional[int] = None + status: Optional[str] = None + prompt_id: Optional[UUID] = None + last_update: datetime + brain_type: BrainType + brain_definition: Optional[ApiBrainDefinitionEntity] = None + connected_brains_ids: Optional[List[UUID]] = None + raw: Optional[bool] = None + jq_instructions: Optional[str] = None + integration: Optional[IntegrationEntity] = None + integration_description: Optional[IntegrationDescriptionEntity] = None + + @property + def id(self) -> UUID: + return self.brain_id + + def dict(self, **kwargs): + data = super().dict( + **kwargs, + ) + data["id"] = self.id + return data + + +class PublicBrain(BaseModel): + id: UUID + name: str + description: Optional[str] = None + number_of_subscribers: int = 0 + last_update: str + brain_type: BrainType + brain_definition: Optional[ApiBrainDefinitionEntity] = None + + +class RoleEnum(str, Enum): + Viewer = "Viewer" + Editor = "Editor" + Owner = "Owner" + + +class BrainUser(BaseModel): + id: UUID + user_id: UUID + rights: RoleEnum + default_brain: bool = False + + +class MinimalUserBrainEntity(BaseModel): + id: UUID + name: str + rights: RoleEnum + status: str + brain_type: BrainType + description: str + integration_logo_url: str + max_files: int diff --git a/backend/core/quivr_core/api/modules/brain/entity/integration_brain.py b/backend/core/quivr_core/api/modules/brain/entity/integration_brain.py new file mode 100644 index 000000000000..1b60b3f3af54 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/entity/integration_brain.py @@ -0,0 +1,46 @@ +from enum import Enum +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + + +class IntegrationType(str, Enum): + CUSTOM = "custom" + SYNC = "sync" + DOC = "doc" + + +class IntegrationBrainTag(str, Enum): + NEW = "new" + RECOMMENDED = "recommended" + MOST_POPULAR = "most_popular" + PREMIUM = "premium" + COMING_SOON = "coming_soon" + COMMUNITY = "community" + DEPRECATED = "deprecated" + + +class IntegrationDescriptionEntity(BaseModel): + id: UUID + integration_name: str + integration_logo_url: Optional[str] = None + connection_settings: Optional[dict] = None + integration_type: IntegrationType + tags: Optional[list[IntegrationBrainTag]] = [] + information: Optional[str] = None + description: str + max_files: int + allow_model_change: bool + integration_display_name: str + onboarding_brain: bool + + +class IntegrationEntity(BaseModel): + id: int + user_id: str + brain_id: str + integration_id: UUID + settings: Optional[dict] = None + credentials: Optional[dict] = None + last_synced: str diff --git a/backend/core/quivr_core/api/modules/brain/repository/__init__.py b/backend/core/quivr_core/api/modules/brain/repository/__init__.py new file mode 100644 index 000000000000..46e80777797f --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/repository/__init__.py @@ -0,0 +1,4 @@ +from .brains import Brains +from .brains_users import BrainsUsers +from .brains_vectors import BrainsVectors +from .integration_brains import IntegrationBrain, IntegrationDescription diff --git a/backend/core/quivr_core/api/modules/brain/repository/brains.py b/backend/core/quivr_core/api/modules/brain/repository/brains.py new file mode 100644 index 000000000000..e3ca02a122e5 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/repository/brains.py @@ -0,0 +1,124 @@ +from uuid import UUID + +from sqlalchemy import text + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import ( + get_embedding_client, + get_pg_database_engine, + get_supabase_client, +) +from quivr_core.api.modules.brain.dto.inputs import BrainUpdatableProperties +from quivr_core.api.modules.brain.entity.brain_entity import BrainEntity, PublicBrain + +logger = get_logger(__name__) + + +class Brains: + def __init__(self): + supabase_client = get_supabase_client() + self.db = supabase_client + pg_engine = get_pg_database_engine() + self.pg_engine = pg_engine + + def create_brain(self, brain): + embeddings = get_embedding_client() + string_to_embed = f"Name: {brain.name} Description: {brain.description}" + brain_meaning = embeddings.embed_query(string_to_embed) + brain_dict = brain.dict( + exclude={ + "brain_definition", + "brain_secrets_values", + "connected_brains_ids", + "integration", + } + ) + brain_dict["meaning"] = brain_meaning + response = (self.db.table("brains").insert(brain_dict)).execute() + + return BrainEntity(**response.data[0]) + + def get_public_brains(self): + response = ( + self.db.from_("brains") + .select( + "id:brain_id, name, description, last_update, brain_type, brain_definition: api_brain_definition(*), number_of_subscribers:brains_users(count)" + ) + .filter("status", "eq", "public") + .execute() + ) + public_brains: list[PublicBrain] = [] + + for item in response.data: + item["number_of_subscribers"] = item["number_of_subscribers"][0]["count"] + if not item["brain_definition"]: + del item["brain_definition"] + else: + item["brain_definition"]["secrets"] = [] + + public_brains.append(PublicBrain(**item)) + return public_brains + + def update_brain_last_update_time(self, brain_id): + try: + with self.pg_engine.begin() as connection: + query = """ + UPDATE brains + SET last_update = now() + WHERE brain_id = '{brain_id}' + """ + connection.execute(text(query.format(brain_id=brain_id))) + except Exception as e: + logger.error(e) + + def get_brain_details(self, brain_id): + with self.pg_engine.begin() as connection: + query = """ + SELECT * FROM brains + WHERE brain_id = '{brain_id}' + """ + response = connection.execute( + text(query.format(brain_id=brain_id)) + ).fetchall() + if len(response) == 0: + return None + return BrainEntity(**response[0]._mapping) + + def delete_brain(self, brain_id: str): + with self.pg_engine.begin() as connection: + results = connection.execute( + text(f"DELETE FROM brains WHERE brain_id = '{brain_id}'") + ) + + return results + + def update_brain_by_id( + self, brain_id: UUID, brain: BrainUpdatableProperties + ) -> BrainEntity | None: + embeddings = get_embedding_client() + string_to_embed = f"Name: {brain.name} Description: {brain.description}" + brain_meaning = embeddings.embed_query(string_to_embed) + brain_dict = brain.dict(exclude_unset=True) + brain_dict["meaning"] = brain_meaning + update_brain_response = ( + self.db.table("brains") + .update(brain_dict) + .match({"brain_id": brain_id}) + .execute() + ).data + + if len(update_brain_response) == 0: + return None + + return BrainEntity(**update_brain_response[0]) + + def get_brain_by_id(self, brain_id: UUID) -> BrainEntity | None: + # TODO: merge this method with get_brain_details + with self.pg_engine.begin() as connection: + response = connection.execute( + text(f"SELECT * FROM brains WHERE brain_id = '{brain_id}'") + ).fetchall() + + if len(response) == 0: + return None + return BrainEntity(**response[0]._mapping) diff --git a/backend/core/quivr_core/api/modules/brain/repository/brains_users.py b/backend/core/quivr_core/api/modules/brain/repository/brains_users.py new file mode 100644 index 000000000000..29be9cfeced8 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/repository/brains_users.py @@ -0,0 +1,204 @@ +from uuid import UUID + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import get_embedding_client, get_supabase_client +from quivr_core.api.modules.brain.entity.brain_entity import ( + BrainUser, + MinimalUserBrainEntity, +) + +logger = get_logger(__name__) + + +class BrainsUsers: + def __init__(self): + supabase_client = get_supabase_client() + self.db = supabase_client + + def update_meaning(self, brain: MinimalUserBrainEntity): + embeddings = get_embedding_client() + string_to_embed = f"Name: {brain.name} Description: {brain.description}" + brain_meaning = embeddings.embed_query(string_to_embed) + brain_dict = {"meaning": brain_meaning} + response = ( + self.db.table("brains") + .update(brain_dict) + .match({"brain_id": brain.id}) + .execute() + ).data + + if len(response) == 0: + return False + + return True + + def get_user_brains(self, user_id) -> list[MinimalUserBrainEntity]: + response = ( + self.db.from_("brains_users") + .select( + "id:brain_id, rights, brains (brain_id, name, status, brain_type, description, meaning, integrations_user (brain_id, integration_id, integrations (id, integration_name, integration_logo_url, max_files)))" + ) + .filter("user_id", "eq", user_id) + .execute() + ) + user_brains: list[MinimalUserBrainEntity] = [] + for item in response.data: + integration_logo_url = "" + max_files = 5000 + if item["brains"]["brain_type"] == "integration": + if "integrations_user" in item["brains"]: + for integration_user in item["brains"]["integrations_user"]: + if "integrations" in integration_user: + integration_logo_url = integration_user["integrations"][ + "integration_logo_url" + ] + max_files = integration_user["integrations"]["max_files"] + + user_brains.append( + MinimalUserBrainEntity( + id=item["brains"]["brain_id"], + name=item["brains"]["name"], + rights=item["rights"], + status=item["brains"]["status"], + brain_type=item["brains"]["brain_type"], + description=( + item["brains"]["description"] + if item["brains"]["description"] is not None + else "" + ), + integration_logo_url=str(integration_logo_url), + max_files=max_files, + ) + ) + user_brains[-1].rights = item["rights"] + if item["brains"]["meaning"] is None: + self.update_meaning(user_brains[-1]) + + return user_brains + + def get_brain_for_user(self, user_id, brain_id): + response = ( + self.db.from_("brains_users") + .select( + "id:brain_id, rights, brains (id: brain_id, status, name, brain_type, description)" + ) + .filter("user_id", "eq", user_id) + .filter("brain_id", "eq", brain_id) + .execute() + ) + if len(response.data) == 0: + return None + brain_data = response.data[0] + + return MinimalUserBrainEntity( + id=brain_data["brains"]["id"], + name=brain_data["brains"]["name"], + rights=brain_data["rights"], + status=brain_data["brains"]["status"], + brain_type=brain_data["brains"]["brain_type"], + description=( + brain_data["brains"]["description"] + if brain_data["brains"]["description"] is not None + else "" + ), + integration_logo_url="", + max_files=100, + ) + + def delete_brain_user_by_id( + self, + user_id: UUID, + brain_id: UUID, + ): + results = ( + self.db.table("brains_users") + .delete() + .match({"brain_id": str(brain_id), "user_id": str(user_id)}) + .execute() + ) + return results.data + + def delete_brain_users(self, brain_id: str): + results = ( + self.db.table("brains_users") + .delete() + .match({"brain_id": brain_id}) + .execute() + ) + + return results + + def create_brain_user(self, user_id: UUID, brain_id, rights, default_brain: bool): + response = ( + self.db.table("brains_users") + .insert( + { + "brain_id": str(brain_id), + "user_id": str(user_id), + "rights": rights, + "default_brain": default_brain, + } + ) + .execute() + ) + return response + + def get_user_default_brain_id(self, user_id: UUID) -> UUID | None: + response = ( + self.db.from_("brains_users") + .select("brain_id") + .filter("user_id", "eq", str(user_id)) + .filter("default_brain", "eq", "True") + .execute() + ).data + if len(response) == 0: + return None + return UUID(response[0].get("brain_id")) + + def get_brain_users(self, brain_id: UUID) -> list[BrainUser]: + response = ( + self.db.table("brains_users") + .select("id:brain_id, *") + .filter("brain_id", "eq", str(brain_id)) + .execute() + ) + + return [BrainUser(**item) for item in response.data] + + def delete_brain_subscribers(self, brain_id: UUID): + results = ( + self.db.table("brains_users") + .delete() + .match({"brain_id": str(brain_id)}) + .match({"rights": "Viewer"}) + .execute() + ).data + + return results + + def get_brain_subscribers_count(self, brain_id: UUID) -> int: + response = ( + self.db.from_("brains_users") + .select( + "count", + ) + .filter("brain_id", "eq", str(brain_id)) + .execute() + ).data + if len(response) == 0: + raise ValueError(f"Brain with id {brain_id} does not exist.") + return response[0]["count"] + + def update_brain_user_default_status( + self, user_id: UUID, brain_id: UUID, default_brain: bool + ): + self.db.table("brains_users").update({"default_brain": default_brain}).match( + {"brain_id": brain_id, "user_id": user_id} + ).execute() + + def update_brain_user_rights( + self, user_id: UUID, brain_id: UUID, rights: str + ) -> None: + self.db.table("brains_users").update({"rights": rights}).match( + {"brain_id": brain_id, "user_id": user_id} + ).execute() diff --git a/backend/core/quivr_core/api/modules/brain/repository/brains_vectors.py b/backend/core/quivr_core/api/modules/brain/repository/brains_vectors.py new file mode 100644 index 000000000000..dfc551337b96 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/repository/brains_vectors.py @@ -0,0 +1,102 @@ +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import get_supabase_client + +logger = get_logger(__name__) + + +class BrainsVectors: + def __init__(self): + supabase_client = get_supabase_client() + self.db = supabase_client + + def create_brain_vector(self, brain_id, vector_id, file_sha1): + response = ( + self.db.table("brains_vectors") + .insert( + { + "brain_id": str(brain_id), + "vector_id": str(vector_id), + "file_sha1": file_sha1, + } + ) + .execute() + ) + return response.data + + def get_vector_ids_from_file_sha1(self, file_sha1: str): + # move to vectors class + vectorsResponse = ( + self.db.table("vectors") + .select("id") + .filter("file_sha1", "eq", file_sha1) + .execute() + ) + return vectorsResponse.data + + def get_brain_vector_ids(self, brain_id): + """ + Retrieve unique brain data (i.e. uploaded files and crawled websites). + """ + + response = ( + self.db.from_("brains_vectors") + .select("vector_id") + .filter("brain_id", "eq", brain_id) + .execute() + ) + + vector_ids = [item["vector_id"] for item in response.data] + + if len(vector_ids) == 0: + return [] + + return vector_ids + + def delete_file_from_brain(self, brain_id, file_name: str): + # First, get the vector_ids associated with the file_name + # TODO: filter by brain_id + file_vectors = ( + self.db.table("vectors") + .select("id") + .filter("metadata->>file_name", "eq", file_name) + .execute() + ) + + file_vectors_ids = [item["id"] for item in file_vectors.data] + + # remove current file vectors from brain vectors + self.db.table("brains_vectors").delete().filter( + "vector_id", "in", f"({','.join(map(str, file_vectors_ids))})" + ).filter("brain_id", "eq", brain_id).execute() + + vectors_used_by_another_brain = ( + self.db.table("brains_vectors") + .select("vector_id") + .filter("vector_id", "in", f"({','.join(map(str, file_vectors_ids))})") + .filter("brain_id", "neq", brain_id) + .execute() + ) + + vectors_used_by_another_brain_ids = [ + item["vector_id"] for item in vectors_used_by_another_brain.data + ] + + vectors_no_longer_used_ids = [ + id for id in file_vectors_ids if id not in vectors_used_by_another_brain_ids + ] + + self.db.table("vectors").delete().filter( + "id", "in", f"({','.join(map(str, vectors_no_longer_used_ids))})" + ).execute() + + return {"message": f"File {file_name} in brain {brain_id} has been deleted."} + + def delete_brain_vector(self, brain_id: str): + results = ( + self.db.table("brains_vectors") + .delete() + .match({"brain_id": brain_id}) + .execute() + ) + + return results diff --git a/backend/core/quivr_core/api/modules/brain/repository/integration_brains.py b/backend/core/quivr_core/api/modules/brain/repository/integration_brains.py new file mode 100644 index 000000000000..9e39dc486b7f --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/repository/integration_brains.py @@ -0,0 +1,136 @@ +from typing import List +from uuid import UUID + +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.brain.entity.integration_brain import ( + IntegrationDescriptionEntity, + IntegrationEntity, +) + + +class IntegrationBrain: + """This is all the methods to interact with the integration brain. + + Args: + IntegrationBrainInterface (_type_): _description_ + """ + + def __init__(self): + self.db = get_supabase_client() + + def get_integration_brain(self, brain_id, user_id=None): + query = ( + self.db.table("integrations_user") + .select("*") + .filter("brain_id", "eq", brain_id) + ) + + if user_id: + query.filter("user_id", "eq", user_id) + + response = query.execute() + + if len(response.data) == 0: + return None + + return IntegrationEntity(**response.data[0]) + + def update_last_synced(self, brain_id, user_id): + response = ( + self.db.table("integrations_user") + .update({"last_synced": "now()"}) + .filter("brain_id", "eq", str(brain_id)) + .filter("user_id", "eq", str(user_id)) + .execute() + ) + if len(response.data) == 0: + return None + return IntegrationEntity(**response.data[0]) + + def add_integration_brain(self, brain_id, user_id, integration_id, settings): + response = ( + self.db.table("integrations_user") + .insert( + [ + { + "brain_id": str(brain_id), + "user_id": str(user_id), + "integration_id": str(integration_id), + "settings": settings, + } + ] + ) + .execute() + ) + if len(response.data) == 0: + return None + return IntegrationEntity(**response.data[0]) + + def update_integration_brain(self, brain_id, user_id, integration_brain): + response = ( + self.db.table("integrations_user") + .update(integration_brain.dict(exclude={"brain_id", "user_id"})) + .filter("brain_id", "eq", str(brain_id)) + .filter("user_id", "eq", str(user_id)) + .execute() + ) + if len(response.data) == 0: + return None + return IntegrationEntity(**response.data[0]) + + def delete_integration_brain(self, brain_id, user_id): + self.db.table("integrations_user").delete().filter( + "brain_id", "eq", str(brain_id) + ).filter("user_id", "eq", str(user_id)).execute() + return None + + def get_integration_brain_by_type_integration( + self, integration_name + ) -> List[IntegrationEntity] | None: + response = ( + self.db.table("integrations_user") + .select("*, integrations ()") + .filter("integrations.integration_name", "eq", integration_name) + .execute() + ) + if len(response.data) == 0: + return None + + return [IntegrationEntity(**data) for data in response.data] + + +class IntegrationDescription: + def __init__(self): + self.db = get_supabase_client() + + def get_integration_description(self, integration_id: UUID): + response = ( + self.db.table("integrations") + .select("*") + .filter("id", "eq", str(integration_id)) + .execute() + ) + if len(response.data) == 0: + return None + + return IntegrationDescriptionEntity(**response.data[0]) + + def get_integration_description_by_user_brain_id( + self, brain_id: UUID, user_id: UUID + ): + response = ( + self.db.table("integrations_user") + .select("*") + .filter("brain_id", "eq", str(brain_id)) + .filter("user_id", "eq", str(user_id)) + .execute() + ) + if len(response.data) == 0: + return None + + integration_id = response.data[0]["integration_id"] + return self.get_integration_description(integration_id) + + def get_all_integration_descriptions(self): + response = self.db.table("integrations").select("*").execute() + return [IntegrationDescriptionEntity(**data) for data in response.data] diff --git a/backend/core/quivr_core/api/modules/brain/service/__init__.py b/backend/core/quivr_core/api/modules/brain/service/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/brain/service/brain_service.py b/backend/core/quivr_core/api/modules/brain/service/brain_service.py new file mode 100644 index 000000000000..8b5fe10e0486 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/brain_service.py @@ -0,0 +1,220 @@ +from typing import Optional, Tuple +from uuid import UUID + +from fastapi import HTTPException + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.dto.inputs import ( + BrainUpdatableProperties, + CreateBrainProperties, +) +from quivr_core.api.modules.brain.entity.brain_entity import ( + BrainEntity, + BrainType, + PublicBrain, +) +from quivr_core.api.modules.brain.entity.integration_brain import IntegrationEntity +from quivr_core.api.modules.brain.repository import ( + Brains, + BrainsUsers, + BrainsVectors, + IntegrationBrain, + IntegrationDescription, +) +from quivr_core.api.modules.knowledge.service.knowledge_service import KnowledgeService +from quivr_core.api.packages.vectorstore.supabase import CustomSupabaseVectorStore + +logger = get_logger(__name__) + +knowledge_service = KnowledgeService() + + +class BrainService: + def __init__(self): + self.brain_repository: Brains = Brains() + self.brain_user_repository = BrainsUsers() + self.brain_vector = BrainsVectors() + self.integration_brains_repository = IntegrationBrain() + self.integration_description_repository = IntegrationDescription() + + def get_brain_by_id(self, brain_id: UUID): + return self.brain_repository.get_brain_by_id(brain_id) + + def get_integration_brain(self, brain_id) -> IntegrationEntity | None: + return self.integration_brains_repository.get_integration_brain(brain_id) + + def find_brain_from_question( + self, + brain_id: UUID, + question: str, + user, + chat_id: UUID, + history, + vector_store: CustomSupabaseVectorStore, + ) -> Tuple[Optional[BrainEntity], dict[str, str]]: + """Find the brain to use for a question. + + Args: + brain_id (UUID): ID of the brain to use if exists + question (str): Question for which to find the brain + user (UserEntity): User asking the question + chat_id (UUID): ID of the chat + + Returns: + Optional[BrainEntity]: Returns the brain to use for the question + """ + metadata = {} + + # Init + + brain_id_to_use = brain_id + brain_to_use = None + + # Get the first question from the chat_question + + question = question + + list_brains = [] # To return + + if history and not brain_id_to_use: + question = history[0].user_message + brain_id_to_use = history[0].brain_id + brain_to_use = self.get_brain_by_id(brain_id_to_use) + + # If a brain_id is provided, use it + if brain_id_to_use and not brain_to_use: + brain_to_use = self.get_brain_by_id(brain_id_to_use) + + else: + # Calculate the closest brains to the question + list_brains = vector_store.find_brain_closest_query(user.id, question) + + unique_list_brains = [] + seen_brain_ids = set() + + for brain in list_brains: + if brain["id"] not in seen_brain_ids: + unique_list_brains.append(brain) + seen_brain_ids.add(brain["id"]) + + metadata["close_brains"] = unique_list_brains[:5] + + if list_brains and not brain_to_use: + brain_id_to_use = list_brains[0]["id"] + brain_to_use = self.get_brain_by_id(brain_id_to_use) + + return brain_to_use, metadata + + def create_brain( + self, + user_id: UUID, + brain: Optional[CreateBrainProperties], + ) -> BrainEntity: + if brain is None: + brain = CreateBrainProperties() # type: ignore model and brain_definition + + if brain.brain_type == BrainType.integration: + return self.create_brain_integration(user_id, brain) + + created_brain = self.brain_repository.create_brain(brain) + return created_brain + + def create_brain_integration( + self, + user_id: UUID, + brain: CreateBrainProperties, + ) -> BrainEntity: + created_brain = self.brain_repository.create_brain(brain) + if brain.integration is not None: + self.integration_brains_repository.add_integration_brain( + user_id=user_id, + brain_id=created_brain.brain_id, + integration_id=brain.integration.integration_id, + settings=brain.integration.settings, + ) + return created_brain + + def delete_brain(self, brain_id: UUID) -> dict[str, str]: + brain_to_delete = self.get_brain_by_id(brain_id=brain_id) + if brain_to_delete is None: + raise HTTPException(status_code=404, detail="Brain not found.") + + knowledge_service.remove_brain_all_knowledge(brain_id) + + self.brain_vector.delete_brain_vector(str(brain_id)) + self.brain_user_repository.delete_brain_users(str(brain_id)) + self.brain_repository.delete_brain(str(brain_id)) # type: ignore + + return {"message": "Brain deleted."} + + def get_brain_prompt_id(self, brain_id: UUID) -> UUID | None: + brain = self.get_brain_by_id(brain_id) + prompt_id = brain.prompt_id if brain else None + + return prompt_id + + def update_brain_by_id( + self, brain_id: UUID, brain_new_values: BrainUpdatableProperties + ) -> BrainEntity: + """Update a prompt by id""" + + existing_brain = self.brain_repository.get_brain_by_id(brain_id) + + if existing_brain is None: + raise HTTPException( + status_code=404, + detail=f"Brain with id {brain_id} not found", + ) + brain_update_answer = self.brain_repository.update_brain_by_id( + brain_id, + brain=BrainUpdatableProperties( + **brain_new_values.dict( + exclude={"brain_definition", "connected_brains_ids", "integration"} + ) + ), + ) + + if brain_update_answer is None: + raise HTTPException( + status_code=404, + detail=f"Brain with id {brain_id} not found", + ) + + if brain_update_answer is None: + raise HTTPException( + status_code=404, + detail=f"Brain with id {brain_id} not found", + ) + + self.brain_repository.update_brain_last_update_time(brain_id) + return brain_update_answer + + def update_brain_last_update_time(self, brain_id: UUID): + self.brain_repository.update_brain_last_update_time(brain_id) + + def get_brain_details( + self, brain_id: UUID, user_id: UUID | None = None + ) -> BrainEntity | None: + brain = self.brain_repository.get_brain_details(brain_id) + if brain is None: + return None + + # TODO: N+1 here !! + if brain.brain_type == BrainType.integration: + brain.integration = ( + self.integration_brains_repository.get_integration_brain( + brain_id, user_id + ) + ) + + if brain.integration: + brain.integration_description = ( + self.integration_description_repository.get_integration_description( + brain.integration.integration_id + ) + ) + + return brain + + def get_public_brains(self) -> list[PublicBrain]: + return self.brain_repository.get_public_brains() diff --git a/backend/core/quivr_core/api/modules/brain/service/brain_user_service.py b/backend/core/quivr_core/api/modules/brain/service/brain_user_service.py new file mode 100644 index 000000000000..ac528e262ae0 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/brain_user_service.py @@ -0,0 +1,79 @@ +from typing import List +from uuid import UUID + +from fastapi import HTTPException + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.entity.brain_entity import ( + BrainEntity, + BrainUser, + MinimalUserBrainEntity, + RoleEnum, +) +from quivr_core.api.modules.brain.repository.brains import Brains +from quivr_core.api.modules.brain.repository.brains_users import BrainsUsers +from quivr_core.api.modules.brain.service.brain_service import BrainService + +logger = get_logger(__name__) + +brain_service = BrainService() + + +class BrainUserService: + + def __init__(self): + self.brain_repository = Brains() + self.brain_user_repository = BrainsUsers() + + def get_user_default_brain(self, user_id: UUID) -> BrainEntity | None: + brain_id = self.brain_user_repository.get_user_default_brain_id(user_id) + + if brain_id is None: + return None + + return brain_service.get_brain_by_id(brain_id) + + def delete_brain_user(self, user_id: UUID, brain_id: UUID) -> None: + brain_to_delete_user_from = brain_service.get_brain_by_id(brain_id=brain_id) + if brain_to_delete_user_from is None: + raise HTTPException(status_code=404, detail="Brain not found.") + + self.brain_user_repository.delete_brain_user_by_id( + user_id=user_id, + brain_id=brain_id, + ) + + def delete_brain_users(self, brain_id: UUID) -> None: + self.brain_user_repository.delete_brain_subscribers( + brain_id=brain_id, + ) + + def create_brain_user( + self, user_id: UUID, brain_id: UUID, rights: RoleEnum, is_default_brain: bool + ): + self.brain_user_repository.create_brain_user( + user_id=user_id, + brain_id=brain_id, + rights=rights, + default_brain=is_default_brain, + ) + + def get_brain_for_user(self, user_id: UUID, brain_id: UUID): + return self.brain_user_repository.get_brain_for_user(user_id, brain_id) # type: ignore + + def get_user_brains(self, user_id: UUID) -> list[MinimalUserBrainEntity]: + results = self.brain_user_repository.get_user_brains(user_id) # type: ignore + + return results # type: ignore + + def get_brain_users(self, brain_id: UUID) -> List[BrainUser]: + return self.brain_user_repository.get_brain_users(brain_id) + + def update_brain_user_rights( + self, brain_id: UUID, user_id: UUID, rights: str + ) -> None: + self.brain_user_repository.update_brain_user_rights( + brain_id=brain_id, + user_id=user_id, + rights=rights, + ) diff --git a/backend/core/quivr_core/api/modules/brain/service/brain_vector_service.py b/backend/core/quivr_core/api/modules/brain/service/brain_vector_service.py new file mode 100644 index 000000000000..bed5e3a82de2 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/brain_vector_service.py @@ -0,0 +1,57 @@ +from uuid import UUID + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.repository.brains_vectors import BrainsVectors +from quivr_core.api.modules.knowledge.repository.storage import Storage +from quivr_core.api.packages.embeddings.vectors import get_unique_files_from_vector_ids + +logger = get_logger(__name__) + + +class BrainVectorService: + + def __init__(self, brain_id: UUID): + self.repository = BrainsVectors() + self.id = brain_id + self.files = [] + + def create_brain_vector(self, vector_id, file_sha1): + return self.repository.create_brain_vector(self.id, vector_id, file_sha1) # type: ignore + + def update_brain_with_file(self, file_sha1: str): + # not used + vector_ids = self.repository.get_vector_ids_from_file_sha1(file_sha1) + if vector_ids is None or len(vector_ids) == 0: + logger.info(f"No vector ids found for file {file_sha1}") + return + + for vector_id in vector_ids: + self.create_brain_vector(vector_id, file_sha1) + + def get_unique_brain_files(self): + """ + Retrieve unique brain data (i.e. uploaded files and crawled websites). + """ + + vector_ids = self.repository.get_brain_vector_ids(self.id) # type: ignore + self.files = get_unique_files_from_vector_ids(vector_ids) + + return self.files + + def delete_file_from_brain(self, file_name: str, only_vectors: bool = False): + file_name_with_brain_id = f"{self.id}/{file_name}" + storage = Storage() + if not only_vectors: + storage.remove_file(file_name_with_brain_id) + return self.repository.delete_file_from_brain(self.id, file_name) # type: ignore + + def delete_file_url_from_brain(self, file_name: str): + return self.repository.delete_file_from_brain(self.id, file_name) # type: ignore + + @property + def brain_size(self): + # TODO: change the calculation of the brain size, calculate the size stored for the embeddings + what's in the storage + self.get_unique_brain_files() + current_brain_size = sum(float(doc["size"]) for doc in self.files) + + return current_brain_size diff --git a/backend/core/quivr_core/api/modules/brain/service/get_question_context_from_brain.py b/backend/core/quivr_core/api/modules/brain/service/get_question_context_from_brain.py new file mode 100644 index 000000000000..7b88efe0a084 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/get_question_context_from_brain.py @@ -0,0 +1,66 @@ +from uuid import UUID + +from attr import dataclass + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import get_embedding_client, get_supabase_client +from quivr_core.api.modules.upload.service.generate_file_signed_url import ( + generate_file_signed_url, +) +from quivr_core.api.packages.vectorstore.supabase import CustomSupabaseVectorStore + +logger = get_logger(__name__) + + +@dataclass +class DocumentAnswer: + file_name: str + file_sha1: str + file_size: int + file_url: str = "" + file_id: str = "" + file_similarity: float = 0.0 + + +def get_question_context_from_brain(brain_id: UUID, question: str) -> list[str]: + """Finds the best brain to answer the question based on the question's meaning. + + Args: + brain_id (UUID): Id of the brain to search in + question (str): Question to search for in the vector store + + Returns: + str: _descripton_ + """ + # TODO: Move to AnswerGenerator service + supabase_client = get_supabase_client() + embeddings = get_embedding_client() + + vector_store = CustomSupabaseVectorStore( + supabase_client, + embeddings, + table_name="vectors", + brain_id=brain_id, + number_docs=20, + ) + documents = vector_store.similarity_search(question, k=20, threshold=0.8) + + answers = [] + file_sha1s = [] + for document in documents: + if document.metadata["file_sha1"] not in file_sha1s: + file_sha1s.append(document.metadata["file_sha1"]) + file_path_in_storage = f"{brain_id}/{document.metadata['file_name']}" + signed_url = generate_file_signed_url(file_path_in_storage) + answers.append( + DocumentAnswer( + file_name=document.metadata["file_name"], + file_sha1=document.metadata["file_sha1"], + file_size=document.metadata["file_size"], + file_id=document.metadata["id"], + file_similarity=document.metadata["similarity"], + file_url=signed_url.get("signedURL", "") if signed_url else "", + ), + ) + + return answers diff --git a/backend/core/quivr_core/api/modules/brain/service/integration_brain_service.py b/backend/core/quivr_core/api/modules/brain/service/integration_brain_service.py new file mode 100644 index 000000000000..4ce8569e67f0 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/integration_brain_service.py @@ -0,0 +1,27 @@ +from quivr_core.api.modules.brain.entity.integration_brain import ( + IntegrationDescriptionEntity, +) +from quivr_core.api.modules.brain.repository.integration_brains import ( + IntegrationDescription, +) + + +class IntegrationBrainDescriptionService: + + def __init__(self): + self.repository = IntegrationDescription() + + def get_all_integration_descriptions(self) -> list[IntegrationDescriptionEntity]: + return self.repository.get_all_integration_descriptions() + + def get_integration_description( + self, integration_id + ) -> IntegrationDescriptionEntity | None: + return self.repository.get_integration_description(integration_id) + + def get_integration_description_by_user_brain_id( + self, brain_id, user_id + ) -> IntegrationDescriptionEntity | None: + return self.repository.get_integration_description_by_user_brain_id( + brain_id, user_id + ) diff --git a/backend/core/quivr_core/api/modules/brain/service/utils/__init__.py b/backend/core/quivr_core/api/modules/brain/service/utils/__init__.py new file mode 100644 index 000000000000..1b5b3a524cf5 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/utils/__init__.py @@ -0,0 +1 @@ +from .validate_brain import validate_api_brain diff --git a/backend/core/quivr_core/api/modules/brain/service/utils/format_chat_history.py b/backend/core/quivr_core/api/modules/brain/service/utils/format_chat_history.py new file mode 100644 index 000000000000..3e3019432bdf --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/utils/format_chat_history.py @@ -0,0 +1,31 @@ +from typing import Dict, List, Tuple + +from langchain.schema import AIMessage, BaseMessage, HumanMessage, SystemMessage + +from quivr_core.api.modules.chat.dto.outputs import GetChatHistoryOutput + + +def format_chat_history( + history: List[GetChatHistoryOutput], +) -> List[Dict[str, str]]: + """Format the chat history into a list of HumanMessage and AIMessage""" + formatted_history = [] + for chat in history: + if chat.user_message: + formatted_history.append(HumanMessage(content=chat.user_message)) + if chat.assistant: + formatted_history.append(AIMessage(content=chat.assistant)) + return formatted_history + + +def format_history_to_openai_mesages( + tuple_history: List[Tuple[str, str]], system_message: str, question: str +) -> List[BaseMessage]: + """Format the chat history into a list of Base Messages""" + messages = [] + messages.append(SystemMessage(content=system_message)) + for human, ai in tuple_history: + messages.append(HumanMessage(content=human)) + messages.append(AIMessage(content=ai)) + messages.append(HumanMessage(content=question)) + return messages diff --git a/backend/core/quivr_core/api/modules/brain/service/utils/get_prompt_to_use_id.py b/backend/core/quivr_core/api/modules/brain/service/utils/get_prompt_to_use_id.py new file mode 100644 index 000000000000..f27f88fd5f17 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/utils/get_prompt_to_use_id.py @@ -0,0 +1,21 @@ +from typing import Optional +from uuid import UUID + +from quivr_core.api.modules.brain.service.brain_service import BrainService + +brain_service = BrainService() + + +def get_prompt_to_use_id( + brain_id: Optional[UUID], prompt_id: Optional[UUID] +) -> Optional[UUID]: + if brain_id is None and prompt_id is None: + return None + + return ( + prompt_id + if prompt_id + else brain_service.get_brain_prompt_id(brain_id) + if brain_id + else None + ) diff --git a/backend/core/quivr_core/api/modules/brain/service/utils/validate_brain.py b/backend/core/quivr_core/api/modules/brain/service/utils/validate_brain.py new file mode 100644 index 000000000000..faad3b819ca8 --- /dev/null +++ b/backend/core/quivr_core/api/modules/brain/service/utils/validate_brain.py @@ -0,0 +1,14 @@ +from fastapi import HTTPException + +from quivr_core.api.modules.brain.dto.inputs import CreateBrainProperties + + +def validate_api_brain(brain: CreateBrainProperties): + if brain.brain_definition is None: + raise HTTPException(status_code=404, detail="Brain definition not found") + + if brain.brain_definition.url is None: + raise HTTPException(status_code=404, detail="Brain url not found") + + if brain.brain_definition.method is None: + raise HTTPException(status_code=404, detail="Brain method not found") diff --git a/backend/core/quivr_core/api/modules/brain/tests/test_brain_service.py b/backend/core/quivr_core/api/modules/brain/tests/test_brain_service.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/chat/__init__.py b/backend/core/quivr_core/api/modules/chat/__init__.py new file mode 100644 index 000000000000..21252557e6b7 --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/__init__.py @@ -0,0 +1,6 @@ +# noqa: +from quivr_core.api.modules.brain.entity.brain_entity import Brain +from quivr_core.api.modules.prompt.entity.prompt import Prompt +from quivr_core.api.modules.user.entity.user_identity import User + +__all__ = ["Brain", "User", "Prompt"] diff --git a/backend/core/quivr_core/api/modules/chat/controller/__init__.py b/backend/core/quivr_core/api/modules/chat/controller/__init__.py new file mode 100644 index 000000000000..0ec06ba92835 --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/controller/__init__.py @@ -0,0 +1 @@ +from .chat_routes import chat_router diff --git a/backend/core/quivr_core/api/modules/chat/controller/chat_routes.py b/backend/core/quivr_core/api/modules/chat/controller/chat_routes.py new file mode 100644 index 000000000000..8c0fba1705af --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/controller/chat_routes.py @@ -0,0 +1,225 @@ +from typing import Annotated, List, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.responses import StreamingResponse + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.service.brain_service import BrainService +from quivr_core.api.modules.chat.dto.chats import ChatItem, ChatQuestion +from quivr_core.api.modules.chat.dto.inputs import ( + ChatMessageProperties, + ChatUpdatableProperties, + CreateChatProperties, + QuestionAndAnswer, +) +from quivr_core.api.modules.chat.entity.chat import Chat +from quivr_core.api.modules.chat.service.chat_service import ChatService +from quivr_core.api.modules.dependencies import get_current_user, get_service +from quivr_core.api.modules.knowledge.repository.knowledges import KnowledgeRepository +from quivr_core.api.modules.prompt.service.prompt_service import PromptService +from quivr_core.api.modules.rag.rag_service import RAGService +from quivr_core.api.modules.user.entity.user_identity import UserIdentity +from quivr_core.api.packages.utils.telemetry import maybe_send_telemetry + +logger = get_logger(__name__) + +brain_service = BrainService() +knowledge_service = KnowledgeRepository() + + +ChatServiceDep = Annotated[ChatService, Depends(get_service(ChatService))] +UserIdentityDep = Annotated[UserIdentity, Depends(get_current_user)] +PromptServiceDep = Annotated[PromptService, Depends(get_service(PromptService))] + + +chat_router = APIRouter() + + +@chat_router.get("/chat/healthz", tags=["Health"]) +async def healthz(): + return {"status": "ok"} + + +# get all chats +@chat_router.get("/chat", tags=["Chat"]) +async def get_chats(current_user: UserIdentityDep, chat_service: ChatServiceDep): + """ + Retrieve all chats for the current user. + + - `current_user`: The current authenticated user. + - Returns a list of all chats for the user. + + This endpoint retrieves all the chats associated with the current authenticated user. It returns a list of chat objects + containing the chat ID and chat name for each chat. + """ + chats = await chat_service.get_user_chats(current_user.id) + return {"chats": chats} + + +# delete one chat +@chat_router.delete("/chat/{chat_id}", tags=["Chat"]) +async def delete_chat(chat_id: UUID, chat_service: ChatServiceDep): + """ + Delete a specific chat by chat ID. + """ + + chat_service.delete_chat_from_db(chat_id) + return {"message": f"{chat_id} has been deleted."} + + +# update existing chat metadata +@chat_router.put("/chat/{chat_id}/metadata", tags=["Chat"]) +async def update_chat_metadata_handler( + chat_data: ChatUpdatableProperties, + chat_id: UUID, + current_user: UserIdentityDep, + chat_service: ChatServiceDep, +): + """ + Update chat attributes + """ + + chat = await chat_service.get_chat_by_id(chat_id) + if str(current_user.id) != chat.user_id: + raise HTTPException( + status_code=403, # pyright: ignore reportPrivateUsage=none + detail="You should be the owner of the chat to update it.", # pyright: ignore reportPrivateUsage=none + ) + return chat_service.update_chat(chat_id=chat_id, chat_data=chat_data) + + +# update existing message +@chat_router.put("/chat/{chat_id}/{message_id}", tags=["Chat"]) +async def update_chat_message( + chat_message_properties: ChatMessageProperties, + chat_id: UUID, + message_id: UUID, + current_user: UserIdentityDep, + chat_service: ChatServiceDep, +): + chat = await chat_service.get_chat_by_id(chat_id) + if str(current_user.id) != chat.user_id: + raise HTTPException( + status_code=403, detail="You should be the owner of the chat to update it." + ) + return chat_service.update_chat_message( + chat_id=chat_id, + message_id=message_id, + chat_message_properties=chat_message_properties, + ) + + +# create new chat +@chat_router.post("/chat", tags=["Chat"]) +async def create_chat_handler( + chat_data: CreateChatProperties, + current_user: UserIdentityDep, + chat_service: ChatServiceDep, +): + """ + Create a new chat with initial chat messages. + """ + + return await chat_service.create_chat( + user_id=current_user.id, new_chat_data=chat_data + ) + + +# add new question to chat +@chat_router.post( + "/chat/{chat_id}/question", + tags=["Chat"], +) +async def create_question_handler( + request: Request, + chat_question: ChatQuestion, + chat_id: UUID, + current_user: UserIdentityDep, + chat_service: ChatServiceDep, + prompt_service: PromptServiceDep, + brain_id: Annotated[UUID | None, Query()] = None, +): + try: + rag_service = RAGService( + current_user, + brain_id, + chat_id, + brain_service, + prompt_service, + chat_service, + knowledge_service, + ) + chat_answer = await rag_service.generate_answer(chat_question.question) + + maybe_send_telemetry("question_asked", {"streaming": False}, request) + return chat_answer + + except AssertionError: + raise HTTPException( + status_code=422, + detail="inprocessable entity", + ) + except HTTPException as e: + raise e + + +# stream new question response from chat +@chat_router.post( + "/chat/{chat_id}/question/stream", + tags=["Chat"], +) +async def create_stream_question_handler( + request: Request, + chat_question: ChatQuestion, + chat_id: UUID, + chat_service: ChatServiceDep, + current_user: UserIdentityDep, + prompt_service: PromptServiceDep, + brain_id: Annotated[UUID | None, Query()] = None, +) -> StreamingResponse: + logger.info( + f"Creating question for chat {chat_id} with brain {brain_id} of type {type(brain_id)}" + ) + + rag_service = RAGService( + current_user, + brain_id, + chat_id, + brain_service, + prompt_service, + chat_service, + knowledge_service, + ) + maybe_send_telemetry("question_asked", {"streaming": True}, request) + + return StreamingResponse( + rag_service.generate_answer_stream(chat_question.question), + media_type="text/event-stream", + ) + + +# get chat history +@chat_router.get("/chat/{chat_id}/history", tags=["Chat"]) +async def get_chat_history_handler( + chat_id: UUID, + chat_service: ChatServiceDep, +) -> List[ChatItem]: + return await chat_service.get_chat_history_with_notifications(chat_id) + + +@chat_router.post( + "/chat/{chat_id}/question/answer", + tags=["Chat"], +) +async def add_question_and_answer_handler( + chat_id: UUID, + question_and_answer: QuestionAndAnswer, + chat_service: ChatServiceDep, +) -> Optional[Chat]: + """ + Add a new question and anwser to the chat. + """ + history = await chat_service.add_question_and_answer(chat_id, question_and_answer) + # TODO(@aminediro) : Do we need to return the chat ?? + return history.chat diff --git a/backend/core/quivr_core/api/modules/chat/dto/__init__.py b/backend/core/quivr_core/api/modules/chat/dto/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/chat/dto/chats.py b/backend/core/quivr_core/api/modules/chat/dto/chats.py new file mode 100644 index 000000000000..f499cf006f80 --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/dto/chats.py @@ -0,0 +1,47 @@ +from enum import Enum +from typing import List, Optional, Tuple, Union +from uuid import UUID + +from pydantic import BaseModel + +from quivr_core.api.modules.chat.dto.outputs import GetChatHistoryOutput +from quivr_core.api.modules.notification.entity.notification import Notification + + +class ChatMessage(BaseModel): + model: str + question: str + # A list of tuples where each tuple is (speaker, text) + history: List[Tuple[str, str]] + temperature: float = 0.0 + max_tokens: int = 256 + use_summarization: bool = False + chat_id: Optional[UUID] = None + chat_name: Optional[str] = None + + +class ChatQuestion(BaseModel): + question: str + model: Optional[str] = None + temperature: Optional[float] = None + max_tokens: Optional[int] = None + brain_id: Optional[UUID] = None + prompt_id: Optional[UUID] = None + + +class Sources(BaseModel): + name: str + source_url: str + type: str + original_file_name: str + citation: str + + +class ChatItemType(Enum): + MESSAGE = "MESSAGE" + NOTIFICATION = "NOTIFICATION" + + +class ChatItem(BaseModel): + item_type: ChatItemType + body: Union[GetChatHistoryOutput, Notification] diff --git a/backend/core/quivr_core/api/modules/chat/dto/inputs.py b/backend/core/quivr_core/api/modules/chat/dto/inputs.py new file mode 100644 index 000000000000..d3acab722a2a --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/dto/inputs.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + + +class CreateChatHistory(BaseModel): + chat_id: UUID + user_message: str + assistant: str + prompt_id: Optional[UUID] = None + brain_id: Optional[UUID] = None + metadata: Optional[dict] = {} + + +class QuestionAndAnswer(BaseModel): + question: str + answer: str + + +@dataclass +class CreateChatProperties: + name: str + + def __init__(self, name: str): + self.name = name + + +@dataclass +class ChatUpdatableProperties: + chat_name: Optional[str] = None + + def __init__(self, chat_name: Optional[str]): + self.chat_name = chat_name + + +class ChatMessageProperties(BaseModel, extra="ignore"): + thumbs: Optional[bool] + + def dict(self, *args, **kwargs): + chat_dict = super().dict(*args, **kwargs) + if chat_dict.get("thumbs"): + # Set thumbs to boolean value or None if not present + chat_dict["thumbs"] = bool(chat_dict["thumbs"]) + return chat_dict diff --git a/backend/core/quivr_core/api/modules/chat/dto/outputs.py b/backend/core/quivr_core/api/modules/chat/dto/outputs.py new file mode 100644 index 000000000000..86ee1fd6adba --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/dto/outputs.py @@ -0,0 +1,54 @@ +from datetime import datetime +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class GetChatHistoryOutput(BaseModel): + chat_id: UUID + message_id: UUID + user_message: str + message_time: datetime + assistant: str | None = None + prompt_title: str | None = None + brain_name: str | None = None + brain_id: UUID | None = None # string because UUID is not JSON serializable + metadata: Optional[dict] | None = None + thumbs: Optional[bool] | None = None + + def dict(self, *args, **kwargs): + chat_history = super().dict(*args, **kwargs) + chat_history["chat_id"] = str(chat_history.get("chat_id")) + chat_history["message_id"] = str(chat_history.get("message_id")) + + return chat_history + + +class FunctionCall(BaseModel): + arguments: str + name: str + + +class ChatCompletionMessageToolCall(BaseModel): + id: str + function: FunctionCall + type: str = "function" + + +class CompletionMessage(BaseModel): + # = "assistant" | "user" | "system" | "tool" + role: str + content: str | None = None + tool_calls: Optional[List[ChatCompletionMessageToolCall]] = None + + +class CompletionResponse(BaseModel): + finish_reason: str + message: CompletionMessage + + +class BrainCompletionOutput(BaseModel): + messages: List[CompletionMessage] + question: str + response: CompletionResponse diff --git a/backend/core/quivr_core/api/modules/chat/entity/__init__.py b/backend/core/quivr_core/api/modules/chat/entity/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/chat/entity/chat.py b/backend/core/quivr_core/api/modules/chat/entity/chat.py new file mode 100644 index 000000000000..53720f344325 --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/entity/chat.py @@ -0,0 +1,81 @@ +from datetime import datetime +from typing import List +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncAttrs +from sqlmodel import JSON, TIMESTAMP, Column, Field, Relationship, SQLModel, text +from sqlmodel import UUID as PGUUID + +from quivr_core.api.modules.brain.entity.brain_entity import Brain +from quivr_core.api.modules.user.entity.user_identity import User + + +class Chat(SQLModel, table=True): + __tablename__ = "chats" # type: ignore + chat_id: UUID | None = Field( + default=None, + sa_column=Column( + PGUUID, + server_default=text("uuid_generate_v4()"), + primary_key=True, + ), + ) + chat_name: str | None + creation_time: datetime | None = Field( + default=None, + sa_column=Column( + TIMESTAMP(timezone=False), + server_default=text("CURRENT_TIMESTAMP"), + ), + ) + user_id: UUID | None = Field(default=None, foreign_key="users.id") + user: User | None = Relationship(back_populates="chats") # type: ignore + chat_history: List["ChatHistory"] | None = Relationship(back_populates="chat") # type: ignore + + +class ChatHistory(AsyncAttrs, SQLModel, table=True): + __tablename__ = "chat_history" # type: ignore # type : ignore + + message_id: UUID | None = Field( + default=None, + sa_column=Column( + PGUUID, + server_default=text("uuid_generate_v4()"), + primary_key=True, + ), + ) + chat_id: UUID | None = Field( + default=None, + foreign_key="chats.chat_id", + primary_key=True, + nullable=False, # Added nullable constraint + ) + chat: Chat | None = Relationship( + back_populates="chat_history", sa_relationship_kwargs={"lazy": "select"} + ) # type: ignore + user_message: str | None = None + assistant: str | None = None + message_time: datetime | None = Field( + default=None, + sa_column=Column( + TIMESTAMP(timezone=False), + server_default=text("CURRENT_TIMESTAMP"), + ), + ) + metadata_: dict | None = Field( + default=None, sa_column=Column("metadata", JSON, default=None) + ) + prompt_id: UUID | None = Field(default=None, foreign_key="prompts.id") + brain_id: UUID | None = Field( + default=None, + foreign_key="brains.brain_id", + ) + + thumbs: bool | None = None + brain: Brain | None = Relationship( + back_populates="brain_chat_history", sa_relationship_kwargs={"lazy": "select"} + ) # type: ignore + + class Config: + # Note: Pydantic can't generate schema for arbitrary types + arbitrary_types_allowed = True diff --git a/backend/core/quivr_core/api/modules/chat/repository/__init__.py b/backend/core/quivr_core/api/modules/chat/repository/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/chat/repository/chats.py b/backend/core/quivr_core/api/modules/chat/repository/chats.py new file mode 100644 index 000000000000..4754be796f69 --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/repository/chats.py @@ -0,0 +1,127 @@ +from typing import Sequence +from uuid import UUID + +from sqlalchemy import exc +from sqlmodel import select +from sqlmodel.ext.asyncio.session import AsyncSession + +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.chat.dto.inputs import ( + ChatMessageProperties, + QuestionAndAnswer, +) +from quivr_core.api.modules.chat.entity.chat import Chat, ChatHistory +from quivr_core.api.modules.dependencies import BaseRepository + + +class ChatRepository(BaseRepository): + def __init__(self, session: AsyncSession): + super().__init__(session) + # TODO: for now use it instead of session + self.db = get_supabase_client() + + async def get_user_chats(self, user_id: UUID) -> Sequence[Chat]: + query = select(Chat).where(Chat.user_id == user_id) + response = await self.session.exec(query) + return response.all() + + async def create_chat(self, new_chat: Chat) -> Chat: + try: + self.session.add(new_chat) + await self.session.commit() + except exc.IntegrityError: + await self.session.rollback() + # TODO(@aminediro): Custom exceptions + raise Exception() + + await self.session.refresh(new_chat) + return new_chat + + async def get_chat_by_id(self, chat_id: UUID): + query = select(Chat).where(Chat.chat_id == chat_id) + response = await self.session.exec(query) + return response.one() + + async def get_chat_history(self, chat_id: UUID) -> Sequence[ChatHistory]: + query = ( + select(ChatHistory) + .where(ChatHistory.chat_id == chat_id) + # TODO: type hints of sqlmodel arent stable for order_by + .order_by(ChatHistory.message_time) # type: ignore + ) + response = await self.session.exec(query) + return response.all() + + async def add_question_and_answer( + self, chat_id: UUID, question_and_answer: QuestionAndAnswer + ) -> ChatHistory: + chat = ChatHistory( + chat_id=chat_id, + user_message=question_and_answer.question, + assistant=question_and_answer.answer, + ) + try: + self.session.add(chat) + await self.session.commit() + except exc.IntegrityError: + await self.session.rollback() + # TODO(@aminediro) : for now, build an exception system + raise Exception("can't create chat_history ") + await self.session.refresh(chat) + return chat + + def update_chat_history(self, chat_history): + response = ( + self.db.table("chat_history") + .insert( + { + "chat_id": str(chat_history.chat_id), + "user_message": chat_history.user_message, + "assistant": chat_history.assistant, + "prompt_id": ( + str(chat_history.prompt_id) if chat_history.prompt_id else None + ), + "brain_id": ( + str(chat_history.brain_id) if chat_history.brain_id else None + ), + "metadata": chat_history.metadata if chat_history.metadata else {}, + } + ) + .execute() + ) + return response + + def update_chat(self, chat_id, updates): + response = ( + self.db.table("chats").update(updates).match({"chat_id": chat_id}).execute() + ) + + return response + + def update_message_by_id(self, message_id, updates): + response = ( + self.db.table("chat_history") + .update(updates) + .match({"message_id": message_id}) + .execute() + ) + + return response + + def delete_chat(self, chat_id): + self.db.table("chats").delete().match({"chat_id": chat_id}).execute() + + def delete_chat_history(self, chat_id): + self.db.table("chat_history").delete().match({"chat_id": chat_id}).execute() + + def update_chat_message( + self, chat_id, message_id, chat_message_properties: ChatMessageProperties + ): + response = ( + self.db.table("chat_history") + .update(chat_message_properties) + .match({"message_id": message_id, "chat_id": chat_id}) + .execute() + ) + + return response diff --git a/backend/core/quivr_core/api/modules/chat/service/__init__.py b/backend/core/quivr_core/api/modules/chat/service/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/chat/service/chat_service.py b/backend/core/quivr_core/api/modules/chat/service/chat_service.py new file mode 100644 index 000000000000..133161430e31 --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/service/chat_service.py @@ -0,0 +1,207 @@ +import random +from typing import List +from uuid import UUID + +from fastapi import HTTPException + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.entity.brain_entity import Brain +from quivr_core.api.modules.brain.service.brain_service import BrainService +from quivr_core.api.modules.chat.dto.chats import ChatItem +from quivr_core.api.modules.chat.dto.inputs import ( + ChatMessageProperties, + ChatUpdatableProperties, + CreateChatHistory, + CreateChatProperties, + QuestionAndAnswer, +) +from quivr_core.api.modules.chat.dto.outputs import GetChatHistoryOutput +from quivr_core.api.modules.chat.entity.chat import Chat, ChatHistory +from quivr_core.api.modules.chat.repository.chats import ChatRepository +from quivr_core.api.modules.chat.service.utils import ( + merge_chat_history_and_notifications, +) +from quivr_core.api.modules.dependencies import BaseService +from quivr_core.api.modules.notification.service.notification_service import ( + NotificationService, +) +from quivr_core.api.modules.prompt.entity.prompt import Prompt + +logger = get_logger(__name__) + +brain_service = BrainService() +notification_service = NotificationService() + + +class ChatService(BaseService[ChatRepository]): + repository_cls = ChatRepository + + def __init__(self, repository: ChatRepository): + self.repository = repository + + async def create_chat( + self, user_id: UUID, new_chat_data: CreateChatProperties + ) -> Chat: + # Chat is created upon the user's first question asked + logger.info(f"New chat entry in chats table for user {user_id}") + + inserted_chat = await self.repository.create_chat( + Chat(chat_name=new_chat_data.name, user_id=user_id) + ) + logger.info(f"Insert response {inserted_chat}") + + return inserted_chat + + def get_follow_up_question( + self, brain_id: UUID = None, question: str = None + ) -> list[str]: + follow_up = [ + "Summarize the conversation", + "Explain in more detail", + "Explain like I'm 5", + "Provide a list", + "Give examples", + "Use simpler language", + "Elaborate on a specific point", + "Provide pros and cons", + "Break down into steps", + "Illustrate with an image or diagram", + ] + # Return 3 random follow up questions amongs the list + random3 = random.sample(follow_up, 3) + return random3 + + async def add_question_and_answer( + self, chat_id: UUID, question_and_answer: QuestionAndAnswer + ) -> ChatHistory: + return await self.repository.add_question_and_answer( + chat_id, question_and_answer + ) + + async def get_chat_by_id(self, chat_id: UUID) -> Chat: + chat = await self.repository.get_chat_by_id(chat_id) + return chat + + async def get_chat_history(self, chat_id: UUID) -> List[GetChatHistoryOutput]: + history = await self.repository.get_chat_history(chat_id) + enriched_history: List[GetChatHistoryOutput] = [] + if len(history) == 0: + return enriched_history + brain: Brain = await history[0].awaitable_attrs.brain + prompt: Prompt = await brain.awaitable_attrs.prompt + for message in history: + enriched_history.append( + # TODO : WHY bother with having ids here ?? + GetChatHistoryOutput( + chat_id=(message.chat_id), + message_id=message.message_id, + user_message=message.user_message, + assistant=message.assistant, + message_time=message.message_time, + brain_name=brain.name if brain else None, + brain_id=brain.brain_id if brain else None, + prompt_title=(prompt.title if prompt else None), + metadata=message.metadata_, + thumbs=message.thumbs, + ) + ) + return enriched_history + + async def get_chat_history_with_notifications( + self, + chat_id: UUID, + ) -> List[ChatItem]: + chat_history = await self.get_chat_history(chat_id) + chat_notifications = [] + return merge_chat_history_and_notifications(chat_history, chat_notifications) + + async def get_user_chats(self, user_id: UUID) -> List[Chat]: + return list(await self.repository.get_user_chats(user_id)) + + def update_chat_history(self, chat_history: CreateChatHistory) -> ChatHistory: + response: List[ChatHistory] = ( + self.repository.update_chat_history(chat_history) + ).data + if len(response) == 0: + raise HTTPException( + status_code=500, + detail="An exception occurred while updating chat history.", + ) + return ChatHistory(**response[0]) # pyright: ignore reportPrivateUsage=none + + def update_chat(self, chat_id, chat_data: ChatUpdatableProperties) -> Chat: + if not chat_id: + logger.error("No chat_id provided") + return # pyright: ignore reportPrivateUsage=none + + updates = {} + + if chat_data.chat_name is not None: + updates["chat_name"] = chat_data.chat_name + + updated_chat = None + + if updates: + updated_chat = (self.repository.update_chat(chat_id, updates)).data[0] + logger.info(f"Chat {chat_id} updated") + else: + logger.info(f"No updates to apply for chat {chat_id}") + return updated_chat # pyright: ignore reportPrivateUsage=none + + def update_message_by_id( + self, + message_id: str, + user_message: str = None, # pyright: ignore reportPrivateUsage=none + assistant: str = None, # pyright: ignore reportPrivateUsage=none + metadata: dict = None, # pyright: ignore reportPrivateUsage=none + ) -> ChatHistory: + if not message_id: + logger.error("No message_id provided") + return # pyright: ignore reportPrivateUsage=none + + updates = {} + + if user_message is not None: + updates["user_message"] = user_message + + if assistant is not None: + updates["assistant"] = assistant + + if metadata is not None: + updates["metadata"] = metadata + + updated_message = None + + if updates: + updated_message = ( + self.repository.update_message_by_id(message_id, updates) + ).data[ # type: ignore + 0 + ] + logger.info(f"Message {message_id} updated") + else: + logger.info(f"No updates to apply for message {message_id}") + return ChatHistory(**updated_message) # pyright: ignore reportPrivateUsage=none + + def delete_chat_from_db(self, chat_id): + try: + self.repository.delete_chat_history(chat_id) + except Exception as e: + print(e) + pass + try: + self.repository.delete_chat(chat_id) + except Exception as e: + print(e) + pass + + def update_chat_message( + self, chat_id, message_id, chat_message_properties: ChatMessageProperties + ): + try: + return self.repository.update_chat_message( + chat_id, message_id, chat_message_properties + ).data + except Exception as e: + print(e) + pass diff --git a/backend/core/quivr_core/api/modules/chat/service/utils.py b/backend/core/quivr_core/api/modules/chat/service/utils.py new file mode 100644 index 000000000000..10a629d096c1 --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/service/utils.py @@ -0,0 +1,35 @@ +from typing import List + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.chat.dto.chats import ChatItem, ChatItemType +from quivr_core.api.modules.chat.dto.outputs import GetChatHistoryOutput +from quivr_core.api.modules.notification.entity.notification import Notification + +logger = get_logger(__name__) + + +def merge_chat_history_and_notifications( + chat_history: List[GetChatHistoryOutput], notifications: List[Notification] +) -> List[ChatItem]: + chat_history_and_notifications = chat_history + notifications + + chat_history_and_notifications.sort( + key=lambda x: ( + x.message_time + if isinstance(x, GetChatHistoryOutput) and x.message_time + else x.datetime + ) + ) + + transformed_data = [] + for item in chat_history_and_notifications: + if isinstance(item, GetChatHistoryOutput): + item_type = ChatItemType.MESSAGE + body = item + else: + item_type = ChatItemType.NOTIFICATION + body = item + transformed_item = ChatItem(item_type=item_type, body=body) + transformed_data.append(transformed_item) + + return transformed_data diff --git a/backend/core/quivr_core/api/modules/chat/tests/conftest.py b/backend/core/quivr_core/api/modules/chat/tests/conftest.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/chat/tests/test_chats.py b/backend/core/quivr_core/api/modules/chat/tests/test_chats.py new file mode 100644 index 000000000000..26a0039b2d9e --- /dev/null +++ b/backend/core/quivr_core/api/modules/chat/tests/test_chats.py @@ -0,0 +1,177 @@ +import asyncio +import os +from typing import List, Tuple +from uuid import uuid4 + +import pytest +import pytest_asyncio +import sqlalchemy +from sqlalchemy.ext.asyncio import create_async_engine +from sqlmodel import create_engine, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from quivr_core.api.modules.brain.entity.brain_entity import Brain, BrainType +from quivr_core.api.modules.chat.dto.inputs import QuestionAndAnswer +from quivr_core.api.modules.chat.entity.chat import Chat, ChatHistory +from quivr_core.api.modules.chat.repository.chats import ChatRepository +from quivr_core.api.modules.chat.service.chat_service import ChatService +from quivr_core.api.modules.user.entity.user_identity import User + +pg_database_url = "postgres:postgres@localhost:54322/postgres" + + +@pytest.fixture(scope="session", autouse=True) +def db_setup(): + # setup + sync_engine = create_engine( + "postgresql://" + pg_database_url, + echo=True if os.getenv("ORM_DEBUG") else False, + ) + # TODO(@amine) : for now don't drop anything + yield sync_engine + # teardown + # NOTE: For now we rely on Supabase migrations for defining schemas + # SQLModel.metadata.create_all(sync_engine, checkfirst=True) + # SQLModel.metadata.drop_all(sync_engine) + + +@pytest_asyncio.fixture(scope="session") +async def async_engine(): + engine = create_async_engine( + "postgresql+asyncpg://" + pg_database_url, + echo=True if os.getenv("ORM_DEBUG") else False, + ) + yield engine + + +@pytest.fixture(scope="session") +def event_loop(request: pytest.FixtureRequest): + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + +@pytest_asyncio.fixture() +async def session(async_engine): + async with async_engine.connect() as conn: + await conn.begin() + await conn.begin_nested() + async_session = AsyncSession(conn, expire_on_commit=False) + + @sqlalchemy.event.listens_for( + async_session.sync_session, "after_transaction_end" + ) + def end_savepoint(session, transaction): + if conn.closed: + return + if not conn.in_nested_transaction(): + conn.sync_connection.begin_nested() + + yield async_session + + +TestData = Tuple[Brain, User, List[Chat], List[ChatHistory]] + + +@pytest_asyncio.fixture() +async def test_data( + session: AsyncSession, +) -> TestData: + # User data + user_1 = ( + await session.exec(select(User).where(User.email == "admin@quivr.app")) + ).one() + # Brain data + brain_1 = Brain(name="test_brain", description="this is a test brain") + # Chat data + chat_1 = Chat(chat_name="chat1", user=user_1) + chat_2 = Chat(chat_name="chat2", user=user_1) + + chat_history_1 = ChatHistory( + user_message="Hello", + assistant="Hello! How can I assist you today?", + chat=chat_1, + brain=brain_1, + ) + chat_history_2 = ChatHistory( + user_message="Hello", + assistant="Hello! How can I assist you today?", + chat=chat_1, + brain=brain_1, + ) + session.add(brain_1) + session.add(chat_1) + session.add(chat_2) + session.add(chat_history_1) + session.add(chat_history_2) + + await session.refresh(user_1) + await session.commit() + return brain_1, user_1, [chat_1, chat_2], [chat_history_1, chat_history_2] + + +@pytest.mark.asyncio +async def test_get_user_chats_empty(session): + repo = ChatRepository(session) + chats = await repo.get_user_chats(user_id=uuid4()) + assert len(chats) == 0 + + +@pytest.mark.asyncio +async def test_get_user_chats(session: AsyncSession, test_data: TestData): + _, local_user, chats, _ = test_data + repo = ChatRepository(session) + assert local_user.id is not None + query_chats = await repo.get_user_chats(local_user.id) + assert len(query_chats) == len(chats) + + +@pytest.mark.asyncio +async def test_get_chat_history(session: AsyncSession, test_data: TestData): + brain_1, _, chats, chat_history = test_data + assert chats[0].chat_id + assert len(chat_history) > 0 + assert chat_history[-1].message_time + assert chat_history[0].message_time + + repo = ChatRepository(session) + query_chat_history = await repo.get_chat_history(chats[0].chat_id) + assert chat_history == query_chat_history + assert query_chat_history[-1].message_time + assert query_chat_history[0].message_time + assert query_chat_history[-1].message_time >= query_chat_history[0].message_time + + # TODO: Should be tested in test_brain_repository + # Checks that brain is correct + assert query_chat_history[-1].brain is not None + assert query_chat_history[-1].brain.brain_type == BrainType.integration + + +@pytest.mark.asyncio +async def test_add_qa(session: AsyncSession, test_data: TestData): + _, _, [chat, *_], __ = test_data + assert chat.chat_id + qa = QuestionAndAnswer(question="question", answer="answer") + repo = ChatRepository(session) + resp_chat = await repo.add_question_and_answer(chat.chat_id, qa) + + assert resp_chat.chat_id == chat.chat_id + assert resp_chat.user_message == qa.question + assert resp_chat.assistant == qa.answer + + +## CHAT SERVICE + + +@pytest.mark.asyncio +async def test_service_get_chat_history(session: AsyncSession, test_data: TestData): + brain, _, [chat, *_], __ = test_data + assert chat.chat_id + repo = ChatRepository(session) + service = ChatService(repo) + history = await service.get_chat_history(chat.chat_id) + + assert len(history) > 0 + assert all(h.chat_id == chat.chat_id for h in history) + assert history[0].brain_name == brain.name + assert history[0].brain_id == brain.brain_id diff --git a/backend/core/quivr_core/api/modules/dependencies.py b/backend/core/quivr_core/api/modules/dependencies.py new file mode 100644 index 000000000000..fc34b560bfb7 --- /dev/null +++ b/backend/core/quivr_core/api/modules/dependencies.py @@ -0,0 +1,70 @@ +import os +from typing import AsyncGenerator, Callable, Generic, Type, TypeVar +from uuid import uuid4 + +from fastapi import Depends +from sqlalchemy.ext.asyncio import create_async_engine +from sqlmodel.ext.asyncio.session import AsyncSession + +from quivr_core.api.models.settings import settings +from quivr_core.api.modules.user.entity.user_identity import UserIdentity + + +class BaseRepository: + def __init__(self, session: AsyncSession): + self.session = session + + +R = TypeVar("R", bound=BaseRepository) + + +class BaseService(Generic[R]): + # associated repository type + repository_cls: Type[R] + + def __init__(self, repository: R): + self.repository = repository + + @classmethod + def get_repository_cls(cls) -> Type[R]: + return cls.repository_cls # type: ignore + + +S = TypeVar("S", bound=BaseService) + +async_engine = create_async_engine( + settings.pg_database_async_url, + echo=True if os.getenv("ORM_DEBUG") else False, + future=True, +) + + +async def get_async_session() -> AsyncGenerator[AsyncSession, None]: + async with AsyncSession(async_engine) as session: + yield session + + +def get_repository(repository_model: Type[R]) -> Callable[..., R]: + def _get_repository(session: AsyncSession = Depends(get_async_session)) -> R: + return repository_model(session) + + return _get_repository + + +def get_service(service: Type[S]) -> Callable[..., S]: + def _get_service( + repository: BaseRepository = Depends( + get_repository(service.get_repository_cls()) + ), + ) -> S: + return service(repository) + + return _get_service + + +def get_current_user() -> UserIdentity: + # TODO: get it one time from db + return UserIdentity( + id=uuid4(), + email="admin@quivr.app", + ) diff --git a/backend/core/quivr_core/api/modules/knowledge/__init__.py b/backend/core/quivr_core/api/modules/knowledge/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/knowledge/controller/__init__.py b/backend/core/quivr_core/api/modules/knowledge/controller/__init__.py new file mode 100644 index 000000000000..911883cdc250 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/controller/__init__.py @@ -0,0 +1 @@ +from .knowledge_routes import knowledge_router diff --git a/backend/core/quivr_core/api/modules/knowledge/controller/knowledge_routes.py b/backend/core/quivr_core/api/modules/knowledge/controller/knowledge_routes.py new file mode 100644 index 000000000000..3c3187b17a3e --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/controller/knowledge_routes.py @@ -0,0 +1,86 @@ +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.service.brain_vector_service import BrainVectorService +from quivr_core.api.modules.dependencies import get_current_user +from quivr_core.api.modules.knowledge.service.knowledge_service import KnowledgeService +from quivr_core.api.modules.upload.service.generate_file_signed_url import ( + generate_file_signed_url, +) +from quivr_core.api.modules.user.entity.user_identity import UserIdentity + +knowledge_router = APIRouter() +logger = get_logger(__name__) + +knowledge_service = KnowledgeService() + + +@knowledge_router.get("/knowledge", tags=["Knowledge"]) +async def list_knowledge_in_brain_endpoint( + brain_id: UUID = Query(..., description="The ID of the brain"), + current_user: UserIdentity = Depends(get_current_user), +): + """ + Retrieve and list all the knowledge in a brain. + """ + + knowledges = knowledge_service.get_all_knowledge(brain_id) + + return {"knowledges": knowledges} + + +@knowledge_router.delete( + "/knowledge/{knowledge_id}", + tags=["Knowledge"], +) +async def delete_endpoint( + knowledge_id: UUID, + current_user: UserIdentity = Depends(get_current_user), + brain_id: UUID = Query(..., description="The ID of the brain"), +): + """ + Delete a specific knowledge from a brain. + """ + + knowledge = knowledge_service.get_knowledge(knowledge_id) + file_name = knowledge.file_name if knowledge.file_name else knowledge.url + knowledge_service.remove_knowledge(knowledge_id) + + brain_vector_service = BrainVectorService(brain_id) + if knowledge.file_name: + brain_vector_service.delete_file_from_brain(knowledge.file_name) + elif knowledge.url: + brain_vector_service.delete_file_url_from_brain(knowledge.url) + + return { + "message": f"{file_name} of brain {brain_id} has been deleted by user {current_user.email}." + } + + +@knowledge_router.get( + "/knowledge/{knowledge_id}/signed_download_url", + tags=["Knowledge"], +) +async def generate_signed_url_endpoint( + knowledge_id: UUID, + current_user: UserIdentity = Depends(get_current_user), +): + """ + Generate a signed url to download the file from storage. + """ + + knowledge = knowledge_service.get_knowledge(knowledge_id) + + if knowledge.file_name == None: + raise HTTPException( + status_code=404, + detail=f"Knowledge with id {knowledge_id} is not a file.", + ) + + file_path_in_storage = f"{knowledge.brain_id}/{knowledge.file_name}" + + file_signed_url = generate_file_signed_url(file_path_in_storage) + + return file_signed_url diff --git a/backend/core/quivr_core/api/modules/knowledge/dto/__init__.py b/backend/core/quivr_core/api/modules/knowledge/dto/__init__.py new file mode 100644 index 000000000000..4f3a4b9f7bb5 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/dto/__init__.py @@ -0,0 +1,2 @@ +from .inputs import CreateKnowledgeProperties +from .outputs import DeleteKnowledgeResponse diff --git a/backend/core/quivr_core/api/modules/knowledge/dto/inputs.py b/backend/core/quivr_core/api/modules/knowledge/dto/inputs.py new file mode 100644 index 000000000000..b6d008c35f24 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/dto/inputs.py @@ -0,0 +1,18 @@ +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + + +class CreateKnowledgeProperties(BaseModel): + brain_id: UUID + file_name: Optional[str] = None + url: Optional[str] = None + extension: str = "txt" + integration: Optional[str] = None + integration_link: Optional[str] = None + + def dict(self, *args, **kwargs): + knowledge_dict = super().dict(*args, **kwargs) + knowledge_dict["brain_id"] = str(knowledge_dict.get("brain_id")) + return knowledge_dict diff --git a/backend/core/quivr_core/api/modules/knowledge/dto/outputs.py b/backend/core/quivr_core/api/modules/knowledge/dto/outputs.py new file mode 100644 index 000000000000..905da7cf0d7f --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/dto/outputs.py @@ -0,0 +1,8 @@ +from uuid import UUID + +from pydantic import BaseModel + + +class DeleteKnowledgeResponse(BaseModel): + status: str = "delete" + knowledge_id: UUID diff --git a/backend/core/quivr_core/api/modules/knowledge/entity/__init__.py b/backend/core/quivr_core/api/modules/knowledge/entity/__init__.py new file mode 100644 index 000000000000..6f7620c446d2 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/entity/__init__.py @@ -0,0 +1 @@ +from .knowledge import Knowledge diff --git a/backend/core/quivr_core/api/modules/knowledge/entity/knowledge.py b/backend/core/quivr_core/api/modules/knowledge/entity/knowledge.py new file mode 100644 index 000000000000..504162d0c644 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/entity/knowledge.py @@ -0,0 +1,12 @@ +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + + +class Knowledge(BaseModel): + id: UUID + brain_id: UUID + file_name: Optional[str] = None + url: Optional[str] = None + extension: str = "txt" diff --git a/backend/core/quivr_core/api/modules/knowledge/repository/__init__.py b/backend/core/quivr_core/api/modules/knowledge/repository/__init__.py new file mode 100644 index 000000000000..80f13aa5f77f --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/repository/__init__.py @@ -0,0 +1 @@ +from .knowledges import KnowledgeRepository diff --git a/backend/core/quivr_core/api/modules/knowledge/repository/knowledge_interface.py b/backend/core/quivr_core/api/modules/knowledge/repository/knowledge_interface.py new file mode 100644 index 000000000000..7d36e3b02bcc --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/repository/knowledge_interface.py @@ -0,0 +1,58 @@ +from abc import ABC, abstractmethod +from typing import List +from uuid import UUID + +from quivr_core.api.modules.knowledge.dto.inputs import CreateKnowledgeProperties +from quivr_core.api.modules.knowledge.dto.outputs import DeleteKnowledgeResponse +from quivr_core.api.modules.knowledge.entity.knowledge import Knowledge + + +class KnowledgeInterface(ABC): + @abstractmethod + def insert_knowledge(self, knowledge: CreateKnowledgeProperties) -> Knowledge: + """ + Add a knowledge + """ + pass + + @abstractmethod + def remove_knowledge_by_id( + # todo: update remove brain endpoints to first delete the knowledge + self, + knowledge_id: UUID, + ) -> DeleteKnowledgeResponse: + """ + Args: + knowledge_id (UUID): The id of the knowledge + + Returns: + str: Status message + """ + pass + + @abstractmethod + def get_knowledge_by_id(self, knowledge_id: UUID) -> Knowledge: + """ + Get a knowledge by its id + Args: + brain_id (UUID): The id of the brain + """ + pass + + @abstractmethod + def get_all_knowledge_in_brain(self, brain_id: UUID) -> List[Knowledge]: + """ + Get all the knowledge in a brain + Args: + brain_id (UUID): The id of the brain + """ + pass + + @abstractmethod + def remove_brain_all_knowledge(self, brain_id: UUID) -> None: + """ + Remove all knowledge in a brain + Args: + brain_id (UUID): The id of the brain + """ + pass diff --git a/backend/core/quivr_core/api/modules/knowledge/repository/knowledges.py b/backend/core/quivr_core/api/modules/knowledge/repository/knowledges.py new file mode 100644 index 000000000000..b6896f1ee315 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/repository/knowledges.py @@ -0,0 +1,114 @@ +from uuid import UUID + +from fastapi import HTTPException + +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.knowledge.dto.outputs import DeleteKnowledgeResponse +from quivr_core.api.modules.knowledge.entity.knowledge import Knowledge +from quivr_core.api.modules.knowledge.repository.knowledge_interface import ( + KnowledgeInterface, +) + + +class KnowledgeRepository(KnowledgeInterface): + def __init__(self): + supabase_client = get_supabase_client() + self.db = supabase_client + + def insert_knowledge(self, knowledge): + """ + Add a knowledge + """ + # Check if the knowledge already exists + knowledge_exists = ( + self.db.from_("knowledge") + .select("*") + .filter("brain_id", "eq", knowledge.brain_id) + .filter("file_name", "eq", knowledge.file_name) + .execute() + ).data + + if knowledge_exists: + return Knowledge(**knowledge_exists[0]) # TODO fix this + + response = (self.db.from_("knowledge").insert(knowledge.dict()).execute()).data + return Knowledge(**response[0]) + + def remove_knowledge_by_id( + # todo: update remove brain endpoints to first delete the knowledge + self, + knowledge_id, + ): + """ + Args: + knowledge_id (UUID): The id of the knowledge + + Returns: + str: Status message + """ + response = ( + self.db.from_("knowledge") + .delete() + .filter("id", "eq", knowledge_id) + .execute() + .data + ) + + if response == []: + raise HTTPException(404, "Knowledge not found") + + return DeleteKnowledgeResponse( + # change to response[0].brain_id and knowledge_id[0].brain_id + status="deleted", + knowledge_id=knowledge_id, + ) + + def get_knowledge_by_id(self, knowledge_id): + """ + Get a knowledge by its id + Args: + brain_id (UUID): The id of the brain + """ + knowledge = ( + self.db.from_("knowledge") + .select("*") + .filter("id", "eq", str(knowledge_id)) + .execute() + ).data + + return Knowledge(**knowledge[0]) + + def get_all_knowledge_in_brain(self, brain_id: UUID) -> list[Knowledge]: + """ + Get all the knowledge in a brain + Args: + brain_id (UUID): The id of the brain + """ + all_knowledge = ( + self.db.from_("knowledge") + .select("*") + .filter("brain_id", "eq", str(brain_id)) + .execute() + ).data + + return [Knowledge(**knowledge) for knowledge in all_knowledge] + + def remove_brain_all_knowledge(self, brain_id): + """ + Remove all knowledge in a brain + Args: + brain_id (UUID): The id of the brain + """ + all_knowledge = self.get_all_knowledge_in_brain(brain_id) + knowledge_to_delete_list = [] + + for knowledge in all_knowledge: + if knowledge.file_name: + knowledge_to_delete_list.append(f"{brain_id}/{knowledge.file_name}") + + if knowledge_to_delete_list: + self.db.storage.from_("quivr").remove(knowledge_to_delete_list) + + self.db.from_("knowledge").delete().filter( + "brain_id", "eq", str(brain_id) + ).execute() diff --git a/backend/core/quivr_core/api/modules/knowledge/repository/storage.py b/backend/core/quivr_core/api/modules/knowledge/repository/storage.py new file mode 100644 index 000000000000..b98b041a1753 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/repository/storage.py @@ -0,0 +1,30 @@ +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.knowledge.repository.storage_interface import ( + StorageInterface, +) + +logger = get_logger(__name__) + + +class Storage(StorageInterface): + def __init__(self): + supabase_client = get_supabase_client() + self.db = supabase_client + + def upload_file(self, file_name: str): + """ + Upload file to storage + """ + self.db.storage.from_("quivr").download(file_name) + + def remove_file(self, file_name: str): + """ + Remove file from storage + """ + try: + response = self.db.storage.from_("quivr").remove([file_name]) + return response + except Exception as e: + logger.error(e) + # raise e diff --git a/backend/core/quivr_core/api/modules/knowledge/repository/storage_interface.py b/backend/core/quivr_core/api/modules/knowledge/repository/storage_interface.py new file mode 100644 index 000000000000..228c998276e3 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/repository/storage_interface.py @@ -0,0 +1,10 @@ +from abc import ABC, abstractmethod + + +class StorageInterface(ABC): + @abstractmethod + def remove_file(self, file_name: str): + """ + Remove file from storage + """ + pass diff --git a/backend/core/quivr_core/api/modules/knowledge/service/__init__.py b/backend/core/quivr_core/api/modules/knowledge/service/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/knowledge/service/knowledge_service.py b/backend/core/quivr_core/api/modules/knowledge/service/knowledge_service.py new file mode 100644 index 000000000000..6798f6f24a85 --- /dev/null +++ b/backend/core/quivr_core/api/modules/knowledge/service/knowledge_service.py @@ -0,0 +1,45 @@ +from uuid import UUID + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.knowledge.dto.inputs import CreateKnowledgeProperties +from quivr_core.api.modules.knowledge.entity.knowledge import Knowledge +from quivr_core.api.modules.knowledge.repository.knowledge_interface import ( + KnowledgeInterface, +) +from quivr_core.api.modules.knowledge.repository.knowledges import KnowledgeRepository + +logger = get_logger(__name__) + + +class KnowledgeService: + repository: KnowledgeInterface + + def __init__(self): + self.repository = KnowledgeRepository() + + def add_knowledge(self, knowledge_to_add: CreateKnowledgeProperties): + knowledge = self.repository.insert_knowledge(knowledge_to_add) + + return knowledge + + def get_all_knowledge(self, brain_id: UUID): + knowledges = self.repository.get_all_knowledge_in_brain(brain_id) + + return knowledges + + def get_knowledge(self, knowledge_id: UUID) -> Knowledge: + knowledge = self.repository.get_knowledge_by_id(knowledge_id) + + return knowledge + + def remove_brain_all_knowledge(self, brain_id: UUID) -> None: + self.repository.remove_brain_all_knowledge(brain_id) + + logger.info( + f"All knowledge in brain {brain_id} removed successfully from table" + ) + + def remove_knowledge(self, knowledge_id: UUID): + message = self.repository.remove_knowledge_by_id(knowledge_id) + + return message diff --git a/backend/core/quivr_core/api/modules/notification/__init__.py b/backend/core/quivr_core/api/modules/notification/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/notification/controller/__init__.py b/backend/core/quivr_core/api/modules/notification/controller/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/notification/dto/__init__.py b/backend/core/quivr_core/api/modules/notification/dto/__init__.py new file mode 100644 index 000000000000..2d81927d4b12 --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/dto/__init__.py @@ -0,0 +1 @@ +from .inputs import NotificationUpdatableProperties diff --git a/backend/core/quivr_core/api/modules/notification/dto/inputs.py b/backend/core/quivr_core/api/modules/notification/dto/inputs.py new file mode 100644 index 000000000000..8c4d22ca25f2 --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/dto/inputs.py @@ -0,0 +1,28 @@ +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel +from quivr_core.api.modules.notification.entity.notification import ( + NotificationsStatusEnum, +) + + +class CreateNotification(BaseModel): + """Properties that can be received on notification creation""" + + user_id: UUID + status: NotificationsStatusEnum + title: str + description: Optional[str] = None + + def model_dump(self, *args, **kwargs): + notification_dict = super().model_dump(*args, **kwargs) + notification_dict["user_id"] = str(notification_dict["user_id"]) + return notification_dict + + +class NotificationUpdatableProperties(BaseModel): + """Properties that can be received on notification update""" + + status: Optional[NotificationsStatusEnum] + description: Optional[str] diff --git a/backend/core/quivr_core/api/modules/notification/entity/__init__.py b/backend/core/quivr_core/api/modules/notification/entity/__init__.py new file mode 100644 index 000000000000..209eacf8425f --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/entity/__init__.py @@ -0,0 +1 @@ +from .notification import Notification diff --git a/backend/core/quivr_core/api/modules/notification/entity/notification.py b/backend/core/quivr_core/api/modules/notification/entity/notification.py new file mode 100644 index 000000000000..4a1378b8d65f --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/entity/notification.py @@ -0,0 +1,24 @@ +from datetime import datetime +from enum import Enum +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel + + +class NotificationsStatusEnum(str, Enum): + INFO = "info" + SUCCESS = "success" + WARNING = "warning" + ERROR = "error" + + +class Notification(BaseModel): + id: UUID + user_id: UUID + status: NotificationsStatusEnum + title: str + description: Optional[str] + archived: Optional[bool] = False + read: Optional[bool] = False + datetime: Optional[datetime] # timestamp diff --git a/backend/core/quivr_core/api/modules/notification/repository/__init__.py b/backend/core/quivr_core/api/modules/notification/repository/__init__.py new file mode 100644 index 000000000000..2d49d9356a1e --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/repository/__init__.py @@ -0,0 +1 @@ +from .notifications import Notifications diff --git a/backend/core/quivr_core/api/modules/notification/repository/notifications.py b/backend/core/quivr_core/api/modules/notification/repository/notifications.py new file mode 100644 index 000000000000..75b36e8db4c6 --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/repository/notifications.py @@ -0,0 +1,68 @@ +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.notification.dto.inputs import CreateNotification +from quivr_core.api.modules.notification.entity.notification import Notification +from quivr_core.api.modules.notification.repository.notifications_interface import ( + NotificationInterface, +) + +logger = get_logger(__name__) + + +class Notifications(NotificationInterface): + def __init__(self, supabase_client): + self.db = supabase_client + + def add_notification(self, notification: CreateNotification): + """ + Add a notification + """ + response = ( + self.db.from_("notifications").insert(notification.model_dump()).execute() + ).data + return Notification(**response[0]) + + def update_notification_by_id( + self, + notification_id, + notification, + ): + if notification_id is None: + logger.info("Notification id is required") + return None + + """Update a notification by id""" + response = ( + self.db.from_("notifications") + .update(notification.model_dump(exclude_unset=True)) + .filter("id", "eq", notification_id) + .execute() + ).data + + if response == []: + logger.info(f"Notification with id {notification_id} not found") + return None + + return Notification(**response[0]) + + def remove_notification_by_id(self, notification_id): + """ + Remove a notification by id + Args: + notification_id (UUID): The id of the notification + + Returns: + str: Status message + """ + response = ( + self.db.from_("notifications") + .delete() + .filter("id", "eq", notification_id) + .execute() + .data + ) + + if response == []: + logger.info(f"Notification with id {notification_id} not found") + return None + + return {"status": "success"} diff --git a/backend/core/quivr_core/api/modules/notification/repository/notifications_interface.py b/backend/core/quivr_core/api/modules/notification/repository/notifications_interface.py new file mode 100644 index 000000000000..423e46df1917 --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/repository/notifications_interface.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from uuid import UUID + +from quivr_core.api.modules.notification.dto.inputs import ( + CreateNotification, + NotificationUpdatableProperties, +) +from quivr_core.api.modules.notification.entity.notification import Notification + + +class NotificationInterface(ABC): + @abstractmethod + def add_notification(self, notification: CreateNotification) -> Notification: + """ + Add a notification + """ + pass + + @abstractmethod + def update_notification_by_id( + self, notification_id: UUID, notification: NotificationUpdatableProperties + ) -> Notification: + """Update a notification by id""" + pass + + @abstractmethod + def remove_notification_by_id(self, notification_id: UUID): + """ + Remove a notification by id + Args: + notification_id (UUID): The id of the notification + + Returns: + str: Status message + """ + pass diff --git a/backend/core/quivr_core/api/modules/notification/service/__init__.py b/backend/core/quivr_core/api/modules/notification/service/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/notification/service/notification_service.py b/backend/core/quivr_core/api/modules/notification/service/notification_service.py new file mode 100644 index 000000000000..7fd86bd5ef8b --- /dev/null +++ b/backend/core/quivr_core/api/modules/notification/service/notification_service.py @@ -0,0 +1,34 @@ +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.notification.dto.inputs import ( + CreateNotification, + NotificationUpdatableProperties, +) +from quivr_core.api.modules.notification.repository.notifications import Notifications +from quivr_core.api.modules.notification.repository.notifications_interface import ( + NotificationInterface, +) + + +class NotificationService: + repository: NotificationInterface + + def __init__(self): + supabase_client = get_supabase_client() + self.repository = Notifications(supabase_client) + + def add_notification(self, notification: CreateNotification): + """ + Add a notification + """ + return self.repository.add_notification(notification) + + def update_notification_by_id( + self, notification_id, notification: NotificationUpdatableProperties + ): + """ + Update a notification + """ + if notification: + return self.repository.update_notification_by_id( + notification_id, notification + ) diff --git a/backend/core/quivr_core/api/modules/notification/tests/test_notification.py b/backend/core/quivr_core/api/modules/notification/tests/test_notification.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/prompt/__init__.py b/backend/core/quivr_core/api/modules/prompt/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/prompt/controller/__init__.py b/backend/core/quivr_core/api/modules/prompt/controller/__init__.py new file mode 100644 index 000000000000..225300b6f7bc --- /dev/null +++ b/backend/core/quivr_core/api/modules/prompt/controller/__init__.py @@ -0,0 +1 @@ +from .prompt_routes import prompt_router diff --git a/backend/core/quivr_core/api/modules/prompt/controller/prompt_routes.py b/backend/core/quivr_core/api/modules/prompt/controller/prompt_routes.py new file mode 100644 index 000000000000..f87757c25c7d --- /dev/null +++ b/backend/core/quivr_core/api/modules/prompt/controller/prompt_routes.py @@ -0,0 +1,63 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends +from quivr_core.api.modules.dependencies import get_service +from quivr_core.api.modules.prompt.entity.prompt import ( + CreatePromptProperties, + Prompt, + PromptUpdatableProperties, +) +from quivr_core.api.modules.prompt.service import PromptService + +prompt_router = APIRouter() + + +PromptServiceDep = Annotated[PromptService, Depends(get_service(PromptService))] + + +@prompt_router.get("/prompts", tags=["Prompt"]) +async def get_prompts( + prompt_service: PromptServiceDep, +) -> list[Prompt]: + """ + Retrieve all public prompt + """ + return prompt_service.get_public_prompts() + + +@prompt_router.get("/prompts/{prompt_id}", tags=["Prompt"]) +async def get_prompt( + prompt_id: UUID, + prompt_service: PromptServiceDep, +) -> Prompt | None: + """ + Retrieve a prompt by its id + """ + + return prompt_service.get_prompt_by_id(prompt_id) + + +@prompt_router.put("/prompts/{prompt_id}", tags=["Prompt"]) +async def update_prompt( + prompt_id: UUID, + prompt: PromptUpdatableProperties, + prompt_service: PromptServiceDep, +) -> Prompt | None: + """ + Update a prompt by its id + """ + + return prompt_service.update_prompt_by_id(prompt_id, prompt) + + +@prompt_router.post("/prompts", tags=["Prompt"]) +async def create_prompt_route( + prompt: CreatePromptProperties, + prompt_service: PromptServiceDep, +) -> Prompt | None: + """ + Create a prompt by its id + """ + + return prompt_service.create_prompt(prompt) diff --git a/backend/core/quivr_core/api/modules/prompt/entity/__init__.py b/backend/core/quivr_core/api/modules/prompt/entity/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/prompt/entity/prompt.py b/backend/core/quivr_core/api/modules/prompt/entity/prompt.py new file mode 100644 index 000000000000..2e91ee7bd4ce --- /dev/null +++ b/backend/core/quivr_core/api/modules/prompt/entity/prompt.py @@ -0,0 +1,53 @@ +from enum import Enum +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel +from sqlmodel import UUID as PGUUID +from sqlmodel import Column, Field, Relationship, SQLModel, text + + +class PromptStatusEnum(str, Enum): + private = "private" + public = "public" + + +class Prompt(SQLModel, table=True): + __tablename__ = "prompts" # type: ignore + id: UUID | None = Field( + default=None, + sa_column=Column( + PGUUID, + server_default=text("uuid_generate_v4()"), + primary_key=True, + ), + ) + content: str | None = None + title: str | None = Field(default=None, max_length=255) + status: str = Field(default="private", max_length=255) + brain: List["Brain"] = Relationship( # noqa: F821 + back_populates="prompt", sa_relationship_kwargs={"lazy": "joined"} + ) + + +class CreatePromptProperties(BaseModel): + """Properties that can be received on prompt creation""" + + title: str + content: str + status: PromptStatusEnum = PromptStatusEnum.private + + +class PromptUpdatableProperties(BaseModel): + """Properties that can be received on prompt update""" + + title: Optional[str] = None + content: Optional[str] = None + status: Optional[PromptStatusEnum] = None + + +class DeletePromptResponse(BaseModel): + """Response when deleting a prompt""" + + status: str = "delete" + prompt_id: UUID diff --git a/backend/core/quivr_core/api/modules/prompt/repository/__init__.py b/backend/core/quivr_core/api/modules/prompt/repository/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/prompt/repository/prompts.py b/backend/core/quivr_core/api/modules/prompt/repository/prompts.py new file mode 100644 index 000000000000..bffde769dfb2 --- /dev/null +++ b/backend/core/quivr_core/api/modules/prompt/repository/prompts.py @@ -0,0 +1,104 @@ +from uuid import UUID + +from fastapi import HTTPException +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.dependencies import BaseRepository +from quivr_core.api.modules.prompt.entity.prompt import DeletePromptResponse, Prompt + + +class PromptRepository(BaseRepository): + def __init__(self, session): + super().__init__(session) + self.db = get_supabase_client() + + def create_prompt(self, prompt): + """ + Create a prompt + """ + + response = (self.db.from_("prompts").insert(prompt.dict()).execute()).data + + return Prompt(**response[0]) + + def delete_prompt_by_id(self, prompt_id: UUID): + """ + Delete a prompt by id + Args: + prompt_id (UUID): The id of the prompt + + Returns: + A dictionary containing the status of the delete and prompt_id of the deleted prompt + """ + + # Update brains where prompt_id is equal to the value to NULL + self.db.from_("brains").update({"prompt_id": None}).filter( + "prompt_id", "eq", str(prompt_id) + ).execute() + + # Update chat_history where prompt_id is equal to the value to NULL + self.db.from_("chat_history").update({"prompt_id": None}).filter( + "prompt_id", "eq", str(prompt_id) + ).execute() + + # Delete the prompt + response = ( + self.db.from_("prompts") + .delete() + .filter("id", "eq", str(prompt_id)) + .execute() + .data + ) + + if response == []: + raise HTTPException(404, "Prompt not found") + + return DeletePromptResponse(status="deleted", prompt_id=prompt_id) + + def get_prompt_by_id(self, prompt_id: UUID): + """ + Get a prompt by its id + + Args: + prompt_id (UUID): The id of the prompt + + Returns: + Prompt: The prompt + """ + + response = ( + self.db.from_("prompts") + .select("*") + .filter("id", "eq", str(prompt_id)) + .execute() + ).data + + if response == []: + return None + return Prompt(**response[0]) + + def get_public_prompts(self): + """ + List all public prompts + """ + + return ( + self.db.from_("prompts") + .select("*") + .filter("status", "eq", "public") + .execute() + ).data + + def update_prompt_by_id(self, prompt_id: UUID, prompt): + """Update a prompt by id""" + + response = ( + self.db.from_("prompts") + .update(prompt.dict(exclude_unset=True)) + .filter("id", "eq", str(prompt_id)) + .execute() + ).data + + if response == []: + raise HTTPException(404, "Prompt not found") + + return Prompt(**response[0]) diff --git a/backend/core/quivr_core/api/modules/prompt/service/__init__.py b/backend/core/quivr_core/api/modules/prompt/service/__init__.py new file mode 100644 index 000000000000..04be1eb3c873 --- /dev/null +++ b/backend/core/quivr_core/api/modules/prompt/service/__init__.py @@ -0,0 +1 @@ +from .prompt_service import PromptService diff --git a/backend/core/quivr_core/api/modules/prompt/service/prompt_service.py b/backend/core/quivr_core/api/modules/prompt/service/prompt_service.py new file mode 100644 index 000000000000..c6c033823fc1 --- /dev/null +++ b/backend/core/quivr_core/api/modules/prompt/service/prompt_service.py @@ -0,0 +1,71 @@ +from typing import List +from uuid import UUID + +from quivr_core.api.modules.brain.service.utils.get_prompt_to_use_id import ( + get_prompt_to_use_id, +) +from quivr_core.api.modules.dependencies import BaseService +from quivr_core.api.modules.prompt.entity.prompt import ( + CreatePromptProperties, + DeletePromptResponse, + Prompt, + PromptUpdatableProperties, +) +from quivr_core.api.modules.prompt.repository.prompts import PromptRepository + + +class PromptService(BaseService[PromptRepository]): + repository_cls = PromptRepository + + def __init__(self, repository: PromptRepository): + self.repository = repository + + def create_prompt(self, prompt: CreatePromptProperties) -> Prompt: + return self.repository.create_prompt(prompt) + + def delete_prompt_by_id(self, prompt_id: UUID) -> DeletePromptResponse: + """ + Delete a prompt by id + Args: + prompt_id (UUID): The id of the prompt + + Returns: + Prompt: The prompt + """ + return self.repository.delete_prompt_by_id(prompt_id) + + def get_prompt_by_id(self, prompt_id: UUID) -> Prompt | None: + """ + Get a prompt by its id + + Args: + prompt_id (UUID): The id of the prompt + + Returns: + Prompt: The prompt + """ + return self.repository.get_prompt_by_id(prompt_id) + + def get_public_prompts(self) -> List[Prompt]: + """ + List all public prompts + """ + + return self.repository.get_public_prompts() + + def update_prompt_by_id( + self, prompt_id: UUID, prompt: PromptUpdatableProperties + ) -> Prompt: + """Update a prompt by id""" + + return self.repository.update_prompt_by_id(prompt_id, prompt) + + def get_prompt_to_use( + self, brain_id: UUID | None, prompt_id: UUID | None + ) -> Prompt | None: + prompt_to_use_id = get_prompt_to_use_id(brain_id, prompt_id) + + if prompt_to_use_id is None: + return None + + return self.get_prompt_by_id(prompt_to_use_id) diff --git a/backend/core/quivr_core/api/modules/prompt/tests/test_prompt.py b/backend/core/quivr_core/api/modules/prompt/tests/test_prompt.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/rag/__init__.py b/backend/core/quivr_core/api/modules/rag/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/rag/rag_service.py b/backend/core/quivr_core/api/modules/rag/rag_service.py new file mode 100644 index 000000000000..ab0f79cf1c66 --- /dev/null +++ b/backend/core/quivr_core/api/modules/rag/rag_service.py @@ -0,0 +1,293 @@ +import datetime +from uuid import UUID, uuid4 + +from langchain_community.chat_models import ChatLiteLLM + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import ( + get_embedding_client, + get_supabase_client, + settings, +) +from quivr_core.api.modules.brain.entity.brain_entity import BrainEntity +from quivr_core.api.modules.brain.service.brain_service import BrainService +from quivr_core.api.modules.brain.service.utils.format_chat_history import ( + format_chat_history, +) +from quivr_core.api.modules.chat.dto.inputs import CreateChatHistory +from quivr_core.api.modules.chat.dto.outputs import GetChatHistoryOutput +from quivr_core.api.modules.chat.service.chat_service import ChatService +from quivr_core.api.modules.knowledge.repository.knowledges import KnowledgeRepository +from quivr_core.api.modules.prompt.entity.prompt import Prompt +from quivr_core.api.modules.prompt.service.prompt_service import PromptService +from quivr_core.api.modules.user.entity.user_identity import UserIdentity +from quivr_core.api.packages.vectorstore.supabase import CustomSupabaseVectorStore +from quivr_core.config import RAGConfig +from quivr_core.models import ParsedRAGResponse, RAGResponseMetadata +from quivr_core.quivr_rag import QuivrQARAG + +from .utils import generate_source + +logger = get_logger(__name__) + + +class RAGService: + def __init__( + self, + current_user: UserIdentity, + brain_id: UUID | None, + chat_id: UUID, + brain_service: BrainService, + prompt_service: PromptService, + chat_service: ChatService, + knowledge_service: KnowledgeRepository, + ): + # Services + self.brain_service = brain_service + self.prompt_service = prompt_service + self.chat_service = chat_service + self.knowledge_service = knowledge_service + + # Base models + self.current_user = current_user + self.chat_id = chat_id + self.brain = self.get_or_create_brain(brain_id, self.current_user.id) + self.prompt = self.get_brain_prompt(self.brain) + + # check at init time + # TODO: how to get model ? + self.model_to_use = "" + # self.model_to_use = "" self.check_and_update_user_usage( + # self.current_user, self.brain + # ) + + def get_brain_prompt(self, brain: BrainEntity) -> Prompt | None: + return ( + self.prompt_service.get_prompt_by_id(brain.prompt_id) + if brain.prompt_id + else None + ) + + def get_llm(self, rag_config: RAGConfig): + api_base = ( + settings.ollama_api_base_url + if settings.ollama_api_base_url and rag_config.model.startswith("ollama") + else None + ) + return ChatLiteLLM( + temperature=rag_config.temperature, + max_tokens=rag_config.max_tokens, + model=rag_config.model, + streaming=rag_config.streaming, + verbose=False, + api_base=api_base, + ) # pyright: ignore reportPrivateUsage=none + + def get_or_create_brain(self, brain_id: UUID | None, user_id: UUID) -> BrainEntity: + brain = None + if brain_id is not None: + brain = self.brain_service.get_brain_details(brain_id, user_id) + + # TODO: Create if doesn't exist + assert brain + + if brain.integration: + assert brain.integration.user_id == user_id + return brain + + # def check_and_update_user_usage(self, user: UserIdentity, brain: BrainEntity): + # """Check user limits and raises if user reached his limits: + # 1. Raise if one of the conditions : + # - User doesn't have access to brains + # - Model of brain is not is user_settings.models + # - Latest sum_30d(user_daily_user) < user_settings.max_monthly_usage + # - Check sum(user_settings.daily_user_count)+ model_price < user_settings.monthly_chat_credits + # 2. Updates user usage + # """ + # # TODO(@aminediro) : THIS is bug prone, should retrieve it from DB here + # user_usage = UserUsage(id=user.id, email=user.email) + # user_settings = user_usage.get_user_settings() + # all_models = user_usage.get_models() + + # # TODO(@aminediro): refactor this function + # model_to_use = find_model_and_generate_metadata( + # brain.model, + # user_settings, + # all_models, + # ) + # cost = compute_cost(model_to_use, all_models) + # # Raises HTTP if user usage exceeds limits + # update_user_usage(user_usage, user_settings, cost) # noqa: F821 + # return model_to_use + + def create_vector_store( + self, brain_id: UUID, max_input: int + ) -> CustomSupabaseVectorStore: + supabase_client = get_supabase_client() + embeddings = get_embedding_client() + return CustomSupabaseVectorStore( + supabase_client, + embeddings, + table_name="vectors", + brain_id=brain_id, + max_input=max_input, + ) + + def save_answer(self, question: str, answer: ParsedRAGResponse): + return self.chat_service.update_chat_history( + CreateChatHistory( + **{ + "chat_id": self.chat_id, + "user_message": question, + "assistant": answer.answer, + "brain_id": self.brain.brain_id, + # TODO: prompt_id should always be not None + "prompt_id": self.prompt.id if self.prompt else None, + "metadata": answer.metadata.model_dump() if answer.metadata else {}, + } + ) + ) + + async def generate_answer( + self, + question: str, + ): + logger.info( + f"Creating question for chat {self.chat_id} with brain {self.brain.brain_id} " + ) + rag_config = RAGConfig( + model=self.model_to_use.name, + temperature=self.brain.temperature, + max_input=self.model_to_use.max_input, + max_tokens=self.brain.max_tokens, + prompt=self.prompt.content if self.prompt else None, + streaming=False, + ) + history = await self.chat_service.get_chat_history(self.chat_id) + # Get list of files + list_files = self.knowledge_service.get_all_knowledge_in_brain( + self.brain.brain_id + ) + # Build RAG dependencies to inject + vector_store = self.create_vector_store( + self.brain.brain_id, rag_config.max_input + ) + llm = self.get_llm(rag_config) + # Initialize the RAG pipline + rag_pipeline = QuivrQARAG( + rag_config=rag_config, llm=llm, vector_store=vector_store + ) + # Format the history, sanitize the input + transformed_history = format_chat_history(history) + + parsed_response = rag_pipeline.answer(question, transformed_history, list_files) + + # Save the answer to db + new_chat_entry = self.save_answer(question, parsed_response) + + # Format output to be correct + return GetChatHistoryOutput( + **{ + "chat_id": self.chat_id, + "user_message": question, + "assistant": parsed_response.answer, + "message_time": new_chat_entry.message_time, + "prompt_title": (self.prompt.title if self.prompt else None), + "brain_name": self.brain.name if self.brain else None, + "message_id": new_chat_entry.message_id, + "brain_id": str(self.brain.brain_id) if self.brain else None, + "metadata": ( + parsed_response.metadata.model_dump() + if parsed_response.metadata + else {} + ), + } + ) + + async def generate_answer_stream( + self, + question: str, + ): + logger.info( + f"Creating question for chat {self.chat_id} with brain {self.brain.brain_id} " + ) + # Build the rag config + rag_config = RAGConfig( + model=self.model_to_use.name, + temperature=self.brain.temperature, + max_input=self.model_to_use.max_input, + max_tokens=self.brain.max_tokens, + prompt=self.prompt.content if self.prompt else "", + streaming=True, + ) + # Getting chat history + history = await self.chat_service.get_chat_history(self.chat_id) + # Format the history, sanitize the input + transformed_history = format_chat_history(history) + + # Get list of files urls + # TODO: Why do we get ALL the files ? + list_files = self.knowledge_service.get_all_knowledge_in_brain( + self.brain.brain_id + ) + llm = self.get_llm(rag_config) + vector_store = self.create_vector_store( + self.brain.brain_id, rag_config.max_input + ) + # Initialize the rag pipline + rag_pipeline = QuivrQARAG( + rag_config=rag_config, llm=llm, vector_store=vector_store + ) + + full_answer = "" + + message_metadata = { + "chat_id": self.chat_id, + "message_id": uuid4(), # do we need it ?, + "user_message": question, # TODO: define result + "message_time": datetime.datetime.now(), # TODO: define result + "prompt_title": (self.prompt.title if self.prompt else ""), + "brain_name": self.brain.name if self.brain else None, + "brain_id": self.brain.brain_id if self.brain else None, + } + + async for response in rag_pipeline.answer_astream( + question, transformed_history, list_files + ): + # Format output to be correct servicedf;j + if not response.last_chunk: + streamed_chat_history = GetChatHistoryOutput( + assistant=response.answer, + metadata=response.metadata.model_dump(), + **message_metadata, + ) + full_answer += response.answer + yield f"data: {streamed_chat_history.model_dump_json()}" + + # For last chunk parse the sources, and the full answer + streamed_chat_history = GetChatHistoryOutput( + assistant=response.answer, + metadata=response.metadata.model_dump(), + **message_metadata, + ) + + sources_urls = generate_source( + response.metadata.sources, + self.brain.brain_id, + ( + streamed_chat_history.metadata["citations"] + if streamed_chat_history.metadata + else None + ), + ) + if streamed_chat_history.metadata: + streamed_chat_history.metadata["sources"] = sources_urls + + self.save_answer( + question, + ParsedRAGResponse( + answer=full_answer, + metadata=RAGResponseMetadata(**streamed_chat_history.metadata), + ), + ) + yield f"data: {streamed_chat_history.model_dump_json()}" diff --git a/backend/core/quivr_core/api/modules/rag/utils.py b/backend/core/quivr_core/api/modules/rag/utils.py new file mode 100644 index 000000000000..908fa2b69545 --- /dev/null +++ b/backend/core/quivr_core/api/modules/rag/utils.py @@ -0,0 +1,91 @@ +from typing import Any, List +from uuid import UUID + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.chat.dto.chats import Sources +from quivr_core.api.modules.upload.service.generate_file_signed_url import ( + generate_file_signed_url, +) + +logger = get_logger(__name__) + + +def generate_source( + source_documents: List[Any] | None, + brain_id: UUID, + citations: List[int] | None = None, +) -> List[Sources]: + """ + Generate the sources list for the answer + It takes in a list of sources documents and citations that points to the docs index that was used in the answer + """ + # Initialize an empty list for sources + sources_list: List[Sources] = [] + + # Initialize a dictionary for storing generated URLs + generated_urls = {} + + # remove duplicate sources with same name and create a list of unique sources + sources_url_cache = {} + + # Get source documents from the result, default to an empty list if not found + # If source documents exist + if source_documents: + logger.info(f"Citations {citations}") + for index, doc in enumerate(source_documents): + logger.info(f"Processing source document {doc.metadata['file_name']}") + if citations is not None: + if index not in citations: + logger.info(f"Skipping source document {doc.metadata['file_name']}") + continue + # Check if 'url' is in the document metadata + is_url = ( + "original_file_name" in doc.metadata + and doc.metadata["original_file_name"] is not None + and doc.metadata["original_file_name"].startswith("http") + ) + + # Determine the name based on whether it's a URL or a file + name = ( + doc.metadata["original_file_name"] + if is_url + else doc.metadata["file_name"] + ) + + # Determine the type based on whether it's a URL or a file + type_ = "url" if is_url else "file" + + # Determine the source URL based on whether it's a URL or a file + if is_url: + source_url = doc.metadata["original_file_name"] + else: + file_path = f"{brain_id}/{doc.metadata['file_name']}" + # Check if the URL has already been generated + if file_path in generated_urls: + source_url = generated_urls[file_path] + else: + # Generate the URL + if file_path in sources_url_cache: + source_url = sources_url_cache[file_path] + else: + generated_url = generate_file_signed_url(file_path) + if generated_url is not None: + source_url = generated_url.get("signedURL", "") + else: + source_url = "" + # Store the generated URL + generated_urls[file_path] = source_url + + # Append a new Sources object to the list + sources_list.append( + Sources( + name=name, + type=type_, + source_url=source_url, + original_file_name=name, + citation=doc.page_content, + ) + ) + else: + logger.info("No source documents found or source_documents is not a list.") + return sources_list diff --git a/backend/core/quivr_core/api/modules/upload/__init__.py b/backend/core/quivr_core/api/modules/upload/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/upload/controller/__init__.py b/backend/core/quivr_core/api/modules/upload/controller/__init__.py new file mode 100644 index 000000000000..56030aa683eb --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/controller/__init__.py @@ -0,0 +1 @@ +from .upload_routes import upload_router diff --git a/backend/core/quivr_core/api/modules/upload/controller/upload_routes.py b/backend/core/quivr_core/api/modules/upload/controller/upload_routes.py new file mode 100644 index 000000000000..605a22935f62 --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/controller/upload_routes.py @@ -0,0 +1,97 @@ +import os +from typing import Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile + +from quivr_core.api.celery_worker import process_file_and_notify +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.dependencies import get_current_user +from quivr_core.api.modules.knowledge.dto.inputs import CreateKnowledgeProperties +from quivr_core.api.modules.knowledge.service.knowledge_service import KnowledgeService +from quivr_core.api.modules.notification.dto.inputs import ( + CreateNotification, + NotificationUpdatableProperties, +) +from quivr_core.api.modules.notification.entity.notification import ( + NotificationsStatusEnum, +) +from quivr_core.api.modules.notification.service.notification_service import ( + NotificationService, +) +from quivr_core.api.modules.upload.service.upload_file import upload_file_storage +from quivr_core.api.modules.user.entity.user_identity import UserIdentity +from quivr_core.api.packages.utils.telemetry import maybe_send_telemetry + +logger = get_logger(__name__) +upload_router = APIRouter() + +notification_service = NotificationService() +knowledge_service = KnowledgeService() + + +@upload_router.get("/upload/healthz", tags=["Health"]) +async def healthz(): + return {"status": "ok"} + + +@upload_router.post("/upload", tags=["Upload"]) +async def upload_file( + uploadFile: UploadFile, + brain_id: UUID = Query(..., description="The ID of the brain"), + chat_id: Optional[UUID] = Query(None, description="The ID of the chat"), + current_user: UserIdentity = Depends(get_current_user), +): + upload_notification = notification_service.add_notification( + CreateNotification( + user_id=current_user.id, + status=NotificationsStatusEnum.INFO, + title=f"Processing File {uploadFile.filename}", + ) + ) + + maybe_send_telemetry("upload_file", {"file_name": uploadFile.filename}) + + file_content = await uploadFile.read() + filename_with_brain_id = str(brain_id) + "/" + str(uploadFile.filename) + + try: + upload_file_storage(file_content, filename_with_brain_id) + + except Exception as e: + print(e) + + notification_service.update_notification_by_id( + upload_notification.id if upload_notification else None, + NotificationUpdatableProperties( + status=NotificationsStatusEnum.ERROR, + description=f"There was an error uploading the file: {e}", + ), + ) + if "The resource already exists" in str(e): + raise HTTPException( + status_code=403, + detail=f"File {uploadFile.filename} already exists in storage.", + ) + else: + raise HTTPException( + status_code=500, detail=f"Failed to upload file to storage. {e}" + ) + + knowledge_to_add = CreateKnowledgeProperties( + brain_id=brain_id, + file_name=uploadFile.filename, + extension=os.path.splitext( + uploadFile.filename # pyright: ignore reportPrivateUsage=none + )[-1].lower(), + ) + + knowledge_service.add_knowledge(knowledge_to_add) + + process_file_and_notify.delay( + file_name=filename_with_brain_id, + file_original_name=uploadFile.filename, + brain_id=brain_id, + notification_id=upload_notification.id, + ) + return {"message": "File processing has started."} diff --git a/backend/core/quivr_core/api/modules/upload/service/__init__.py b/backend/core/quivr_core/api/modules/upload/service/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/upload/service/generate_file_signed_url.py b/backend/core/quivr_core/api/modules/upload/service/generate_file_signed_url.py new file mode 100644 index 000000000000..aaaf4b727473 --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/service/generate_file_signed_url.py @@ -0,0 +1,27 @@ +from multiprocessing import get_logger + +from supabase.client import Client + +from quivr_core.api.models.settings import get_supabase_client + +logger = get_logger() + +SIGNED_URL_EXPIRATION_PERIOD_IN_SECONDS = 3600 + + +def generate_file_signed_url(path): + supabase_client: Client = get_supabase_client() + + try: + response = supabase_client.storage.from_("quivr").create_signed_url( + path, + SIGNED_URL_EXPIRATION_PERIOD_IN_SECONDS, + options={ + "download": True, + "transform": None, + }, + ) + logger.info("RESPONSE SIGNED URL", response) + return response + except Exception as e: + logger.error(e) diff --git a/backend/core/quivr_core/api/modules/upload/service/list_files.py b/backend/core/quivr_core/api/modules/upload/service/list_files.py new file mode 100644 index 000000000000..1a9c3faf38ce --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/service/list_files.py @@ -0,0 +1,17 @@ +from multiprocessing import get_logger + +from supabase.client import Client + +from quivr_core.api.models.settings import get_supabase_client + +logger = get_logger() + + +def list_files_from_storage(path): + supabase_client: Client = get_supabase_client() + + try: + response = supabase_client.storage.from_("quivr").list(path) + return response + except Exception as e: + logger.error(e) diff --git a/backend/core/quivr_core/api/modules/upload/service/upload_file.py b/backend/core/quivr_core/api/modules/upload/service/upload_file.py new file mode 100644 index 000000000000..d3733fab8a06 --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/service/upload_file.py @@ -0,0 +1,134 @@ +import json +import os + +from langchain.pydantic_v1 import Field +from langchain.schema import Document +from supabase.client import Client + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import get_supabase_client + +logger = get_logger(__name__) + + +# Mapping of file extensions to MIME types +mime_types = { + ".txt": "text/plain", + ".csv": "text/csv", + ".md": "text/markdown", + ".markdown": "text/markdown", + ".telegram": "application/x-telegram", + ".m4a": "audio/mp4", + ".mp3": "audio/mpeg", + ".webm": "audio/webm", + ".mp4": "video/mp4", + ".mpga": "audio/mpeg", + ".wav": "audio/wav", + ".mpeg": "video/mpeg", + ".pdf": "application/pdf", + ".html": "text/html", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".odt": "application/vnd.oasis.opendocument.text", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".xls": "application/vnd.ms-excel", + ".epub": "application/epub+zip", + ".ipynb": "application/x-ipynb+json", + ".py": "text/x-python", +} + + +def check_file_exists(brain_id: str, file_identifier: str) -> bool: + supabase_client: Client = get_supabase_client() + try: + # Check if the file exists + logger.info(f"Checking if file {file_identifier} exists.") + # This needs to be converted into a file_identifier that is safe for a URL + + response = supabase_client.storage.from_("quivr").list(brain_id) + + # Check if the file_identifier is in the response + file_exists = any( + file["name"].split(".")[0] == file_identifier.split(".")[0] + for file in response + ) + logger.info(f"File identifier: {file_identifier}") + logger.info(f"File exists: {file_exists}") + if file_exists: + logger.info(f"File {file_identifier} exists.") + return True + else: + logger.info(f"File {file_identifier} does not exist.") + return False + except Exception as e: + logger.error(f"An error occurred while checking the file: {e}") + return True + + +def upload_file_storage(file, file_identifier: str, upsert: str = "false"): + supabase_client: Client = get_supabase_client() + response = None + + try: + # Get the file extension + _, file_extension = os.path.splitext(file_identifier) + + # Get the MIME type for the file extension + mime_type = mime_types.get(file_extension, "text/html") + + response = supabase_client.storage.from_("quivr").upload( + file_identifier, + file, + file_options={ + "content-type": mime_type, + "upsert": upsert, + "cache-control": "3600", + }, + ) + + return response + except Exception as e: + if "The resource already exists" in str(e) and upsert == "true": + response = supabase_client.storage.from_("quivr").update( + file_identifier, + file, + file_options={ + "content-type": mime_type, + "upsert": upsert, + "cache-control": "3600", + }, + ) + else: + raise e + + +class DocumentSerializable(Document): + """Class for storing a piece of text and associated metadata.""" + + page_content: str + metadata: dict = Field(default_factory=dict) + + @property + def lc_serializable(self) -> bool: + return True + + def __repr__(self): + return f"Document(page_content='{self.page_content[:50]}...', metadata={self.metadata})" + + def __str__(self): + return self.__repr__() + + def to_json(self) -> str: + """Convert the Document object to a JSON string.""" + return json.dumps( + { + "page_content": self.page_content, + "metadata": self.metadata, + } + ) + + @classmethod + def from_json(cls, json_str: str): + """Create a Document object from a JSON string.""" + data = json.loads(json_str) + return cls(page_content=data["page_content"], metadata=data["metadata"]) diff --git a/backend/core/quivr_core/api/modules/upload/tests/test_files/test.bib b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.bib new file mode 100644 index 000000000000..14b61838bdf3 --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.bib @@ -0,0 +1,6 @@ +@Article{citekey, + author = "", + title = "", + journal = "", + year = "", +} diff --git a/backend/core/quivr_core/api/modules/upload/tests/test_files/test.csv b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.csv new file mode 100644 index 000000000000..a025eb598153 --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.csv @@ -0,0 +1,17 @@ +quivrhq/quivr,Sat May 13 2023 02:20:09 GMT+0200 (heure d’été d’Europe centrale),0 +quivrhq/quivr,Tue May 16 2023 18:03:49 GMT+0200 (heure d’été d’Europe centrale),660 +quivrhq/quivr,Thu May 18 2023 03:04:23 GMT+0200 (heure d’été d’Europe centrale),1380 +quivrhq/quivr,Thu May 18 2023 23:04:11 GMT+0200 (heure d’été d’Europe centrale),2070 +quivrhq/quivr,Sat May 20 2023 04:44:40 GMT+0200 (heure d’été d’Europe centrale),2790 +quivrhq/quivr,Sun May 21 2023 03:19:46 GMT+0200 (heure d’été d’Europe centrale),3510 +quivrhq/quivr,Mon May 22 2023 08:03:18 GMT+0200 (heure d’été d’Europe centrale),4230 +quivrhq/quivr,Tue May 23 2023 16:57:58 GMT+0200 (heure d’été d’Europe centrale),4950 +quivrhq/quivr,Sat May 27 2023 02:18:31 GMT+0200 (heure d’été d’Europe centrale),5640 +quivrhq/quivr,Thu Jun 01 2023 18:45:27 GMT+0200 (heure d’été d’Europe centrale),6360 +quivrhq/quivr,Thu Jun 08 2023 16:33:57 GMT+0200 (heure d’été d’Europe centrale),7080 +quivrhq/quivr,Mon Jun 19 2023 12:58:34 GMT+0200 (heure d’été d’Europe centrale),7800 +quivrhq/quivr,Tue Jun 27 2023 14:45:52 GMT+0200 (heure d’été d’Europe centrale),8520 +quivrhq/quivr,Fri Jun 30 2023 11:43:51 GMT+0200 (heure d’été d’Europe centrale),9210 +quivrhq/quivr,Fri Jul 07 2023 23:08:23 GMT+0200 (heure d’été d’Europe centrale),9930 +quivrhq/quivr,Mon Jul 10 2023 08:13:07 GMT+0200 (heure d’été d’Europe centrale),10650 +quivrhq/quivr,Wed Jul 12 2023 09:40:29 GMT+0200 (heure d’été d’Europe centrale),13837 diff --git a/backend/core/quivr_core/api/modules/upload/tests/test_files/test.pdf b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.pdf new file mode 100644 index 000000000000..e256f71d70b5 Binary files /dev/null and b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.pdf differ diff --git a/backend/core/quivr_core/api/modules/upload/tests/test_files/test.txt b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.txt new file mode 100644 index 000000000000..90bfcb510602 --- /dev/null +++ b/backend/core/quivr_core/api/modules/upload/tests/test_files/test.txt @@ -0,0 +1 @@ +this is a test diff --git a/backend/core/quivr_core/api/modules/upload/tests/test_upload.py b/backend/core/quivr_core/api/modules/upload/tests/test_upload.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/user/__init__.py b/backend/core/quivr_core/api/modules/user/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/user/controller/__init__.py b/backend/core/quivr_core/api/modules/user/controller/__init__.py new file mode 100644 index 000000000000..d02e1c740e5c --- /dev/null +++ b/backend/core/quivr_core/api/modules/user/controller/__init__.py @@ -0,0 +1 @@ +from .user_controller import user_router diff --git a/backend/core/quivr_core/api/modules/user/controller/user_controller.py b/backend/core/quivr_core/api/modules/user/controller/user_controller.py new file mode 100644 index 000000000000..413f4c5b08b5 --- /dev/null +++ b/backend/core/quivr_core/api/modules/user/controller/user_controller.py @@ -0,0 +1,93 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends + +from quivr_core.api.modules.brain.service.brain_user_service import BrainUserService +from quivr_core.api.modules.dependencies import get_current_user, get_service +from quivr_core.api.modules.user.dto.inputs import UserUpdatableProperties +from quivr_core.api.modules.user.entity.user_identity import UserIdentity +from quivr_core.api.modules.user.service.user_service import UserService + +user_router = APIRouter() +brain_user_service = BrainUserService() + +UserServiceDep = Annotated[UserService, Depends(get_service(UserService))] + +UserIdentityDep = Annotated[UserIdentity, Depends(get_current_user)] + + +@user_router.get("/user", tags=["User"]) +async def get_user_endpoint( + current_user: UserIdentityDep, user_service: UserServiceDep +): + """ + Get user information and statistics. + + - `current_user`: The current authenticated user. + - Returns the user's email, maximum brain size, current brain size, maximum requests number, requests statistics, and the current date. + + This endpoint retrieves information and statistics about the authenticated user. It includes the user's email, maximum brain size, + current brain size, maximum requests number, requests statistics, and the current date. The brain size is calculated based on the + user's uploaded vectors, and the maximum brain size is obtained from the environment variables. The requests statistics provide + information about the user's API usage. + """ + + user_settings = user_service.get_user_settings(current_user.id) + + return { + "email": current_user.email, + "current_brain_size": 0, + "models": user_settings.get("models", []), + "id": current_user.id, + } + + +@user_router.put( + "/user/identity", + tags=["User"], +) +def update_user_identity_route( + user_identity_updatable_properties: UserUpdatableProperties, + current_user: UserIdentityDep, + user_service: UserServiceDep, +) -> UserIdentity: + """ + Update user identity. + """ + return user_service.update_user_properties( + current_user.id, user_identity_updatable_properties + ) + + +@user_router.get( + "/user/identity", + tags=["User"], +) +def get_user_identity_route( + current_user: UserIdentityDep, + user_service: UserServiceDep, +) -> UserIdentity: + """ + Get user identity. + """ + return user_service.get_user_identity(current_user.id) + + +@user_router.delete( + "/user_data", + tags=["User"], +) +async def delete_user_data_route( + current_user: UserIdentityDep, + user_service: UserServiceDep, +): + """ + Delete a user. + - `user_id`: The ID of the user to delete. + + This endpoint deletes a user from the system. + """ + + user_service.delete_user_data(current_user.id) + + return {"message": "User deleted successfully"} diff --git a/backend/core/quivr_core/api/modules/user/dto/inputs.py b/backend/core/quivr_core/api/modules/user/dto/inputs.py new file mode 100644 index 000000000000..348e99af68b8 --- /dev/null +++ b/backend/core/quivr_core/api/modules/user/dto/inputs.py @@ -0,0 +1,12 @@ +from typing import Optional + +from pydantic import BaseModel + + +class UserUpdatableProperties(BaseModel): + # Nothing for now + username: Optional[str] = None + company: Optional[str] = None + onboarded: Optional[bool] = None + company_size: Optional[str] = None + usage_purpose: Optional[str] = None diff --git a/backend/core/quivr_core/api/modules/user/entity/user_identity.py b/backend/core/quivr_core/api/modules/user/entity/user_identity.py new file mode 100644 index 000000000000..8feaccfe0c2e --- /dev/null +++ b/backend/core/quivr_core/api/modules/user/entity/user_identity.py @@ -0,0 +1,28 @@ +from typing import List, Optional +from uuid import UUID, uuid4 + +from pydantic import BaseModel +from sqlmodel import Field, Relationship, SQLModel + + +class User(SQLModel, table=True): + __tablename__ = "users" # type: ignore + + id: UUID | None = Field( + primary_key=True, + nullable=False, + default_factory=uuid4, + ) + email: str + onboarded: bool | None = None + chats: List["Chat"] | None = Relationship(back_populates="user") # type: ignore + + +class UserIdentity(BaseModel): + id: UUID + email: Optional[str] = None + username: Optional[str] = None + company: Optional[str] = None + onboarded: Optional[bool] = None + company_size: Optional[str] = None + usage_purpose: Optional[str] = None diff --git a/backend/core/quivr_core/api/modules/user/repository/__init__.py b/backend/core/quivr_core/api/modules/user/repository/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/modules/user/repository/user_repository.py b/backend/core/quivr_core/api/modules/user/repository/user_repository.py new file mode 100644 index 000000000000..099d9831c723 --- /dev/null +++ b/backend/core/quivr_core/api/modules/user/repository/user_repository.py @@ -0,0 +1,146 @@ +from uuid import UUID + +from sqlmodel.ext.asyncio.session import AsyncSession + +from quivr_core.api.models.settings import get_supabase_client +from quivr_core.api.modules.dependencies import BaseRepository +from quivr_core.api.modules.user.dto.inputs import UserUpdatableProperties +from quivr_core.api.modules.user.entity.user_identity import UserIdentity + + +class UserRepository(BaseRepository): + def __init__(self, session: AsyncSession): + super().__init__(session) + self.db = get_supabase_client() + + def create_user_identity(self, id): + response = ( + self.db.from_("user_identity") + .insert( + { + "user_id": str(id), + } + ) + .execute() + ) + user_identity = response.data[0] + return UserIdentity(id=user_identity.get("user_id")) + + def update_user_properties( + self, + user_id: UUID, + user_identity_updatable_properties: UserUpdatableProperties, + ): + response = ( + self.db.from_("user_identity") + .update(user_identity_updatable_properties.__dict__) + .filter("user_id", "eq", user_id) # type: ignore + .execute() + ) + + if len(response.data) == 0: + return self.create_user_identity(user_id) + + user_identity = response.data[0] + + print("USER_IDENTITY", user_identity) + return UserIdentity(id=user_id) + + def get_user_identity(self, user_id): + response = ( + self.db.from_("user_identity") + .select("*, users (email)") + .filter("user_id", "eq", str(user_id)) + .execute() + ) + + if len(response.data) == 0: + return self.create_user_identity(user_id) + + user_identity = response.data[0] + + user_identity["id"] = user_id # Add 'id' field to the dictionary + user_identity["email"] = user_identity["users"]["email"] + return UserIdentity(**user_identity) + + def get_user_id_by_user_email(self, email): + response = ( + self.db.rpc("get_user_id_by_user_email", {"user_email": email}) + .execute() + .data + ) + if len(response) > 0: + return response[0]["user_id"] + return None + + def get_user_email_by_user_id(self, user_id): + response = self.db.rpc( + "get_user_email_by_user_id", {"user_id": str(user_id)} + ).execute() + return response.data[0]["email"] + + def delete_user_data(self, user_id: UUID): + response = ( + self.db.from_("brains_users") + .select("brain_id") + .filter("rights", "eq", "Owner") + .filter("user_id", "eq", str(user_id)) + .execute() + ) + brain_ids = [row["brain_id"] for row in response.data] + + for brain_id in brain_ids: + self.db.table("brains").delete().filter( + "brain_id", "eq", brain_id + ).execute() + + for brain_id in brain_ids: + self.db.table("brains_vectors").delete().filter( + "brain_id", "eq", brain_id + ).execute() + + for brain_id in brain_ids: + self.db.table("chat_history").delete().filter( + "brain_id", "eq", brain_id + ).execute() + + self.db.table("user_settings").delete().filter( + "user_id", "eq", str(user_id) + ).execute() + self.db.table("user_identity").delete().filter( + "user_id", "eq", str(user_id) + ).execute() + self.db.table("users").delete().filter("id", "eq", str(user_id)).execute() + + def get_user_settings(self, user_id: UUID): + """ + Fetch the user settings from the database + """ + + user_settings_response = ( + self.db.from_("user_settings") + .select("*") + .filter("user_id", "eq", str(user_id)) + .execute() + ).data + + if len(user_settings_response) == 0: + # Create the user settings + user_settings_response = ( + self.db.table("user_settings") + .insert({"user_id": str(user_id)}) + .execute() + ).data + + if len(user_settings_response) == 0: + raise ValueError("User settings could not be created") + + user_settings = user_settings_response[0] + + return user_settings + + def get_models(self): + model_settings_response = (self.db.from_("models").select("*").execute()).data + if len(model_settings_response) == 0: + raise ValueError("An issue occured while fetching the model settings") + return model_settings_response diff --git a/backend/core/quivr_core/api/modules/user/service/__init__.py b/backend/core/quivr_core/api/modules/user/service/__init__.py new file mode 100644 index 000000000000..254962a4866f --- /dev/null +++ b/backend/core/quivr_core/api/modules/user/service/__init__.py @@ -0,0 +1 @@ +from .user_service import UserService diff --git a/backend/core/quivr_core/api/modules/user/service/user_service.py b/backend/core/quivr_core/api/modules/user/service/user_service.py new file mode 100644 index 000000000000..676a24d97124 --- /dev/null +++ b/backend/core/quivr_core/api/modules/user/service/user_service.py @@ -0,0 +1,36 @@ +from uuid import UUID + +from quivr_core.api.modules.dependencies import BaseService +from quivr_core.api.modules.user.dto.inputs import UserUpdatableProperties +from quivr_core.api.modules.user.repository.user_repository import UserRepository + + +class UserService(BaseService[UserRepository]): + repository_cls = UserRepository + + def __init__(self, repository: UserRepository): + self.repository = repository + + def get_user_id_by_email(self, email: str) -> UUID | None: + return self.repository.get_user_id_by_user_email(email) + + def get_user_email_by_user_id(self, user_id: UUID) -> str | None: + return self.repository.get_user_email_by_user_id(user_id) + + def get_user_settings(self, user_id: UUID): + return self.repository.get_user_settings(user_id) + + def get_user_identity(self, user_id: UUID): + return self.repository.get_user_identity(user_id) + + def update_user_properties( + self, + user_id: UUID, + user_identity_updatable_properties: UserUpdatableProperties, + ): + return self.repository.update_user_properties( + user_id, user_identity_updatable_properties + ) + + def delete_user_data(self, user_id: UUID): + return self.repository.delete_user_data(user_id) diff --git a/backend/core/quivr_core/api/modules/user/tests/test_user_controller.py b/backend/core/quivr_core/api/modules/user/tests/test_user_controller.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/__init__.py b/backend/core/quivr_core/api/packages/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/embeddings/__init__.py b/backend/core/quivr_core/api/packages/embeddings/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/embeddings/vectors.py b/backend/core/quivr_core/api/packages/embeddings/vectors.py new file mode 100644 index 000000000000..8c1ab42dbe69 --- /dev/null +++ b/backend/core/quivr_core/api/packages/embeddings/vectors.py @@ -0,0 +1,74 @@ +from concurrent.futures import ThreadPoolExecutor +from typing import List +from uuid import UUID + +from pydantic import BaseModel + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.settings import ( + get_documents_vector_store, + get_embedding_client, +) +from quivr_core.api.models.vectors import Vector + +logger = get_logger(__name__) + + +vector_repository = Vector() + + +# TODO: Create interface for embeddings and implement it for Supabase and OpenAI (current Quivr) +class Neurons(BaseModel): + def create_vector(self, docs): + documents_vector_store = get_documents_vector_store() + + try: + sids = documents_vector_store.add_documents(docs) + if sids and len(sids) > 0: + return sids + + except Exception as e: + logger.error(f"Error creating vector for document {e}") + + def create_embedding(self, content): + embeddings = get_embedding_client() + return embeddings.embed_query(content) + + +def error_callback(exception): + print("An exception occurred:", exception) + + +def process_batch(batch_ids: List[str]): + try: + if len(batch_ids) == 1: + return (vector_repository.get_vectors_by_batch(UUID(batch_ids[0]))).data + else: + return (vector_repository.get_vectors_in_batch(batch_ids)).data + except Exception as e: + logger.error("Error retrieving batched vectors", e) + + +# TODO: move to Knowledge class +def get_unique_files_from_vector_ids(vectors_ids): + # Move into Vectors class + """ + Retrieve unique user data vectors. + """ + + # constants + BATCH_SIZE = 5 + + with ThreadPoolExecutor() as executor: + futures = [] + for i in range(0, len(vectors_ids), BATCH_SIZE): + batch_ids = vectors_ids[i : i + BATCH_SIZE] + future = executor.submit(process_batch, batch_ids) + futures.append(future) + + # Retrieve the results + vectors_responses = [future.result() for future in futures] + + documents = [item for sublist in vectors_responses for item in sublist] + unique_files = [dict(t) for t in set(tuple(d.items()) for d in documents)] + return unique_files diff --git a/backend/core/quivr_core/api/packages/files/__init__.py b/backend/core/quivr_core/api/packages/files/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/files/crawl/__init__.py b/backend/core/quivr_core/api/packages/files/crawl/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/files/crawl/crawler.py b/backend/core/quivr_core/api/packages/files/crawl/crawler.py new file mode 100644 index 000000000000..945968b755e9 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/crawl/crawler.py @@ -0,0 +1,44 @@ +import os +import re +import unicodedata + +from langchain_community.document_loaders import PlaywrightURLLoader +from pydantic import BaseModel + +from quivr_core.api.logger import get_logger + +logger = get_logger(__name__) + + +class CrawlWebsite(BaseModel): + url: str + js: bool = False + depth: int = int(os.getenv("CRAWL_DEPTH", "1")) + max_pages: int = 100 + max_time: int = 60 + + def process(self) -> str: + # Extract and combine content recursively + loader = PlaywrightURLLoader( + urls=[self.url], remove_selectors=["header", "footer"] + ) + + data = loader.load() + # Now turn the data into a string + logger.info(f"Extracted content from {len(data)} pages") + logger.debug(f"Extracted data : {data}") + extracted_content = "" + for page in data: + extracted_content += page.page_content + + return extracted_content + + def checkGithub(self): + return "github.com" in self.url + + +def slugify(text): + text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("utf-8") + text = re.sub(r"[^\w\s-]", "", text).strip().lower() + text = re.sub(r"[-\s]+", "-", text) + return text diff --git a/backend/core/quivr_core/api/packages/files/file.py b/backend/core/quivr_core/api/packages/files/file.py new file mode 100644 index 000000000000..053ce2c76d78 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/file.py @@ -0,0 +1,45 @@ +import hashlib +from io import BytesIO + +from fastapi import UploadFile + + +def convert_bytes(bytes, precision=2): + """Converts bytes into a human-friendly format.""" + abbreviations = ["B", "KB", "MB"] + if bytes <= 0: + return "0 B" + size = bytes + index = 0 + while size >= 1024 and index < len(abbreviations) - 1: + size /= 1024 + index += 1 + return f"{size:.{precision}f} {abbreviations[index]}" + + +def get_file_size(file: UploadFile): + if isinstance(file.file, BytesIO): + # If the file object is a BytesIO object, get the size of the bytes data + file_size = len(file.file.getvalue()) + return file_size + # move the cursor to the end of the file + file.file._file.seek(0, 2) # pyright: ignore reportPrivateUsage=none + file_size = ( + file.file._file.tell() # pyright: ignore reportPrivateUsage=none + ) # Getting the size of the file + # move the cursor back to the beginning of the file + file.file.seek(0) + + return file_size + + +def compute_sha1_from_file(file_path): + with open(file_path, "rb") as file: + bytes = file.read() + readable_hash = compute_sha1_from_content(bytes) + return readable_hash + + +def compute_sha1_from_content(content): + readable_hash = hashlib.sha1(content).hexdigest() + return readable_hash diff --git a/backend/core/quivr_core/api/packages/files/parsers/__init__.py b/backend/core/quivr_core/api/packages/files/parsers/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/files/parsers/audio.py b/backend/core/quivr_core/api/packages/files/parsers/audio.py new file mode 100644 index 000000000000..7b7e08039040 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/audio.py @@ -0,0 +1,52 @@ +import time + +import openai +from langchain.schema import Document +from langchain.text_splitter import RecursiveCharacterTextSplitter + +from quivr_core.api.models.files import File +from quivr_core.api.models.settings import get_documents_vector_store +from quivr_core.api.packages.files.file import compute_sha1_from_content + + +def process_audio(file: File, **kwargs): + dateshort = time.strftime("%Y%m%d-%H%M%S") + file_meta_name = f"audiotranscript_{dateshort}.txt" + documents_vector_store = get_documents_vector_store() + + with open(file.tmp_file_path, "rb") as audio_file: + transcript = openai.Audio.transcribe("whisper-1", audio_file) + + file_sha = compute_sha1_from_content( + transcript.text.encode("utf-8") # pyright: ignore reportPrivateUsage=none + ) + file_size = len( + transcript.text.encode("utf-8") # pyright: ignore reportPrivateUsage=none + ) + + chunk_size = 500 + chunk_overlap = 0 + + text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=chunk_size, chunk_overlap=chunk_overlap + ) + texts = text_splitter.split_text( + transcript.text.encode("utf-8") # pyright: ignore reportPrivateUsage=none + ) + + docs_with_metadata = [ + Document( + page_content=text, + metadata={ + "file_sha1": file_sha, + "file_size": file_size, + "file_name": file_meta_name, + "chunk_size": chunk_size, + "chunk_overlap": chunk_overlap, + "date": dateshort, + }, + ) + for text in texts + ] + + documents_vector_store.add_documents(docs_with_metadata) diff --git a/backend/core/quivr_core/api/packages/files/parsers/bibtex.py b/backend/core/quivr_core/api/packages/files/parsers/bibtex.py new file mode 100644 index 000000000000..960bc771d905 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/bibtex.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import BibtexLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_bibtex( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=BibtexLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/code_python.py b/backend/core/quivr_core/api/packages/files/parsers/code_python.py new file mode 100644 index 000000000000..3e7146be8db5 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/code_python.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import PythonLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_python( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=PythonLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/common.py b/backend/core/quivr_core/api/packages/files/parsers/common.py new file mode 100644 index 000000000000..642b8ecc9d48 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/common.py @@ -0,0 +1,118 @@ +import asyncio +import os +import tempfile +import time + +import nest_asyncio +import tiktoken +import uvloop +from langchain.schema import Document +from langchain.text_splitter import RecursiveCharacterTextSplitter +from llama_parse import LlamaParse + +from quivr_core.api.logger import get_logger +from quivr_core.api.models.files import File +from quivr_core.api.modules.brain.service.brain_vector_service import BrainVectorService +from quivr_core.api.modules.upload.service.upload_file import DocumentSerializable +from quivr_core.api.packages.embeddings.vectors import Neurons + +if not isinstance(asyncio.get_event_loop(), uvloop.Loop): + nest_asyncio.apply() + +logger = get_logger(__name__) + + +def process_file( + file: File, + loader_class, + brain_id, + original_file_name, + integration=None, + integration_link=None, +): + dateshort = time.strftime("%Y%m%d") + neurons = Neurons() + + if os.getenv("LLAMA_CLOUD_API_KEY"): + doc = file.file + document_ext = os.path.splitext(doc.filename)[1] + if document_ext in [".pdf", ".docx", ".doc"]: + document_tmp = tempfile.NamedTemporaryFile( + suffix=document_ext, delete=False + ) + # Seek to the beginning of the file + doc.file.seek(0) + document_tmp.write(doc.file.read()) + + parser = LlamaParse( + result_type="markdown", # "markdown" and "text" are available + parsing_instruction="Extract the tables and transform checkboxes into text. Transform tables to key = value. You can duplicates Keys if needed. For example: Productions Fonts = 300 productions Fonts Company Desktop License = Yes for Maximum of 60 Licensed Desktop users For example checkboxes should be: Premium Activated = Yes License Premier = No If a checkbox is present for a table with multiple options. Say Yes for the one activated and no for the one not activated. Format using headers.", + gpt4o_mode=True, + gpt4o_api_key=os.getenv("OPENAI_API_KEY"), + ) + + document_llama_parsed = parser.load_data(document_tmp.name) + document_tmp.close() + document_to_langchain = document_llama_parsed[0].to_langchain_format() + text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=file.chunk_size, chunk_overlap=file.chunk_overlap + ) + document_to_langchain = Document( + page_content=document_to_langchain.page_content + ) + file.documents = text_splitter.split_documents([document_to_langchain]) + else: + file.compute_documents(loader_class) + + metadata = { + "file_sha1": file.file_sha1, + "file_size": file.file_size, + "file_name": file.file_name, + "chunk_size": file.chunk_size, + "chunk_overlap": file.chunk_overlap, + "date": dateshort, + "original_file_name": original_file_name or file.file_name, + "integration": integration or "", + "integration_link": integration_link or "", + } + docs = [] + + enc = tiktoken.get_encoding("cl100k_base") + + if file.documents is not None: + logger.info("Coming here?") + for index, doc in enumerate(file.documents, start=1): # pyright: ignore reportPrivateUsage=none + new_metadata = metadata.copy() + logger.info(f"Processing document {doc}") + # Add filename at beginning of page content + doc.page_content = f"Filename: {new_metadata['original_file_name']} Content: {doc.page_content}" + + doc.page_content = doc.page_content.replace("\u0000", "") + + len_chunk = len(enc.encode(doc.page_content)) + + # Ensure the text is in UTF-8 + doc.page_content = doc.page_content.encode("utf-8", "replace").decode( + "utf-8" + ) + + new_metadata["chunk_size"] = len_chunk + new_metadata["index"] = index + doc_with_metadata = DocumentSerializable( + page_content=doc.page_content, metadata=new_metadata + ) + docs.append(doc_with_metadata) + + created_vector = neurons.create_vector(docs) + + brain_vector_service = BrainVectorService(brain_id) + + if created_vector is not None: + for created_vector_id in created_vector: + result = brain_vector_service.create_brain_vector( + created_vector_id, metadata["file_sha1"] + ) + logger.debug(f"Brain vector created: {result}") + return len(created_vector) + else: + return 0 diff --git a/backend/core/quivr_core/api/packages/files/parsers/csv.py b/backend/core/quivr_core/api/packages/files/parsers/csv.py new file mode 100644 index 000000000000..fc440bf268bc --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/csv.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import CSVLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_csv( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=CSVLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/docx.py b/backend/core/quivr_core/api/packages/files/parsers/docx.py new file mode 100644 index 000000000000..1766d63cc4e0 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/docx.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import Docx2txtLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_docx( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=Docx2txtLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/epub.py b/backend/core/quivr_core/api/packages/files/parsers/epub.py new file mode 100644 index 000000000000..c57374fac3f4 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/epub.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders.epub import UnstructuredEPubLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_epub( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=UnstructuredEPubLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/github.py b/backend/core/quivr_core/api/packages/files/parsers/github.py new file mode 100644 index 000000000000..89575499ef8f --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/github.py @@ -0,0 +1,79 @@ +import os +import time + +from langchain.schema import Document +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain_community.document_loaders import GitLoader + +from quivr_core.api.models.files import File +from quivr_core.api.packages.embeddings.vectors import Neurons +from quivr_core.api.packages.files.file import compute_sha1_from_content + + +def process_github( + repo, + brain_id, +): + random_dir_name = os.urandom(16).hex() + dateshort = time.strftime("%Y%m%d") + loader = GitLoader( + clone_url=repo, + repo_path="/tmp/" + random_dir_name, + ) + documents = loader.load() + os.system("rm -rf /tmp/" + random_dir_name) + + chunk_size = 500 + chunk_overlap = 0 + text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=chunk_size, chunk_overlap=chunk_overlap + ) + + documents = text_splitter.split_documents(documents) + + for doc in documents: + if doc.metadata["file_type"] in [ + ".pyc", + ".png", + ".svg", + ".env", + ".lock", + ".gitignore", + ".gitmodules", + ".gitattributes", + ".gitkeep", + ".git", + ".json", + ]: + continue + metadata = { + "file_sha1": compute_sha1_from_content(doc.page_content.encode("utf-8")), + "file_size": len(doc.page_content) * 8, + "file_name": doc.metadata["file_name"], + "chunk_size": chunk_size, + "chunk_overlap": chunk_overlap, + "date": dateshort, + "original_file_name": doc.metadata["original_file_name"], + } + doc_with_metadata = Document(page_content=doc.page_content, metadata=metadata) + + print(doc_with_metadata.metadata["file_name"]) + + file = File( + file_sha1=compute_sha1_from_content(doc.page_content.encode("utf-8")) + ) + + file_exists = file.file_already_exists() + + if not file_exists: + neurons = Neurons() + created_vector = neurons.create_vector(doc_with_metadata) + + file_exists_in_brain = file.file_already_exists_in_brain(brain_id) + + if not file_exists_in_brain: + file.link_file_to_brain(brain_id) + return { + "message": f"✅ Github with {len(documents)} files has been uploaded.", + "type": "success", + } diff --git a/backend/core/quivr_core/api/packages/files/parsers/html.py b/backend/core/quivr_core/api/packages/files/parsers/html.py new file mode 100644 index 000000000000..afe28e0ab104 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/html.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import UnstructuredHTMLLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_html( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=UnstructuredHTMLLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/markdown.py b/backend/core/quivr_core/api/packages/files/parsers/markdown.py new file mode 100644 index 000000000000..44d75b6de37e --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/markdown.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import UnstructuredMarkdownLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_markdown( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=UnstructuredMarkdownLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/notebook.py b/backend/core/quivr_core/api/packages/files/parsers/notebook.py new file mode 100644 index 000000000000..16b91ba0a0e2 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/notebook.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import NotebookLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_ipnyb( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=NotebookLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/odt.py b/backend/core/quivr_core/api/packages/files/parsers/odt.py new file mode 100644 index 000000000000..f6b10695a8b8 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/odt.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import UnstructuredPDFLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_odt( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=UnstructuredPDFLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/pdf.py b/backend/core/quivr_core/api/packages/files/parsers/pdf.py new file mode 100644 index 000000000000..653f6abf39f0 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/pdf.py @@ -0,0 +1,22 @@ +from langchain_community.document_loaders import UnstructuredPDFLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_pdf( + file: File, + brain_id, + original_file_name, + integration=None, + integration_link=None, +): + return process_file( + file=file, + loader_class=UnstructuredPDFLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/powerpoint.py b/backend/core/quivr_core/api/packages/files/parsers/powerpoint.py new file mode 100644 index 000000000000..57502cb4eda6 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/powerpoint.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import UnstructuredFileLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_powerpoint( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=UnstructuredFileLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/telegram.py b/backend/core/quivr_core/api/packages/files/parsers/telegram.py new file mode 100644 index 000000000000..c18322510f37 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/telegram.py @@ -0,0 +1,17 @@ +from quivr_core.api.models.files import File +from quivr_core.api.packages.files.loaders.telegram import TelegramChatFileLoader + +from .common import process_file + + +def process_telegram( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=TelegramChatFileLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/txt.py b/backend/core/quivr_core/api/packages/files/parsers/txt.py new file mode 100644 index 000000000000..46734c768503 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/txt.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import TextLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_txt( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=TextLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/parsers/xlsx.py b/backend/core/quivr_core/api/packages/files/parsers/xlsx.py new file mode 100644 index 000000000000..53005f7b4ad6 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/parsers/xlsx.py @@ -0,0 +1,18 @@ +from langchain_community.document_loaders import UnstructuredExcelLoader + +from quivr_core.api.models.files import File + +from .common import process_file + + +def process_xlsx( + file: File, brain_id, original_file_name, integration=None, integration_link=None +): + return process_file( + file=file, + loader_class=UnstructuredExcelLoader, + brain_id=brain_id, + original_file_name=original_file_name, + integration=integration, + integration_link=integration_link, + ) diff --git a/backend/core/quivr_core/api/packages/files/processors.py b/backend/core/quivr_core/api/packages/files/processors.py new file mode 100644 index 000000000000..76e47019f444 --- /dev/null +++ b/backend/core/quivr_core/api/packages/files/processors.py @@ -0,0 +1,106 @@ +from quivr_core.api.modules.brain.service.brain_service import BrainService + +from .parsers.audio import process_audio +from .parsers.bibtex import process_bibtex +from .parsers.code_python import process_python +from .parsers.csv import process_csv +from .parsers.docx import process_docx +from .parsers.epub import process_epub +from .parsers.html import process_html +from .parsers.markdown import process_markdown +from .parsers.notebook import process_ipnyb +from .parsers.odt import process_odt +from .parsers.pdf import process_pdf +from .parsers.powerpoint import process_powerpoint +from .parsers.txt import process_txt +from .parsers.xlsx import process_xlsx + +file_processors = { + ".txt": process_txt, + ".csv": process_csv, + ".md": process_markdown, + ".markdown": process_markdown, + ".m4a": process_audio, + ".mp3": process_audio, + ".webm": process_audio, + ".mp4": process_audio, + ".mpga": process_audio, + ".wav": process_audio, + ".mpeg": process_audio, + ".pdf": process_pdf, + ".html": process_html, + ".bib": process_bibtex, + ".pptx": process_powerpoint, + ".docx": process_docx, + ".odt": process_odt, + ".xlsx": process_xlsx, + ".xls": process_xlsx, + ".epub": process_epub, + ".ipynb": process_ipnyb, + ".py": process_python, +} + + +def create_response(message, type): + return {"message": message, "type": type} + + +brain_service = BrainService() + + +# TODO: Move filter_file to a file service to avoid circular imports from quivr_core.api.models/files.py for File class +def filter_file( + file, + brain_id, + original_file_name=None, +): + file_exists = file.file_already_exists() + file_exists_in_brain = file.file_already_exists_in_brain(brain_id) + using_file_name = file.file_name + + brain = brain_service.get_brain_by_id(brain_id) + if brain is None: + raise Exception("It seems like you're uploading knowledge to an unknown brain.") + + if file_exists_in_brain: + return create_response( + f"🤔 {using_file_name} already exists in brain {brain.name}.", # pyright: ignore reportPrivateUsage=none + "warning", + ) + elif file.file_is_empty(): + return create_response( + f"❌ {original_file_name} is empty.", # pyright: ignore reportPrivateUsage=none + "error", # pyright: ignore reportPrivateUsage=none + ) + elif file_exists: + file.link_file_to_brain(brain_id) + return create_response( + f"✅ {using_file_name} has been uploaded to brain {brain.name}.", # pyright: ignore reportPrivateUsage=none + "success", + ) + + if file.file_extension in file_processors: + try: + result = file_processors[file.file_extension]( + file=file, + brain_id=brain_id, + original_file_name=original_file_name, + ) + if result is None or result == 0: + return create_response( + f"? {using_file_name} has been uploaded to brain. There might have been an error while reading it, please make sure the file is not illformed or just an image", # pyright: ignore reportPrivateUsage=none + "warning", + ) + return create_response( + f"✅ {using_file_name} has been uploaded to brain {brain.name} in {result} chunks", # pyright: ignore reportPrivateUsage=none + "success", + ) + except Exception as e: + # Add more specific exceptions as needed. + print(f"Error processing file: {e}") + raise e + + return create_response( + f"❌ {using_file_name} is not supported.", # pyright: ignore reportPrivateUsage=none + "error", + ) diff --git a/backend/core/quivr_core/api/packages/quivr_core/__init__.py b/backend/core/quivr_core/api/packages/quivr_core/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/quivr_core/config.py b/backend/core/quivr_core/api/packages/quivr_core/config.py new file mode 100644 index 000000000000..1f6c5619b661 --- /dev/null +++ b/backend/core/quivr_core/api/packages/quivr_core/config.py @@ -0,0 +1,23 @@ +from pydantic import BaseModel, field_validator + + +class RAGConfig(BaseModel): + model: str = "gpt-3.5-turbo-0125" # pyright: ignore reportPrivateUsage=none + temperature: float | None = 0.1 + max_tokens: int | None = 2000 + max_input: int = 2000 + streaming: bool = False + max_files: int = 20 + prompt: str | None = None + + @field_validator("temperature", mode="before") + def set_default_temperature(cls, v): + if v is None: + return 0.1 + return v + + @field_validator("max_tokens", mode="before") + def set_default_max_tokens(cls, v): + if v is None: + return 2000 + return v diff --git a/backend/core/quivr_core/api/packages/quivr_core/models.py b/backend/core/quivr_core/api/packages/quivr_core/models.py new file mode 100644 index 000000000000..56ce0185f5cd --- /dev/null +++ b/backend/core/quivr_core/api/packages/quivr_core/models.py @@ -0,0 +1,58 @@ +from typing import Any + +from langchain_core.pydantic_v1 import BaseModel as BaseModelV1 +from langchain_core.pydantic_v1 import Field as FieldV1 +from pydantic import BaseModel +from typing_extensions import TypedDict + + +class cited_answer(BaseModelV1): + """Answer the user question based only on the given sources, and cite the sources used.""" + + answer: str = FieldV1( + ..., + description="The answer to the user question, which is based only on the given sources.", + ) + thoughts: str = FieldV1( + ..., + description="""Description of the thought process, based only on the given sources. + Cite the text as much as possible and give the document name it appears in. In the format : 'Doc_name states : cited_text'. Be the most + procedural as possible. Write all the steps needed to find the answer until you find it.""", + ) + citations: list[int] = FieldV1( + ..., + description="The integer IDs of the SPECIFIC sources which justify the answer.", + ) + + followup_questions: list[str] = FieldV1( + ..., + description="Generate up to 3 follow-up questions that could be asked based on the answer given or context provided.", + ) + + +class RawRAGChunkResponse(TypedDict): + answer: dict[str, Any] + docs: dict[str, Any] + + +class RawRAGResponse(TypedDict): + answer: dict[str, Any] + docs: dict[str, Any] + + +class RAGResponseMetadata(BaseModel): + citations: list[int] | None = None + thoughts: str | list[str] | None = None + followup_questions: list[str] | None = None + sources: list[Any] | None = None + + +class ParsedRAGResponse(BaseModel): + answer: str + metadata: RAGResponseMetadata | None = None + + +class ParsedRAGChunkResponse(BaseModel): + answer: str + metadata: RAGResponseMetadata + last_chunk: bool = False diff --git a/backend/core/quivr_core/api/packages/quivr_core/prompts.py b/backend/core/quivr_core/api/packages/quivr_core/prompts.py new file mode 100644 index 000000000000..d0fb80cca13c --- /dev/null +++ b/backend/core/quivr_core/api/packages/quivr_core/prompts.py @@ -0,0 +1,58 @@ +import datetime + +from langchain.prompts import HumanMessagePromptTemplate, SystemMessagePromptTemplate +from langchain_core.prompts import ChatPromptTemplate, PromptTemplate + +# First step is to create the Rephrasing Prompt +_template = """Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. Keep as much details as possible from previous messages. Keep entity names and all. + +Chat History: +{chat_history} +Follow Up Input: {question} +Standalone question:""" +CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(_template) + +# Next is the answering prompt + +template_answer = """ +Context: +{context} + +User Question: {question} +Answer: +""" + +today_date = datetime.datetime.now().strftime("%B %d, %Y") + +system_message_template = ( + f"Your name is Quivr. You're a helpful assistant. Today's date is {today_date}." +) + +system_message_template += """ +When answering use markdown. +Use markdown code blocks for code snippets. +Answer in a concise and clear manner. +Use the following pieces of context from files provided by the user to answer the users. +Answer in the same language as the user question. +If you don't know the answer with the context provided from the files, just say that you don't know, don't try to make up an answer. +Don't cite the source id in the answer objects, but you can use the source to answer the question. +You have access to the files to answer the user question (limited to first 20 files): +{files} + +If not None, User instruction to follow to answer: {custom_instructions} +Don't cite the source id in the answer objects, but you can use the source to answer the question. +""" + + +ANSWER_PROMPT = ChatPromptTemplate.from_messages( + [ + SystemMessagePromptTemplate.from_template(system_message_template), + HumanMessagePromptTemplate.from_template(template_answer), + ] +) + + +# How we format documents +DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template( + template="Source: {index} \n {page_content}" +) diff --git a/backend/core/quivr_core/api/packages/quivr_core/quivr_rag.py b/backend/core/quivr_core/api/packages/quivr_core/quivr_rag.py new file mode 100644 index 000000000000..c61e1efe5a87 --- /dev/null +++ b/backend/core/quivr_core/api/packages/quivr_core/quivr_rag.py @@ -0,0 +1,225 @@ +import logging +import os +from operator import itemgetter +from typing import AsyncGenerator + +from flashrank import Ranker +from langchain.retrievers import ContextualCompressionRetriever +from langchain.retrievers.document_compressors.flashrank_rerank import FlashrankRerank +from langchain_cohere import CohereRerank +from langchain_community.chat_models import ChatLiteLLM +from langchain_core.messages.ai import AIMessageChunk +from langchain_core.output_parsers import StrOutputParser +from langchain_core.runnables import RunnableLambda, RunnablePassthrough +from langchain_core.vectorstores import VectorStore +from langchain_openai import ChatOpenAI + +from quivr_core.api.modules.knowledge.entity.knowledge import Knowledge +from quivr_core.config import RAGConfig +from quivr_core.models import ParsedRAGChunkResponse, ParsedRAGResponse, cited_answer +from quivr_core.prompts import ANSWER_PROMPT, CONDENSE_QUESTION_PROMPT +from quivr_core.utils import ( + combine_documents, + format_file_list, + get_chunk_metadata, + model_supports_function_calling, + parse_chunk_response, + parse_response, +) + +logger = logging.getLogger(__name__) + + +class QuivrQARAG: + def __init__( + self, + *, + rag_config: RAGConfig, + llm: ChatLiteLLM, + vector_store: VectorStore, + ): + self.rag_config = rag_config + self.vector_store = vector_store + self.llm = llm + self.reranker = self._create_reranker() + self.supports_func_calling = model_supports_function_calling( + self.rag_config.model + ) + + @property + def retriever(self): + return self.vector_store.as_retriever() + + def _create_reranker(self): + # TODO: reranker config + if os.getenv("COHERE_API_KEY"): + compressor = CohereRerank(top_n=20) + else: + ranker_model_name = "ms-marco-TinyBERT-L-2-v2" + flashrank_client = Ranker(model_name=ranker_model_name) + compressor = FlashrankRerank( + client=flashrank_client, model=ranker_model_name, top_n=20 + ) + return compressor + + # TODO : refactor and simplify + def filter_history( + self, chat_history, max_history: int = 10, max_tokens: int = 2000 + ): + """ + Filter out the chat history to only include the messages that are relevant to the current question + + Takes in a chat_history= [HumanMessage(content='Qui est Chloé ? '), AIMessage(content="Chloé est une salariée travaillant pour l'entreprise Quivr en tant qu'AI Engineer, sous la direction de son supérieur hiérarchique, Stanislas Girard."), HumanMessage(content='Dis moi en plus sur elle'), AIMessage(content=''), HumanMessage(content='Dis moi en plus sur elle'), AIMessage(content="Désolé, je n'ai pas d'autres informations sur Chloé à partir des fichiers fournis.")] + Returns a filtered chat_history with in priority: first max_tokens, then max_history where a Human message and an AI message count as one pair + a token is 4 characters + """ + chat_history = chat_history[::-1] + total_tokens = 0 + total_pairs = 0 + filtered_chat_history = [] + for i in range(0, len(chat_history), 2): + if i + 1 < len(chat_history): + human_message = chat_history[i] + ai_message = chat_history[i + 1] + message_tokens = ( + len(human_message.content) + len(ai_message.content) + ) // 4 + if ( + total_tokens + message_tokens > max_tokens + or total_pairs >= max_history + ): + break + filtered_chat_history.append(human_message) + filtered_chat_history.append(ai_message) + total_tokens += message_tokens + total_pairs += 1 + chat_history = filtered_chat_history[::-1] + + return chat_history + + def build_chain(self, files: str): + compression_retriever = ContextualCompressionRetriever( + base_compressor=self.reranker, base_retriever=self.retriever + ) + + loaded_memory = RunnablePassthrough.assign( + chat_history=RunnableLambda( + lambda x: self.filter_history(x["chat_history"]), + ), + question=lambda x: x["question"], + ) + + standalone_question = { + "standalone_question": { + "question": lambda x: x["question"], + "chat_history": itemgetter("chat_history"), + } + | CONDENSE_QUESTION_PROMPT + | self.llm + | StrOutputParser(), + } + + # Now we retrieve the documents + retrieved_documents = { + "docs": itemgetter("standalone_question") | compression_retriever, + "question": lambda x: x["standalone_question"], + "custom_instructions": lambda x: self.rag_config.prompt, + } + + final_inputs = { + "context": lambda x: combine_documents(x["docs"]), + "question": itemgetter("question"), + "custom_instructions": itemgetter("custom_instructions"), + "files": lambda _: files, # TODO: shouldn't be here + } + + # Override llm if we have a OpenAI model + llm = self.llm + if self.supports_func_calling: + if self.rag_config.temperature: + llm_function = ChatOpenAI( + max_tokens=self.rag_config.max_tokens, + model=self.rag_config.model, + temperature=self.rag_config.temperature, + ) + else: + llm_function = ChatOpenAI( + max_tokens=self.rag_config.max_tokens, + model=self.rag_config.model, + ) + + llm = llm_function.bind_tools( + [cited_answer], + tool_choice="any", + ) + + answer = { + "answer": final_inputs | ANSWER_PROMPT | llm, + "docs": itemgetter("docs"), + } + + return loaded_memory | standalone_question | retrieved_documents | answer + + def answer( + self, + question: str, + history: list[dict[str, str]], + list_files: list[Knowledge], + metadata: dict[str, str] = {}, + ) -> ParsedRAGResponse: + concat_list_files = format_file_list(list_files, self.rag_config.max_files) + conversational_qa_chain = self.build_chain(concat_list_files) + raw_llm_response = conversational_qa_chain.invoke( + { + "question": question, + "chat_history": history, + "custom_instructions": (self.rag_config.prompt), + }, + config={"metadata": metadata}, + ) + response = parse_response(raw_llm_response, self.rag_config.model) + return response + + async def answer_astream( + self, + question: str, + history: list[dict[str, str]], + list_files: list[Knowledge], + metadata: dict[str, str] = {}, + ) -> AsyncGenerator[ParsedRAGChunkResponse, ParsedRAGChunkResponse]: + concat_list_files = format_file_list(list_files, self.rag_config.max_files) + conversational_qa_chain = self.build_chain(concat_list_files) + + rolling_message = AIMessageChunk(content="") + sources = [] + + async for chunk in conversational_qa_chain.astream( + { + "question": question, + "chat_history": history, + "custom_personality": (self.rag_config.prompt), + }, + config={"metadata": metadata}, + ): + # Could receive this anywhere so we need to save it for the last chunk + if "docs" in chunk: + sources = chunk["docs"] if "docs" in chunk else [] + + if "answer" in chunk: + rolling_message, parsed_chunk = parse_chunk_response( + rolling_message, + chunk, + self.supports_func_calling, + ) + + if self.supports_func_calling and len(parsed_chunk.answer) > 0: + yield parsed_chunk + else: + yield parsed_chunk + + # Last chunk provies + yield ParsedRAGChunkResponse( + answer="", + metadata=get_chunk_metadata(rolling_message, sources), + last_chunk=True, + ) diff --git a/backend/core/quivr_core/api/packages/quivr_core/rag_factory.py b/backend/core/quivr_core/api/packages/quivr_core/rag_factory.py new file mode 100644 index 000000000000..7e88652e8322 --- /dev/null +++ b/backend/core/quivr_core/api/packages/quivr_core/rag_factory.py @@ -0,0 +1,32 @@ +from typing import Type + +from quivr_core.api.logger import get_logger +from quivr_core.api.modules.brain.entity.integration_brain import IntegrationEntity +from quivr_core.api.modules.brain.integrations.Big.Brain import BigBrain +from quivr_core.api.modules.brain.integrations.GPT4.Brain import GPT4Brain +from quivr_core.api.modules.brain.integrations.Multi_Contract.Brain import ( + MultiContractBrain, +) +from quivr_core.api.modules.brain.integrations.Notion.Brain import NotionBrain +from quivr_core.api.modules.brain.integrations.Proxy.Brain import ProxyBrain +from quivr_core.api.modules.brain.integrations.Self.Brain import SelfBrain +from quivr_core.api.modules.brain.integrations.SQL.Brain import SQLBrain +from quivr_core.api.modules.brain.knowledge_brain_qa import KnowledgeBrainQA + +logger = get_logger(__name__) + + +class RAGServiceFactory: + integration_list: dict[str, Type[KnowledgeBrainQA]] = { + "notion": NotionBrain, + "gpt4": GPT4Brain, + "sql": SQLBrain, + "big": BigBrain, + "doc": KnowledgeBrainQA, + "proxy": ProxyBrain, + "self": SelfBrain, + "multi-contract": MultiContractBrain, + } + + def get_brain_cls(self, integration: IntegrationEntity): + pass diff --git a/backend/core/quivr_core/api/packages/quivr_core/utils.py b/backend/core/quivr_core/api/packages/quivr_core/utils.py new file mode 100644 index 000000000000..29e1f965c7d8 --- /dev/null +++ b/backend/core/quivr_core/api/packages/quivr_core/utils.py @@ -0,0 +1,272 @@ +import logging +from typing import Any, Dict, List, Tuple +from uuid import UUID + +from langchain.schema import ( + AIMessage, + BaseMessage, + HumanMessage, + SystemMessage, + format_document, +) +from langchain_core.messages.ai import AIMessageChunk + +from quivr_core.api.modules.chat.dto.chats import Sources +from quivr_core.api.modules.chat.dto.outputs import GetChatHistoryOutput +from quivr_core.api.modules.knowledge.entity.knowledge import Knowledge +from quivr_core.api.modules.upload.service.generate_file_signed_url import ( + generate_file_signed_url, +) +from quivr_core.api.packages.quivr_core.models import ( + ParsedRAGChunkResponse, + ParsedRAGResponse, + RAGResponseMetadata, + RawRAGResponse, +) +from quivr_core.api.packages.quivr_core.prompts import DEFAULT_DOCUMENT_PROMPT + +# TODO(@aminediro): define a types packages where we clearly define IO types +# This should be used for serialization/deseriallization later + + +logger = logging.getLogger(__name__) + + +def model_supports_function_calling(model_name: str): + models_supporting_function_calls = [ + "gpt-4", + "gpt-4-1106-preview", + "gpt-4-0613", + "gpt-3.5-turbo-0125", + "gpt-3.5-turbo-1106", + "gpt-3.5-turbo-0613", + "gpt-4-0125-preview", + "gpt-3.5-turbo", + "gpt-4-turbo", + "gpt-4o", + ] + return model_name in models_supporting_function_calls + + +def format_chat_history( + history: List[GetChatHistoryOutput], +) -> List[Dict[str, str]]: + """Format the chat history into a list of HumanMessage and AIMessage""" + formatted_history = [] + for chat in history: + if chat.user_message: + formatted_history.append(HumanMessage(content=chat.user_message)) + if chat.assistant: + formatted_history.append(AIMessage(content=chat.assistant)) + return formatted_history + + +def format_history_to_openai_mesages( + tuple_history: List[Tuple[str, str]], system_message: str, question: str +) -> List[BaseMessage]: + """Format the chat history into a list of Base Messages""" + messages = [] + messages.append(SystemMessage(content=system_message)) + for human, ai in tuple_history: + messages.append(HumanMessage(content=human)) + messages.append(AIMessage(content=ai)) + messages.append(HumanMessage(content=question)) + return messages + + +def cited_answer_filter(tool): + return tool["name"] == "cited_answer" + + +def get_prev_message_str(msg: AIMessageChunk) -> str: + if msg.tool_calls: + cited_answer = next(x for x in msg.tool_calls if cited_answer_filter(x)) + if "args" in cited_answer and "answer" in cited_answer["args"]: + return cited_answer["args"]["answer"] + return "" + + +def get_chunk_metadata( + msg: AIMessageChunk, sources: list[Any] = [] +) -> RAGResponseMetadata: + # Initiate the source + metadata = {"sources": sources} + if msg.tool_calls: + cited_answer = next(x for x in msg.tool_calls if cited_answer_filter(x)) + + if "args" in cited_answer: + gathered_args = cited_answer["args"] + if "citations" in gathered_args: + citations = gathered_args["citations"] + metadata["citations"] = citations + + if "followup_questions" in gathered_args: + followup_questions = gathered_args["followup_questions"] + metadata["followup_questions"] = followup_questions + + if "thoughts" in gathered_args: + thoughts = gathered_args["thoughts"] + metadata["thoughts"] = thoughts + + return RAGResponseMetadata(**metadata) + + +# TODO: CONVOLUTED LOGIC ! +# TODO(@aminediro): redo this +def parse_chunk_response( + gathered_msg: AIMessageChunk, + raw_chunk: dict[str, Any], + supports_func_calling: bool, +) -> Tuple[AIMessageChunk, ParsedRAGChunkResponse]: + # Init with sources + answer_str = "" + # Get the previously parsed answer + prev_answer = get_prev_message_str(gathered_msg) + + if supports_func_calling: + gathered_msg += raw_chunk["answer"] + if gathered_msg.tool_calls: + cited_answer = next( + x for x in gathered_msg.tool_calls if cited_answer_filter(x) + ) + if "args" in cited_answer: + gathered_args = cited_answer["args"] + if "answer" in gathered_args: + # Only send the difference between answer and response_tokens which was the previous answer + gathered_answer = gathered_args["answer"] + answer_str: str = gathered_answer[len(prev_answer) :] + + return gathered_msg, ParsedRAGChunkResponse( + answer=answer_str, metadata=RAGResponseMetadata() + ) + else: + return gathered_msg, ParsedRAGChunkResponse( + answer=raw_chunk["answer"].content, metadata=RAGResponseMetadata() + ) + + +def parse_response(raw_response: RawRAGResponse, model_name: str) -> ParsedRAGResponse: + answer = raw_response["answer"].content + sources = raw_response["docs"] or [] + + metadata = {"sources": sources} + + if model_supports_function_calling(model_name): + if raw_response["answer"].tool_calls: + citations = raw_response["answer"].tool_calls[-1]["args"]["citations"] + metadata["citations"] = citations + followup_questions = raw_response["answer"].tool_calls[-1]["args"][ + "followup_questions" + ] + thoughts = raw_response["answer"].tool_calls[-1]["args"]["thoughts"] + if followup_questions: + metadata["followup_questions"] = followup_questions + if thoughts: + metadata["thoughts"] = thoughts + answer = raw_response["answer"].tool_calls[-1]["args"]["answer"] + + parsed_response = ParsedRAGResponse( + answer=answer, metadata=RAGResponseMetadata(**metadata) + ) + return parsed_response + + +def combine_documents( + docs, document_prompt=DEFAULT_DOCUMENT_PROMPT, document_separator="\n\n" +): + # for each docs, add an index in the metadata to be able to cite the sources + for doc, index in zip(docs, range(len(docs))): + doc.metadata["index"] = index + doc_strings = [format_document(doc, document_prompt) for doc in docs] + return document_separator.join(doc_strings) + + +def format_file_list(list_files_array: list[Knowledge], max_files: int = 20) -> str: + list_files = [file.file_name or file.url for file in list_files_array] + files: list[str] = list(filter(lambda n: n is not None, list_files)) # type: ignore + files = files[:max_files] + + files_str = "\n".join(files) if list_files_array else "None" + return files_str + + +# TODO: REFACTOR THIS, it does call the DB , so maybe in a service +def generate_source( + source_documents: List[Any] | None, + brain_id: UUID, + citations: List[int] | None = None, +) -> List[Sources]: + """ + Generate the sources list for the answer + It takes in a list of sources documents and citations that points to the docs index that was used in the answer + """ + # Initialize an empty list for sources + sources_list: List[Sources] = [] + + # Initialize a dictionary for storing generated URLs + generated_urls = {} + + # remove duplicate sources with same name and create a list of unique sources + sources_url_cache = {} + + # Get source documents from the result, default to an empty list if not found + # If source documents exist + if source_documents: + logger.info(f"Citations {citations}") + for index, doc in enumerate(source_documents): + logger.info(f"Processing source document {doc.metadata['file_name']}") + if citations is not None: + if index not in citations: + logger.info(f"Skipping source document {doc.metadata['file_name']}") + continue + # Check if 'url' is in the document metadata + is_url = ( + "original_file_name" in doc.metadata + and doc.metadata["original_file_name"] is not None + and doc.metadata["original_file_name"].startswith("http") + ) + + # Determine the name based on whether it's a URL or a file + name = ( + doc.metadata["original_file_name"] + if is_url + else doc.metadata["file_name"] + ) + + # Determine the type based on whether it's a URL or a file + type_ = "url" if is_url else "file" + + # Determine the source URL based on whether it's a URL or a file + if is_url: + source_url = doc.metadata["original_file_name"] + else: + file_path = f"{brain_id}/{doc.metadata['file_name']}" + # Check if the URL has already been generated + if file_path in generated_urls: + source_url = generated_urls[file_path] + else: + # Generate the URL + if file_path in sources_url_cache: + source_url = sources_url_cache[file_path] + else: + generated_url = generate_file_signed_url(file_path) + if generated_url is not None: + source_url = generated_url.get("signedURL", "") + else: + source_url = "" + # Store the generated URL + generated_urls[file_path] = source_url + + # Append a new Sources object to the list + sources_list.append( + Sources( + name=name, + type=type_, + source_url=source_url, + original_file_name=name, + citation=doc.page_content, + ) + ) + else: + logger.info("No source documents found or source_documents is not a list.") + return sources_list diff --git a/backend/core/quivr_core/api/packages/utils/__init__.py b/backend/core/quivr_core/api/packages/utils/__init__.py new file mode 100644 index 000000000000..c9e648c9004c --- /dev/null +++ b/backend/core/quivr_core/api/packages/utils/__init__.py @@ -0,0 +1,2 @@ +from .handle_request_validation_error import handle_request_validation_error +from .parse_message_time import parse_message_time diff --git a/backend/core/quivr_core/api/packages/utils/handle_request_validation_error.py b/backend/core/quivr_core/api/packages/utils/handle_request_validation_error.py new file mode 100644 index 000000000000..454f9f8ef25b --- /dev/null +++ b/backend/core/quivr_core/api/packages/utils/handle_request_validation_error.py @@ -0,0 +1,24 @@ +from fastapi import FastAPI, Request, status +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from quivr_core.api.logger import get_logger + +logger = get_logger(__name__) + + +def handle_request_validation_error(app: FastAPI): + @app.exception_handler(RequestValidationError) + async def validation_exception_handler( + request: Request, exc: RequestValidationError + ): + exc_str = f"{exc}".replace("\n", " ").replace(" ", " ") + logger.error(request, exc_str) + content = { + "status_code": status.HTTP_422_UNPROCESSABLE_ENTITY, + "message": exc_str, + "data": None, + } + return JSONResponse( + content=content, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY + ) diff --git a/backend/core/quivr_core/api/packages/utils/parse_message_time.py b/backend/core/quivr_core/api/packages/utils/parse_message_time.py new file mode 100644 index 000000000000..4aa18a3b8759 --- /dev/null +++ b/backend/core/quivr_core/api/packages/utils/parse_message_time.py @@ -0,0 +1,5 @@ +from datetime import datetime + + +def parse_message_time(message_time_str: str): + return datetime.strptime(message_time_str, "%Y-%m-%dT%H:%M:%S.%f") diff --git a/backend/core/quivr_core/api/packages/utils/telemetry.py b/backend/core/quivr_core/api/packages/utils/telemetry.py new file mode 100644 index 000000000000..c8e45baf3ee8 --- /dev/null +++ b/backend/core/quivr_core/api/packages/utils/telemetry.py @@ -0,0 +1,60 @@ +import hashlib +import json +import os +import threading + +import httpx +from fastapi import Request + +from quivr_core.api.logger import get_logger + +logger = get_logger(__name__) + +# Assume these are your Supabase Function endpoint and any necessary headers +TELEMETRY_URL = "https://ovbvcnwemowuuuaebizd.supabase.co/functions/v1/telemetry" +HEADERS = { + "Content-Type": "application/json", +} + + +def generate_machine_key(): + # Get the OpenAI API key from the environment variables + seed = os.getenv("OPENAI_API_KEY") + + # Use SHA-256 hash to generate a unique key from the seed + unique_key = hashlib.sha256(seed.encode()).hexdigest() + + return unique_key + + +def send_telemetry(event_name: str, event_data: dict, request: Request = None): + # Generate a unique machine key + machine_key = generate_machine_key() + domain = None + if request: + domain = request.url.hostname + logger.info(f"Domain: {domain}") + event_data = {**event_data, "domain": domain} + # Prepare the payload + payload = json.dumps( + { + "anonymous_identifier": machine_key, + "event_name": event_name, + "event_data": event_data, + } + ) + + # Send the telemetry data + with httpx.Client() as client: + _ = client.post(TELEMETRY_URL, headers=HEADERS, data=payload) + + +def maybe_send_telemetry(event_name: str, event_data: dict, request: Request = None): + enable_telemetry = os.getenv("TELEMETRY_ENABLED", "false") + + if enable_telemetry.lower() != "true": + return + + threading.Thread( + target=send_telemetry, args=(event_name, event_data, request) + ).start() diff --git a/backend/core/quivr_core/api/packages/vectorstore/__init__.py b/backend/core/quivr_core/api/packages/vectorstore/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/api/packages/vectorstore/supabase.py b/backend/core/quivr_core/api/packages/vectorstore/supabase.py new file mode 100644 index 000000000000..980c3a90a5f6 --- /dev/null +++ b/backend/core/quivr_core/api/packages/vectorstore/supabase.py @@ -0,0 +1,105 @@ +from typing import Any, List +from uuid import UUID + +from langchain.docstore.document import Document +from langchain.embeddings.base import Embeddings +from langchain_community.vectorstores import SupabaseVectorStore +from supabase.client import Client + +from quivr_core.api.logger import get_logger + +logger = get_logger(__name__) + + +class CustomSupabaseVectorStore(SupabaseVectorStore): + """A custom vector store that uses the match_vectors table instead of the vectors table.""" + + def __init__( + self, + client: Client, + embedding: Embeddings, + table_name: str, + brain_id: UUID | None = None, + user_id: UUID | None = None, + number_docs: int = 35, + max_input: int = 2000, + ): + super().__init__(client, embedding, table_name) + self.brain_id = brain_id + self.user_id = user_id + self.number_docs = number_docs + self.max_input = max_input + + def find_brain_closest_query( + self, + user_id: str, + query: str, + k: int = 6, + table: str = "match_brain", + threshold: float = 0.5, + ) -> list[dict[str, Any]]: + vectors = self._embedding.embed_documents([query]) + query_embedding = vectors[0] + + res = self._client.rpc( + table, + { + "query_embedding": query_embedding, + "match_count": self.number_docs, + "p_user_id": str(self.user_id), + }, + ).execute() + + # Get the brain_id of the brain that is most similar to the query + # Get the brain_id and name of the brains that are most similar to the query + brain_details = [ + { + "id": item.get("id", None), + "name": item.get("name", None), + "similarity": item.get("similarity", 0.0), + } + for item in res.data + ] + return brain_details + + def similarity_search( + self, + query: str, + k: int = 40, + table: str = "match_vectors", + threshold: float = 0.5, + **kwargs: Any, + ) -> List[Document]: + vectors = self._embedding.embed_documents([query]) + query_embedding = vectors[0] + res = self._client.rpc( + table, + { + "query_embedding": query_embedding, + "max_chunk_sum": self.max_input, + "p_brain_id": str(self.brain_id), + }, + ).execute() + + match_result = [ + Document( + metadata={ + **search.get("metadata", {}), + "id": search.get("id", ""), + "similarity": search.get("similarity", 0.0), + }, + page_content=search.get("content", ""), + ) + for search in res.data + if search.get("content") + ] + + sorted_match_result_by_file_name_metadata = sorted( + match_result, + key=lambda x: ( + x.metadata.get("file_name", ""), + x.metadata.get("index", float("inf")), + ), + ) + + return sorted_match_result_by_file_name_metadata diff --git a/backend/core/quivr_core/api/tests/conftest.py b/backend/core/quivr_core/api/tests/conftest.py new file mode 100644 index 000000000000..47621f5e72d4 --- /dev/null +++ b/backend/core/quivr_core/api/tests/conftest.py @@ -0,0 +1,61 @@ +import asyncio +import os + +import pytest +import pytest_asyncio +import sqlalchemy +from sqlalchemy.ext.asyncio import create_async_engine +from sqlmodel import create_engine +from sqlmodel.ext.asyncio.session import AsyncSession + +pg_database_url = "" + + +@pytest.fixture(scope="session", autouse=True) +def db_setup(): + # setup + sync_engine = create_engine( + "postgresql://" + pg_database_url, + echo=True if os.getenv("ORM_DEBUG") else False, + ) + # TODO(@amine) : for now don't drop anything + # SQLModel.metadata.create_all(sync_engine, checkfirst=True) + yield sync_engine + # teardown + # NOTE: For now we rely on Supabase migrations for defining schemas + # SQLModel.metadata.drop_all(sync_engine) + + +@pytest_asyncio.fixture(scope="session") +async def async_engine(): + engine = create_async_engine( + "postgresql+asyncpg://" + pg_database_url, + echo=True if os.getenv("ORM_DEBUG") else False, + ) + yield engine + + +@pytest.fixture(scope="session") +def event_loop(request: pytest.FixtureRequest): + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + +@pytest_asyncio.fixture() +async def session(async_engine): + async with async_engine.connect() as conn: + await conn.begin() + await conn.begin_nested() + async_session = AsyncSession(conn, expire_on_commit=False) + + @sqlalchemy.event.listens_for( + async_session.sync_session, "after_transaction_end" + ) + def end_savepoint(session, transaction): + if conn.closed: + return + if not conn.in_nested_transaction(): + conn.sync_connection.begin_nested() + + yield async_session diff --git a/backend/core/quivr_core/storage/__init__.py b/backend/core/quivr_core/storage/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/core/quivr_core/storage/local_storage.py b/backend/core/quivr_core/storage/local_storage.py new file mode 100644 index 000000000000..cf3e38260016 --- /dev/null +++ b/backend/core/quivr_core/storage/local_storage.py @@ -0,0 +1,60 @@ +import mmap +import os +from io import BytesIO +from pathlib import Path +from typing import BinaryIO +from uuid import UUID, uuid4 + +import aiofiles + +from quivr_core.storage.storage_base import StorageBase + + +class QuivrFile: + __slots__ = ["fd", "filename", "brain_id", "file_size"] + + def __init__( + self, fd: BinaryIO, filename: str, brain_id: UUID, file_size: int | None = None + ) -> None: + self.fd = fd + self.brain_id = brain_id + self.filename = filename + self.file_size = file_size + + @classmethod + def from_path(cls, path: str): + fd = os.open(path, os.O_RDONLY) + file_size = os.stat(path).st_size + mmapped_file = mmap.mmap(fd, file_size, access=mmap.ACCESS_READ) + # TODO: parse file_name and brain_id + file_name = "" + return cls( + fd=BytesIO(mmapped_file), + brain_id=uuid4(), + filename=file_name, + file_size=file_size, + ) + + def local_filepath(self) -> str: + return os.path.join(str(self.brain_id), self.filename) + + +class LocalStorage(StorageBase): + def __init__(self, dir_path: Path | None = None): + if dir_path is None: + self.dir_path = os.getenv("QUIVR_LOCAL_STORAGE", "~/.cache/quivr/files") + else: + self.dir_path = dir_path + os.makedirs(self.dir_path, exist_ok=True) + + async def upload_file(self, file: QuivrFile, exists_ok: bool = False): + path = os.path.join(self.dir_path, file.local_filepath()) + if os.path.exists(path) and not exists_ok: + raise FileExistsError("file already exists") + + async with aiofiles.open(path, "wb") as f: + await f.write(file.fd.read()) + + # TODO: + async def remove_file(self, file_name: str): + pass diff --git a/backend/core/quivr_core/storage/storage_base.py b/backend/core/quivr_core/storage/storage_base.py new file mode 100644 index 000000000000..265980bc4d57 --- /dev/null +++ b/backend/core/quivr_core/storage/storage_base.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod + +from quivr_core.storage.local_storage import QuivrFile + + +class StorageBase(ABC): + @abstractmethod + async def upload_file(self, file: QuivrFile, exists_ok: bool = False): + raise Exception("Unimplemented upload_file method") + + @abstractmethod + async def remove_file(self, file_name: str): + raise Exception("Unimplemented remove_file method")