diff --git a/.env.example b/.env.example index 48a8e376..096915ff 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,30 @@ ################# DEBUG= -# Proxy + +############ +# Postgres # +############ +RISKI__DB__HOSTNAME=db +RISKI__DB__NAME=example_db +RISKI__DB__PASSWORD=password +RISKI__DB__USER=postgres +RISKI__DB__PORT=5432 +RISKI__DB__BATCH_SIZE=100 + +######### +# Kafka # +######### +RISKI__KAFKA__SERVER= +RISKI__KAFKA__SECURITY=true +RISKI__KAFKA__TOPIC= +RISKI__KAFKA__CA_B64= +RISKI__KAFKA__PKCS12_DATA= +RISKI__KAFKA__PKCS12_PW= + +######### +# Proxy # +######### HTTP_PROXY= HTTPS_PROXY= NO_PROXY= diff --git a/.github/workflows/extractor-tests.yml b/.github/workflows/extractor-tests.yml index 702cc5b1..124eba02 100644 --- a/.github/workflows/extractor-tests.yml +++ b/.github/workflows/extractor-tests.yml @@ -38,6 +38,16 @@ jobs: OPENAI_API_BASE: https://example.de OPENAI_API_VERSION: 2024-08-01-preview + TIKTOKEN_CACHE_DIR: tiktoken_cache + RISKI_OPENAI_EMBEDDING_MODEL: text-embedding-3-large + RISKI_EXTRACTOR__TEST__DB_NAME: test_db + RISKI_EXTRACTOR__TEST__DB_USER: postgres + RISKI_EXTRACTOR__TEST__DB_PASSWORD: password + RISKI_EXTRACTOR__TEST__DB_HOSTNAME: 127.0.0.1 + RISKI_EXTRACTOR__TEST__DB_PORT: 5432 + RISKI__KAFKA__SERVER: + RISKI__KAFKA__SECURITY: false + defaults: run: working-directory: riski-extractor diff --git a/riski-core/src/core/kafka/__init__.py b/riski-core/src/core/kafka/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/riski-core/src/core/kafka/security.py b/riski-core/src/core/kafka/security.py new file mode 100644 index 00000000..99e9f06b --- /dev/null +++ b/riski-core/src/core/kafka/security.py @@ -0,0 +1,81 @@ +"""Security helpers for configuring Kafka TLS/mTLS.""" + +from base64 import b64decode +from logging import Logger +from pathlib import Path +from ssl import SSLContext, create_default_context +from tempfile import TemporaryDirectory + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs12 +from faststream.security import BaseSecurity + +from src.logtools import getLogger + +logger: Logger = getLogger(__name__) + + +def setup_security(kafka_pkcs12_data: str, kafka_pkcs12_pw: str, kafka_ca_b64: str) -> BaseSecurity: + """Set up Kafka security configuration based on application settings. + + Returns: + BaseSecurity: The configured security object for Kafka. + + Raises: + ValueError: If required environment variables are not set or if there are issues + loading the PKCS#12 file. + """ + with TemporaryDirectory() as tempdir: + tempdir_path: Path = Path(tempdir) + logger.debug("Setting up Kafka mTLS security using environment variables.") + + # Unpack CA file + ca_data: str | None = kafka_ca_b64 + ca_data = b64decode(s=ca_data).decode(encoding="utf-8") + + # Unpack PKCS#12 file + pkcs12_data: str | None = kafka_pkcs12_data + pkcs12_bytes = b64decode(s=pkcs12_data) + + # Unpack PKCS#12 password + pkcs12_pw: str | None = kafka_pkcs12_pw + pkcs12_pw_bytes: bytes = b64decode(s=pkcs12_pw) + + # Extract the private key and certificate from the PKCS#12 file + try: + private_key, certificate, _ = pkcs12.load_key_and_certificates( + data=pkcs12_bytes, + password=pkcs12_pw_bytes, + ) + except Exception as e: + raise ValueError(f"Failed to load PKCS#12 file: {e}") + + if private_key is None or certificate is None: + raise ValueError("PKCS#12 file does not contain a private key and certificate.") + + # Write cert and key to temporary files + cert_file: Path = tempdir_path / "kafka.cert" + key_file: Path = tempdir_path / "kafka.key" + with open(file=cert_file, mode="wb") as f: + f.write(certificate.public_bytes(encoding=serialization.Encoding.PEM)) + with open(file=key_file, mode="wb") as f: + f.write( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + # Create SSL context + ssl_context: SSLContext = create_default_context( + cadata=ca_data, + ) + + # Load client cert and key + ssl_context.load_cert_chain( + certfile=cert_file, + keyfile=key_file, + ) + + return BaseSecurity(ssl_context=ssl_context, use_ssl=True) diff --git a/riski-core/src/core/settings/core.py b/riski-core/src/core/settings/core.py index 8db0e806..3ad36440 100644 --- a/riski-core/src/core/settings/core.py +++ b/riski-core/src/core/settings/core.py @@ -1,5 +1,6 @@ from core.settings.db import DatabaseSettings from core.settings.genai import GenAISettings +from core.settings.kafka import KafkaSettings from core.settings.testdb import TestDBSettings from pydantic import BaseModel, Field @@ -15,6 +16,8 @@ class CoreSettings(BaseModel): default_factory=lambda: GenAISettings(), ) + kafka: KafkaSettings = Field(default_factory=lambda: KafkaSettings(), description="Kafka related settings") + testdb: TestDBSettings = Field( description="Test DB related settings", default_factory=lambda: TestDBSettings(), diff --git a/riski-core/src/core/settings/kafka.py b/riski-core/src/core/settings/kafka.py new file mode 100644 index 00000000..f7824743 --- /dev/null +++ b/riski-core/src/core/settings/kafka.py @@ -0,0 +1,31 @@ +from pydantic import BaseModel, Field + + +class KafkaSettings(BaseModel): + """ + Kafka configuration settings. + """ + + # === Kafka Settings === + server: str = Field( + description="Kafka Server URL", + ) + topic: str = Field( + default="lhm-riski-parse", + description="Kafka Topic Name", + ) + security: bool = Field( + description="Enable mTLS security for accessing Kafka Server", + ) + ca_b64: str | None = Field( + default=None, + description="Kafka Server CA (B64 Encoded)", + ) + pkcs12_data: str | None = Field( + default=None, + description="Kafka P12 (B64 Encoded)", + ) + pkcs12_pw: str | None = Field( + default=None, + description="Kafka P12 Password (B64 Encoded)", + ) diff --git a/riski-extractor/app.py b/riski-extractor/app.py index 7fea4699..7b3361dd 100644 --- a/riski-extractor/app.py +++ b/riski-extractor/app.py @@ -4,6 +4,8 @@ from config.config import Config, get_config from core.db.db import create_db_and_tables, init_db +from core.kafka.security import setup_security +from faststream.kafka import KafkaBroker from src.extractor.city_council_faction_extractor import CityCouncilFactionExtractor from src.extractor.city_council_meeting_extractor import CityCouncilMeetingExtractor from src.extractor.city_council_meeting_template_extractor import CityCouncilMeetingTemplateExtractor @@ -23,7 +25,7 @@ async def main(): config = get_config() config.print_config() - logger = getLogger() + logger = getLogger(__name__) version = get_version() init_db(config.core.db.database_url) @@ -63,8 +65,13 @@ async def main(): city_council_motion_extractor.run() logger.info("Extracted City Council Motions") - async with Filehandler() as filehandler: - await filehandler.download_and_persist_files(batch_size=config.core.db.batch_size) + broker = await createKafkaBroker(config, logger) + try: + async with Filehandler(kafkaBroker=broker) as filehandler: + await filehandler.download_and_persist_files(batch_size=config.core.db.batch_size) + finally: + await broker.stop() + logger.info("Broker closed.") confidential_file_deleter = ConfidentialFileDeleter() confidential_file_deleter.delete_confidential_files() @@ -75,5 +82,22 @@ async def main(): return 0 +async def createKafkaBroker(config: Config, logger: Logger) -> KafkaBroker: + security = None + if config.core.kafka.security: + security = setup_security(config.core.kafka.pkcs12_data, config.core.kafka.pkcs12_pw, config.core.kafka.ca_b64) + + # Kafka Broker and FastStream app setup + broker = KafkaBroker(bootstrap_servers=config.core.kafka.server, security=security) + logger.debug("Connecting to Broker...") + try: + await broker.connect() + logger.info("Broker connected.") + return broker + except Exception as e: + logger.exception(f"Failed to connect to broker. - {e}") + raise + + if __name__ == "__main__": sys.exit(asyncio.run(main())) diff --git a/riski-extractor/logconf.yaml b/riski-extractor/logconf.yaml index a91b45a9..bb992eb7 100644 --- a/riski-extractor/logconf.yaml +++ b/riski-extractor/logconf.yaml @@ -27,7 +27,7 @@ loggers: handlers: [ console ] propagate: no riski-extractor: - level: INFO + level: DEBUG handlers: [ console ] propagate: no httpx: diff --git a/riski-extractor/pyproject.toml b/riski-extractor/pyproject.toml index 17153c04..0f46bfda 100644 --- a/riski-extractor/pyproject.toml +++ b/riski-extractor/pyproject.toml @@ -15,6 +15,8 @@ dependencies = [ "tokenizers==0.21.2", "torch==2.8.0", "torchvision==0.23.0", + "cryptography>=46.0.5", + "faststream[kafka]>=0.6.5", "pytest-asyncio>=1.3.0", "truststore>=0.10.4", "gitpython>=3.1.45", @@ -42,7 +44,7 @@ dev = [ "langchain-docling>=1.1.0", "langchain-openai>=0.3.32", "pytest==8.4.2", - "faststream[kafka]==0.6.1", + "faststream[kafka]==0.6.5", "ruff>=0.14.11", "python-dotenv>=1.2.1", ] diff --git a/riski-extractor/src/extractor/base_extractor.py b/riski-extractor/src/extractor/base_extractor.py index 312099a8..3e3c5149 100644 --- a/riski-extractor/src/extractor/base_extractor.py +++ b/riski-extractor/src/extractor/base_extractor.py @@ -37,7 +37,7 @@ def __init__( else: self.client = Client(timeout=config.request_timeout) - self.logger = getLogger() + self.logger = getLogger(__name__) self.base_url = base_url self.base_path = base_path self.parser = parser @@ -78,7 +78,7 @@ def _get_object_html(self, link: str) -> str: def _get_sanitized_url(self, unsanitized_path: str) -> str: return f"{self.base_url}/{unsanitized_path.lstrip('./')}" - def run(self) -> list[T]: + def run(self): try: # Initial request for cookies, sessionID etc. self._initial_request() @@ -105,8 +105,9 @@ def run(self) -> list[T]: break self._get_next_page(path=results_per_page_redirect_path, next_page_link=nav_top_next_link) - except Exception: - self.logger.exception("Error extracting objects") + + except Exception as e: + self.logger.exception(f"Error extracting objects. - {e}") def _parse_objects_from_links(self, object_links: list[str]): for link in object_links: diff --git a/riski-extractor/src/filehandler/filehandler.py b/riski-extractor/src/filehandler/filehandler.py index 8eb2475e..1b51b213 100644 --- a/riski-extractor/src/filehandler/filehandler.py +++ b/riski-extractor/src/filehandler/filehandler.py @@ -7,7 +7,9 @@ from config.config import Config, get_config from core.db.db_access import request_batch, update_file_content from core.model.data_models import File +from faststream.kafka import KafkaBroker from httpx import AsyncClient +from src.kafka.message import Message from src.logtools import getLogger @@ -18,14 +20,17 @@ class Filehandler: logger: Logger client: AsyncClient - def __init__(self) -> None: - self.logger = getLogger() + def __init__(self, kafkaBroker: KafkaBroker) -> None: + self.logger = getLogger(__name__) if config.https_proxy or config.http_proxy: limits = httpx.Limits(max_keepalive_connections=5, max_connections=250) self.client = AsyncClient(proxy=config.https_proxy or config.http_proxy, timeout=config.request_timeout, limits=limits) else: self.client = AsyncClient(timeout=config.request_timeout) + self.broker = kafkaBroker + self.logger.info("Filehandler created.") + async def __aenter__(self): return self @@ -34,10 +39,11 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def download_and_persist_files(self, batch_size: int = 100): self.logger.info("Persisting content of all scraped files to database.") - + tasks = [] offset = 0 while True: files: list[File] = request_batch(File, offset=offset, limit=batch_size) + if not files or len(files) < 1: break semaphore = asyncio.Semaphore(batch_size) @@ -64,6 +70,7 @@ async def download_and_persist_file(self, file: File): response = await self.client.get(url=file.id) response.raise_for_status() content = response.content + self.logger.debug(f"Checking necessity of inserting/updating file {file.name} to database.") if file.content is None or content != file.content: content_disposition = response.headers.get("content-disposition") if content_disposition: @@ -83,3 +90,14 @@ async def download_and_persist_file(self, file: File): fileName = file.name self.logger.debug(f"Saving content of file {file.name} to database.") update_file_content(file.db_id, content, fileName) + self.logger.debug(f"Saved content of file {file.name} to database.") + msg = Message(content=str(file.db_id)) + self.logger.debug(f"Publishing: {msg}.") + try: + await self.broker.publish(msg, topic=config.core.kafka.topic) + self.logger.debug(f"Published: {msg}.") + except Exception as e: + # If Kafka Broker is unavailable rollback the file download to ensure + # All Documents that have content, are published to the Kafka Queue + update_file_content(file.db_id, b"", "") + self.logger.error(f"Publishing failed. Rolled file download back: {file.db_id}. - {e}") diff --git a/riski-extractor/src/kafka/message.py b/riski-extractor/src/kafka/message.py new file mode 100644 index 00000000..0d0d2011 --- /dev/null +++ b/riski-extractor/src/kafka/message.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class Message(BaseModel): + content: str + republished: bool = False diff --git a/riski-extractor/src/logtools.py b/riski-extractor/src/logtools.py index 7f25312d..a0eac35d 100644 --- a/riski-extractor/src/logtools.py +++ b/riski-extractor/src/logtools.py @@ -5,8 +5,10 @@ from yaml import safe_load +defautlName: str = "riski-extractor" -def getLogger(name: str = "riski-extractor") -> logging.Logger: + +def getLogger(name: str = None) -> logging.Logger: """Configures logging and returns a logger with the specified name. Parameters: @@ -19,7 +21,9 @@ def getLogger(name: str = "riski-extractor") -> logging.Logger: log_config = safe_load(file) logging.config.dictConfig(log_config) - return logging.getLogger(name) + + logger_name = f"{defautlName}.{name}" if name else defautlName + return logging.getLogger(logger_name) class JsonFormatter(logging.Formatter): diff --git a/riski-extractor/src/parser/base_parser.py b/riski-extractor/src/parser/base_parser.py index 88bd18f7..4bc68b44 100644 --- a/riski-extractor/src/parser/base_parser.py +++ b/riski-extractor/src/parser/base_parser.py @@ -13,7 +13,7 @@ class BaseParser(ABC, Generic[T]): def __init__(self) -> None: - self.logger = getLogger() + self.logger = getLogger(__name__) if platform.system() == "Windows": # For Windows, use the specific code page that works locale.setlocale(locale.LC_TIME, "German_Germany.1252") diff --git a/riski-extractor/src/parser/city_council_meeting_parser.py b/riski-extractor/src/parser/city_council_meeting_parser.py index 3d57d7d5..4fd04e99 100644 --- a/riski-extractor/src/parser/city_council_meeting_parser.py +++ b/riski-extractor/src/parser/city_council_meeting_parser.py @@ -82,8 +82,8 @@ def parse(self, url: str, html: str) -> Meeting: temp_file = get_or_insert_object_to_database(temp_file) auxiliaryFile.append(temp_file) self.logger.debug(f"Saved Document to DB: {doc_title} ({doc_url})") - except Exception: - self.logger.exception(f"Could not save File: {doc_url}") + except Exception as e: + self.logger.exception(f"Could not save File: {doc_url}. - {e}") # --- Remaining Fields --- deleted = False diff --git a/riski-extractor/src/version.py b/riski-extractor/src/version.py index c3ba0121..888b9f7c 100644 --- a/riski-extractor/src/version.py +++ b/riski-extractor/src/version.py @@ -5,7 +5,7 @@ from src.logtools import getLogger -logger = getLogger() +logger = getLogger(__name__) def get_version() -> str: diff --git a/riski-extractor/test/test_filehandler.py b/riski-extractor/test/test_filehandler.py index 003c177c..1819cf5a 100644 --- a/riski-extractor/test/test_filehandler.py +++ b/riski-extractor/test/test_filehandler.py @@ -12,7 +12,8 @@ def mock_file(): @pytest.fixture def filehandler_instance(): - instance = Filehandler() + kafkaBroker = AsyncMock() + instance = Filehandler(kafkaBroker) instance.client = AsyncMock() instance.logger = MagicMock() return instance @@ -20,7 +21,8 @@ def filehandler_instance(): @pytest.mark.asyncio async def test_download_and_persist_file_updates_filename(filehandler_instance, mock_file): - mock_response = MagicMock() + mock_response = AsyncMock() + mock_response.content = b"test content" mock_response.headers = {"content-disposition": 'inline; filename="test_file.txt"'} @@ -30,11 +32,13 @@ async def test_download_and_persist_file_updates_filename(filehandler_instance, await filehandler_instance.download_and_persist_file(mock_file) mock_update.assert_called_once_with(ANY, b"test content", "test_file.txt") + filehandler_instance.broker.publish.assert_called_once() @pytest.mark.asyncio async def test_download_and_persist_file_updates_filename_urlencoding(filehandler_instance, mock_file): - mock_response = MagicMock() + mock_response = AsyncMock() + mock_response.content = b"test content" mock_response.headers = {"content-disposition": 'inline; filename="test%20file.txt"'} @@ -43,11 +47,13 @@ async def test_download_and_persist_file_updates_filename_urlencoding(filehandle await filehandler_instance.download_and_persist_file(mock_file) mock_update.assert_called_once_with(ANY, b"test content", "test file.txt") + filehandler_instance.broker.publish.assert_called_once() @pytest.mark.asyncio async def test_download_and_persist_file_not_updates_filename_when_unchanged_file(filehandler_instance, mock_file): - mock_response = MagicMock() + mock_response = AsyncMock() + mock_response.content = b"test" mock_response.headers = {"content-disposition": 'inline; filename="test_file.txt"'} @@ -58,3 +64,4 @@ async def test_download_and_persist_file_not_updates_filename_when_unchanged_fil with patch("src.filehandler.filehandler.update_file_content") as mock_update: await filehandler_instance.download_and_persist_file(mock_file) mock_update.assert_not_called() + filehandler_instance.broker.publish.assert_not_called() diff --git a/riski-extractor/uv.lock b/riski-extractor/uv.lock index 2a0f1d65..76d688bd 100644 --- a/riski-extractor/uv.lock +++ b/riski-extractor/uv.lock @@ -342,6 +342,45 @@ requires-dist = [ { name = "truststore", specifier = ">=0.10.4" }, ] +[[package]] +name = "cryptography" +version = "46.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/04/ee2a9e8542e4fa2773b81771ff8349ff19cdd56b7258a0cc442639052edb/cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", size = 750064, upload-time = "2026-02-10T19:18:38.255Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/b0bb27f2ba931a65409c6b8a8b358a7f03c0e46eceacddff55f7c84b1f3b/cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", size = 7176289, upload-time = "2026-02-10T19:17:08.274Z" }, + { url = "https://files.pythonhosted.org/packages/ff/9e/6b4397a3e3d15123de3b1806ef342522393d50736c13b20ec4c9ea6693a6/cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", size = 4275637, upload-time = "2026-02-10T19:17:10.53Z" }, + { url = "https://files.pythonhosted.org/packages/63/e7/471ab61099a3920b0c77852ea3f0ea611c9702f651600397ac567848b897/cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", size = 4424742, upload-time = "2026-02-10T19:17:12.388Z" }, + { url = "https://files.pythonhosted.org/packages/37/53/a18500f270342d66bf7e4d9f091114e31e5ee9e7375a5aba2e85a91e0044/cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", size = 4277528, upload-time = "2026-02-10T19:17:13.853Z" }, + { url = "https://files.pythonhosted.org/packages/22/29/c2e812ebc38c57b40e7c583895e73c8c5adb4d1e4a0cc4c5a4fdab2b1acc/cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", size = 4947993, upload-time = "2026-02-10T19:17:15.618Z" }, + { url = "https://files.pythonhosted.org/packages/6b/e7/237155ae19a9023de7e30ec64e5d99a9431a567407ac21170a046d22a5a3/cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", size = 4456855, upload-time = "2026-02-10T19:17:17.221Z" }, + { url = "https://files.pythonhosted.org/packages/2d/87/fc628a7ad85b81206738abbd213b07702bcbdada1dd43f72236ef3cffbb5/cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", size = 3984635, upload-time = "2026-02-10T19:17:18.792Z" }, + { url = "https://files.pythonhosted.org/packages/84/29/65b55622bde135aedf4565dc509d99b560ee4095e56989e815f8fd2aa910/cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", size = 4277038, upload-time = "2026-02-10T19:17:20.256Z" }, + { url = "https://files.pythonhosted.org/packages/bc/36/45e76c68d7311432741faf1fbf7fac8a196a0a735ca21f504c75d37e2558/cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", size = 4912181, upload-time = "2026-02-10T19:17:21.825Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1a/c1ba8fead184d6e3d5afcf03d569acac5ad063f3ac9fb7258af158f7e378/cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", size = 4456482, upload-time = "2026-02-10T19:17:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e5/3fb22e37f66827ced3b902cf895e6a6bc1d095b5b26be26bd13c441fdf19/cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", size = 4405497, upload-time = "2026-02-10T19:17:26.66Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/9d58bb32b1121a8a2f27383fabae4d63080c7ca60b9b5c88be742be04ee7/cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", size = 4667819, upload-time = "2026-02-10T19:17:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/325d2a490c5e94038cdb0117da9397ece1f11201f425c4e9c57fe5b9f08b/cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", size = 3028230, upload-time = "2026-02-10T19:17:30.518Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5a/ac0f49e48063ab4255d9e3b79f5def51697fce1a95ea1370f03dc9db76f6/cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", size = 3480909, upload-time = "2026-02-10T19:17:32.083Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fa/a66aa722105ad6a458bebd64086ca2b72cdd361fed31763d20390f6f1389/cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", size = 7170514, upload-time = "2026-02-10T19:17:56.267Z" }, + { url = "https://files.pythonhosted.org/packages/0f/04/c85bdeab78c8bc77b701bf0d9bdcf514c044e18a46dcff330df5448631b0/cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", size = 4275349, upload-time = "2026-02-10T19:17:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/32/9b87132a2f91ee7f5223b091dc963055503e9b442c98fc0b8a5ca765fab0/cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", size = 4420667, upload-time = "2026-02-10T19:18:00.619Z" }, + { url = "https://files.pythonhosted.org/packages/a1/a6/a7cb7010bec4b7c5692ca6f024150371b295ee1c108bdc1c400e4c44562b/cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", size = 4276980, upload-time = "2026-02-10T19:18:02.379Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7c/c4f45e0eeff9b91e3f12dbd0e165fcf2a38847288fcfd889deea99fb7b6d/cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", size = 4939143, upload-time = "2026-02-10T19:18:03.964Z" }, + { url = "https://files.pythonhosted.org/packages/37/19/e1b8f964a834eddb44fa1b9a9976f4e414cbb7aa62809b6760c8803d22d1/cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", size = 4453674, upload-time = "2026-02-10T19:18:05.588Z" }, + { url = "https://files.pythonhosted.org/packages/db/ed/db15d3956f65264ca204625597c410d420e26530c4e2943e05a0d2f24d51/cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", size = 3978801, upload-time = "2026-02-10T19:18:07.167Z" }, + { url = "https://files.pythonhosted.org/packages/41/e2/df40a31d82df0a70a0daf69791f91dbb70e47644c58581d654879b382d11/cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", size = 4276755, upload-time = "2026-02-10T19:18:09.813Z" }, + { url = "https://files.pythonhosted.org/packages/33/45/726809d1176959f4a896b86907b98ff4391a8aa29c0aaaf9450a8a10630e/cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", size = 4901539, upload-time = "2026-02-10T19:18:11.263Z" }, + { url = "https://files.pythonhosted.org/packages/99/0f/a3076874e9c88ecb2ecc31382f6e7c21b428ede6f55aafa1aa272613e3cd/cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", size = 4452794, upload-time = "2026-02-10T19:18:12.914Z" }, + { url = "https://files.pythonhosted.org/packages/02/ef/ffeb542d3683d24194a38f66ca17c0a4b8bf10631feef44a7ef64e631b1a/cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", size = 4404160, upload-time = "2026-02-10T19:18:14.375Z" }, + { url = "https://files.pythonhosted.org/packages/96/93/682d2b43c1d5f1406ed048f377c0fc9fc8f7b0447a478d5c65ab3d3a66eb/cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", size = 4667123, upload-time = "2026-02-10T19:18:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/45/2d/9c5f2926cb5300a8eefc3f4f0b3f3df39db7f7ce40c8365444c49363cbda/cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", size = 3010220, upload-time = "2026-02-10T19:18:17.361Z" }, + { url = "https://files.pythonhosted.org/packages/48/ef/0c2f4a8e31018a986949d34a01115dd057bf536905dca38897bacd21fac3/cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", size = 3467050, upload-time = "2026-02-10T19:18:18.899Z" }, +] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -567,6 +606,8 @@ source = { editable = "." } dependencies = [ { name = "bs4" }, { name = "core" }, + { name = "cryptography" }, + { name = "faststream", extra = ["kafka"] }, { name = "gitpython" }, { name = "httpx" }, { name = "legacy-cgi" }, @@ -607,6 +648,8 @@ requires-dist = [ { name = "asyncpg", marker = "extra == 'pgvector'", specifier = "==0.30.0" }, { name = "bs4", specifier = ">=0.0.2" }, { name = "core", directory = "../riski-core" }, + { name = "cryptography", specifier = ">=46.0.5" }, + { name = "faststream", extras = ["kafka"], specifier = ">=0.6.5" }, { name = "gitpython", specifier = ">=3.1.45" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "langchain-postgres", marker = "extra == 'pgvector'", specifier = ">=0.0.15" }, @@ -623,7 +666,7 @@ provides-extras = ["pgvector"] [package.metadata.requires-dev] dev = [ - { name = "faststream", extras = ["kafka"], specifier = "==0.6.1" }, + { name = "faststream", extras = ["kafka"], specifier = "==0.6.5" }, { name = "ipykernel", specifier = "==7.0.1" }, { name = "langchain", specifier = ">=0.3.27" }, { name = "langchain-community", specifier = ">=0.3.28" }, @@ -668,16 +711,16 @@ pydantic = [ [[package]] name = "faststream" -version = "0.6.1" +version = "0.6.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "fast-depends", extra = ["pydantic"] }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/04/59ae2e5dfc2ca91a6fd5b71e9931b68e80b050980cbc965f89a3b2cddca1/faststream-0.6.1.tar.gz", hash = "sha256:d19b3fa416ee127fa45422661077b9bc8ee339da5029f811c1cc80d8a396b0c7", size = 292698, upload-time = "2025-10-14T18:00:39.263Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/55/fc2a34405ac63aaf973a33884c5f77f87564bbb9a4343c1d103cbf8c87f5/faststream-0.6.5.tar.gz", hash = "sha256:ddef9e85631edf1aba87e81c8886067bd94ee752f41a09f1d1cd6f75f7e4fade", size = 302206, upload-time = "2025-12-29T16:44:04.85Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/1a/948af05d46b49937b25eb098c4c9335f47695b9e569520eb81c183eff1ea/faststream-0.6.1-py3-none-any.whl", hash = "sha256:5b51d61b9a7663a4897d1000be9221bc5ff00eb2dd05ef9055f988ddb36b0aad", size = 496455, upload-time = "2025-10-14T18:00:37.696Z" }, + { url = "https://files.pythonhosted.org/packages/e2/1c/f60c0b15a8bce42ed7256d2eca6ea713bac259d6684ff392a907f95ca345/faststream-0.6.5-py3-none-any.whl", hash = "sha256:714b13b84cdbe2bdcf0b2b8a5e2b04648cb7683784a5297d445adfee9f2b4f7e", size = 507108, upload-time = "2025-12-29T16:44:03.297Z" }, ] [package.optional-dependencies]