Skip to content

Commit c1e012b

Browse files
ekneg54ekneg54
authored
revise security notes (#870)
* adapt docstring and add security-best-practices for: - restart counter - error backlog size - auth for getters - kafka auth end enc - domain resolver - domain resolver - generic adder - generic resolver - list comparison and labelser - pre_detector - pseudonymizer - template replacer * refactor template_replacer
1 parent 622d28e commit c1e012b

File tree

19 files changed

+401
-164
lines changed

19 files changed

+401
-164
lines changed

‎doc/source/development/coding_examples.rst‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,14 @@ Processor Case Examples
1818
notebooks/processor_examples/string_splitter.ipynb
1919
notebooks/processor_examples/timestamper.ipynb
2020
notebooks/processor_examples/timestamp_differ.ipynb
21+
notebooks/pseudonymization/pseudo_tools.ipynb
22+
notebooks/new_architecture_examples/1_event_metadata.ipynb
23+
notebooks/new_architecture_examples/10_pipeline.ipynb
24+
notebooks/new_architecture_examples/2_event_state.ipynb
25+
notebooks/new_architecture_examples/3_event_class.ipynb
26+
notebooks/new_architecture_examples/4_further_event_classes.ipynb
27+
notebooks/new_architecture_examples/5_event_backlog.ipynb
28+
notebooks/new_architecture_examples/6_opensearch_output.ipynb
29+
notebooks/new_architecture_examples/7_confluentkafka_output.ipynb
30+
notebooks/new_architecture_examples/8_processors.ipynb
31+
notebooks/new_architecture_examples/9_confluentkafka_input.ipynb

‎doc/source/development/notebooks/new_architecture_examples/5_event_backlog.ipynb‎

Lines changed: 111 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,59 @@
11
{
22
"cells": [
33
{
4-
"metadata": {},
54
"cell_type": "markdown",
6-
"source": "# The Event Backlog",
7-
"id": "f623967c8652cdf4"
5+
"id": "f623967c8652cdf4",
6+
"metadata": {},
7+
"source": [
8+
"# The Event Backlog"
9+
]
810
},
911
{
10-
"metadata": {},
1112
"cell_type": "markdown",
12-
"source": "Offers a container for managing events via insertion, retrieval, and deletion.",
13-
"id": "1f16fbe04726f9ff"
13+
"id": "1f16fbe04726f9ff",
14+
"metadata": {},
15+
"source": [
16+
"Offers a container for managing events via insertion, retrieval, and deletion."
17+
]
1418
},
1519
{
16-
"metadata": {},
1720
"cell_type": "markdown",
21+
"id": "df7e0e801a342646",
22+
"metadata": {},
1823
"source": [
1924
"--------------------------------------\n",
2025
"Following code fragment demonstrates:\n",
2126
"- Create Backlog\n",
2227
"- Add Events to Backlog"
23-
],
24-
"id": "df7e0e801a342646"
28+
]
2529
},
2630
{
31+
"cell_type": "code",
32+
"execution_count": 1,
33+
"id": "9389501c425f1789",
2734
"metadata": {
2835
"ExecuteTime": {
2936
"end_time": "2025-07-18T11:17:12.781922Z",
3037
"start_time": "2025-07-18T11:17:12.771765Z"
3138
}
3239
},
33-
"cell_type": "code",
40+
"outputs": [
41+
{
42+
"name": "stdout",
43+
"output_type": "stream",
44+
"text": [
45+
"\n",
46+
"Len: 0\n",
47+
"\n",
48+
"📦 Events in Backlog:\n",
49+
"> event.data={'id': 0, 'message': 'Test Event 0'}, state: receiving\n",
50+
"> event.data={'id': 1, 'message': 'Test Event 1'}, state: processing\n",
51+
"> event.data={'id': 2, 'message': 'Test Event 2'}, state: acked\n",
52+
"\n",
53+
"Len: 3\n"
54+
]
55+
}
56+
],
3457
"source": [
3558
"from logprep.ng.event.set_event_backlog import SetEventBacklog\n",
3659
"from logprep.ng.event.event_state import EventStateType\n",
@@ -63,58 +86,28 @@
6386
" print(f\"> {event.data=}, state: {event.state}\")\n",
6487
"\n",
6588
"print(f\"\\nLen: {len(backlog.backlog)}\")"
66-
],
67-
"id": "9389501c425f1789",
68-
"outputs": [
69-
{
70-
"name": "stdout",
71-
"output_type": "stream",
72-
"text": [
73-
"\n",
74-
"Len: 0\n",
75-
"\n",
76-
"📦 Events in Backlog:\n",
77-
"> event.data={'id': 0, 'message': 'Test Event 0'}, state: receiving\n",
78-
"> event.data={'id': 1, 'message': 'Test Event 1'}, state: processing\n",
79-
"> event.data={'id': 2, 'message': 'Test Event 2'}, state: acked\n",
80-
"\n",
81-
"Len: 3\n"
82-
]
83-
}
84-
],
85-
"execution_count": 1
89+
]
8690
},
8791
{
88-
"metadata": {},
8992
"cell_type": "markdown",
93+
"id": "c578efe4a6b51f16",
94+
"metadata": {},
9095
"source": [
9196
"--------------------------------------\n",
9297
"Following code fragment demonstrates:\n",
9398
"- Getting Event from Backlog"
94-
],
95-
"id": "c578efe4a6b51f16"
99+
]
96100
},
97101
{
102+
"cell_type": "code",
103+
"execution_count": 2,
104+
"id": "2535f8366062db32",
98105
"metadata": {
99106
"ExecuteTime": {
100107
"end_time": "2025-07-18T11:19:21.166388Z",
101108
"start_time": "2025-07-18T11:19:21.162878Z"
102109
}
103110
},
104-
"cell_type": "code",
105-
"source": [
106-
"processing_events = backlog.get(EventStateType.PROCESSING)\n",
107-
"\n",
108-
"print(f\"Len: {len(backlog.backlog)}\")\n",
109-
"\n",
110-
"print(\"\\n📥 Events in PROCESSING state:\")\n",
111-
"\n",
112-
"for event in processing_events:\n",
113-
" print(f\"> {event.data=}, state: {event.state}\")\n",
114-
"\n",
115-
"print(f\"\\nLen: {len(backlog.backlog)}\")\n"
116-
],
117-
"id": "2535f8366062db32",
118111
"outputs": [
119112
{
120113
"name": "stdout",
@@ -129,46 +122,39 @@
129122
]
130123
}
131124
],
132-
"execution_count": 2
125+
"source": [
126+
"processing_events = backlog.get(EventStateType.PROCESSING)\n",
127+
"\n",
128+
"print(f\"Len: {len(backlog.backlog)}\")\n",
129+
"\n",
130+
"print(\"\\n📥 Events in PROCESSING state:\")\n",
131+
"\n",
132+
"for event in processing_events:\n",
133+
" print(f\"> {event.data=}, state: {event.state}\")\n",
134+
"\n",
135+
"print(f\"\\nLen: {len(backlog.backlog)}\")\n"
136+
]
133137
},
134138
{
135-
"metadata": {},
136139
"cell_type": "markdown",
140+
"id": "5a492e823c0f21e8",
141+
"metadata": {},
137142
"source": [
138143
"--------------------------------------\n",
139144
"Following code fragment demonstrates:\n",
140145
"- Removing Events from Backlog"
141-
],
142-
"id": "5a492e823c0f21e8"
146+
]
143147
},
144148
{
149+
"cell_type": "code",
150+
"execution_count": 3,
151+
"id": "16ac2474df41292a",
145152
"metadata": {
146153
"ExecuteTime": {
147154
"end_time": "2025-07-18T11:20:08.601957Z",
148155
"start_time": "2025-07-18T11:20:08.598028Z"
149156
}
150157
},
151-
"cell_type": "code",
152-
"source": [
153-
"print(f\"Len: {len(backlog.backlog)}\")\n",
154-
"\n",
155-
"acked_events = backlog.unregister(EventStateType.ACKED)\n",
156-
"print(\"\\n🧹 Unregistered events with state ACKED:\")\n",
157-
"\n",
158-
"for event in acked_events:\n",
159-
" print(f\"> {event.data=}, state: {event.state}\")\n",
160-
"\n",
161-
"\n",
162-
"print(\"\\n📦 Remaining events in backlog:\")\n",
163-
"\n",
164-
"for state in EventStateType:\n",
165-
" still_in_backlog = backlog.get(state)\n",
166-
" for event in still_in_backlog:\n",
167-
" print(f\"> {event.data=}, state: {event.state}\")\n",
168-
"\n",
169-
"print(f\"\\nLen: {len(backlog.backlog)}\")\n"
170-
],
171-
"id": "16ac2474df41292a",
172158
"outputs": [
173159
{
174160
"name": "stdout",
@@ -187,26 +173,67 @@
187173
]
188174
}
189175
],
190-
"execution_count": 3
176+
"source": [
177+
"print(f\"Len: {len(backlog.backlog)}\")\n",
178+
"\n",
179+
"acked_events = backlog.unregister(EventStateType.ACKED)\n",
180+
"print(\"\\n🧹 Unregistered events with state ACKED:\")\n",
181+
"\n",
182+
"for event in acked_events:\n",
183+
" print(f\"> {event.data=}, state: {event.state}\")\n",
184+
"\n",
185+
"\n",
186+
"print(\"\\n📦 Remaining events in backlog:\")\n",
187+
"\n",
188+
"for state in EventStateType:\n",
189+
" still_in_backlog = backlog.get(state)\n",
190+
" for event in still_in_backlog:\n",
191+
" print(f\"> {event.data=}, state: {event.state}\")\n",
192+
"\n",
193+
"print(f\"\\nLen: {len(backlog.backlog)}\")\n"
194+
]
191195
},
192196
{
193-
"metadata": {},
194197
"cell_type": "markdown",
198+
"id": "405d697678bd50e9",
199+
"metadata": {},
195200
"source": [
196201
"--------------------------------------\n",
197202
"Following code fragment demonstrates:\n",
198203
"- FAILING Removing Events from Backlog"
199-
],
200-
"id": "405d697678bd50e9"
204+
]
201205
},
202206
{
207+
"cell_type": "code",
208+
"execution_count": 4,
209+
"id": "802e191d811ad8be",
203210
"metadata": {
204211
"ExecuteTime": {
205212
"end_time": "2025-07-18T11:22:10.567396Z",
206213
"start_time": "2025-07-18T11:22:10.562906Z"
207214
}
208215
},
209-
"cell_type": "code",
216+
"outputs": [
217+
{
218+
"name": "stdout",
219+
"output_type": "stream",
220+
"text": [
221+
"> Failed: unregister(receiving)\n",
222+
"> Failed: unregister(received)\n",
223+
"> Failed: unregister(processing)\n",
224+
"> Failed: unregister(processed)\n",
225+
"> Failed: unregister(stored_in_output)\n",
226+
"> Failed: unregister(stored_in_error)\n",
227+
"> Failed: unregister(delivered)\n",
228+
"Expected Failing: 7\n",
229+
"Failed: 7\n",
230+
"\n",
231+
"Should work without exceptions\n",
232+
"> Ok: unregister(failed)\n",
233+
"> Ok: unregister(acked)\n"
234+
]
235+
}
236+
],
210237
"source": [
211238
"fail_unregister_states = [\n",
212239
" EventStateType.RECEIVING,\n",
@@ -238,43 +265,20 @@
238265
"for state in success_unregister_states:\n",
239266
" backlog.unregister(state)\n",
240267
" print(f\"> Ok: unregister({state})\")\n"
241-
],
242-
"id": "802e191d811ad8be",
243-
"outputs": [
244-
{
245-
"name": "stdout",
246-
"output_type": "stream",
247-
"text": [
248-
"> Failed: unregister(receiving)\n",
249-
"> Failed: unregister(received)\n",
250-
"> Failed: unregister(processing)\n",
251-
"> Failed: unregister(processed)\n",
252-
"> Failed: unregister(stored_in_output)\n",
253-
"> Failed: unregister(stored_in_error)\n",
254-
"> Failed: unregister(delivered)\n",
255-
"Expected Failing: 7\n",
256-
"Failed: 7\n",
257-
"\n",
258-
"Should work without exceptions\n",
259-
"> Ok: unregister(failed)\n",
260-
"> Ok: unregister(acked)\n"
261-
]
262-
}
263-
],
264-
"execution_count": 4
268+
]
265269
},
266270
{
267-
"metadata": {},
268271
"cell_type": "code",
269-
"outputs": [],
270272
"execution_count": null,
271-
"source": "",
272-
"id": "dc699f2391e39ba7"
273+
"id": "dc699f2391e39ba7",
274+
"metadata": {},
275+
"outputs": [],
276+
"source": []
273277
}
274278
],
275279
"metadata": {
276280
"kernelspec": {
277-
"display_name": "Python 3",
281+
"display_name": ".venv",
278282
"language": "python",
279283
"name": "python3"
280284
},
@@ -288,7 +292,7 @@
288292
"name": "python",
289293
"nbconvert_exporter": "python",
290294
"pygments_lexer": "ipython2",
291-
"version": "2.7.6"
295+
"version": "3.11.11"
292296
}
293297
},
294298
"nbformat": 4,

‎logprep/connector/confluent_kafka/input.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,16 @@ class Config(Input.Config):
256256
.. datatemplate:import-module:: logprep.connector.confluent_kafka.input
257257
:template: defaults-renderer.tmpl
258258
259+
.. security-best-practice::
260+
:title: Kafka Input Consumer Authentication and Encryption
261+
262+
Kafka authentication is a critical aspect of securing your data pipeline.
263+
Ensure that you have the following configurations in place:
264+
265+
- Use SSL/mTLS encryption for data in transit.
266+
- Configure SASL or mTLS authentication for your Kafka clients.
267+
- Regularly rotate your Kafka credentials and secrets.
268+
259269
"""
260270

261271
_last_valid_record: Message

‎logprep/connector/confluent_kafka/output.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,15 @@ class Config(Output.Config):
183183
.. datatemplate:import-module:: logprep.connector.confluent_kafka.output
184184
:template: defaults-renderer.tmpl
185185
186+
.. security-best-practice::
187+
:title: Kafka Output Producer Authentication and Encryption
188+
189+
Kafka authentication is a critical aspect of securing your data pipeline.
190+
Ensure that you have the following configurations in place:
191+
192+
- Use SSL/mTLS encryption for data in transit.
193+
- Configure SASL or mTLS authentication for your Kafka clients.
194+
- Regularly rotate your Kafka credentials and secrets.
186195
"""
187196

188197
@property

‎logprep/connector/dummy/output.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
type: dummy_output
1818
"""
1919

20-
from logging import Logger
2120
from typing import TYPE_CHECKING, List
2221

2322
from attr import define, field

0 commit comments

Comments
 (0)