|
| 1 | +from collections.abc import AsyncIterable |
| 2 | +from pynumaflow.sinker import Datum, Responses, Response, Sinker, Message |
| 3 | +from pynumaflow.sinker import SinkAsyncServer |
| 4 | +import logging |
| 5 | +import random |
| 6 | + |
| 7 | +logging.basicConfig(level=logging.DEBUG) |
| 8 | +logger = logging.getLogger(__name__) |
| 9 | + |
| 10 | + |
| 11 | +class UserDefinedSink(Sinker): |
| 12 | + async def handler(self, datums: AsyncIterable[Datum]) -> Responses: |
| 13 | + responses = Responses() |
| 14 | + async for msg in datums: |
| 15 | + if primary_sink_write_status(): |
| 16 | + logger.info( |
| 17 | + "Write to User Defined Sink succeeded, writing %s to onSuccess sink", |
| 18 | + msg.value.decode("utf-8"), |
| 19 | + ) |
| 20 | + # create a message to be sent to onSuccess sink |
| 21 | + on_success_message = Response.as_on_success( |
| 22 | + msg.id, |
| 23 | + Message(msg.value, ["on_success"], msg.user_metadata), |
| 24 | + ) |
| 25 | + responses.append(on_success_message) |
| 26 | + # Sending `None`, on the other hand, specifies that simply send |
| 27 | + # the original message to the onSuccess sink |
| 28 | + # `responses.append(Response.as_on_success(msg.id, None))` |
| 29 | + else: |
| 30 | + logger.info( |
| 31 | + "Write to User Defined Sink failed, writing %s to fallback sink", |
| 32 | + msg.value.decode("utf-8"), |
| 33 | + ) |
| 34 | + responses.append(Response.as_fallback(msg.id)) |
| 35 | + return responses |
| 36 | + |
| 37 | + |
| 38 | +async def udsink_handler(datums: AsyncIterable[Datum]) -> Responses: |
| 39 | + responses = Responses() |
| 40 | + async for msg in datums: |
| 41 | + if primary_sink_write_status(): |
| 42 | + logger.info( |
| 43 | + "Write to User Defined Sink succeeded, writing %s to onSuccess sink", |
| 44 | + msg.value.decode("utf-8"), |
| 45 | + ) |
| 46 | + # create a message to be sent to onSuccess sink |
| 47 | + on_success_message = Response.as_on_success( |
| 48 | + msg.id, |
| 49 | + Message(msg.value).with_keys(["on_success"]).with_user_metadata(msg.user_metadata), |
| 50 | + ) |
| 51 | + responses.append(on_success_message) |
| 52 | + # Sending `None`, on the other hand, specifies that simply send |
| 53 | + # the original message to the onSuccess sink |
| 54 | + # `responses.append(Response.as_on_success(msg.id, None))` |
| 55 | + else: |
| 56 | + logger.info( |
| 57 | + "Write to User Defined Sink failed, writing %s to fallback sink", |
| 58 | + msg.value.decode("utf-8"), |
| 59 | + ) |
| 60 | + responses.append(Response.as_fallback(msg.id)) |
| 61 | + return responses |
| 62 | + |
| 63 | + |
| 64 | +def primary_sink_write_status(): |
| 65 | + # simulate writing to primary sink and return status of the same |
| 66 | + # return True if writing to primary sink succeeded |
| 67 | + # return False if writing to primary sink failed |
| 68 | + return random.randint(0, 1) == 1 |
| 69 | + |
| 70 | + |
| 71 | +if __name__ == "__main__": |
| 72 | + sink_handler = UserDefinedSink() |
| 73 | + grpc_server = SinkAsyncServer(sink_handler) |
| 74 | + grpc_server.start() |
0 commit comments