Skip to content

Commit 31c50a2

Browse files
committed
basic lambda
1 parent 6ed6f1b commit 31c50a2

19 files changed

+152
-1004
lines changed

id_sync/poetry.lock

Lines changed: 129 additions & 129 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

id_sync/src/clients.py

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
import os
22
import logging
3-
import redis
43
from boto3 import client as boto3_client
54

6-
75
logging.basicConfig(level="INFO")
86
logger = logging.getLogger()
97
logger.setLevel("INFO")
@@ -13,8 +11,5 @@
1311
REGION_NAME = os.getenv("AWS_REGION", "eu-west-2")
1412
REDIS_HOST = os.getenv("REDIS_HOST", "")
1513
REDIS_PORT = os.getenv("REDIS_PORT", 6379)
16-
1714
s3_client = boto3_client("s3", region_name=REGION_NAME)
1815
firehose_client = boto3_client("firehose", region_name=REGION_NAME)
19-
logger.info(f"Connecting to Redis at {REDIS_HOST}:{REDIS_PORT}")
20-
redis_client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)

id_sync/src/constants.py

Lines changed: 0 additions & 3 deletions
This file was deleted.

id_sync/src/event_read.py

Lines changed: 0 additions & 15 deletions
This file was deleted.
Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
from clients import logger
2-
from s3_event import S3Event
3-
from record_processor import process_record
4-
from event_read import read_event
52
from log_decorator import logging_decorator
6-
from clients import redis_client
3+
from record_processor import process_record
4+
75
'''
86
Event Processor
97
The Business Logic for the Redis Sync Lambda Function.
@@ -14,21 +12,17 @@
1412
def handler(event, _):
1513

1614
try:
17-
# check if the event requires a read, ie {"read": "my-hashmap"}
18-
if "read" in event:
19-
return read_event(redis_client, event, logger)
20-
elif "Records" in event:
15+
if "Records" in event:
2116
logger.info("Processing S3 event with %d records", len(event.get('Records', [])))
22-
s3_event = S3Event(event)
23-
record_count = len(s3_event.get_s3_records())
17+
record_count = len(event.get('Records', []))
2418
if record_count == 0:
2519
logger.info("No records found in event")
2620
return {"status": "success", "message": "No records found in event"}
2721
else:
2822
error_count = 0
2923
file_keys = []
30-
for record in s3_event.get_s3_records():
31-
record_result = process_record(record)
24+
for record in event.get('Records', []):
25+
record_result = process_record(record, None)
3226
file_keys.append(record_result["file_key"])
3327
if record_result["status"] == "error":
3428
error_count += 1

id_sync/src/record_processor.py

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,7 @@
1-
from clients import logger
2-
from s3_event import S3EventRecord
3-
from redis_cacher import RedisCacher
41
'''
5-
Record Processor
6-
This module processes individual S3 records from an event.
7-
It is used to upload data to Redis ElastiCache.
2+
record Processor
83
'''
94

105

11-
def process_record(record: S3EventRecord) -> dict:
12-
try:
13-
logger.info("Processing S3 r bucket: %s, key: %s",
14-
record.get_bucket_name(), record.get_object_key())
15-
bucket_name = record.get_bucket_name()
16-
file_key = record.get_object_key()
17-
18-
base_log_data = {
19-
"file_key": file_key
20-
}
21-
22-
try:
23-
result = RedisCacher.upload(bucket_name, file_key)
24-
result.update(base_log_data)
25-
return result
26-
27-
except Exception as error: # pylint: disable=broad-except
28-
logger.exception("Error uploading to cache for filename '%s'", file_key)
29-
error_data = {"status": "error", "message": str(error)}
30-
error_data.update(base_log_data)
31-
return error_data
32-
33-
except Exception: # pylint: disable=broad-except
34-
msg = "Error processing record"
35-
logger.exception(msg)
36-
return {"status": "error", "message": msg}
6+
def process_record(record, _):
7+
return "hello world"

id_sync/src/redis_cacher.py

Lines changed: 0 additions & 46 deletions
This file was deleted.

id_sync/src/s3_event.py

Lines changed: 0 additions & 27 deletions
This file was deleted.

id_sync/src/s3_reader.py

Lines changed: 0 additions & 22 deletions
This file was deleted.

id_sync/src/transform_configs.py

Lines changed: 0 additions & 29 deletions
This file was deleted.

0 commit comments

Comments
 (0)