|
| 1 | +from datetime import datetime, timedelta, timezone |
| 2 | +from typing import Any |
| 3 | + |
| 4 | +import boto3 |
| 5 | +import fire |
| 6 | + |
| 7 | +from nrlf.consumer.fhir.r4.model import DocumentReference |
| 8 | +from nrlf.core.logger import logger |
| 9 | +from nrlf.core.validators import DocumentReferenceValidator |
| 10 | + |
| 11 | +dynamodb = boto3.client("dynamodb") |
| 12 | +paginator = dynamodb.get_paginator("scan") |
| 13 | + |
| 14 | +logger.setLevel("ERROR") |
| 15 | + |
| 16 | + |
| 17 | +def _validate_document(document: str): |
| 18 | + docref = DocumentReference.model_validate_json(document) |
| 19 | + |
| 20 | + validator = DocumentReferenceValidator() |
| 21 | + result = validator.validate(data=docref) |
| 22 | + |
| 23 | + if not result.is_valid: |
| 24 | + raise Exception("Failed to validate document: " + str(result.issues)) |
| 25 | + |
| 26 | + |
| 27 | +def _find_invalid_pointers(table_name: str) -> dict[str, float]: |
| 28 | + """ |
| 29 | + Find pointers in the given table that are invalid. |
| 30 | + Parameters: |
| 31 | + - table_name: The name of the pointers table to use. |
| 32 | + """ |
| 33 | + |
| 34 | + print(f"Finding invalid pointers in table {table_name}....") # noqa |
| 35 | + |
| 36 | + params: dict[str, Any] = { |
| 37 | + "TableName": table_name, |
| 38 | + "PaginationConfig": {"PageSize": 50}, |
| 39 | + } |
| 40 | + |
| 41 | + invalid_pointers = [] |
| 42 | + total_scanned_count = 0 |
| 43 | + |
| 44 | + start_time = datetime.now(tz=timezone.utc) |
| 45 | + |
| 46 | + for page in paginator.paginate(**params): |
| 47 | + for item in page["Items"]: |
| 48 | + id = item.get("id", {}).get("S") |
| 49 | + document = item.get("document", {}).get("S", "") |
| 50 | + try: |
| 51 | + _validate_document(document) |
| 52 | + except Exception as exc: |
| 53 | + invalid_pointers.append((id, exc)) |
| 54 | + |
| 55 | + total_scanned_count += page["ScannedCount"] |
| 56 | + |
| 57 | + if total_scanned_count % 1000 == 0: |
| 58 | + print(".", end="", flush=True) # noqa |
| 59 | + |
| 60 | + if total_scanned_count % 100000 == 0: |
| 61 | + print( # noqa |
| 62 | + f"scanned={total_scanned_count} invalid={len(invalid_pointers)}" |
| 63 | + ) |
| 64 | + |
| 65 | + end_time = datetime.now(tz=timezone.utc) |
| 66 | + |
| 67 | + print("Writing invalid_pointers to file ./invalid_pointers.txt ...") # noqa |
| 68 | + with open("invalid_pointers.txt", "w") as f: |
| 69 | + for id, err in invalid_pointers: |
| 70 | + f.write(f"{id}: {err}\n") |
| 71 | + |
| 72 | + print(" Done") # noqa |
| 73 | + return { |
| 74 | + "invalid_pointers": len(invalid_pointers), |
| 75 | + "scanned_count": total_scanned_count, |
| 76 | + "took-secs": timedelta.total_seconds(end_time - start_time), |
| 77 | + } |
| 78 | + |
| 79 | + |
| 80 | +if __name__ == "__main__": |
| 81 | + fire.Fire(_find_invalid_pointers) |
0 commit comments