Possible helper app to list unique IDs from log

[gregbert42]

Apologies if this isn't done the right way - but i created a python script that takes the output of rtl listen all command, and then parses out the unique endpointIDs and IDs. This was helpful after a long night of listening to see what unique broadcasters I had.

So for instance, i run

docker run --rm -ti -e LISTEN_ONLY=yes --device=/dev/bus/usb:/dev/bus/usb allangood/rtlamr2mqtt > ~/logfile

and then i run the program below, python3 extractid.py ~/logfile

import json
import sys
import re

def parse_log_file(file_path):
    extracted_data = []
    rejected_lines = []

    # Regular expression to strip the timestamp and other prefixes
    json_line_pattern = re.compile(r"^\[.*?\]\s*(\{.*\})$")

    with open(file_path, 'r') as file:
        for line_number, line in enumerate(file, start=1):
            line = line.strip()
            match = json_line_pattern.match(line)
            if match:
                json_part = match.group(1)
                try:
                    data = json.loads(json_part)
                    # Extract 'ID' and 'EndpointID' if they exist in the JSON
                    id_field = data.get("Message", {}).get("ID")
                    endpoint_id_field = data.get("Message", {}).get("EndpointID")

                    # Only add the data if at least one of the fields is present
                    if id_field is not None or endpoint_id_field is not None:
                        extracted_data.append({
                            "ID": id_field,
                            "EndpointID": endpoint_id_field
                        })
                    else:
                        rejected_lines.append((line_number, line))
                except json.JSONDecodeError:
                    # Record lines that are not valid JSON
                    rejected_lines.append((line_number, line))
            else:
                # Record lines that do not match the expected pattern
                rejected_lines.append((line_number, line))

    return extracted_data, rejected_lines

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python script.py <file_path>")
        sys.exit(1)

    file_path = sys.argv[1]

    # Parse the log file and extract the relevant fields
    parsed_data, rejected_lines = parse_log_file(file_path)

    # Remove duplicate entries
    unique_data = list({(entry["ID"], entry["EndpointID"]): entry for entry in parsed_data}.values())

    # Print the extracted data
    print("Extracted Data (Unique):")
    for entry in unique_data:
        print(entry)

    # Print rejected lines
    if rejected_lines:
        print("\nRejected Lines:")
        for line_number, line in rejected_lines:
            print(f"Line {line_number}: {line}")