eprints-hyku.py

#!/usr/bin/env python3
"""
Compare OAI-PMH records from Eprints and Hyku repositories.
Checks for missing records and file count mismatches.
"""

import argparse
import sys
from collections import defaultdict
from typing import Dict, List, Set, Tuple
from sickle import Sickle
from sickle.oaiexceptions import NoRecordsMatch
from urllib.parse import urlparse

def is_valid_url(url:str) -> bool:
    """
    checks if url provided is valid url
    code snippet from https://stackoverflow.com/a/38020041
    """
    try:
        res = urlparse(url)
        return all([res.scheme, res.netloc])
    except Exception as e:
        log.debug(f"{url} is not a url")
        return False

def extract_eprints_id(oai_identifier: str) -> str:
    """Extract numeric ID from Eprints OAI identifier (format: oai:domain:123)."""
    parts = oai_identifier.split(':')
    if len(parts) >= 3:
        return parts[-1]
    return None


def extract_file_from_uri(uri: str) -> str:
    """Extract filename from URI (last component after final slash)."""
    return uri.rstrip('/').split('/')[-1]


def extract_hyku_identifier(oai_identifier: str) -> str:
    """Extract UUID from Hyku OAI identifier (format: oai:domain:uuid)."""
    parts = oai_identifier.split(':')
    if len(parts) >= 3:
        return parts[-1]
    return None


def harvest_eprints(base_url: str) -> Dict[str, Tuple[List[str], int]]:
    """
    Harvest Eprints repository via OAI-PMH.
    
    Returns:
        Dict mapping numeric_id -> (list_of_files, file_count)
    """
    print(f"Harvesting Eprints from: {base_url}")
    sickle = Sickle(base_url)
    eprints_data = {}
    
    try:
        records = sickle.ListRecords(metadataPrefix='oai_dc')
        
        for record in records:
            # Extract numeric ID from OAI identifier
            oai_id = record.header.identifier
            numeric_id = extract_eprints_id(oai_id)
            
            if not numeric_id:
                print(f"Warning: Could not extract numeric ID from {oai_id}")
                continue

            if not hasattr(record, "metadata"):
                continue
            
            # Extract file identifiers from metadata
            files = []
            metadata = record.metadata
            
            # Look for identifier elements (URIs)
            if 'identifier' in metadata:
                identifiers = metadata['identifier']
                if not isinstance(identifiers, list):
                    identifiers = [identifiers]
                
                for identifier in identifiers:
                    if identifier and is_valid_url(identifier):
                        filename = extract_file_from_uri(identifier)
                        files.append(filename)
            
            file_count = len(files)
            eprints_data[numeric_id] = (files, file_count)
            
            if len(eprints_data) % 100 == 0:
                print(f"  Processed {len(eprints_data)} Eprints records...")
    
    except NoRecordsMatch:
        print("No records found in Eprints repository")
    except Exception as e:
        print(f"Error harvesting Eprints: {e}")
        sys.exit(1)
    
    print(f"Harvested {len(eprints_data)} Eprints records")
    return eprints_data


def harvest_hyku(base_url: str) -> Dict[str, Tuple[str, int]]:
    """
    Harvest Hyku repository via OAI-PMH.
    
    Returns:
        Dict mapping numeric_id -> (uuid, file_count)
    """
    print(f"\nHarvesting Hyku from: {base_url}")
    sickle = Sickle(base_url)
    hyku_data = {}
    skipped = 0
    
    try:
        records = sickle.ListRecords(metadataPrefix='oai_hyku')
        
        for record in records:
            # Extract UUID from OAI identifier
            oai_id = record.header.identifier
            uuid = extract_hyku_identifier(oai_id)
            
            if not uuid:
                print(f"Warning: Could not extract UUID from {oai_id}")
                continue
            
            metadata = record.metadata
            
            # Check for identifier element
            if 'identifier' not in metadata:
                skipped += 1
                continue
            
            # Extract numeric ID from identifier URI
            identifiers = metadata['identifier']
            if not isinstance(identifiers, list):
                identifiers = [identifiers]
            
            numeric_id = None
            for identifier in identifiers:
                if identifier and is_valid_url(identifier):
                    numeric_id = extract_file_from_uri(identifier)
                    break
            
            if not numeric_id:
                skipped += 1
                continue
            
            # Count file_url elements
            file_count = 0
            if 'file_url' in metadata:
                file_urls = metadata['file_url']
                if isinstance(file_urls, list):
                    file_count = len(file_urls)
                else:
                    file_count = 1 if file_urls else 0
            
            hyku_data[numeric_id] = (uuid, file_count)
            
            if len(hyku_data) % 100 == 0:
                print(f"  Processed {len(hyku_data)} Hyku records...")
    
    except NoRecordsMatch:
        print("No records found in Hyku repository")
    except Exception as e:
        print(f"Error harvesting Hyku: {e}")
        sys.exit(1)
    
    print(f"Harvested {len(hyku_data)} Hyku records (skipped {skipped} without identifier)")
    return hyku_data


def compare_repositories(eprints_data: Dict, hyku_data: Dict):
    """Compare Eprints and Hyku data and report discrepancies."""
    print("\n" + "="*80)
    print("COMPARISON REPORT")
    print("="*80)
    
    missing_in_hyku = []
    file_count_mismatches = []
    
    for numeric_id, (eprints_files, eprints_count) in eprints_data.items():
        if numeric_id not in hyku_data:
            missing_in_hyku.append({
                'numeric_id': numeric_id,
                'eprints_files': eprints_files,
                'eprints_count': eprints_count
            })
        else:
            hyku_uuid, hyku_count = hyku_data[numeric_id]
            if eprints_count != hyku_count:
                file_count_mismatches.append({
                    'numeric_id': numeric_id,
                    'eprints_files': eprints_files,
                    'eprints_count': eprints_count,
                    'hyku_uuid': hyku_uuid,
                    'hyku_count': hyku_count
                })
    
    # Report missing records
    print(f"\nRecords in Eprints but not in Hyku: {len(missing_in_hyku)}")
    if missing_in_hyku:
        print("\nDETAILS OF MISSING RECORDS:")
        print("-" * 80)
        for record in missing_in_hyku:
            print(f"\nEprints ID: {record['numeric_id']}")
            print(f"  File count: {record['eprints_count']}")
            print(f"  Files: {', '.join(record['eprints_files']) if record['eprints_files'] else 'None'}")
    
    # Report file count mismatches
    print(f"\n\nRecords with file count mismatches: {len(file_count_mismatches)}")
    if file_count_mismatches:
        print("\nDETAILS OF FILE COUNT MISMATCHES:")
        print("-" * 80)
        for record in file_count_mismatches:
            print(f"\nEprints ID: {record['numeric_id']}")
            print(f"  Eprints file count: {record['eprints_count']}")
            print(f"  Eprints files: {', '.join(record['eprints_files']) if record['eprints_files'] else 'None'}")
            print(f"  Hyku UUID: {record['hyku_uuid']}")
            print(f"  Hyku file count: {record['hyku_count']}")
    
    # Summary statistics
    print("\n" + "="*80)
    print("SUMMARY")
    print("="*80)
    print(f"Total Eprints records: {len(eprints_data)}")
    print(f"Total Hyku records: {len(hyku_data)}")
    print(f"Matched records: {len(eprints_data) - len(missing_in_hyku)}")
    print(f"Missing in Hyku: {len(missing_in_hyku)}")
    print(f"File count mismatches: {len(file_count_mismatches)}")
    
    total_issues = len(missing_in_hyku) + len(file_count_mismatches)
    print(f"\nTotal issues found: {total_issues}")


def main():
    parser = argparse.ArgumentParser(
        description='Compare OAI-PMH records from Eprints and Hyku repositories'
    )
    parser.add_argument(
        '--eprints-url',
        required=True,
        help='Base URL for Eprints OAI-PMH endpoint'
    )
    parser.add_argument(
        '--hyku-url',
        required=True,
        help='Base URL for Hyku OAI-PMH endpoint'
    )
    
    args = parser.parse_args()
    
    # Harvest both repositories
    eprints_data = harvest_eprints(args.eprints_url)
    hyku_data = harvest_hyku(args.hyku_url)
    
    # Compare and report
    compare_repositories(eprints_data, hyku_data)


if __name__ == '__main__':
    main()