|
| 1 | +""" |
| 2 | +This script generates the essential assets (embedding index and metadata) required for Digital Collections Explorer |
| 3 | +by converting `beto_idx.pt` provided by Mahowald and Lee (https://zenodo.org/records/11538437) into `item_ids.pt` and `metadata.json` |
| 4 | +
|
| 5 | +The output assets can be placed directly in the `data/embeddings` folder, allowing the FastAPI server to access them |
| 6 | +in the same way as we do in our public demo at https://digital-collections-explorer.com/ |
| 7 | +""" |
| 8 | + |
| 9 | +import pandas as pd |
| 10 | +import torch |
| 11 | +import json |
| 12 | +import base64 |
| 13 | + |
| 14 | +ORIGINAL_INDEX_PATH = 'input/beto_idx.pt' |
| 15 | +CSV_PATH = 'input/merged_files.csv' |
| 16 | +FINAL_METADATA_PATH = 'output/metadata.json' |
| 17 | +FINAL_INDEX_PATH = 'output/item_ids.pt' |
| 18 | + |
| 19 | +def generate_assets(): |
| 20 | + # --- 1. Load the original beto_idx.pt file --- |
| 21 | + original_idx = torch.load(ORIGINAL_INDEX_PATH) |
| 22 | + total_items = len(original_idx) |
| 23 | + print(f"Found {total_items} entries in the original index.") |
| 24 | + |
| 25 | + # --- 2. Build a lookup table from merged_files.csv --- |
| 26 | + df = pd.read_csv(CSV_PATH) |
| 27 | + df.dropna(subset=['p1_item_id', 'file_url'], inplace=True) |
| 28 | + df['iiif_id'] = df['file_url'].apply(lambda url: url.split('/')[5] if isinstance(url, str) else None) |
| 29 | + df.dropna(subset=['iiif_id'], inplace=True) |
| 30 | + iiif_to_p1_lookup = pd.Series(df.p1_item_id.values, index=df.iiif_id).to_dict() |
| 31 | + |
| 32 | + # --- 3. Generate new index and metadata --- |
| 33 | + final_metadata = {} |
| 34 | + final_beto_idx = [] |
| 35 | + |
| 36 | + for image_url in original_idx: |
| 37 | + # a. Extract iiif_id |
| 38 | + try: |
| 39 | + iiif_id = image_url.split('/')[5] |
| 40 | + except IndexError: |
| 41 | + b64_key = base64.urlsafe_b64encode(f"ERROR_PARSING_{len(final_beto_idx)}".encode('utf-8')).decode('utf-8') |
| 42 | + final_beto_idx.append(b64_key) |
| 43 | + final_metadata[b64_key] = {'error': f'Could not parse iiif_id from URL: {image_url}'} |
| 44 | + continue |
| 45 | + |
| 46 | + # b. Generate Base64 key |
| 47 | + b64_key = base64.urlsafe_b64encode(iiif_id.encode('utf-8')).decode('utf-8') |
| 48 | + |
| 49 | + # c. Append key to the new index |
| 50 | + final_beto_idx.append(b64_key) |
| 51 | + |
| 52 | + # d. Find p1_item_id |
| 53 | + p1_item_id = iiif_to_p1_lookup.get(iiif_id, "p1_item_id_not_found") |
| 54 | + |
| 55 | + # e. Assemble the new metadata object |
| 56 | + url_base = f"https://tile.loc.gov/image-services/iiif/{iiif_id}" |
| 57 | + paths = { |
| 58 | + 'original': f"{url_base}/full/pct:100/0/default.jpg", |
| 59 | + 'processed': f"{url_base}/full/2000,/0/default.jpg", |
| 60 | + 'thumbnail': f"{url_base}/full/400,/0/default.jpg" |
| 61 | + } |
| 62 | + final_metadata[b64_key] = { |
| 63 | + 'type': 'image', |
| 64 | + 'iiif_id': iiif_id, |
| 65 | + 'url': p1_item_id, |
| 66 | + 'paths': paths |
| 67 | + } |
| 68 | + |
| 69 | + # --- 4. Final Save and Validation --- |
| 70 | + with open(FINAL_METADATA_PATH, 'w') as f: |
| 71 | + json.dump(final_metadata, f, indent=4) |
| 72 | + print(f"Successfully saved {FINAL_METADATA_PATH} with {len(final_metadata)} entries.") |
| 73 | + |
| 74 | + torch.save(final_beto_idx, FINAL_INDEX_PATH) |
| 75 | + print(f"Successfully saved {FINAL_INDEX_PATH} with {len(final_beto_idx)} entries.") |
| 76 | + |
| 77 | + assert len(original_idx) == len(final_beto_idx), "CRITICAL: Final index length does not match original!" |
| 78 | + |
| 79 | +if __name__ == '__main__': |
| 80 | + generate_assets() |
0 commit comments