|
| 1 | +""" |
| 2 | +Offchain storage utilities for uploading refined data. |
| 3 | +
|
| 4 | +Uploads go through the relay service which handles authentication |
| 5 | +and forwards to the offchain storage service. |
| 6 | +""" |
| 7 | +import json |
| 8 | +import logging |
| 9 | +import os |
| 10 | +import requests |
| 11 | +from refiner.config import settings |
| 12 | + |
| 13 | + |
| 14 | +def upload_file_to_storage(file_path=None): |
| 15 | + """ |
| 16 | + Uploads a file to offchain storage via the relay service. |
| 17 | +
|
| 18 | + :param file_path: Path to the file to upload (defaults to encrypted database) |
| 19 | + :return: Dict with hash and url from storage service |
| 20 | + """ |
| 21 | + if file_path is None: |
| 22 | + # Default to the encrypted database file |
| 23 | + file_path = os.path.join(settings.OUTPUT_DIR, "db.libsql.pgp") |
| 24 | + |
| 25 | + if not os.path.exists(file_path): |
| 26 | + raise FileNotFoundError(f"File not found: {file_path}") |
| 27 | + |
| 28 | + if not settings.RELAY_UPLOAD_URL: |
| 29 | + raise Exception( |
| 30 | + "Error: RELAY_UPLOAD_URL not configured. " |
| 31 | + "This should be provided by the relay service via env_vars." |
| 32 | + ) |
| 33 | + |
| 34 | + try: |
| 35 | + with open(file_path, 'rb') as file: |
| 36 | + files = { |
| 37 | + 'file': (os.path.basename(file_path), file, 'application/octet-stream') |
| 38 | + } |
| 39 | + response = requests.post( |
| 40 | + settings.RELAY_UPLOAD_URL, |
| 41 | + files=files |
| 42 | + ) |
| 43 | + |
| 44 | + response.raise_for_status() |
| 45 | + result = response.json() |
| 46 | + |
| 47 | + # Response format: { url, ipfsHash (which is now SHA-256 hash), size } |
| 48 | + content_hash = result.get('ipfsHash') or result.get('hash') |
| 49 | + url = result.get('url') |
| 50 | + |
| 51 | + logging.info(f"Successfully uploaded file to storage - hash: {content_hash[:16]}...") |
| 52 | + |
| 53 | + return { |
| 54 | + 'hash': content_hash, |
| 55 | + 'url': url, |
| 56 | + 'size': result.get('size', os.path.getsize(file_path)) |
| 57 | + } |
| 58 | + |
| 59 | + except requests.exceptions.RequestException as e: |
| 60 | + logging.error(f"An error occurred while uploading file to storage: {e}") |
| 61 | + raise e |
| 62 | + |
| 63 | + |
| 64 | +def upload_json_to_storage(data): |
| 65 | + """ |
| 66 | + Uploads JSON data to offchain storage via the relay service. |
| 67 | +
|
| 68 | + :param data: JSON data to upload (dictionary or list) |
| 69 | + :return: Dict with hash and url from storage service |
| 70 | + """ |
| 71 | + if not settings.RELAY_UPLOAD_URL: |
| 72 | + raise Exception( |
| 73 | + "Error: RELAY_UPLOAD_URL not configured. " |
| 74 | + "This should be provided by the relay service via env_vars." |
| 75 | + ) |
| 76 | + |
| 77 | + try: |
| 78 | + # Convert JSON to bytes |
| 79 | + json_bytes = json.dumps(data).encode('utf-8') |
| 80 | + |
| 81 | + files = { |
| 82 | + 'file': ('data.json', json_bytes, 'application/json') |
| 83 | + } |
| 84 | + response = requests.post( |
| 85 | + settings.RELAY_UPLOAD_URL, |
| 86 | + files=files |
| 87 | + ) |
| 88 | + |
| 89 | + response.raise_for_status() |
| 90 | + result = response.json() |
| 91 | + |
| 92 | + content_hash = result.get('ipfsHash') or result.get('hash') |
| 93 | + url = result.get('url') |
| 94 | + |
| 95 | + logging.info(f"Successfully uploaded JSON to storage - hash: {content_hash[:16]}...") |
| 96 | + |
| 97 | + return { |
| 98 | + 'hash': content_hash, |
| 99 | + 'url': url, |
| 100 | + 'size': result.get('size', len(json_bytes)) |
| 101 | + } |
| 102 | + |
| 103 | + except requests.exceptions.RequestException as e: |
| 104 | + logging.error(f"An error occurred while uploading JSON to storage: {e}") |
| 105 | + raise e |
| 106 | + |
| 107 | + |
| 108 | +# Backward compatibility aliases |
| 109 | +def upload_file_to_ipfs(file_path=None): |
| 110 | + """ |
| 111 | + Backward compatible alias for upload_file_to_storage. |
| 112 | + Returns just the hash string for compatibility with existing code. |
| 113 | + """ |
| 114 | + result = upload_file_to_storage(file_path) |
| 115 | + return result['hash'] |
| 116 | + |
| 117 | + |
| 118 | +def upload_json_to_ipfs(data): |
| 119 | + """ |
| 120 | + Backward compatible alias for upload_json_to_storage. |
| 121 | + Returns just the hash string for compatibility with existing code. |
| 122 | + """ |
| 123 | + result = upload_json_to_storage(data) |
| 124 | + return result['hash'] |
| 125 | + |
| 126 | + |
| 127 | +# Test with: python -m refiner.utils.storage |
| 128 | +if __name__ == "__main__": |
| 129 | + # Test file upload |
| 130 | + test_file = os.path.join(settings.OUTPUT_DIR, "db.libsql.pgp") |
| 131 | + if os.path.exists(test_file): |
| 132 | + result = upload_file_to_storage(test_file) |
| 133 | + print(f"File uploaded - hash: {result['hash']}") |
| 134 | + print(f"URL: {result['url']}") |
| 135 | + |
| 136 | + # Test JSON upload |
| 137 | + test_json = {"test": "data", "timestamp": "2024-01-01"} |
| 138 | + result = upload_json_to_storage(test_json) |
| 139 | + print(f"JSON uploaded - hash: {result['hash']}") |
| 140 | + print(f"URL: {result['url']}") |
0 commit comments