|
| 1 | +import collections.abc |
| 2 | +import io |
| 3 | +from typing import Union |
| 4 | +from zipfile import ZipFile |
| 5 | + |
| 6 | +import httpx |
| 7 | +from loguru import logger |
| 8 | + |
| 9 | +from bioimageio.spec._io import load_description |
| 10 | + |
| 11 | +from ._description import ( |
| 12 | + InvalidDescr, |
| 13 | + ResourceDescr, |
| 14 | + build_description, |
| 15 | +) |
| 16 | +from ._internal._settings import settings |
| 17 | +from ._internal.common_nodes import ResourceDescrBase |
| 18 | +from ._internal.io import BioimageioYamlContent, get_reader |
| 19 | +from ._internal.io_basics import BIOIMAGEIO_YAML |
| 20 | +from ._internal.io_utils import write_yaml |
| 21 | +from ._internal.validation_context import get_validation_context |
| 22 | +from ._package import get_resource_package_content |
| 23 | +from .common import HttpUrl, PermissiveFileSource |
| 24 | + |
| 25 | + |
| 26 | +# TODO: remove alpha stage warning |
| 27 | +def upload( |
| 28 | + source: Union[PermissiveFileSource, ZipFile, ResourceDescr, BioimageioYamlContent], |
| 29 | + /, |
| 30 | +) -> HttpUrl: |
| 31 | + """Upload a new resource description (version) to the hypha server to be shared at bioimage.io. |
| 32 | + To edit an existing resource **version**, please login to https://bioimage.io and use the web interface. |
| 33 | +
|
| 34 | + WARNING: This upload function is in alpha stage and might change in the future. |
| 35 | +
|
| 36 | + Args: |
| 37 | + source: The resource description to upload. |
| 38 | +
|
| 39 | + Returns: |
| 40 | + A URL to the uploaded resource description. |
| 41 | + Note: It might take some time until the resource is processed and available for download from the returned URL. |
| 42 | + """ |
| 43 | + |
| 44 | + if settings.hypha_upload_token is None: |
| 45 | + raise ValueError( |
| 46 | + """ |
| 47 | +Upload token is not set. Please set BIOIMAGEIO_HYPHA_UPLOAD_TOKEN in your environment variables. |
| 48 | +By setting this token you agree to our terms of service at https://bioimage.io/#/toc. |
| 49 | +
|
| 50 | +How to obtain a token: |
| 51 | + 1. Login to https://bioimage.io |
| 52 | + 2. Generate a new token at https://bioimage.io/#/api?tab=hypha-rpc |
| 53 | +""" |
| 54 | + ) |
| 55 | + |
| 56 | + if isinstance(source, ResourceDescrBase): |
| 57 | + # If source is already a ResourceDescr, we can use it directly |
| 58 | + descr = source |
| 59 | + elif isinstance(source, dict): |
| 60 | + descr = build_description(source) |
| 61 | + else: |
| 62 | + descr = load_description(source) |
| 63 | + |
| 64 | + if isinstance(descr, InvalidDescr): |
| 65 | + raise ValueError("Uploading invalid resource descriptions is not allowed.") |
| 66 | + |
| 67 | + if descr.type != "model": |
| 68 | + raise NotImplementedError( |
| 69 | + f"For now, only model resources can be uploaded (got type={descr.type})." |
| 70 | + ) |
| 71 | + |
| 72 | + if descr.id is not None: |
| 73 | + raise ValueError( |
| 74 | + "You cannot upload a resource with an id. Please remove the id from the description and make sure to upload a new non-existing resource. To edit an existing resource, please use the web interface at https://bioimage.io." |
| 75 | + ) |
| 76 | + |
| 77 | + content = get_resource_package_content(descr) |
| 78 | + |
| 79 | + metadata = content[BIOIMAGEIO_YAML] |
| 80 | + assert isinstance(metadata, dict) |
| 81 | + manifest = dict(metadata) |
| 82 | + |
| 83 | + # only admins can upload a resource with a version |
| 84 | + artifact_version = "stage" # if descr.version is None else str(descr.version) |
| 85 | + |
| 86 | + # Create new model |
| 87 | + r = httpx.post( |
| 88 | + settings.hypha_upload, |
| 89 | + json={ |
| 90 | + "parent_id": "bioimage-io/bioimage.io", |
| 91 | + "alias": ( |
| 92 | + descr.id or "{animal_adjective}-{animal}" |
| 93 | + ), # TODO: adapt for non-model uploads, |
| 94 | + "type": descr.type, |
| 95 | + "manifest": manifest, |
| 96 | + "version": artifact_version, |
| 97 | + }, |
| 98 | + headers=( |
| 99 | + headers := { |
| 100 | + "Authorization": f"Bearer {settings.hypha_upload_token}", |
| 101 | + "Content-Type": "application/json", |
| 102 | + } |
| 103 | + ), |
| 104 | + ) |
| 105 | + |
| 106 | + response = r.json() |
| 107 | + artifact_id = response.get("id") |
| 108 | + if artifact_id is None: |
| 109 | + try: |
| 110 | + logger.error("Response detail: {}", "".join(response["detail"])) |
| 111 | + except Exception: |
| 112 | + logger.error("Response: {}", response) |
| 113 | + |
| 114 | + raise RuntimeError(f"Upload did not return resource id: {response}") |
| 115 | + else: |
| 116 | + logger.info("Uploaded resource description {}", artifact_id) |
| 117 | + |
| 118 | + for file_name, file_source in content.items(): |
| 119 | + # Get upload URL for a file |
| 120 | + response = httpx.post( |
| 121 | + settings.hypha_upload.replace("/create", "/put_file"), |
| 122 | + json={ |
| 123 | + "artifact_id": artifact_id, |
| 124 | + "file_path": file_name, |
| 125 | + }, |
| 126 | + headers=headers, |
| 127 | + ) |
| 128 | + upload_url = response.raise_for_status().json() |
| 129 | + |
| 130 | + # Upload file to the provided URL |
| 131 | + if isinstance(file_source, collections.abc.Mapping): |
| 132 | + buf = io.BytesIO() |
| 133 | + write_yaml(file_source, buf) |
| 134 | + files = {file_name: buf} |
| 135 | + else: |
| 136 | + files = {file_name: get_reader(file_source)} |
| 137 | + |
| 138 | + response = httpx.put( |
| 139 | + upload_url, |
| 140 | + files=files, # pyright: ignore[reportArgumentType] |
| 141 | + # TODO: follow up on https://github.com/encode/httpx/discussions/3611 |
| 142 | + headers={"Content-Type": ""}, # Important for S3 uploads |
| 143 | + ) |
| 144 | + logger.info("Uploaded '{}' successfully", file_name) |
| 145 | + |
| 146 | + # Update model status |
| 147 | + manifest["status"] = "request-review" |
| 148 | + response = httpx.post( |
| 149 | + settings.hypha_upload.replace("/create", "/edit"), |
| 150 | + json={ |
| 151 | + "artifact_id": artifact_id, |
| 152 | + "version": artifact_version, |
| 153 | + "manifest": manifest, |
| 154 | + }, |
| 155 | + headers=headers, |
| 156 | + ) |
| 157 | + logger.info( |
| 158 | + "Updated status of {}/{} to 'request-review'", artifact_id, artifact_version |
| 159 | + ) |
| 160 | + logger.warning( |
| 161 | + "Upload successfull. Please note that the uploaded resource might not be available for download immediately." |
| 162 | + ) |
| 163 | + with get_validation_context().replace(perform_io_checks=False): |
| 164 | + return HttpUrl( |
| 165 | + f"https://hypha.aicell.io/bioimage-io/artifacts/{artifact_id}/files/rdf.yaml?version={artifact_version}" |
| 166 | + ) |
0 commit comments