Skip to content

Commit 2185c63

Browse files
aiden-yguaiden-ygu
authored
Merge pull request #1 from ivantarapov/aiden-ygu-patch-1
Update deploy-medimageparse3d.md
2 parents 30d349d + 11620eb commit 2185c63

File tree

1 file changed

+98
-55
lines changed

1 file changed

+98
-55
lines changed

‎articles/ai-foundry/how-to/healthcare-ai/deploy-medimageparse3d.md

Lines changed: 98 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ Consume the MedImageParse 3D segmentation model as a REST API, using simple GET
6262

6363
```python
6464
from azure.ai.ml import MLClient
65-
from azure.identity import DeviceCodeCredential
65+
from azure.identity import DefaultAzureCredential
6666

6767
credential = DefaultAzureCredential()
6868

@@ -75,17 +75,21 @@ In the deployment configuration, you get to choose authentication method. This e
7575

7676
Once the model is deployed, use the following code to send data and retrieve segmentation masks.
7777

78-
TODO: the example here follows MedImageParse (2D) where it uses `ml_client_workspace.online_endpoints.invoke` instead of `urllib.request.urlopen` as in this [notebook](https://dev.azure.com/msazuredev/HLS%20AI%20Platform/_git/3dMedImageParseDeployment?path=/notebooks/03.model.endpoint.api.call.ipynb&version=GBmain&line=192&lineEnd=193&lineStartColumn=1&lineEndColumn=1&lineStyle=plain&_a=contents). Verify the correct call pattern.
79-
8078
```python
8179
import base64
8280
import json
81+
import urllib.request
82+
import matplotlib.pyplot as plt
83+
import nibabel as nib
84+
import tempfile
8385
import os
8486

85-
sample_image = "example.nii.gz"
87+
# Replace with the path to your NIfTI input file
88+
sample_image = "./examples/amos_0308.nii.gz"
8689
with open(sample_image, "rb") as image_file:
8790
base64_image = base64.b64encode(image_file.read()).decode('utf-8')
8891

92+
# Prepare data payload
8993
data = {
9094
"input_data": {
9195
"columns": [ "image", "text" ],
@@ -99,21 +103,39 @@ data = {
99103
}
100104
}
101105
data_json = json.dumps(data)
106+
body = str.encode(data_json)
107+
108+
# Add your endpoint URL and API key
109+
url = "<your-endpoint-url>"
110+
api_key = "<your-api-key>"
111+
112+
headers = {
113+
'Content-Type': 'application/json',
114+
'Authorization': 'Bearer ' + api_key
115+
}
116+
117+
req = urllib.request.Request(url, body, headers)
118+
119+
# Make sure decode_base64_to_nifti() and plot_segmentation_masks() are defined above
120+
try:
121+
response = urllib.request.urlopen(req)
122+
result = response.read()
123+
result_list = json.loads(result)
124+
125+
# Extract and decode NIfTI segmentation output
126+
nifti_file_str = result_list[0]["nifti_file"]
127+
nifti_file_data = decode_base64_to_nifti(nifti_file_str)
102128

103-
# Create request json
104-
request_file_name = "sample_request_data.json"
105-
with open(request_file_name, "w") as request_file:
106-
json.dump(data, request_file)
129+
print(nifti_file_data.shape)
130+
plot_segmentation_masks(nifti_file_data)
107131

108-
response = ml_client_workspace.online_endpoints.invoke(
109-
endpoint_name=endpoint_name,
110-
deployment_name=deployment_name,
111-
request_file=request_file_name,
112-
)
132+
except urllib.error.HTTPError as error:
133+
print("The request failed with status code: " + str(error.code))
134+
print(error.info())
135+
print(error.read().decode("utf8", 'ignore'))
113136
```
114137

115138
## Use MedImageParse 3D REST API
116-
TODO: verify all contents in this section
117139

118140
MedImageParse 3D model assumes a simple single-turn interaction where one request produces one response.
119141

@@ -123,19 +145,19 @@ Request payload is a JSON formatted string containing the following parameters:
123145

124146
| Key | Type | Required/Default | Description |
125147
| ------------- | -------------- | :-----------------:| ----------------- |
126-
| `input_data` | `[object]` | Y | An object containing the input data payload |
148+
| `input_data` | `[object]` | Yes | An object containing the input data |
127149

128150
The `input_data` object contains the following fields:
129151

130152
| Key | Type | Required/Default | Allowed values | Description |
131153
| ------------- | -------------- | :-----------------:| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
132-
| `columns` | `list[string]` | Y | `"image"`, `"text"` | An object containing the strings mapping data to inputs passed to the model.|
133-
| `index` | `integer` | Y | 0 - 256 | Count of inputs passed to the model. You're limited by how much data can be passed in a single POST request, which depends on the size of your images. Therefore, it's reasonable to keep this number in the dozens. |
134-
| `data` | `list[list[string]]` | Y | "" | The list contains the items passed to the model which is defined by the index parameter. Each item is a list of two strings. The order is defined by the `columns` parameter. The `text` string contains the prompt text. The `image` string is the input volume in NIfTI format encoded using base64 and decoded as utf-8 string. The input text is a string containing the target (e.g., organ) to be segmented. |
154+
| `columns` | `list[string]` | Yes | `"image"`, `"text"` | An object containing the strings mapping data to inputs passed to the model.|
155+
| `index` | `integer` | Yes | 0 - 256 | Count of inputs passed to the model. You're limited by how much data can be passed in a single POST request, which depends on the size of your images. Therefore, it's reasonable to keep this number in the dozens. |
156+
| `data` | `list[list[string]]` | Yes | Base64 image + text prompt | The list contains the items passed to the model which is defined by the index parameter. Each item is a list of two strings. The order is defined by the `columns` parameter. The `text` string contains the prompt text. The `image` string is the input volume in NIfTI format encoded using base64 and decoded as utf-8 string. The input text is a string containing the target (e.g., organ) to be segmented. |
135157

136158
### Request example
137159

138-
**Requesting segmentation of all cells in a pathology image**
160+
**Requesting segmentation of pancreas**
139161
```JSON
140162
{
141163
"input_data": {
@@ -154,66 +176,87 @@ The `input_data` object contains the following fields:
154176

155177
### Response schema
156178

157-
Response payload is a list of JSON-formatted strings, each corresponding to a submitted volume. Each string contains a `segmentation_object` object.
179+
The response is a list of objects. Each object contains the segmentation result for one input. The segmentation mask is encoded as a Base64 string inside a serialized JSON object under the key `nifti_file`.
158180

159-
`segmentation_object` contains the following fields:
160-
161-
| Key | Type | Description |
162-
| ------------- | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
163-
| `image_features` | `segmentation_mask` | An object representing the segmentation masks for a given image |
164-
| `text_features` | `list[string]` | List of strings, one per each submitted text string, classifying the segmentation masks into one of 16 biomedical segmentation categories each: `liver`, `lung`, `kidney`, `pancreas`, `heart anatomies`, `brain anatomies`, `eye anatomies`, `vessel`, `other organ`, `tumor`, `infection`, `other lesion`, `fluid disturbance`, `other abnormality`, `histology structure`, `other` |
165-
166-
`segmentation_mask` contains the following fields:
167-
168-
| Key | Type | Description |
169-
| ------------- | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
170-
| `data` | `string` | A base64-encoded NumPy array containing the one-hot encoded segmentation mask. There could be multiple instances of objects in the returned array. Decode and use `np.frombuffer` to deserialize. The array contains a three-dimensional matrix. The array's size is `1024x1024` (matching the input image dimensions), with the third dimension representing the number of input sentences provided. See the provided [sample notebooks](#learn-more-from-samples) for decoding and usage examples. |
171-
| `shape` | `list[int]` | A list representing the shape of the array (typically `[NUM_PROMPTS, 1024, 1024]`) |
172-
| `dtype` | `string` | An instance of the [NumPy dtype class](https://numpy.org/doc/stable/reference/arrays.dtypes.html) serialized to a string. Describes the data packing in the data array. |
181+
| Key | Type | Description |
182+
| -------------- | ------ | --------------------------------------------------------------------------- |
183+
| `nifti_file` | string | JSON-formatted string containing the base64-encoded NIfTI segmentation mask |
173184

174185
### Response example
175-
The requested segmentation mask is stored in NIfTI, represented by an encoded string.
176186

177-
TODO: verify the value of nifti_file is a string or a json object (without the quote).
178-
```JSON
187+
```json
179188
[
180189
{
181-
"nifti_file": "{'data': 'H4sIAAAAAAAE...'}"
190+
"nifti_file": "{\"data\": \"H4sIAAAAAAAE...\"}"
182191
}
183192
]
184193
```
185194

186-
TODO: In an [example notebook](https://dev.azure.com/msazuredev/HLS%20AI%20Platform/_git/3dMedImageParseDeployment?path=/notebooks/01.model.packaging.ipynb&version=GBmain&line=314&lineEnd=315&lineStartColumn=1&lineEndColumn=1&lineStyle=plain&_a=contents), `temp_file.flush()` and `os.unlink(temp_file.name)` are commented out. Are these lines needed?
195+
The `nifti_file` field is a **stringified JSON object**. To decode:
187196

188-
The NIfTI file can be obtained by decoding the returned string using a code like
189197
```python
190-
def decode_base64_to_nifti(base64_string: str) -> nib.Nifti1Image:
198+
import json
199+
import base64
200+
import tempfile
201+
import nibabel as nib
202+
import os
203+
204+
def decode_base64_to_nifti(base64_string: str):
191205
"""
192206
Decode a Base64 string back to a NIfTI image.
193-
207+
194208
Args:
195-
base64_string (str): Base64 encoded string of NIfTI image
196-
209+
base64_string (str): Base64 encoded string of NIfTI image, wrapped in a JSON string
210+
197211
Returns:
198-
nib.Nifti1Image: Decoded NIfTI image object
212+
np.ndarray: Decoded NIfTI image data as a NumPy array
199213
"""
214+
# Parse the inner JSON object to extract the 'data' field
200215
base64_string = json.loads(base64_string)["data"]
201-
# Decode Base64 string to bytes
216+
217+
# Decode the Base64 string to raw bytes
202218
byte_data = base64.b64decode(base64_string)
203-
204-
# Create a temporary file to load the NIfTI image
219+
220+
# Write the decoded bytes to a temporary .nii.gz file
205221
with tempfile.NamedTemporaryFile(suffix='.nii.gz', delete=False) as temp_file:
206-
temp_file.write(byte_data)
207-
temp_file.flush()
208-
# Load NIfTI image from the temporary file
209-
nifti_image = nib.load(temp_file.name)
210-
211-
# Remove temporary file
222+
temp_file.write(byte_data) # Write bytes to file
223+
temp_file.flush() # Ensure it's fully written to disk
224+
nifti_image = nib.load(temp_file.name) # Load the image with nibabel
225+
226+
# Clean up the temporary file to avoid clutter
212227
os.unlink(temp_file.name)
213-
228+
229+
# Return the image as a NumPy array
214230
return nifti_image.get_fdata()
215231
```
232+
To plot segmentation mask
233+
```python
234+
import matplotlib.pyplot as plt
235+
236+
def plot_segmentation_masks(segmentation_masks):
237+
"""
238+
Plot a series of 2D slices from a 3D segmentation mask volume.
239+
Only slices with non-zero masks are displayed.
216240
241+
Args:
242+
segmentation_masks (np.ndarray): A 3D NumPy array of shape (H, W, D),
243+
where D is the number of slices.
244+
"""
245+
index = 1
246+
plt.figure(figsize=(15, 15))
247+
248+
# Loop through each slice (along the depth axis)
249+
for i in range(segmentation_masks.shape[2]):
250+
# Only show slices that contain non-zero segmentation
251+
if segmentation_masks[:, :, i].sum() > 0:
252+
plt.subplot(4, 4, index) # Adjust grid size if needed
253+
plt.imshow(segmentation_masks[:, :, i], cmap='gray')
254+
plt.axis('off')
255+
index += 1
256+
257+
plt.tight_layout()
258+
plt.show()
259+
```
217260
### Supported input formats
218261

219262
The deployed model API supports volumes encoded in NIfTI format.

0 commit comments

Comments
 (0)