Skip to content

Commit dfb7c21

Browse files
daquinteroflexdaquinteroflex
committed
Testing batch endpoints
1 parent 266fe84 commit dfb7c21

File tree

3 files changed

+163
-42
lines changed

3 files changed

+163
-42
lines changed

‎tidy3d/plugins/smatrix/component_modelers/base.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,8 @@ def _warn_rf_license(cls, val):
9090
def get_task_name(port: Port, mode_index: Optional[int] = None) -> str:
9191
"""The name of a task, determined by the port of the source and mode index, if given."""
9292
if mode_index is not None:
93-
return f"smatrix_{port.name}_{mode_index}"
94-
return f"smatrix_{port.name}"
93+
return f"{port.name}_{mode_index}"
94+
return f"{port.name}"
9595

9696
def get_port_by_name(self, port_name: str) -> Port:
9797
"""Get the port from the name."""

‎tidy3d/plugins/smatrix/web/api/webapi.py

Lines changed: 74 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ def run(
176176
Monitor progress of each of the running tasks.
177177
"""
178178
if isinstance(simulation, ComponentModelerType):
179+
modeler = simulation
179180
batch_task = upload_modeler(
180181
simulation=simulation,
181182
task_name=task_name,
@@ -188,23 +189,52 @@ def run(
188189
solver_version=solver_version,
189190
reduce_simulation=reduce_simulation,
190191
)
191-
from tidy3d.plugins.smatrix.run import create_batch
192-
batch = create_batch(
193-
modeler=simulation,
194-
folder_name=folder_name,
195-
solver_version=solver_version,
196-
parent_batch_id=batch_task.batch_id,
197-
group_id=batch_task.group_id
192+
print(batch_task.batch_task_name_map)
193+
print(batch_task.batch_task_name_map.values())
194+
for port, task_id_i in batch_task.batch_task_name_map.items():
195+
simulation_i = modeler.sim_dict[port]
196+
task_id_i_r = upload(
197+
simulation=simulation_i,
198+
task_id=task_id_i,
199+
task_name=task_name,
200+
folder_name=folder_name,
201+
callback_url=callback_url,
202+
verbose=verbose,
203+
progress_callback=progress_callback_upload,
204+
simulation_type=simulation_type,
205+
parent_tasks=parent_tasks,
206+
solver_version=solver_version,
207+
reduce_simulation=reduce_simulation,
208+
)
209+
print(task_id_i)
210+
print(task_id_i_r)
211+
from ..core.http_util import http
212+
# resp = http.post(
213+
# f"tidy3d/projects/{batch_task.batch_id}/batch-check",
214+
# {
215+
# "batchType": "RF_SWEEP",
216+
# "solverVersion": "dario-rf-0.0.0",
217+
# "protocolVersion": "2.10.0rc2"
218+
# },
219+
# )
220+
# print(resp)
221+
# resp = http.post(
222+
# f"tidy3d/projects/{batch_task.batch_id}/batch-submit",
223+
# {
224+
# "batchType": "RF_SWEEP",
225+
# "solverVersion": "dario-rf-0.0.0",
226+
# "protocolVersion": "2.10.0rc2"
227+
# },
228+
# )
229+
resp = http.get(
230+
f"tidy3d/tasks/{batch_task.batch_id}/batch-detail?batchType=RF_SWEEP",
231+
{
232+
"batchType": "RF_SWEEP",
233+
"solverVersion": "dario-rf-0.0.0",
234+
"protocolVersion": "2.10.0rc2"
235+
},
198236
)
199-
batch.run()
200-
# postprocess(
201-
# batch_id,
202-
# solver_version=solver_version,
203-
# worker_group=worker_group,
204-
# pay_type=pay_type,
205-
# priority=priority,
206-
# ) # TODO FIX
207-
237+
print(resp)
208238
from ..core.http_util import http
209239
resp = http.post(
210240
f"tidy3d/projects/{batch_task.batch_id}/postprocess",
@@ -215,6 +245,25 @@ def run(
215245
},
216246
)
217247
print(resp)
248+
249+
# from tidy3d.plugins.smatrix.run import create_batch
250+
# batch = create_batch(
251+
# modeler=simulation,
252+
# folder_name=folder_name,
253+
# solver_version=solver_version,
254+
# parent_batch_id=batch_task.batch_id,
255+
# group_id=batch_task.group_id
256+
# )
257+
# batch.run()
258+
# postprocess(
259+
# batch_id,
260+
# solver_version=solver_version,
261+
# worker_group=worker_group,
262+
# pay_type=pay_type,
263+
# priority=priority,
264+
# ) # TODO FIX
265+
266+
218267
# monitor(task_id, verbose=verbose)
219268
# data = load(
220269
# task_id=task_id, path=path, verbose=verbose, progress_callback=progress_callback_download
@@ -257,6 +306,7 @@ def upload(
257306
simulation: SimulationType,
258307
task_name: str,
259308
folder_name: str = "default",
309+
task_id: Optional[str] = None,
260310
callback_url: Optional[str] = None,
261311
verbose: bool = True,
262312
progress_callback: Optional[Callable[[float], None]] = None,
@@ -330,9 +380,14 @@ def upload(
330380
if (task_type == "COMPONENT_MODELER") or (task_type == "TERMINAL_COMPONENT_MODELER"): # TODO REMOVE
331381
task_type = "RF"
332382

333-
task = SimulationTask.create(
334-
task_type, task_name, folder_name, callback_url, simulation_type, parent_tasks, file_type="Gz", port_name_list=port_name_list
335-
)
383+
if task_id:
384+
task = SimulationTask.get(
385+
task_id=task_id
386+
)
387+
else:
388+
task = SimulationTask.create(
389+
task_type, task_name, folder_name, callback_url, simulation_type, parent_tasks, file_type="Gz", port_name_list=port_name_list
390+
)
336391
print(task)
337392
if verbose:
338393
console = get_logging_console()

‎tidy3d/plugins/smatrix/web/core/task_core.py

Lines changed: 87 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,79 @@
2828
from .types import PayType, Queryable, ResourceLifecycle, Submittable, Tidy3DResource
2929
from tidy3d.plugins.smatrix.web.core.types import TaskType
3030

31+
32+
def upload_simulation(
33+
resource_id: str,
34+
stub: TaskStub,
35+
verbose: bool = True,
36+
progress_callback: Optional[Callable[[float], None]] = None,
37+
remote_sim_file: str = SIM_FILE_HDF5_GZ,
38+
) -> None:
39+
"""Upload :class:`.Simulation` object to Server.
40+
41+
Parameters
42+
----------
43+
stub: :class:`TaskStub`
44+
An instance of TaskStub.
45+
verbose: bool = True
46+
Whether to display progress bars.
47+
progress_callback : Callable[[float], None] = None
48+
Optional callback function called while uploading the data.
49+
"""
50+
if not resource_id:
51+
raise WebError("Expected field 'resource_id' is unset.")
52+
if not stub:
53+
raise WebError("Expected field 'simulation' is unset.")
54+
# Also upload hdf5.gz containing all data.
55+
file, file_name = tempfile.mkstemp()
56+
os.close(file)
57+
try:
58+
# upload simulation
59+
# compress .hdf5 to .hdf5.gz
60+
stub.to_hdf5_gz(file_name)
61+
upload_local_file(
62+
resource_id,
63+
file_name,
64+
remote_sim_file,
65+
verbose=verbose,
66+
progress_callback=progress_callback,
67+
)
68+
finally:
69+
os.unlink(file_name)
70+
71+
72+
def upload_local_file(
73+
resource_id: str,
74+
local_file: str,
75+
remote_filename: str,
76+
verbose: bool = True,
77+
progress_callback: Optional[Callable[[float], None]] = None,
78+
) -> None:
79+
"""
80+
Upload file to platform. Using this method when the json file is too large to parse
81+
as :class".simulation".
82+
Parameters
83+
----------
84+
local_file: str
85+
local file path.
86+
remote_filename: str
87+
file name on the server
88+
verbose: bool = True
89+
Whether to display progress bars.
90+
progress_callback : Callable[[float], None] = None
91+
Optional callback function called while uploading the data.
92+
"""
93+
if not resource_id:
94+
raise WebError("Expected field 'resource_id' is unset.")
95+
96+
upload_file(
97+
resource_id,
98+
local_file,
99+
remote_filename,
100+
verbose=verbose,
101+
progress_callback=progress_callback,
102+
)
103+
31104
class Folder(Tidy3DResource, Queryable, extra=Extra.allow):
32105
"""Tidy3D Folder."""
33106

@@ -432,27 +505,19 @@ def upload_simulation(
432505
raise WebError("Expected fields 'task_id' or 'batch_id' are unset.")
433506
if not stub:
434507
raise WebError("Expected field 'simulation' is unset.")
435-
# Also upload hdf5.gz containing all data.
436-
file, file_name = tempfile.mkstemp()
437-
os.close(file)
438-
try:
439-
# upload simulation
440-
# compress .hdf5 to .hdf5.gz
441-
stub.to_hdf5_gz(file_name)
442-
if (self.task_type == "RF") and (self.task_id is None):
443-
resource_id = self.batch_id
444-
else:
445-
resource_id = self.task_id
446508

447-
upload_file(
448-
resource_id,
449-
file_name,
450-
remote_sim_file,
451-
verbose=verbose,
452-
progress_callback=progress_callback,
453-
)
454-
finally:
455-
os.unlink(file_name)
509+
if (self.task_type == "RF") and (self.task_id is None):
510+
resource_id = self.batch_id
511+
else:
512+
resource_id = self.task_id
513+
514+
upload_simulation(
515+
resource_id=resource_id,
516+
stub=stub,
517+
verbose=verbose,
518+
progress_callback=progress_callback,
519+
remote_sim_file=remote_sim_file
520+
)
456521

457522
def upload_file(
458523
self,
@@ -483,7 +548,7 @@ def upload_file(
483548
else:
484549
resource_id = self.task_id
485550

486-
upload_file(
551+
upload_local_file(
487552
resource_id,
488553
local_file,
489554
remote_filename,
@@ -865,3 +930,4 @@ def validate_post_upload(self, parent_tasks: Optional[list[str]] = None):
865930

866931
except Exception as e:
867932
raise WebError(f"Provided 'parent_tasks' failed validation: {e!s}") from e
933+

0 commit comments

Comments
 (0)