Skip to content

Commit 85aaa1a

Browse files
Ken LippoldKen Lippold
Ken Lippold
authored and
Ken Lippold
committed
Updated 359 from main
2 parents 3136942 + de76bc3 commit 85aaa1a

File tree

5 files changed

+79
-14
lines changed

5 files changed

+79
-14
lines changed

‎src/hydroserverpy/api/models/base.py‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,10 +117,16 @@ def __init__(
117117

118118
self.filters = data.get("filters")
119119
self.order_by = data.get("order_by")
120-
self.page = data.get("page") or (int(response.headers.get("X-Page")) if response else None)
121-
self.page_size = data.get("page_size") or (int(response.headers.get("X-Page-Size")) if response else None)
122-
self.total_pages = data.get("total_pages") or (int(response.headers.get("X-Total-Pages")) if response else None)
123-
self.total_count = data.get("total_count") or (int(response.headers.get("X-Total-Count")) if response else None)
120+
self.page = self._resolve_int_metadata("page", "X-Page", response, data)
121+
self.page_size = self._resolve_int_metadata(
122+
"page_size", "X-Page-Size", response, data
123+
)
124+
self.total_pages = self._resolve_int_metadata(
125+
"total_pages", "X-Total-Pages", response, data
126+
)
127+
self.total_count = self._resolve_int_metadata(
128+
"total_count", "X-Total-Count", response, data
129+
)
124130

125131
if "items" in data:
126132
self.items = data["items"]
@@ -129,6 +135,24 @@ def __init__(
129135
else:
130136
self.items = []
131137

138+
@staticmethod
139+
def _resolve_int_metadata(
140+
field_name: str,
141+
header_name: str,
142+
response: Optional[Response],
143+
data: dict,
144+
) -> Optional[int]:
145+
field_value = data.get(field_name)
146+
if field_value is not None:
147+
return int(field_value)
148+
149+
if response:
150+
header_value = response.headers.get(header_name)
151+
if header_value is not None:
152+
return int(header_value)
153+
154+
return None
155+
132156
@property
133157
def service(self) -> "HydroServerBaseService":
134158
return self._service

‎src/hydroserverpy/api/models/etl/task.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@
1616

1717
class Task(HydroServerBaseModel):
1818
name: str = Field(..., max_length=255)
19+
task_type: Literal["ETL", "Aggregation"] = Field("ETL", alias="type")
1920
extractor_settings: dict = Field(default_factory=dict, alias="extractorSettings")
2021
transformer_settings: dict = Field(default_factory=dict, alias="transformerSettings")
2122
loader_settings: dict = Field(default_factory=dict, alias="loaderSettings")
22-
data_connection_id: uuid.UUID = Field(
23+
data_connection_id: Optional[uuid.UUID] = Field(
2324
None, validation_alias=AliasChoices("dataConnectionId", AliasPath("dataConnection", "id"))
2425
)
2526
orchestration_system_id: uuid.UUID = Field(
@@ -41,6 +42,7 @@ class Task(HydroServerBaseModel):
4142

4243
_editable_fields: ClassVar[set[str]] = {
4344
"name",
45+
"task_type",
4446
"extractor_settings",
4547
"transformer_settings",
4648
"loader_settings",
@@ -72,6 +74,8 @@ def orchestration_system(self) -> Optional[OrchestrationSystem]:
7274

7375
@cached_property
7476
def data_connection(self) -> Optional[DataConnection]:
77+
if not self.data_connection_id:
78+
return None
7579
return self.client.dataconnections.get(uid=self.data_connection_id)
7680

7781
def get_task_runs(

‎src/hydroserverpy/api/models/sta/observation.py‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,21 @@ def __init__(
2525
**data
2626
):
2727
self.filters = data.get("filters")
28-
self.order_by = data.get("order_by")
29-
self.page = data.get("page") or (int(response.headers.get("X-Page")) if response else None)
30-
self.page_size = data.get("page_size") or (int(response.headers.get("X-Page-Size")) if response else None)
31-
self.total_pages = data.get("total_pages") or (int(response.headers.get("X-Total-Pages")) if response else None)
32-
self.total_count = data.get("total_count") or (int(response.headers.get("X-Total-Count")) if response else None)
28+
raw_order_by = data.get("order_by")
29+
if isinstance(raw_order_by, str):
30+
self.order_by = [item for item in raw_order_by.split(",") if item]
31+
else:
32+
self.order_by = raw_order_by
33+
self.page = self._resolve_int_metadata("page", "X-Page", response, data)
34+
self.page_size = self._resolve_int_metadata(
35+
"page_size", "X-Page-Size", response, data
36+
)
37+
self.total_pages = self._resolve_int_metadata(
38+
"total_pages", "X-Total-Pages", response, data
39+
)
40+
self.total_count = self._resolve_int_metadata(
41+
"total_count", "X-Total-Count", response, data
42+
)
3343
self.datastream = datastream
3444

3545
if "dataframe" in data:
@@ -44,6 +54,24 @@ def __init__(
4454
else:
4555
self.dataframe = pd.DataFrame()
4656

57+
@staticmethod
58+
def _resolve_int_metadata(
59+
field_name: str,
60+
header_name: str,
61+
response: Optional[Response],
62+
data: dict,
63+
) -> Optional[int]:
64+
field_value = data.get(field_name)
65+
if field_value is not None:
66+
return int(field_value)
67+
68+
if response:
69+
header_value = response.headers.get(header_name)
70+
if header_value is not None:
71+
return int(header_value)
72+
73+
return None
74+
4775
def next_page(self):
4876
"""Fetches the next page of data from HydroServer."""
4977

@@ -89,7 +117,10 @@ def fetch_all(self) -> "ObservationCollection":
89117

90118
page_num += 1
91119

92-
merged_dataframe = pd.concat(all_dataframes, ignore_index=True)
120+
if not all_dataframes:
121+
merged_dataframe = self.dataframe.iloc[0:0].copy()
122+
else:
123+
merged_dataframe = pd.concat(all_dataframes, ignore_index=True)
93124

94125
return self.__class__(
95126
dataframe=merged_dataframe,

‎src/hydroserverpy/api/services/etl/task.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ def list(
2222
page_size: int = ...,
2323
order_by: List[str] = ...,
2424
workspace: Optional[Union["Workspace", UUID, str]] = ...,
25+
task_type: str = ...,
2526
orchestration_system: Optional[Union["OrchestrationSystem", UUID, str]] = ...,
2627
orchestration_system_type: str = ...,
2728
data_connection: Union["Workspace", UUID, str] = ...,
@@ -50,6 +51,7 @@ def list(
5051
page_size=page_size,
5152
order_by=order_by,
5253
workspace_id=normalize_uuid(workspace),
54+
type=task_type,
5355
orchestration_system_id=normalize_uuid(orchestration_system),
5456
orchestration_system_type=orchestration_system_type,
5557
data_connection_id=normalize_uuid(data_connection),
@@ -76,8 +78,9 @@ def create(
7678
self,
7779
name: str,
7880
workspace: Union["Workspace", UUID, str],
79-
data_connection: Union["DataConnection", UUID, str],
8081
orchestration_system: Union["OrchestrationSystem", UUID, str],
82+
data_connection: Optional[Union["DataConnection", UUID, str]] = None,
83+
task_type: str = "ETL",
8184
extractor_variables: Optional[dict] = None,
8285
transformer_variables: Optional[dict] = None,
8386
loader_variables: Optional[dict] = None,
@@ -95,6 +98,7 @@ def create(
9598
body = {
9699
"id": normalize_uuid(uid),
97100
"name": name,
101+
"type": task_type,
98102
"workspaceId": normalize_uuid(workspace),
99103
"dataConnectionId": normalize_uuid(data_connection),
100104
"orchestrationSystemId": normalize_uuid(orchestration_system),
@@ -121,8 +125,9 @@ def create(
121125
def update(
122126
self,
123127
uid: Union[UUID, str],
128+
task_type: str = ...,
124129
name: str = ...,
125-
data_connection: Union["DataConnection", UUID, str] = ...,
130+
data_connection: Optional[Union["DataConnection", UUID, str]] = ...,
126131
orchestration_system: Union["OrchestrationSystem", UUID, str] = ...,
127132
extractor_variables: dict = ...,
128133
transformer_variables: dict = ...,
@@ -138,6 +143,7 @@ def update(
138143
"""Update an ETL task."""
139144

140145
body: Dict[str, Any] = {
146+
"type": task_type,
141147
"name": name,
142148
"dataConnectionId": normalize_uuid(data_connection),
143149
"orchestrationSystemId": normalize_uuid(orchestration_system),

‎src/hydroserverpy/api/services/sta/datastream.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ def get_observations(
253253
collection = ObservationCollection(
254254
datastream=datastream,
255255
response=response,
256-
order_by=params.get("order_by"),
256+
order_by=order_by if order_by is not ... else None,
257257
filters={k: v for k, v in params.items() if k not in ["page", "page_size", "order_by", "format"]},
258258
)
259259
if fetch_all is True:

0 commit comments

Comments
 (0)