Skip to content

Commit 33ca4c2

Browse files
Ken LippoldKen Lippold
authored andcommitted
Fixed datastream ETL start load time calculation
1 parent d6353f4 commit 33ca4c2

File tree

1 file changed

+4
-1
lines changed

1 file changed

+4
-1
lines changed

src/hydroserverpy/api/models/etl/loaders/hydroserver_loader.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,12 @@ def load(self, data: pd.DataFrame, payload: Payload) -> None:
2828
begin_date = self.earliest_begin_date(payload)
2929
new_data = data[data["timestamp"] > begin_date]
3030
for col in new_data.columns.difference(["timestamp"]):
31+
datastream = self.client.datastreams.get(
32+
uid=str(col)
33+
)
3134
df = (
3235
new_data[["timestamp", col]]
36+
.loc[lambda d: d["timestamp"] > datastream.phenomenon_end_time]
3337
.rename(columns={col: "value"})
3438
.dropna(subset=["value"])
3539
)
@@ -67,7 +71,6 @@ def load(self, data: pd.DataFrame, payload: Payload) -> None:
6771
start,
6872
end - 1,
6973
)
70-
break
7174
raise
7275

7376
def _fetch_earliest_begin(

0 commit comments

Comments
 (0)