Skip to content

Commit 0800284

Browse files
committed
Add logging to filtered_hyper
1 parent a70e3f3 commit 0800284

File tree

1 file changed

+6
-5
lines changed

1 file changed

+6
-5
lines changed

src/lamp_py/tableau/jobs/filtered_hyper.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from typing import Callable
2-
from datetime import date, timedelta
2+
from datetime import date, datetime, timedelta
33
import pyarrow
44
import pyarrow.parquet as pq
55
import pyarrow.dataset as pd
@@ -83,8 +83,11 @@ def create_tableau_parquet(self, partition_template: str) -> bool:
8383
-------
8484
True if parquet created, False otherwise
8585
"""
86+
process_logger = ProcessLogger("filtered_hyper_create_parquet")
8687
if isinstance(self.start_date, int):
87-
self.end_date = date.today()
88+
end_datetime = datetime.now()
89+
process_logger.add_metadata(now=end_datetime)
90+
self.end_date = end_datetime.date()
8891
self.start_date = self.end_date - timedelta(days=self.start_date)
8992

9093
if self.start_date is not None and self.end_date is not None:
@@ -112,9 +115,7 @@ def create_tableau_parquet(self, partition_template: str) -> bool:
112115
format="parquet",
113116
filesystem=S3FileSystem(),
114117
)
115-
process_logger = ProcessLogger(
116-
"filtered_hyper_create_parquet", start_date=self.start_date, end_date=self.end_date
117-
)
118+
process_logger.add_metadata(start_date=self.start_date, end_date=self.end_date)
118119
process_logger.log_start()
119120
if len(ds_paths) == 0:
120121
process_logger.add_metadata(n_paths_zero=len(ds_paths))

0 commit comments

Comments
 (0)