From 8e9ef5ebb305fafd1b1b158d1cecbab7bbf85b3b Mon Sep 17 00:00:00 2001 From: tomADC443 <61840572+tomADC443@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:58:11 +0100 Subject: [PATCH 01/24] feat: check that time range is valid for S2 harmonized collection --- backend/src/routes/ndvi_router.py | 16 +++++++++------- backend/src/validation/utils.py | 18 +++++++++++++++--- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/src/routes/ndvi_router.py b/backend/src/routes/ndvi_router.py index e60e9ec..0fbeb61 100644 --- a/backend/src/routes/ndvi_router.py +++ b/backend/src/routes/ndvi_router.py @@ -13,7 +13,7 @@ from src.utils.temporal import get_optimistic_rounding from src.validation.models import NDVIResponse from src.validation.utils import ( - validate_timestamp_in_range, + validate_timestamp_in_range_of_S2_imagery, validate_timestamp_start_date_before_end_date, ) @@ -22,18 +22,20 @@ @ndvi_router.get("/ndvi", response_model=NDVIResponse) async def get_temperature_data( - startDate: int = Query(..., description="Start date as UNIX timestamp in seconds"), - endDate: int = Query(..., description="End date as UNIX timestamp in seconds"), + startDate: int = Query(..., + description="Start date as UNIX timestamp in seconds"), + endDate: int = Query(..., + description="End date as UNIX timestamp in seconds"), location: LocationName = Query(..., description="Location name"), temporalResolution: TemporalResolution = Query( ..., description="Temporal resolution" ), - aggregation: AggregationMethod = Query(..., description="Aggregation method"), + aggregation: AggregationMethod = Query(..., + description="Aggregation method"), ): - validate_timestamp_in_range(startDate) - validate_timestamp_in_range(endDate) - validate_timestamp_start_date_before_end_date(startDate, endDate) + validate_timestamp_start_date_before_end_date(startDate, endDate) + validate_timestamp_in_range_of_S2_imagery(startDate, endDate) start_date_dt = datetime.utcfromtimestamp(startDate) end_date_dt = datetime.utcfromtimestamp(endDate) diff --git a/backend/src/validation/utils.py b/backend/src/validation/utils.py index 72d3488..d08708e 100644 --- a/backend/src/validation/utils.py +++ b/backend/src/validation/utils.py @@ -1,4 +1,5 @@ from fastapi import HTTPException +import time # Timestamp @@ -6,11 +7,22 @@ def validate_timestamp_in_range(timestamp): min_timestamp = 1388530801 # 01/01/2014 max_timestamp = 2208985201 # 01/01/2040 if not (min_timestamp <= timestamp <= max_timestamp): - raise HTTPException(status_code=400, detail=f"Timestamp must be between {min_timestamp} and {max_timestamp}") + raise HTTPException( + status_code=400, detail=f"Timestamp must be between {min_timestamp} and {max_timestamp}") return timestamp +def validate_timestamp_in_range_of_S2_imagery(start_timestamp, end_timestamp): + min_timestamp = 1491004800 # 01/04/2017 + max_timestamp = int(time.time()) # Current timestamp + if not (min_timestamp <= start_timestamp <= max_timestamp and min_timestamp <= end_timestamp <= max_timestamp): + raise HTTPException( + status_code=400, detail=f"Timestamp must be between {min_timestamp} and now ({max_timestamp})") + return + + def validate_timestamp_start_date_before_end_date(startDate, endDate): if endDate <= startDate: - raise HTTPException(status_code=400, detail="endDate must be after startDate") - return endDate \ No newline at end of file + raise HTTPException( + status_code=400, detail="endDate must be after startDate") + return endDate From 78d8aee0f9535b6d3394aef3f2e7f3e790ec04ea Mon Sep 17 00:00:00 2001 From: tomADC443 <61840572+tomADC443@users.noreply.github.com> Date: Wed, 13 Nov 2024 18:20:53 +0100 Subject: [PATCH 02/24] feat: add preprocessing step to remove unnecessary bands from images --- backend/src/gee/index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/src/gee/index.py b/backend/src/gee/index.py index 1834e48..e45001f 100644 --- a/backend/src/gee/index.py +++ b/backend/src/gee/index.py @@ -19,9 +19,12 @@ def get_preprocessed_imagery( image_collection = get_imagery(aoi, start_date, end_date) # Preprocess the imagery + image_collection = image_collection.select( + "SCL", "B8", "B4") # Removing unused bands mosaicked_collection = get_mosaicked_by_date_collection(image_collection) clipped_collection = mosaicked_collection.map(lambda img: img.clip(aoi)) - cloud_masked_collection = clipped_collection.map(lambda img: get_cloud_masked(img)) + cloud_masked_collection = clipped_collection.map( + lambda img: get_cloud_masked(img)) return cloud_masked_collection From 3e35820d603dc9c44b4b212254ef17fa667abbf4 Mon Sep 17 00:00:00 2001 From: tomADC443 <61840572+tomADC443@users.noreply.github.com> Date: Wed, 13 Nov 2024 18:23:48 +0100 Subject: [PATCH 03/24] refactor: Aggregation and Filler DataFrame operations into two separate functions --- backend/src/service.py | 250 ++++++++++++++++++++++++++++------------- 1 file changed, 175 insertions(+), 75 deletions(-) diff --git a/backend/src/service.py b/backend/src/service.py index 8c4871b..21e4ee1 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -1,77 +1,106 @@ -from datetime import datetime +from datetime import datetime, timezone, timedelta import pandas as pd from src.constants import AggregationMethod, LocationPolygon, TemporalResolution from src.gee.index import get_preprocessed_imagery from src.gee.ndvi import get_ndvi_info +from src.gee.ndvi_cache import ndvi_daily_cache +from typing import List, Dict, Union +import math -def aggregate_time_series( - ndvi_info: list[dict], - temporal_resolution, - aggregation_method, - start_date: datetime, - end_date: datetime, -): - # Generate a complete date range based on temporal resolution - if temporal_resolution == "DAILY": - date_range = pd.date_range(start=start_date, end=end_date, freq="D", tz="UTC") - elif temporal_resolution == "MONTHLY": - date_range = pd.date_range(start=start_date, end=end_date, freq="MS", tz="UTC") - else: - raise ValueError("Unsupported temporal resolution") - - # Create a DataFrame with the full date range, initially filled with None - df = pd.DataFrame(index=date_range) - df.index.name = "timestamp" - df["value"] = None - - # Convert ndvi_info to a DataFrame and set the index - if temporal_resolution == "MONTHLY": - for record in ndvi_info: - record["timestamp"] = record["timestamp"].replace(day=1) - info_df = pd.DataFrame(ndvi_info) - info_df["timestamp"] = pd.to_datetime(info_df["timestamp"], unit="s", utc=True) - - # Align info_df to the temporal resolution - if temporal_resolution == "DAILY": - info_df["timestamp"] = info_df["timestamp"].dt.floor("D") - elif temporal_resolution == "MONTHLY": - info_df["timestamp"] = info_df["timestamp"].dt.to_period("M").dt.to_timestamp() - - info_df.set_index("timestamp", inplace=True) - - # Update the full DataFrame with actual NDVI values - df.loc[info_df.index, "value"] = info_df["value"] - - # Resample the DataFrame based on the temporal resolution - resampled_df = ( - df.resample("D") if temporal_resolution == "DAILY" else df.resample("M") - ) - - # Aggregate the resampled DataFrame based on the aggregation method, ignoring None values - if aggregation_method == "MEAN": - aggregated_df = resampled_df.mean() - elif aggregation_method == "MIN": - aggregated_df = resampled_df.min() - elif aggregation_method == "MAX": - aggregated_df = resampled_df.max() - elif aggregation_method == "MEDIAN": - aggregated_df = resampled_df.median() +def initialize_time_series( + time_series: List[Dict[str, Union[int, float]]], + temporal_resolution: TemporalResolution, + aggregation_method: AggregationMethod +) -> pd.DataFrame: + """ + Initializes a pandas DataFrame from a time series and applies temporal resolution and aggregation. + + Parameters: + time_series (List[Dict[str, Union[int, float]]]): List of dicts with 'timestamp' and 'value'. + temporal_resolution (TemporalResolution): Temporal resolution, either DAILY or MONTHLY. + aggregation_method (AggregationMethod): Aggregation method to use if resolution is MONTHLY. + + Returns: + pd.DataFrame: The resulting DataFrame with applied resolution and aggregation. + """ + # Check if the time_series is empty + if not time_series: + # Return an empty DataFrame with a datetime index and 'value' column in UTC + if temporal_resolution == TemporalResolution.MONTHLY: + empty_index = pd.date_range( + start="1970-01-01", periods=0, freq='MS', tz='UTC') + else: + empty_index = pd.date_range( + start="1970-01-01", periods=0, freq='D', tz='UTC') + + return pd.DataFrame(index=empty_index, columns=['value']) + + # Convert timestamps to datetime in UTC and create DataFrame + df = pd.DataFrame(time_series) + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s', utc=True) + df.set_index('timestamp', inplace=True) + + # Resample based on temporal resolution and apply aggregation if needed + if temporal_resolution == TemporalResolution.MONTHLY: + if aggregation_method == AggregationMethod.MEAN: + df = df.resample('MS').mean() + elif aggregation_method == AggregationMethod.MEDIAN: + df = df.resample('MS').median() + elif aggregation_method == AggregationMethod.MAX: + df = df.resample('MS').max() + elif aggregation_method == AggregationMethod.MIN: + df = df.resample('MS').min() + # If DAILY, do nothing as time series is already in daily format + return df + + +def fill_missing_dates( + df: pd.DataFrame, + start: datetime, + end: datetime, + temporal_resolution: TemporalResolution +) -> pd.DataFrame: + """ + Fills missing entries in the time series, adding NaN for missing days or months. + + Parameters: + df (pd.DataFrame): Input DataFrame with timestamps as index. + start (datetime): Start datetime for filling. + end (datetime): End datetime for filling. + temporal_resolution (TemporalResolution): Temporal resolution, either DAILY or MONTHLY. + + Returns: + pd.DataFrame: DataFrame with missing dates or months filled with NaN values. + """ + # Ensure start and end are in UTC + if start.tzinfo is None: + start = start.replace(tzinfo=timezone.utc) else: - raise ValueError("Unsupported aggregation method") - - # Replace NaNs with None for final output consistency - aggregated_df = aggregated_df.where(pd.notnull(aggregated_df), None) + start = start.astimezone(timezone.utc) - # Convert the aggregated DataFrame back to a list of dicts with ISO format timestamps - aggregated_info = [ - {"timestamp": record["timestamp"].isoformat(), "value": record["value"]} - for record in aggregated_df.reset_index().to_dict(orient="records") - ] + if end.tzinfo is None: + end = end.replace(tzinfo=timezone.utc) + else: + end = end.astimezone(timezone.utc) + + # Generate the complete date range based on the temporal resolution + if temporal_resolution == TemporalResolution.DAILY: + date_range = pd.date_range(start=start, end=end, freq='D', tz='UTC') + elif temporal_resolution == TemporalResolution.MONTHLY: + date_range = pd.date_range(start=start, end=end, freq='MS', tz='UTC') + # If the input DataFrame is empty, create a new one with NaNs for all dates in the range + if df.empty: + df = pd.DataFrame(index=date_range, columns=['value']) + df['value'] = None + else: + # Reindex to the complete date range, filling missing dates with NaN + df = df.reindex(date_range) - return aggregated_info + df.columns = ['value'] + return df def ndvi_service( @@ -81,15 +110,86 @@ def ndvi_service( start_date: datetime, end_date: datetime, ): - masked_images = get_preprocessed_imagery( - LocationPolygon[location.value].value, - start_date, - end_date, - ) - NDVI_time_series = get_ndvi_info( - masked_images, LocationPolygon[location.value].value - ) - aggregated_data_dict = aggregate_time_series( - NDVI_time_series, temporal_resolution, aggregation_method, start_date, end_date - ) - return aggregated_data_dict + # Temporary implementation of GEE Caching strategy + current_cache_end_date = datetime( + 2024, 9, 29, tzinfo=timezone.utc) + if start_date < current_cache_end_date and end_date < current_cache_end_date: # current end of cache + cache_start_date = start_date + cache_end_date = end_date + processing_start_date = None + processing_end_date = None + + elif start_date < current_cache_end_date and end_date > current_cache_end_date: + cache_start_date = start_date + cache_end_date = current_cache_end_date + processing_start_date = current_cache_end_date + timedelta(days=1) + processing_end_date = end_date + + elif start_date > current_cache_end_date: + cache_start_date = None + cache_end_date = None + processing_start_date = start_date + processing_end_date = end_date + + if processing_start_date: + + masked_images = get_preprocessed_imagery( + LocationPolygon[location.value].value, + processing_start_date, + processing_end_date, + ) + NDVI_time_series = get_ndvi_info( + masked_images, LocationPolygon[location.value].value + ) + + if cache_start_date: + cached_data_subset = get_cache_subset(cache_start_date, cache_end_date) + + if processing_start_date and cache_start_date: + ndvi_data = cached_data_subset + NDVI_time_series + else: + ndvi_data = cached_data_subset if cache_start_date else NDVI_time_series + + index_df = initialize_time_series( + ndvi_data, temporal_resolution, aggregation_method) + + filled_df = fill_missing_dates( + index_df, start_date, end_date, temporal_resolution) + + return convert_df_to_list(filled_df) + + +def get_cache_subset(start_date: datetime, end_date: datetime): + subset: list[dict] = [] + for entry in ndvi_daily_cache: + if entry["timestamp"] >= int(start_date.timestamp()) and entry["timestamp"] <= int(end_date.timestamp()): + subset.append(entry) + return subset + + +def convert_df_to_list(df: pd.DataFrame) -> List[Dict[str, Union[int, float, None]]]: + """ + Converts a DataFrame with a datetime index back to a list of dictionaries in the original format. + + Parameters: + df (pd.DataFrame): Input DataFrame with datetime index and a 'value' column. + + Returns: + List[Dict[str, Union[int, float, None]]]: List of dictionaries with 'timestamp' in epoch format and 'value'. + """ + # Convert the DataFrame index to epoch timestamps and reset index + df_reset = df.reset_index() + df_reset['timestamp'] = df_reset['index'].astype(int) // 10**9 + df_reset = df_reset.rename(columns={'value': 'value'}) + + # Convert to list of dictionaries + result = df_reset[['timestamp', 'value']].to_dict(orient='records') + + # Convert NaN to None (needs to handle empty df as well) + for entry in result: + if entry['value'] is None: + entry['value'] = None + elif math.isnan(entry['value']): + entry['value'] = None + + return result From d06b41f143f03e0aa25a4eb03f8c7c0f1aedc445 Mon Sep 17 00:00:00 2001 From: tomADC443 <61840572+tomADC443@users.noreply.github.com> Date: Wed, 13 Nov 2024 18:24:41 +0100 Subject: [PATCH 04/24] refactor: temporal utility functions (cleaner, more readable) --- backend/src/utils/temporal.py | 49 ++++++++++++----------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/backend/src/utils/temporal.py b/backend/src/utils/temporal.py index 5eb73cf..90bc123 100644 --- a/backend/src/utils/temporal.py +++ b/backend/src/utils/temporal.py @@ -1,3 +1,4 @@ +from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta, timezone from src.constants import TemporalResolution @@ -7,46 +8,30 @@ def get_optimistic_rounding( start_date: datetime, end_date: datetime, temporal_resolution: TemporalResolution ) -> (datetime, datetime): """Get optimistic rounding for the start date based on the temporal resolution.""" - start_date = start_date.astimezone(timezone.utc) - end_date = end_date.astimezone(timezone.utc) + if temporal_resolution == TemporalResolution.DAILY: - start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) - end_date = end_date.replace(hour=23, minute=59, second=59, microsecond=999999) + start_date = first_second_of_day(start_date) + end_date = last_second_of_day(end_date) elif temporal_resolution == TemporalResolution.MONTHLY: - start_date = get_start_of_day(start_date) - end_date = last_time_of_month(end_date) - # Set end_date to the last day of the month at 23:59:59.999999 - next_month = end_date.replace(day=1, month=end_date.month % 12 + 1) - end_date = next_month - timedelta(seconds=1) - else: - raise ValueError(f"Temporal resolution {temporal_resolution} not supported.") - print(start_date, end_date) - return start_date, end_date - + start_date = first_second_of_month(start_date) + end_date = last_second_of_month(end_date) -def last_time_of_month(dt: datetime) -> datetime: - # Move to the beginning of the next month - if dt.month == 12: - next_month = datetime(dt.year + 1, 1, 1) - else: - next_month = datetime(dt.year, dt.month + 1, 1) + return start_date, end_date - # Subtract one microsecond to get the last moment of the given month - last_time = next_month - timedelta(microseconds=1) - return last_time +def first_second_of_day(dt: datetime) -> datetime: + return dt.replace(hour=0, minute=0, second=0, microsecond=0) -def get_start_of_day(dt: datetime) -> datetime: - return dt.replace(hour=0, minute=0, second=0, microsecond=0) +def last_second_of_day(dt: datetime) -> datetime: + return dt.replace(hour=23, minute=59, second=59, microsecond=0) -def get_month_start(unix_timestamp: int) -> int: - # Convert Unix timestamp to a datetime object in UTC - dt = datetime.fromtimestamp(unix_timestamp, tz=timezone.utc) +def first_second_of_month(dt: datetime) -> datetime: + return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - # Create a new datetime object for the beginning of the month - month_start = datetime(dt.year, dt.month, 1, tzinfo=timezone.utc) - # Convert back to Unix timestamp - return int(month_start.timestamp()) +def last_second_of_month(dt: datetime) -> datetime: + next_month = dt.replace(day=1, hour=0, minute=0, + second=0, microsecond=0) + relativedelta(months=1) + return (next_month - timedelta(seconds=1)).replace(tzinfo=timezone.utc) From 128b9c414b9695972b3ced6794382b5af95d67b9 Mon Sep 17 00:00:00 2001 From: tomADC443 <61840572+tomADC443@users.noreply.github.com> Date: Wed, 13 Nov 2024 18:26:13 +0100 Subject: [PATCH 05/24] fix: different timezones (now always utc) --- backend/src/gee/ndvi.py | 5 +++-- backend/src/routes/ndvi_router.py | 8 +++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/src/gee/ndvi.py b/backend/src/gee/ndvi.py index 85ccf9a..a571b56 100644 --- a/backend/src/gee/ndvi.py +++ b/backend/src/gee/ndvi.py @@ -53,9 +53,10 @@ def get_ndvi_info( ).getInfo() if len(timestamp_list) != len(index_value_list): - print(f"Timestamps: {len(timestamp_list)}, Values: {len(index_value_list)}") raise ValueError( - "The lists of gee indexing values and timestamps do not have the same size" + "The lists of gee indexing values and timestamps do not have the same size" + + f"Timestamps: {len(timestamp_list)}, Values: {len(index_value_list)}" + ) # Convert the results to a list of dictionaries diff --git a/backend/src/routes/ndvi_router.py b/backend/src/routes/ndvi_router.py index 0fbeb61..0e60ffc 100644 --- a/backend/src/routes/ndvi_router.py +++ b/backend/src/routes/ndvi_router.py @@ -1,8 +1,7 @@ -from datetime import datetime +from datetime import datetime, timezone from fastapi import APIRouter, Query from fastapi.responses import JSONResponse - from src.constants import ( AggregationMethod, LocationName, @@ -36,8 +35,8 @@ async def get_temperature_data( validate_timestamp_start_date_before_end_date(startDate, endDate) validate_timestamp_in_range_of_S2_imagery(startDate, endDate) - start_date_dt = datetime.utcfromtimestamp(startDate) - end_date_dt = datetime.utcfromtimestamp(endDate) + start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc) + end_date_dt = datetime.fromtimestamp(endDate, tz=timezone.utc) rounded_start_date, rounded_end_date = get_optimistic_rounding( start_date_dt, end_date_dt, temporalResolution @@ -63,5 +62,4 @@ async def get_temperature_data( "data": data, } - print(response) return JSONResponse(content=response) From 326e70ab657a9ac548d9cac7fe9b751e1b10be35 Mon Sep 17 00:00:00 2001 From: tomADC443 <61840572+tomADC443@users.noreply.github.com> Date: Wed, 13 Nov 2024 18:26:44 +0100 Subject: [PATCH 06/24] feat: introduce static cache file --- backend/src/gee/ndvi_cache.py | 539 ++++++++++++++++++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 backend/src/gee/ndvi_cache.py diff --git a/backend/src/gee/ndvi_cache.py b/backend/src/gee/ndvi_cache.py new file mode 100644 index 0000000..115cfd9 --- /dev/null +++ b/backend/src/gee/ndvi_cache.py @@ -0,0 +1,539 @@ +ndvi_daily_cache = [ + + {'timestamp': 1493251200, 'value': 0.6265267304234295}, + {'timestamp': 1494720000, 'value': 0.68603163673333}, + {'timestamp': 1494979200, 'value': 0.755257128311451}, + {'timestamp': 1495843200, 'value': 0.7354373002096525}, + {'timestamp': 1496707200, 'value': 0.31356286742809775}, + {'timestamp': 1497571200, 'value': 0.4542143670691183}, + {'timestamp': 1498435200, 'value': 0.5919617512091055}, + {'timestamp': 1499040000, 'value': 0.6539697157884732}, + {'timestamp': 1499299200, 'value': 0.6303952412263235}, + {'timestamp': 1499904000, 'value': 0.3453876749102669}, + {'timestamp': 1501632000, 'value': 0.4050406969946647}, + {'timestamp': 1502755200, 'value': 0.6429425803278062}, + {'timestamp': 1504483200, 'value': 0.5918015606195185}, + {'timestamp': 1505088000, 'value': 0.5199806437043044}, + {'timestamp': 1505952000, 'value': 0.5613072154540079}, + {'timestamp': 1509667200, 'value': 0.355040321129086}, + {'timestamp': 1511136000, 'value': 0.4793029710214357}, + {'timestamp': 1512000000, 'value': 0.5144682745732605}, + {'timestamp': 1512259200, 'value': 0.4812850460359476}, + {'timestamp': 1512864000, 'value': 0.4915769963854998}, + {'timestamp': 1513123200, 'value': 0.5612609179087754}, + {'timestamp': 1513555200, 'value': 0.49397625433694664}, + {'timestamp': 1513728000, 'value': 0.10078945539648299}, + {'timestamp': 1513987200, 'value': -0.05380305031374222}, + # until here its 2017 + {'timestamp': 1515888000, 'value': 0.01634385135231174}, + {'timestamp': 1516579200, 'value': 0.4084963960575644}, + {'timestamp': 1517443200, 'value': 0.5275309839482423}, + {'timestamp': 1517875200, 'value': 0.506521617561721}, + {'timestamp': 1518048000, 'value': 0.490625119426628}, + {'timestamp': 1518480000, 'value': 0.5111775938710303}, + {'timestamp': 1518739200, 'value': 0.46794661547132277}, + {'timestamp': 1519344000, 'value': 0.36935062396906804}, + {'timestamp': 1519776000, 'value': 0.27336033950917427}, + {'timestamp': 1520035200, 'value': 0.37743127832902273}, + {'timestamp': 1521331200, 'value': 0.342227538494118}, + {'timestamp': 1521936000, 'value': 0.3081052442616005}, + {'timestamp': 1522627200, 'value': 0.37092985707956994}, + {'timestamp': 1523059200, 'value': 0.4297057750906333}, + {'timestamp': 1523232000, 'value': 0.4387548022615023}, + {'timestamp': 1523923200, 'value': 0.65378630233386}, + {'timestamp': 1524096000, 'value': 0.633789637457917}, + {'timestamp': 1524355200, 'value': 0.7060110186976779}, + {'timestamp': 1524787200, 'value': 0.7624711921453309}, + {'timestamp': 1524960000, 'value': 0.7497715661757918}, + {'timestamp': 1525392000, 'value': 0.7596553124834161}, + {'timestamp': 1525651200, 'value': 0.754118311745244}, + {'timestamp': 1525824000, 'value': 0.7232205427443583}, + {'timestamp': 1526083200, 'value': 0.3822415269246948}, + {'timestamp': 1526256000, 'value': 0.7340398528120343}, + {'timestamp': 1526515200, 'value': 0.28521040360598626}, + {'timestamp': 1526947200, 'value': 0.6952578447035102}, + {'timestamp': 1527120000, 'value': 0.6232964708384111}, + {'timestamp': 1527552000, 'value': 0.6300091691019815}, + {'timestamp': 1527811200, 'value': 0.5837729773361593}, + {'timestamp': 1528243200, 'value': 0.6409030517490615}, + {'timestamp': 1528416000, 'value': 0.6041067426351739}, + {'timestamp': 1528675200, 'value': 0.5428701270313011}, + {'timestamp': 1529107200, 'value': 0.44278144287635474}, + {'timestamp': 1529280000, 'value': 0.34814687536377137}, + {'timestamp': 1530403200, 'value': 0.5099651844736147}, + {'timestamp': 1530576000, 'value': 0.4806309790189844}, + {'timestamp': 1531008000, 'value': 0.29236687187783805}, + {'timestamp': 1531699200, 'value': 0.4351651719304671}, + {'timestamp': 1531872000, 'value': 0.40396922025585447}, + {'timestamp': 1532304000, 'value': 0.4929734694779009}, + {'timestamp': 1532563200, 'value': 0.5049480674886351}, + {'timestamp': 1532736000, 'value': 0.47175783767312296}, + {'timestamp': 1532995200, 'value': 0.4686040013961713}, + {'timestamp': 1533427200, 'value': 0.3689835971360573}, + {'timestamp': 1533600000, 'value': 0.41796636527900427}, + {'timestamp': 1534032000, 'value': 0.38439341812607486}, + {'timestamp': 1534896000, 'value': 0.3416124871522577}, + {'timestamp': 1535155200, 'value': 0.2764639043940805}, + {'timestamp': 1535760000, 'value': 0.2294651988350261}, + {'timestamp': 1536019200, 'value': 0.2994392537612457}, + {'timestamp': 1536192000, 'value': 0.2836145495281138}, + {'timestamp': 1536451200, 'value': 0.2877444495163961}, + {'timestamp': 1537315200, 'value': 0.2518901279728607}, + {'timestamp': 1537747200, 'value': 0.04412068554646227}, + {'timestamp': 1538179200, 'value': 0.27060413178935944}, + {'timestamp': 1538352000, 'value': 0.218526678880176}, + {'timestamp': 1539216000, 'value': 0.3358831303480651}, + {'timestamp': 1539475200, 'value': 0.34343979370280736}, + {'timestamp': 1539648000, 'value': 0.32026366381651145}, + {'timestamp': 1539907200, 'value': 0.31340923136862847}, + {'timestamp': 1540080000, 'value': 0.36016472321904647}, + {'timestamp': 1540339200, 'value': 0.3460083515305792}, + {'timestamp': 1540944000, 'value': 0.42180934711053264}, + {'timestamp': 1541203200, 'value': 0.3606669028141891}, + {'timestamp': 1541635200, 'value': 0.03708648541774034}, + {'timestamp': 1542499200, 'value': 0.17338581915345114}, + {'timestamp': 1543363200, 'value': 0.4738111853498088}, + {'timestamp': 1543968000, 'value': 0.47951016892420906}, + {'timestamp': 1544227200, 'value': 0.0663400635450533}, + {'timestamp': 1544400000, 'value': 0.22558175917054904}, + {'timestamp': 1545696000, 'value': 0.01207277210298325}, + {'timestamp': 1546128000, 'value': 0.06949516092806259}, + # until here its 2018 + + {'timestamp': 1546387200, 'value': 0.19575113537918123}, + {'timestamp': 1546819200, 'value': 0.004088486747965585}, + {'timestamp': 1547424000, 'value': 0.4628066318766027}, + {'timestamp': 1548115200, 'value': 0.4601170296239082}, + {'timestamp': 1548547200, 'value': 0.16502884449147368}, + {'timestamp': 1549584000, 'value': 0.45229330852409055}, + {'timestamp': 1549843200, 'value': 0.15426414088875784}, + {'timestamp': 1550275200, 'value': 0.4844492269864073}, + {'timestamp': 1550448000, 'value': 0.45415724658453716}, + {'timestamp': 1550707200, 'value': 0.23666395258440662}, + {'timestamp': 1551139200, 'value': 0.07907500390519201}, + {'timestamp': 1551744000, 'value': 0.1974367818867876}, + {'timestamp': 1552435200, 'value': 0.2763185157667595}, + {'timestamp': 1552867200, 'value': -0.0071800199836781695}, + {'timestamp': 1553040000, 'value': 0.44023582421288454}, + {'timestamp': 1553299200, 'value': 0.5616058583472479}, + {'timestamp': 1554163200, 'value': 0.6088354782316263}, + {'timestamp': 1554595200, 'value': 0.6399512199914456}, + {'timestamp': 1554768000, 'value': 0.6646672015472888}, + {'timestamp': 1555459200, 'value': 0.6485335011250782}, + {'timestamp': 1555632000, 'value': 0.6660331815652318}, + {'timestamp': 1555891200, 'value': 0.6974715796915939}, + {'timestamp': 1556064000, 'value': 0.6591090636161403}, + {'timestamp': 1557187200, 'value': 0.6921844436335354}, + {'timestamp': 1557619200, 'value': 0.7237710469613807}, + {'timestamp': 1558051200, 'value': 0.5622858292688305}, + {'timestamp': 1558224000, 'value': 0.6935684926748218}, + {'timestamp': 1559088000, 'value': 0.6245298968235982}, + {'timestamp': 1559347200, 'value': 0.6883854192631761}, + {'timestamp': 1559520000, 'value': 0.7016563575969763}, + {'timestamp': 1559779200, 'value': 0.6850383691684236}, + {'timestamp': 1560211200, 'value': 0.656470990596835}, + {'timestamp': 1560384000, 'value': 0.7238889987682}, + {'timestamp': 1560816000, 'value': 0.7035307391094375}, + {'timestamp': 1561075200, 'value': 0.6196358167172958}, + {'timestamp': 1561248000, 'value': 0.5980658765203083}, + {'timestamp': 1561680000, 'value': 0.5682400437486497}, + {'timestamp': 1561939200, 'value': 0.3400334709746356}, + {'timestamp': 1562371200, 'value': 0.16810500318259972}, + {'timestamp': 1562803200, 'value': 0.5256886782823283}, + {'timestamp': 1562976000, 'value': 0.4828430203411382}, + {'timestamp': 1563408000, 'value': 0.4477864993285167}, + {'timestamp': 1563667200, 'value': 0.43142048885631473}, + {'timestamp': 1563840000, 'value': 0.5123785941775288}, + {'timestamp': 1564099200, 'value': 0.5118334691193546}, + {'timestamp': 1564272000, 'value': 0.47658669387040065}, + {'timestamp': 1564704000, 'value': 0.2993281732463428}, + {'timestamp': 1565395200, 'value': 0.3997237032477171}, + {'timestamp': 1565568000, 'value': 0.3734611545776927}, + {'timestamp': 1566432000, 'value': 0.553340821019502}, + {'timestamp': 1566691200, 'value': 0.5513021771738195}, + {'timestamp': 1566864000, 'value': 0.5463825185642255}, + {'timestamp': 1567123200, 'value': 0.21662176920867515}, + {'timestamp': 1567296000, 'value': 0.43273218662635077}, + {'timestamp': 1567555200, 'value': 0.48772592943947113}, + {'timestamp': 1567728000, 'value': 0.4215565424979785}, + {'timestamp': 1568160000, 'value': 0.49199222075663573}, + {'timestamp': 1568419200, 'value': 0.4546686358405888}, + {'timestamp': 1568851200, 'value': 0.5316299742986982}, + {'timestamp': 1569024000, 'value': 0.5050756284663493}, + {'timestamp': 1569456000, 'value': 0.3714272473547276}, + {'timestamp': 1569715200, 'value': 0.012086493540667431}, + {'timestamp': 1571011200, 'value': 0.5601561861315829}, + {'timestamp': 1572307200, 'value': 0.5356334635640078}, + {'timestamp': 1572912000, 'value': 0.4391747111811623}, + {'timestamp': 1573344000, 'value': 0.6659406925942615}, + {'timestamp': 1574467200, 'value': 0.32388295893529057}, + {'timestamp': 1574899200, 'value': 0.17902818025326842}, + {'timestamp': 1575072000, 'value': 0.4713729520715623}, + {'timestamp': 1575331200, 'value': 0.6132103540678879}, + {'timestamp': 1575504000, 'value': 0.5894704386046352}, + {'timestamp': 1575936000, 'value': 0.6217669074338891}, + {'timestamp': 1576195200, 'value': 0.5687294579081753}, + {'timestamp': 1576368000, 'value': 0.18916815180865942}, + {'timestamp': 1576800000, 'value': 0.6037218544209375}, + # until here its 2019 + {'timestamp': 1577923200, 'value': 0.6106607485321933}, + {'timestamp': 1578355200, 'value': 0.002772249651503641}, + {'timestamp': 1578960000, 'value': 0.039889901477437366}, + {'timestamp': 1579219200, 'value': 0.5498426602606883}, + {'timestamp': 1579392000, 'value': 0.12463008162965945}, + {'timestamp': 1579824000, 'value': 0.5452118568383161}, + {'timestamp': 1580688000, 'value': 0.19285440472844556}, + {'timestamp': 1581120000, 'value': 0.5349233397200529}, + {'timestamp': 1581984000, 'value': 0.41061245567924765}, + {'timestamp': 1582243200, 'value': 0.4987324301884347}, + {'timestamp': 1583107200, 'value': 0.16397724497593927}, + {'timestamp': 1583280000, 'value': 0.32556816604560035}, + {'timestamp': 1583712000, 'value': 0.028160691939547327}, + {'timestamp': 1583971200, 'value': 0.40284770443025025}, + {'timestamp': 1584144000, 'value': 0.6349186440600055}, + {'timestamp': 1584403200, 'value': 0.2791129158697314}, + {'timestamp': 1584576000, 'value': 0.3762696064585376}, + {'timestamp': 1584835200, 'value': 0.6511059563604233}, + {'timestamp': 1585008000, 'value': 0.6329986751370975}, + {'timestamp': 1585267200, 'value': 0.5987580526634962}, + {'timestamp': 1585699200, 'value': 0.6227551970900924}, + {'timestamp': 1585872000, 'value': 0.5347724961569185}, + {'timestamp': 1586131200, 'value': 0.6327020615368077}, + {'timestamp': 1586304000, 'value': 0.6006515508260071}, + {'timestamp': 1586736000, 'value': 0.6522948086132051}, + {'timestamp': 1586995200, 'value': 0.6110782070702317}, + {'timestamp': 1587168000, 'value': 0.6417717497420365}, + {'timestamp': 1587427200, 'value': 0.6438999314473854}, + {'timestamp': 1587600000, 'value': 0.6296070325555527}, + {'timestamp': 1587859200, 'value': 0.6139715295880748}, + {'timestamp': 1588032000, 'value': 0.6016991503602547}, + {'timestamp': 1588291200, 'value': 0.6048452268959833}, + {'timestamp': 1588464000, 'value': 0.3548655861699142}, + {'timestamp': 1588723200, 'value': 0.7106839685170496}, + {'timestamp': 1588896000, 'value': 0.7112474894558717}, + {'timestamp': 1589328000, 'value': 0.6296398878166072}, + {'timestamp': 1589587200, 'value': 0.6739577862539349}, + {'timestamp': 1590192000, 'value': 0.0571304518695734}, + {'timestamp': 1590451200, 'value': 0.7060073637909001}, + {'timestamp': 1590624000, 'value': 0.2973821568775205}, + {'timestamp': 1590883200, 'value': 0.6854767595194105}, + {'timestamp': 1591056000, 'value': 0.6485029921918724}, + {'timestamp': 1591315200, 'value': 0.49870885670099085}, + {'timestamp': 1591488000, 'value': 0.3763445984837111}, + {'timestamp': 1592179200, 'value': 0.6149271317935514}, + {'timestamp': 1592352000, 'value': 0.1414903367375361}, + {'timestamp': 1592784000, 'value': 0.5948858659053022}, + {'timestamp': 1593043200, 'value': 0.4498251464639131}, + {'timestamp': 1593216000, 'value': 0.5272039951120531}, + {'timestamp': 1594080000, 'value': 0.3837704256571959}, + {'timestamp': 1594339200, 'value': 0.2911646173299671}, + {'timestamp': 1594512000, 'value': 0.4977832283157321}, + {'timestamp': 1595203200, 'value': 0.20726488571115734}, + {'timestamp': 1596067200, 'value': 0.49307978331855673}, + {'timestamp': 1596240000, 'value': 0.4566579362561879}, + {'timestamp': 1596499200, 'value': 0.391521811571442}, + {'timestamp': 1596672000, 'value': 0.42554546950592553}, + {'timestamp': 1596931200, 'value': 0.35022409279627237}, + {'timestamp': 1597104000, 'value': 0.38468834195220786}, + {'timestamp': 1597536000, 'value': 0.3412284738505061}, + {'timestamp': 1597795200, 'value': 0.30528961677949606}, + {'timestamp': 1597968000, 'value': 0.32196833124223506}, + {'timestamp': 1598832000, 'value': 0.35366390955095417}, + {'timestamp': 1599091200, 'value': 0.4343526544785653}, + {'timestamp': 1599264000, 'value': 0.03448406973182179}, + {'timestamp': 1599696000, 'value': 0.37114051174253926}, + {'timestamp': 1599955200, 'value': 0.5749910861856561}, + {'timestamp': 1600128000, 'value': 0.584619444298503}, + {'timestamp': 1600387200, 'value': 0.6151645893185002}, + {'timestamp': 1600560000, 'value': 0.5793651023317429}, + {'timestamp': 1600819200, 'value': 0.5599830871951561}, + {'timestamp': 1601251200, 'value': 0.6473475164629133}, + {'timestamp': 1601424000, 'value': 0.632010548709421}, + {'timestamp': 1601683200, 'value': 0.6410922055636433}, + {'timestamp': 1602115200, 'value': 0.4740622153388786}, + {'timestamp': 1602979200, 'value': 0.005711123344876248}, + {'timestamp': 1603584000, 'value': 0.6994891137287609}, + {'timestamp': 1604448000, 'value': 0.608658248994832}, + {'timestamp': 1604707200, 'value': 0.7115847640768033}, + {'timestamp': 1605139200, 'value': 0.08949784970353686}, + {'timestamp': 1605744000, 'value': 0.022849772434432095}, + {'timestamp': 1606435200, 'value': 0.4188422043235015}, + {'timestamp': 1606608000, 'value': 0.2500683303648727}, + {'timestamp': 1606867200, 'value': 0.2604494519978076}, + {'timestamp': 1607040000, 'value': 0.07083324936938587}, + {'timestamp': 1607299200, 'value': 0.002813341091232573}, + {'timestamp': 1607731200, 'value': 0.4187637973213316}, + {'timestamp': 1608336000, 'value': 0.612775111399876}, + {'timestamp': 1608595200, 'value': -0.029898428378950404}, + {'timestamp': 1608768000, 'value': 0.004315493270424968}, + {'timestamp': 1609200000, 'value': 0.4312730098931475}, + # until here its 2020 + {'timestamp': 1609459200, 'value': 0.01435058501894068}, + {'timestamp': 1609891200, 'value': 0.2923432869827817}, + {'timestamp': 1610323200, 'value': 0.0668237159232399}, + {'timestamp': 1611187200, 'value': 0.5369455034572239}, + {'timestamp': 1612051200, 'value': 0.055436857855142414}, + {'timestamp': 1612915200, 'value': 0.014911965149072238}, + {'timestamp': 1613088000, 'value': 0.01704268959229717}, + {'timestamp': 1613952000, 'value': 0.47666126930181013}, + {'timestamp': 1614211200, 'value': 0.40249965224608275}, + {'timestamp': 1614643200, 'value': 0.48217023910839407}, + {'timestamp': 1615248000, 'value': 0.4857652322292789}, + {'timestamp': 1615507200, 'value': 0.23827403158690943}, + {'timestamp': 1616371200, 'value': 0.5281767438543209}, + {'timestamp': 1617235200, 'value': 0.012463302374166159}, + {'timestamp': 1617408000, 'value': 0.36797845579231414}, + {'timestamp': 1617667200, 'value': 0.3413998686870444}, + {'timestamp': 1617840000, 'value': -0.0007593096602570172}, + {'timestamp': 1618099200, 'value': 0.1588694278504701}, + {'timestamp': 1618272000, 'value': 0.2530503929402777}, + {'timestamp': 1618963200, 'value': 0.6782696743923541}, + {'timestamp': 1619136000, 'value': 0.5994564865783631}, + {'timestamp': 1619568000, 'value': 0.6737351778513451}, + {'timestamp': 1620000000, 'value': 0.016924522307793323}, + {'timestamp': 1620259200, 'value': 0.01606474366095763}, + {'timestamp': 1620432000, 'value': 0.6330257332134035}, + {'timestamp': 1620691200, 'value': 0.717899089620151}, + {'timestamp': 1621123200, 'value': 0.42380173968839757}, + {'timestamp': 1621296000, 'value': 0.4033266298501044}, + {'timestamp': 1621555200, 'value': 0.6920621748795662}, + {'timestamp': 1621728000, 'value': 0.6260287142711787}, + {'timestamp': 1621987200, 'value': -0.005978637506810127}, + {'timestamp': 1622419200, 'value': 0.7904180101910425}, + {'timestamp': 1622592000, 'value': 0.7507581742359998}, + {'timestamp': 1622851200, 'value': 0.7340596869750035}, + {'timestamp': 1623024000, 'value': 0.713082625860254}, + {'timestamp': 1623283200, 'value': 0.6162225833638466}, + {'timestamp': 1623888000, 'value': 0.5970207971386315}, + {'timestamp': 1624147200, 'value': 0.5150093669774768}, + {'timestamp': 1624320000, 'value': 0.021000450337780315}, + {'timestamp': 1624579200, 'value': 0.45922488065462064}, + {'timestamp': 1624752000, 'value': 0.4967584306180959}, + {'timestamp': 1625184000, 'value': 0.3931810201670696}, + {'timestamp': 1625443200, 'value': 0.029347269193163553}, + {'timestamp': 1625875200, 'value': 0.4619241476221313}, + {'timestamp': 1626048000, 'value': 0.4226732311911146}, + {'timestamp': 1626307200, 'value': 0.49927513744860536}, + {'timestamp': 1626480000, 'value': 0.18278688072834576}, + {'timestamp': 1626912000, 'value': 0.41382215157983376}, + {'timestamp': 1627171200, 'value': 0.6010501555679375}, + {'timestamp': 1627344000, 'value': 0.6117434219873504}, + {'timestamp': 1627603200, 'value': 0.5208266892285692}, + {'timestamp': 1628035200, 'value': 0.6441416594696726}, + {'timestamp': 1628208000, 'value': 0.6034584964961202}, + {'timestamp': 1628467200, 'value': 0.64590786606526}, + {'timestamp': 1628640000, 'value': 0.4772097732538933}, + {'timestamp': 1628899200, 'value': 0.5880408265167602}, + {'timestamp': 1629072000, 'value': 0.44680911441258525}, + {'timestamp': 1629504000, 'value': 0.109105974373757}, + {'timestamp': 1629763200, 'value': 0.5395000130664906}, + {'timestamp': 1629936000, 'value': 0.458826030047935}, + {'timestamp': 1630195200, 'value': 0.28350180200114466}, + {'timestamp': 1630627200, 'value': 0.5688902152406968}, + {'timestamp': 1630800000, 'value': 0.43670135868371135}, + {'timestamp': 1631059200, 'value': 0.5948213293638251}, + {'timestamp': 1631232000, 'value': 0.554547020677893}, + {'timestamp': 1631923200, 'value': -0.017812937577812448}, + {'timestamp': 1632096000, 'value': 0.0019416418307588795}, + {'timestamp': 1632787200, 'value': 0.027521948760183122}, + {'timestamp': 1633651200, 'value': 0.6992170331018968}, + {'timestamp': 1633824000, 'value': 0.7061244648094013}, + {'timestamp': 1634083200, 'value': 0.6225944600006217}, + {'timestamp': 1634256000, 'value': 0.05714529819266201}, + {'timestamp': 1634947200, 'value': 0.6253835953395109}, + {'timestamp': 1635120000, 'value': 0.646177337901081}, + {'timestamp': 1635379200, 'value': 0.6667752468768585}, + {'timestamp': 1635552000, 'value': 0.6417093892688839}, + {'timestamp': 1636243200, 'value': 0.5549119008730556}, + {'timestamp': 1637539200, 'value': 0.5842417213194347}, + {'timestamp': 1637712000, 'value': -0.02299490551774671}, + {'timestamp': 1638144000, 'value': -0.00758547438802522}, + {'timestamp': 1638403200, 'value': 0.5143914299255691}, + {'timestamp': 1638576000, 'value': -0.1109406938627911}, + {'timestamp': 1639008000, 'value': 0.023029429420381983}, + {'timestamp': 1639267200, 'value': 0.024638896217661042}, + {'timestamp': 1639699200, 'value': -0.014363757133942142}, + {'timestamp': 1639872000, 'value': 0.011914837591533221}, + # until here its 2021 + {'timestamp': 1641427200, 'value': 0.5101549880831526}, + {'timestamp': 1641600000, 'value': 0.3887980581208711}, + {'timestamp': 1642464000, 'value': 0.2908277073649076}, + {'timestamp': 1642723200, 'value': 0.09096975011191563}, + {'timestamp': 1643328000, 'value': 0.369768095625778}, + {'timestamp': 1643760000, 'value': 0.2962413837894405}, + {'timestamp': 1644624000, 'value': 0.46836335847631205}, + {'timestamp': 1645920000, 'value': 0.2082614694292724}, + {'timestamp': 1646611200, 'value': 0.47022668892142844}, + {'timestamp': 1646784000, 'value': 0.45634158198743174}, + {'timestamp': 1647043200, 'value': 0.4641304764728491}, + {'timestamp': 1647216000, 'value': 0.439508319453624}, + {'timestamp': 1647648000, 'value': 0.4221169001419796}, + {'timestamp': 1647907200, 'value': 0.4425966960328659}, + {'timestamp': 1648080000, 'value': 0.4753782294251874}, + {'timestamp': 1648339200, 'value': 0.45627407595897207}, + {'timestamp': 1648512000, 'value': 0.47646076351947864}, + {'timestamp': 1648944000, 'value': 0.38671477286937855}, + {'timestamp': 1649635200, 'value': 0.58796790004507}, + {'timestamp': 1650067200, 'value': 0.6347346101398396}, + {'timestamp': 1650240000, 'value': 0.6263818349781571}, + {'timestamp': 1650672000, 'value': 0.6495226867754919}, + {'timestamp': 1650931200, 'value': 0.6867098674822344}, + {'timestamp': 1651104000, 'value': 0.6295148354799598}, + {'timestamp': 1651536000, 'value': 0.6663085005558228}, + {'timestamp': 1651795200, 'value': 0.6612043296879485}, + {'timestamp': 1651968000, 'value': 0.6902022479410631}, + {'timestamp': 1652227200, 'value': 0.6758318769248401}, + {'timestamp': 1652659200, 'value': 0.6560463825331692}, + {'timestamp': 1652832000, 'value': 0.6743604469746679}, + {'timestamp': 1653091200, 'value': 0.5917613969845943}, + {'timestamp': 1654128000, 'value': 0.6859302960341542}, + {'timestamp': 1654819200, 'value': 0.6118822200279516}, + {'timestamp': 1655856000, 'value': 0.5172830428976952}, + {'timestamp': 1656288000, 'value': 0.43986954061211364}, + {'timestamp': 1656547200, 'value': 0.34190041012449773}, + {'timestamp': 1656720000, 'value': 0.4193678789443398}, + {'timestamp': 1656979200, 'value': 0.393259289566778}, + {'timestamp': 1657584000, 'value': 0.4309890930798414}, + {'timestamp': 1658016000, 'value': 0.2777647567881633}, + {'timestamp': 1658275200, 'value': 0.39715504435734067}, + {'timestamp': 1658707200, 'value': 0.37016542504876004}, + {'timestamp': 1659571200, 'value': 0.34231296403384054}, + {'timestamp': 1659744000, 'value': 0.28926778249008184}, + {'timestamp': 1660003200, 'value': 0.30904499302081206}, + {'timestamp': 1660608000, 'value': 0.23752333515678647}, + {'timestamp': 1661299200, 'value': 0.21692575013363097}, + {'timestamp': 1661472000, 'value': 0.26788093919798717}, + {'timestamp': 1662163200, 'value': 0.5217756707981661}, + {'timestamp': 1662336000, 'value': 0.5977810346275365}, + {'timestamp': 1662768000, 'value': 0.5379871186171694}, + {'timestamp': 1663459200, 'value': 0.7349933166750889}, + {'timestamp': 1663632000, 'value': 0.673641456907865}, + {'timestamp': 1664064000, 'value': 0.6215492199142049}, + {'timestamp': 1664496000, 'value': 0.772380642343471}, + {'timestamp': 1664755200, 'value': 0.6334922152602114}, + {'timestamp': 1664928000, 'value': 0.6156581292973645}, + {'timestamp': 1666483200, 'value': 0.6692634931805789}, + {'timestamp': 1666915200, 'value': 0.7228286861507973}, + {'timestamp': 1667088000, 'value': 0.726920807653625}, + {'timestamp': 1667347200, 'value': 0.7427448875178663}, + {'timestamp': 1668211200, 'value': 0.6738504720290368}, + {'timestamp': 1668384000, 'value': 0.721222057771743}, + {'timestamp': 1669507200, 'value': 0.666140043780319}, + {'timestamp': 1671235200, 'value': 0.039560165834786186}, + {'timestamp': 1672099200, 'value': 0.4181342412037926}, + + # until here its 2022 + {'timestamp': 1672704000, 'value': 0.49311103866901945}, + {'timestamp': 1673827200, 'value': 0.5721801522171637}, + {'timestamp': 1677715200, 'value': 0.40309570282911744}, + {'timestamp': 1678579200, 'value': 0.3285946798542066}, + {'timestamp': 1679875200, 'value': 0.604913363250996}, + {'timestamp': 1680048000, 'value': 0.6241211192214813}, + {'timestamp': 1680739200, 'value': 0.6770332524516463}, + {'timestamp': 1682035200, 'value': 0.7195395171743766}, + {'timestamp': 1683072000, 'value': 0.776642542110257}, + {'timestamp': 1683504000, 'value': 0.7946412386885652}, + {'timestamp': 1683763200, 'value': 0.6962946310451835}, + {'timestamp': 1683936000, 'value': 0.7803666870020902}, + {'timestamp': 1684627200, 'value': 0.744074654951325}, + {'timestamp': 1685059200, 'value': 0.7603736326247179}, + {'timestamp': 1685232000, 'value': 0.7307521424847342}, + {'timestamp': 1685491200, 'value': 0.6595660036736165}, + {'timestamp': 1685664000, 'value': 0.6844185268873992}, + {'timestamp': 1685923200, 'value': 0.6609223570577207}, + {'timestamp': 1686096000, 'value': 0.6243770761357701}, + {'timestamp': 1687392000, 'value': 0.33966505122465546}, + {'timestamp': 1687651200, 'value': 0.46991814590987957}, + {'timestamp': 1687824000, 'value': 0.5637975455700079}, + {'timestamp': 1688256000, 'value': 0.5214708303888645}, + {'timestamp': 1688688000, 'value': 0.6235262779374806}, + {'timestamp': 1688947200, 'value': 0.5294325381096169}, + {'timestamp': 1689120000, 'value': 0.5378739374319399}, + {'timestamp': 1689379200, 'value': 0.5774819523508761}, + {'timestamp': 1689552000, 'value': 0.4300296294988784}, + {'timestamp': 1689984000, 'value': 0.3540605442689726}, + {'timestamp': 1690675200, 'value': 0.5559975830254852}, + {'timestamp': 1691539200, 'value': 0.6153487749947343}, + {'timestamp': 1691712000, 'value': 0.4640778921573764}, + {'timestamp': 1691971200, 'value': 0.6340361324676178}, + {'timestamp': 1692144000, 'value': 0.6643802566638307}, + {'timestamp': 1692403200, 'value': 0.6067618892145015}, + {'timestamp': 1692576000, 'value': 0.4995997577630159}, + {'timestamp': 1692835200, 'value': 0.6413726523764097}, + {'timestamp': 1693008000, 'value': 0.5552765524029254}, + {'timestamp': 1693440000, 'value': 0.5019676160726875}, + {'timestamp': 1693699200, 'value': 0.3575762499142059}, + {'timestamp': 1693872000, 'value': 0.636932829286457}, + {'timestamp': 1694131200, 'value': 0.5860366775305434}, + {'timestamp': 1694304000, 'value': 0.5678766403737106}, + {'timestamp': 1694736000, 'value': 0.5551531160988741}, + {'timestamp': 1695168000, 'value': 0.5267156551730596}, + {'timestamp': 1695600000, 'value': 0.3034994712244936}, + {'timestamp': 1696032000, 'value': 0.4584922258377994}, + {'timestamp': 1696723200, 'value': 0.5904255552176838}, + {'timestamp': 1698192000, 'value': 0.41143405956899415}, + {'timestamp': 1699056000, 'value': 0.6294013230687814}, + {'timestamp': 1699315200, 'value': 0.5625790498918815}, + {'timestamp': 1700352000, 'value': 0.3344593256089601}, + {'timestamp': 1700784000, 'value': 0.22997949110107285}, + {'timestamp': 1703203200, 'value': 0.5129322858835728}, + # until here its 2023 + {'timestamp': 1704672000, 'value': 0.4537940461130249}, + {'timestamp': 1706400000, 'value': 0.4841096797858553}, + {'timestamp': 1708128000, 'value': 0.3511160651300334}, + {'timestamp': 1708819200, 'value': 0.562864984877086}, + {'timestamp': 1709424000, 'value': 0.5528860685305712}, + {'timestamp': 1709683200, 'value': 0.4956625869394157}, + {'timestamp': 1709856000, 'value': 0.5740671639202845}, + {'timestamp': 1710288000, 'value': 0.5754883333185951}, + {'timestamp': 1710720000, 'value': 0.5781645983396947}, + {'timestamp': 1710979200, 'value': 0.6237791569348801}, + {'timestamp': 1711411200, 'value': 0.6805453663367135}, + {'timestamp': 1711843200, 'value': 0.44476014092304844}, + {'timestamp': 1712016000, 'value': 0.7037068616309425}, + {'timestamp': 1713312000, 'value': 0.44905630802780866}, + {'timestamp': 1713571200, 'value': 0.6595994526976465}, + {'timestamp': 1714003200, 'value': 0.6618991807578869}, + {'timestamp': 1714176000, 'value': 0.5086040253726309}, + {'timestamp': 1714435200, 'value': 0.7871618291679429}, + {'timestamp': 1714608000, 'value': 0.7642049393963221}, + {'timestamp': 1715299200, 'value': 0.7507461516420122}, + {'timestamp': 1715731200, 'value': 0.729646566522368}, + {'timestamp': 1715904000, 'value': 0.7007678963716821}, + {'timestamp': 1716163200, 'value': 0.4666880552899499}, + {'timestamp': 1716595200, 'value': 0.5894210184248679}, + {'timestamp': 1717200000, 'value': 0.6822153554153109}, + {'timestamp': 1717632000, 'value': 0.6532572845376999}, + {'timestamp': 1717891200, 'value': 0.70896413011386}, + {'timestamp': 1718496000, 'value': 0.603215338515721}, + {'timestamp': 1719187200, 'value': 0.637128334417188}, + {'timestamp': 1719360000, 'value': 0.6068163184736923}, + {'timestamp': 1719619200, 'value': 0.6288699313084715}, + {'timestamp': 1720224000, 'value': 0.5924314172946463}, + {'timestamp': 1720483200, 'value': 0.5156037447155328}, + {'timestamp': 1720656000, 'value': 0.5877750112506149}, + {'timestamp': 1720915200, 'value': 0.5446529073721998}, + {'timestamp': 1721347200, 'value': 0.5742237606531847}, + {'timestamp': 1721520000, 'value': 0.5641866773489524}, + {'timestamp': 1721779200, 'value': 0.4648095090560619}, + {'timestamp': 1722211200, 'value': 0.5643601252638971}, + {'timestamp': 1722643200, 'value': 0.5854776683005347}, + {'timestamp': 1722816000, 'value': 0.6287237036738268}, + {'timestamp': 1723075200, 'value': 0.36412344373600897}, + {'timestamp': 1723248000, 'value': 0.49056984190497194}, + {'timestamp': 1723507200, 'value': 0.6288169228142412}, + {'timestamp': 1724112000, 'value': 0.5628947382513431}, + {'timestamp': 1724371200, 'value': 0.5533013108657124}, + {'timestamp': 1724803200, 'value': 0.5050495959276382}, + {'timestamp': 1724976000, 'value': 0.4609939427470864}, + {'timestamp': 1725235200, 'value': 0.4467849685147322}, + {'timestamp': 1725408000, 'value': 0.4353219027209249}, + {'timestamp': 1725667200, 'value': 0.42904769396575165}, + {'timestamp': 1726099200, 'value': 0.4045242606090997}, + {'timestamp': 1726531200, 'value': 0.4328800438447044}, + {'timestamp': 1726704000, 'value': 0.43806171404605726}, + {'timestamp': 1726963200, 'value': 0.43907114056947844}, + {'timestamp': 1727395200, 'value': 0.46624384621001974}, + {'timestamp': 1727568000, 'value': 0.46813784900297967} + # Sep 29 2024 until here +] From 43f992c466c87c6ae9fd5bd973b8b4b0d8d1de7c Mon Sep 17 00:00:00 2001 From: wherop Date: Mon, 18 Nov 2024 15:42:19 +0100 Subject: [PATCH 07/24] refactor: delete emply file --- backend/src/gee/utils.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/src/gee/utils.py diff --git a/backend/src/gee/utils.py b/backend/src/gee/utils.py deleted file mode 100644 index e69de29..0000000 From a2bc44f0b36c667a0d94ba08bc8f91de300c2f18 Mon Sep 17 00:00:00 2001 From: wherop Date: Mon, 18 Nov 2024 15:46:18 +0100 Subject: [PATCH 08/24] refactor: delete emply file --- backend/src/routes/ndvi.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 backend/src/routes/ndvi.py diff --git a/backend/src/routes/ndvi.py b/backend/src/routes/ndvi.py deleted file mode 100644 index e69de29..0000000 From df6cc263ba1948ebd19b7323bcfef20d7ba2dc6f Mon Sep 17 00:00:00 2001 From: wherop Date: Tue, 19 Nov 2024 16:40:26 +0100 Subject: [PATCH 09/24] feat: refactor 'ndvi_router' to 'sat_index_router'; add MSAVI route to sat_index_router --- backend/src/constants.py | 1 + .../sat_index_controller.py} | 12 +- backend/src/main.py | 4 +- backend/src/routes/sat_index_router.py | 110 ++++++++++++++++++ backend/src/validation/models.py | 7 ++ 5 files changed, 125 insertions(+), 9 deletions(-) rename backend/src/{routes/ndvi_router.py => controller/sat_index_controller.py} (89%) create mode 100644 backend/src/routes/sat_index_router.py diff --git a/backend/src/constants.py b/backend/src/constants.py index 989b361..15bc264 100644 --- a/backend/src/constants.py +++ b/backend/src/constants.py @@ -29,6 +29,7 @@ class Unit(str, Enum): MM = "mm" PERCENT = "%" M3 = "m³/m³" + DIMENSIONLESS = "Dimensionless" class LocationPolygon(Enum): diff --git a/backend/src/routes/ndvi_router.py b/backend/src/controller/sat_index_controller.py similarity index 89% rename from backend/src/routes/ndvi_router.py rename to backend/src/controller/sat_index_controller.py index 0e60ffc..e768788 100644 --- a/backend/src/routes/ndvi_router.py +++ b/backend/src/controller/sat_index_controller.py @@ -1,8 +1,9 @@ from datetime import datetime, timezone -from fastapi import APIRouter, Query +from fastapi import Query from fastapi.responses import JSONResponse from src.constants import ( + IndexType, AggregationMethod, LocationName, TemporalResolution, @@ -10,17 +11,14 @@ ) from src.service import ndvi_service from src.utils.temporal import get_optimistic_rounding -from src.validation.models import NDVIResponse from src.validation.utils import ( validate_timestamp_in_range_of_S2_imagery, validate_timestamp_start_date_before_end_date, ) -ndvi_router = APIRouter() - -@ndvi_router.get("/ndvi", response_model=NDVIResponse) -async def get_temperature_data( +async def sat_index_controller( + sat_index_type: IndexType, startDate: int = Query(..., description="Start date as UNIX timestamp in seconds"), endDate: int = Query(..., @@ -62,4 +60,4 @@ async def get_temperature_data( "data": data, } - return JSONResponse(content=response) + return JSONResponse(content=response) \ No newline at end of file diff --git a/backend/src/main.py b/backend/src/main.py index 518873d..1f55537 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -1,7 +1,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from src.routes.ndvi_router import ndvi_router +from backend.src.routes.sat_index_router import sat_index_router from .weather.router import router as weather_router @@ -23,4 +23,4 @@ def read_root(): app.include_router(weather_router, prefix="/weather", tags=["Weather Data"]) -app.include_router(ndvi_router, prefix="/index", tags=["NDVI Data"]) +app.include_router(sat_index_router, prefix="/index", tags=["NDVI Data"]) diff --git a/backend/src/routes/sat_index_router.py b/backend/src/routes/sat_index_router.py new file mode 100644 index 0000000..567ce04 --- /dev/null +++ b/backend/src/routes/sat_index_router.py @@ -0,0 +1,110 @@ +from datetime import datetime, timezone + +from fastapi import APIRouter, Query +from fastapi.responses import JSONResponse +from src.constants import ( + AggregationMethod, + LocationName, + TemporalResolution, + Unit, +) +from src.service import ndvi_service +from src.utils.temporal import get_optimistic_rounding +from src.validation.models import NDVIResponse, MSAVIResponse +from src.validation.utils import ( + validate_timestamp_in_range_of_S2_imagery, + validate_timestamp_start_date_before_end_date, +) + +sat_index_router = APIRouter() + + +@sat_index_router.get("/ndvi", response_model=NDVIResponse) +async def get_ndvi_data( + startDate: int = Query(..., + description="Start date as UNIX timestamp in seconds"), + endDate: int = Query(..., + description="End date as UNIX timestamp in seconds"), + location: LocationName = Query(..., description="Location name"), + temporalResolution: TemporalResolution = Query( + ..., description="Temporal resolution" + ), + aggregation: AggregationMethod = Query(..., + description="Aggregation method"), +): + + validate_timestamp_start_date_before_end_date(startDate, endDate) + validate_timestamp_in_range_of_S2_imagery(startDate, endDate) + start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc) + end_date_dt = datetime.fromtimestamp(endDate, tz=timezone.utc) + + rounded_start_date, rounded_end_date = get_optimistic_rounding( + start_date_dt, end_date_dt, temporalResolution + ) + + data = ndvi_service( + location=location, + temporal_resolution=temporalResolution, + aggregation_method=aggregation, + start_date=rounded_start_date, + end_date=rounded_end_date, + ) + + response = { + "meta": { + "location": LocationName[location].value, + "startDate": int(rounded_start_date.timestamp()), + "endDate": int(rounded_end_date.timestamp()), + "temporalResolution": TemporalResolution[temporalResolution].value, + "aggregation": AggregationMethod[aggregation].value, + "unit": Unit.NORMALIZED_DIFFERENCE.value, + }, + "data": data, + } + + return JSONResponse(content=response) + +@sat_index_router.get("/msavi", response_model=MSAVIResponse) +async def get_msavi_data( + startDate: int = Query(..., + description="Start date as UNIX timestamp in seconds"), + endDate: int = Query(..., + description="End date as UNIX timestamp in seconds"), + location: LocationName = Query(..., description="Location name"), + temporalResolution: TemporalResolution = Query( + ..., description="Temporal resolution" + ), + aggregation: AggregationMethod = Query(..., + description="Aggregation method"), +): + + validate_timestamp_start_date_before_end_date(startDate, endDate) + validate_timestamp_in_range_of_S2_imagery(startDate, endDate) + start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc) + end_date_dt = datetime.fromtimestamp(endDate, tz=timezone.utc) + + rounded_start_date, rounded_end_date = get_optimistic_rounding( + start_date_dt, end_date_dt, temporalResolution + ) + + data = ndvi_service( + location=location, + temporal_resolution=temporalResolution, + aggregation_method=aggregation, + start_date=rounded_start_date, + end_date=rounded_end_date, + ) + + response = { + "meta": { + "location": LocationName[location].value, + "startDate": int(rounded_start_date.timestamp()), + "endDate": int(rounded_end_date.timestamp()), + "temporalResolution": TemporalResolution[temporalResolution].value, + "aggregation": AggregationMethod[aggregation].value, + "unit": Unit.NORMALIZED_DIFFERENCE.value, + }, + "data": data, + } + + return JSONResponse(content=response) diff --git a/backend/src/validation/models.py b/backend/src/validation/models.py index feb9bae..ad4d59d 100644 --- a/backend/src/validation/models.py +++ b/backend/src/validation/models.py @@ -34,3 +34,10 @@ class NDVIMetaResponse(BaseMeta): class NDVIResponse(BaseMeta): meta: NDVIMetaResponse data: List[DataPoint] + +class MSAVIMetaResponse(BaseMeta): + unit: Unit = Unit.DIMENSIONLESS + +class MSAVIResponse(BaseMeta): + meta: MSAVIMetaResponse + data: List[DataPoint] \ No newline at end of file From 109c80e225ac00739d633bf99eebffee8eac14ec Mon Sep 17 00:00:00 2001 From: wherop Date: Wed, 20 Nov 2024 13:10:04 +0100 Subject: [PATCH 10/24] refactor: rename image preprocessing file --- backend/src/gee/{index.py => image_preprocessing.py} | 0 backend/src/service.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename backend/src/gee/{index.py => image_preprocessing.py} (100%) diff --git a/backend/src/gee/index.py b/backend/src/gee/image_preprocessing.py similarity index 100% rename from backend/src/gee/index.py rename to backend/src/gee/image_preprocessing.py diff --git a/backend/src/service.py b/backend/src/service.py index 21e4ee1..47030eb 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -3,7 +3,7 @@ import pandas as pd from src.constants import AggregationMethod, LocationPolygon, TemporalResolution -from src.gee.index import get_preprocessed_imagery +from src.gee.image_preprocessing import get_preprocessed_imagery from src.gee.ndvi import get_ndvi_info from src.gee.ndvi_cache import ndvi_daily_cache from typing import List, Dict, Union From 331fdeafb9f66f0bc5bb55a2f3cd78c556a635c2 Mon Sep 17 00:00:00 2001 From: wherop Date: Wed, 20 Nov 2024 13:15:15 +0100 Subject: [PATCH 11/24] refactor: rename gee/ndvi.py to gee/sat_index_info.py --- backend/src/gee/{ndvi.py => sat_index_info.py} | 0 backend/src/service.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename backend/src/gee/{ndvi.py => sat_index_info.py} (100%) diff --git a/backend/src/gee/ndvi.py b/backend/src/gee/sat_index_info.py similarity index 100% rename from backend/src/gee/ndvi.py rename to backend/src/gee/sat_index_info.py diff --git a/backend/src/service.py b/backend/src/service.py index 47030eb..ab2f2b2 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -4,7 +4,7 @@ from src.constants import AggregationMethod, LocationPolygon, TemporalResolution from src.gee.image_preprocessing import get_preprocessed_imagery -from src.gee.ndvi import get_ndvi_info +from src.gee.sat_index_info import get_ndvi_info from src.gee.ndvi_cache import ndvi_daily_cache from typing import List, Dict, Union import math From 437f8647a091af10f4e67f3d94d1cb52fa7b9854 Mon Sep 17 00:00:00 2001 From: wherop Date: Wed, 20 Nov 2024 17:00:50 +0100 Subject: [PATCH 12/24] feat: add function to change index formula --- backend/src/gee/sat_index_info.py | 46 ++++++++++++++++++++++--------- backend/src/service.py | 4 +-- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/backend/src/gee/sat_index_info.py b/backend/src/gee/sat_index_info.py index a571b56..8b2d926 100644 --- a/backend/src/gee/sat_index_info.py +++ b/backend/src/gee/sat_index_info.py @@ -1,19 +1,38 @@ import ee +from constants import IndexType INDEX_FEATURE_LABEL = "indexing_value" TIMESTAMP_FEATURE_LABEL = "start_of_day_timestamp" INDEX_NULL_STRING = "NULL" -def calculate_mean_ndvi_GEE_SERVER(image: ee.Image, aoi: ee.Geometry.Polygon): - ndvi = image.normalizedDifference(["B8", "B4"]).rename("NDVI") - mean_ndvi = ndvi.reduceRegion( +def get_index_image_by_index_type(index_type: IndexType, image: ee.Image): + match index_type: + case IndexType.NDVI: + return image.normalizedDifference(["B8", "B4"]).rename("NDVI") + case IndexType.MSAVI: + return image.expression( + expression="((2 * NIR + 1) - ((2 * NIR + 1)**2 - 8 * (NIR - RED))**0.5) / 2", + opt_map={ + "NIR": image.select("B4"), + "RED": image.select("B8"), + }, + ).rename("MSAVI") + case _: + return None + + +def calculate_mean_index_GEE_SERVER(image: ee.Image, aoi: ee.Geometry.Polygon, index_type: IndexType): + index_image = get_index_image_by_index_type(index_type, image) + mean_index = index_image.reduceRegion( reducer=ee.Reducer.mean(), geometry=aoi, scale=10, maxPixels=1e8 - ).get("NDVI") - mean_ndvi = ee.Algorithms.If( - ee.Algorithms.IsEqual(mean_ndvi, None), INDEX_NULL_STRING, mean_ndvi + ).get(index_type.value) + + mean_index = ee.Algorithms.If( + ee.Algorithms.IsEqual(mean_index, None), INDEX_NULL_STRING, mean_index ) - return image.set(INDEX_FEATURE_LABEL, mean_ndvi) + return image.set(INDEX_FEATURE_LABEL, mean_index) + def calculate_start_of_day_timestamp_GEE_SERVER(image: ee.Image): @@ -27,28 +46,29 @@ def calculate_start_of_day_timestamp_GEE_SERVER(image: ee.Image): return image.set(TIMESTAMP_FEATURE_LABEL, start_of_day) -def get_ndvi_info( +def get_sat_index_info( image_collection: ee.ImageCollection, coordinates: ee.Geometry.Polygon ): aoi = ee.Geometry.Polygon(coordinates) + index_type = IndexType.MSAVI # TODO: Refactor this into a parameter # Setting indexing values Server side - image_collection_with_ndvi = image_collection.map( - lambda img: calculate_mean_ndvi_GEE_SERVER(img, aoi) + image_collection_with_index = image_collection.map( + lambda img: calculate_mean_index_GEE_SERVER(img, aoi, index_type) ) # Setting timestamps Server side - image_collection_with_timestamp_and_ndvi = image_collection_with_ndvi.map( + image_collection_with_timestamp_and_index = image_collection_with_index.map( lambda img: calculate_start_of_day_timestamp_GEE_SERVER(img) ) # Getting indexing values to the client side - index_value_list = image_collection_with_timestamp_and_ndvi.aggregate_array( + index_value_list = image_collection_with_timestamp_and_index.aggregate_array( INDEX_FEATURE_LABEL ).getInfo() # Getting timestamps to the client - timestamp_list = image_collection_with_timestamp_and_ndvi.aggregate_array( + timestamp_list = image_collection_with_timestamp_and_index.aggregate_array( TIMESTAMP_FEATURE_LABEL ).getInfo() diff --git a/backend/src/service.py b/backend/src/service.py index ab2f2b2..885f158 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -4,7 +4,7 @@ from src.constants import AggregationMethod, LocationPolygon, TemporalResolution from src.gee.image_preprocessing import get_preprocessed_imagery -from src.gee.sat_index_info import get_ndvi_info +from src.gee.sat_index_info import get_sat_index_info from src.gee.ndvi_cache import ndvi_daily_cache from typing import List, Dict, Union import math @@ -138,7 +138,7 @@ def ndvi_service( processing_start_date, processing_end_date, ) - NDVI_time_series = get_ndvi_info( + NDVI_time_series = get_sat_index_info( masked_images, LocationPolygon[location.value].value ) From 60295bbf20cd8230f1362934457c42ff666b648c Mon Sep 17 00:00:00 2001 From: wherop Date: Wed, 20 Nov 2024 17:28:24 +0100 Subject: [PATCH 13/24] feat: pass index_type from router to GEE --- backend/src/controller/sat_index_controller.py | 4 ++-- backend/src/gee/sat_index_info.py | 3 +-- backend/src/main.py | 2 +- backend/src/routes/sat_index_router.py | 9 ++++++--- backend/src/service.py | 7 ++++--- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/backend/src/controller/sat_index_controller.py b/backend/src/controller/sat_index_controller.py index 0772cc5..5f51c08 100644 --- a/backend/src/controller/sat_index_controller.py +++ b/backend/src/controller/sat_index_controller.py @@ -9,7 +9,7 @@ TemporalResolution, Unit, ) -from src.service import ndvi_service +from src.service import sat_index_service from src.utils.temporal import get_optimistic_rounding from src.validation.utils import ( validate_timestamp_in_range_of_S2_imagery, @@ -43,7 +43,7 @@ async def sat_index_controller( start_date_dt, end_date_dt, temporalResolution ) - data = ndvi_service( + data = sat_index_service( location=location, temporal_resolution=temporalResolution, aggregation_method=aggregation, diff --git a/backend/src/gee/sat_index_info.py b/backend/src/gee/sat_index_info.py index 8b2d926..b495e63 100644 --- a/backend/src/gee/sat_index_info.py +++ b/backend/src/gee/sat_index_info.py @@ -47,10 +47,9 @@ def calculate_start_of_day_timestamp_GEE_SERVER(image: ee.Image): def get_sat_index_info( - image_collection: ee.ImageCollection, coordinates: ee.Geometry.Polygon + image_collection: ee.ImageCollection, coordinates: ee.Geometry.Polygon, index_type: IndexType ): aoi = ee.Geometry.Polygon(coordinates) - index_type = IndexType.MSAVI # TODO: Refactor this into a parameter # Setting indexing values Server side image_collection_with_index = image_collection.map( diff --git a/backend/src/main.py b/backend/src/main.py index 1f55537..cca6523 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -1,7 +1,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from backend.src.routes.sat_index_router import sat_index_router +from src.routes.sat_index_router import sat_index_router from .weather.router import router as weather_router diff --git a/backend/src/routes/sat_index_router.py b/backend/src/routes/sat_index_router.py index 567ce04..058a419 100644 --- a/backend/src/routes/sat_index_router.py +++ b/backend/src/routes/sat_index_router.py @@ -7,8 +7,9 @@ LocationName, TemporalResolution, Unit, + IndexType ) -from src.service import ndvi_service +from src.service import sat_index_service from src.utils.temporal import get_optimistic_rounding from src.validation.models import NDVIResponse, MSAVIResponse from src.validation.utils import ( @@ -42,12 +43,13 @@ async def get_ndvi_data( start_date_dt, end_date_dt, temporalResolution ) - data = ndvi_service( + data = sat_index_service( location=location, temporal_resolution=temporalResolution, aggregation_method=aggregation, start_date=rounded_start_date, end_date=rounded_end_date, + index_type=IndexType.NDVI ) response = { @@ -87,12 +89,13 @@ async def get_msavi_data( start_date_dt, end_date_dt, temporalResolution ) - data = ndvi_service( + data = sat_index_service( location=location, temporal_resolution=temporalResolution, aggregation_method=aggregation, start_date=rounded_start_date, end_date=rounded_end_date, + index_type=IndexType.MSAVI ) response = { diff --git a/backend/src/service.py b/backend/src/service.py index 885f158..aa27148 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -2,7 +2,7 @@ import pandas as pd -from src.constants import AggregationMethod, LocationPolygon, TemporalResolution +from src.constants import AggregationMethod, LocationPolygon, TemporalResolution, IndexType from src.gee.image_preprocessing import get_preprocessed_imagery from src.gee.sat_index_info import get_sat_index_info from src.gee.ndvi_cache import ndvi_daily_cache @@ -103,12 +103,13 @@ def fill_missing_dates( return df -def ndvi_service( +def sat_index_service( location: LocationPolygon, temporal_resolution: TemporalResolution, aggregation_method: AggregationMethod, start_date: datetime, end_date: datetime, + index_type: IndexType ): # Temporary implementation of GEE Caching strategy current_cache_end_date = datetime( @@ -139,7 +140,7 @@ def ndvi_service( processing_end_date, ) NDVI_time_series = get_sat_index_info( - masked_images, LocationPolygon[location.value].value + masked_images, LocationPolygon[location.value].value, index_type ) if cache_start_date: From b8828d4bc0db8badd585914d44a7f0625e37f471 Mon Sep 17 00:00:00 2001 From: wherop Date: Thu, 21 Nov 2024 13:59:08 +0100 Subject: [PATCH 14/24] refactor: add '__init__.py' to all python package directories --- backend/src/{analysis => controller}/__init__.py | 0 backend/src/routes/{__inti__.py => __init__.py} | 0 backend/src/utils/__init__.py | 0 backend/src/validation/__init__.py | 0 backend/src/weather/__init__.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename backend/src/{analysis => controller}/__init__.py (100%) rename backend/src/routes/{__inti__.py => __init__.py} (100%) create mode 100644 backend/src/utils/__init__.py create mode 100644 backend/src/validation/__init__.py create mode 100644 backend/src/weather/__init__.py diff --git a/backend/src/analysis/__init__.py b/backend/src/controller/__init__.py similarity index 100% rename from backend/src/analysis/__init__.py rename to backend/src/controller/__init__.py diff --git a/backend/src/routes/__inti__.py b/backend/src/routes/__init__.py similarity index 100% rename from backend/src/routes/__inti__.py rename to backend/src/routes/__init__.py diff --git a/backend/src/utils/__init__.py b/backend/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/validation/__init__.py b/backend/src/validation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/weather/__init__.py b/backend/src/weather/__init__.py new file mode 100644 index 0000000..e69de29 From b517f730ba9f7e96c529592112be1cb59546e187 Mon Sep 17 00:00:00 2001 From: wherop Date: Thu, 21 Nov 2024 14:01:12 +0100 Subject: [PATCH 15/24] fix: update fastapi dependency reference to 'fastapi[standard]' --- backend/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 59e081f..144abd5 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,5 +2,5 @@ earthengine-api python-dotenv pandas plotly -fastapi +fastapi[standard] uvicorn \ No newline at end of file From d9c1ec30a22663ad74db29f1fd501f202a6e10ae Mon Sep 17 00:00:00 2001 From: wherop Date: Thu, 21 Nov 2024 14:21:44 +0100 Subject: [PATCH 16/24] fix: correct constants import error --- backend/src/gee/sat_index_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/gee/sat_index_info.py b/backend/src/gee/sat_index_info.py index b495e63..6a28e24 100644 --- a/backend/src/gee/sat_index_info.py +++ b/backend/src/gee/sat_index_info.py @@ -1,5 +1,5 @@ import ee -from constants import IndexType +from ..constants import IndexType INDEX_FEATURE_LABEL = "indexing_value" TIMESTAMP_FEATURE_LABEL = "start_of_day_timestamp" From 128650b14f1bae822b26711513aa8c9a7546daa1 Mon Sep 17 00:00:00 2001 From: wherop Date: Thu, 21 Nov 2024 14:45:14 +0100 Subject: [PATCH 17/24] style: update route labels to be more descriptive --- backend/src/main.py | 3 +- backend/src/routes/sat_index_router.py | 55 ++++++++++++++++---------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/backend/src/main.py b/backend/src/main.py index cca6523..3c6b323 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -17,10 +17,11 @@ allow_headers=["*"], ) + @app.get("/") def read_root(): return {"Hello": "World"} app.include_router(weather_router, prefix="/weather", tags=["Weather Data"]) -app.include_router(sat_index_router, prefix="/index", tags=["NDVI Data"]) +app.include_router(sat_index_router, prefix="/index", tags=["Vegetation Indices"]) diff --git a/backend/src/routes/sat_index_router.py b/backend/src/routes/sat_index_router.py index 058a419..fdd22fa 100644 --- a/backend/src/routes/sat_index_router.py +++ b/backend/src/routes/sat_index_router.py @@ -7,7 +7,7 @@ LocationName, TemporalResolution, Unit, - IndexType + IndexType, ) from src.service import sat_index_service from src.utils.temporal import get_optimistic_rounding @@ -22,18 +22,24 @@ @sat_index_router.get("/ndvi", response_model=NDVIResponse) async def get_ndvi_data( - startDate: int = Query(..., - description="Start date as UNIX timestamp in seconds"), - endDate: int = Query(..., - description="End date as UNIX timestamp in seconds"), - location: LocationName = Query(..., description="Location name"), + startDate: int = Query( + ..., + description="First date of requested date range in UNIX timestamp as seconds", + ), + endDate: int = Query( + ..., + description="Last date of requested date range in UNIX timestamp as seconds", + ), + location: LocationName = Query(..., description="Name of the requested location"), temporalResolution: TemporalResolution = Query( - ..., description="Temporal resolution" + ..., + description="Time interval that a single data point should represent e.g. one month", + ), + aggregation: AggregationMethod = Query( + ..., + description="Method of aggregating available data into a single datapoint to represent the selected time interval e.g. mean average", ), - aggregation: AggregationMethod = Query(..., - description="Aggregation method"), ): - validate_timestamp_start_date_before_end_date(startDate, endDate) validate_timestamp_in_range_of_S2_imagery(startDate, endDate) start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc) @@ -49,7 +55,7 @@ async def get_ndvi_data( aggregation_method=aggregation, start_date=rounded_start_date, end_date=rounded_end_date, - index_type=IndexType.NDVI + index_type=IndexType.NDVI, ) response = { @@ -66,20 +72,27 @@ async def get_ndvi_data( return JSONResponse(content=response) + @sat_index_router.get("/msavi", response_model=MSAVIResponse) async def get_msavi_data( - startDate: int = Query(..., - description="Start date as UNIX timestamp in seconds"), - endDate: int = Query(..., - description="End date as UNIX timestamp in seconds"), - location: LocationName = Query(..., description="Location name"), + startDate: int = Query( + ..., + description="First date of requested date range in UNIX timestamp as seconds", + ), + endDate: int = Query( + ..., + description="Last date of requested date range in UNIX timestamp as seconds", + ), + location: LocationName = Query(..., description="Name of the requested location"), temporalResolution: TemporalResolution = Query( - ..., description="Temporal resolution" + ..., + description="Time interval that a single data point should represent e.g. one month", + ), + aggregation: AggregationMethod = Query( + ..., + description="Method of aggregating available data into a single datapoint to represent the selected time interval e.g. mean average", ), - aggregation: AggregationMethod = Query(..., - description="Aggregation method"), ): - validate_timestamp_start_date_before_end_date(startDate, endDate) validate_timestamp_in_range_of_S2_imagery(startDate, endDate) start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc) @@ -95,7 +108,7 @@ async def get_msavi_data( aggregation_method=aggregation, start_date=rounded_start_date, end_date=rounded_end_date, - index_type=IndexType.MSAVI + index_type=IndexType.MSAVI, ) response = { From d98b02e4ef87bf70f833f3c1349ecb5e92433c19 Mon Sep 17 00:00:00 2001 From: wherop Date: Thu, 21 Nov 2024 15:07:54 +0100 Subject: [PATCH 18/24] feat: add index type to response meta; correct unit for MSAVI meta --- backend/src/routes/sat_index_router.py | 2 +- backend/src/validation/models.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/routes/sat_index_router.py b/backend/src/routes/sat_index_router.py index fdd22fa..7873695 100644 --- a/backend/src/routes/sat_index_router.py +++ b/backend/src/routes/sat_index_router.py @@ -118,7 +118,7 @@ async def get_msavi_data( "endDate": int(rounded_end_date.timestamp()), "temporalResolution": TemporalResolution[temporalResolution].value, "aggregation": AggregationMethod[aggregation].value, - "unit": Unit.NORMALIZED_DIFFERENCE.value, + "unit": Unit.DIMENSIONLESS.value, }, "data": data, } diff --git a/backend/src/validation/models.py b/backend/src/validation/models.py index ad4d59d..c136ecc 100644 --- a/backend/src/validation/models.py +++ b/backend/src/validation/models.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from src.constants import AggregationMethod, LocationName, TemporalResolution, Unit +from src.constants import AggregationMethod, LocationName, TemporalResolution, Unit, IndexType class DataPoint(BaseModel): @@ -28,6 +28,7 @@ class TemperatureResponse(BaseModel): class NDVIMetaResponse(BaseMeta): + indexType: IndexType = IndexType.NDVI unit: Unit = Unit.NORMALIZED_DIFFERENCE @@ -36,6 +37,7 @@ class NDVIResponse(BaseMeta): data: List[DataPoint] class MSAVIMetaResponse(BaseMeta): + indexType: IndexType = IndexType.MSAVI unit: Unit = Unit.DIMENSIONLESS class MSAVIResponse(BaseMeta): From 86e72fecc720ee11966b47d7b2b9095b8308a48d Mon Sep 17 00:00:00 2001 From: wherop Date: Fri, 22 Nov 2024 13:31:53 +0100 Subject: [PATCH 19/24] feat&docs: add comments and print statements to index service --- backend/src/service.py | 84 ++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/backend/src/service.py b/backend/src/service.py index aa27148..78211d4 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -2,7 +2,12 @@ import pandas as pd -from src.constants import AggregationMethod, LocationPolygon, TemporalResolution, IndexType +from src.constants import ( + AggregationMethod, + LocationPolygon, + TemporalResolution, + IndexType, +) from src.gee.image_preprocessing import get_preprocessed_imagery from src.gee.sat_index_info import get_sat_index_info from src.gee.ndvi_cache import ndvi_daily_cache @@ -13,7 +18,7 @@ def initialize_time_series( time_series: List[Dict[str, Union[int, float]]], temporal_resolution: TemporalResolution, - aggregation_method: AggregationMethod + aggregation_method: AggregationMethod, ) -> pd.DataFrame: """ Initializes a pandas DataFrame from a time series and applies temporal resolution and aggregation. @@ -31,28 +36,30 @@ def initialize_time_series( # Return an empty DataFrame with a datetime index and 'value' column in UTC if temporal_resolution == TemporalResolution.MONTHLY: empty_index = pd.date_range( - start="1970-01-01", periods=0, freq='MS', tz='UTC') + start="1970-01-01", periods=0, freq="MS", tz="UTC" + ) else: empty_index = pd.date_range( - start="1970-01-01", periods=0, freq='D', tz='UTC') + start="1970-01-01", periods=0, freq="D", tz="UTC" + ) - return pd.DataFrame(index=empty_index, columns=['value']) + return pd.DataFrame(index=empty_index, columns=["value"]) # Convert timestamps to datetime in UTC and create DataFrame df = pd.DataFrame(time_series) - df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s', utc=True) - df.set_index('timestamp', inplace=True) + df["timestamp"] = pd.to_datetime(df["timestamp"], unit="s", utc=True) + df.set_index("timestamp", inplace=True) # Resample based on temporal resolution and apply aggregation if needed if temporal_resolution == TemporalResolution.MONTHLY: if aggregation_method == AggregationMethod.MEAN: - df = df.resample('MS').mean() + df = df.resample("MS").mean() elif aggregation_method == AggregationMethod.MEDIAN: - df = df.resample('MS').median() + df = df.resample("MS").median() elif aggregation_method == AggregationMethod.MAX: - df = df.resample('MS').max() + df = df.resample("MS").max() elif aggregation_method == AggregationMethod.MIN: - df = df.resample('MS').min() + df = df.resample("MS").min() # If DAILY, do nothing as time series is already in daily format return df @@ -61,7 +68,7 @@ def fill_missing_dates( df: pd.DataFrame, start: datetime, end: datetime, - temporal_resolution: TemporalResolution + temporal_resolution: TemporalResolution, ) -> pd.DataFrame: """ Fills missing entries in the time series, adding NaN for missing days or months. @@ -88,18 +95,18 @@ def fill_missing_dates( # Generate the complete date range based on the temporal resolution if temporal_resolution == TemporalResolution.DAILY: - date_range = pd.date_range(start=start, end=end, freq='D', tz='UTC') + date_range = pd.date_range(start=start, end=end, freq="D", tz="UTC") elif temporal_resolution == TemporalResolution.MONTHLY: - date_range = pd.date_range(start=start, end=end, freq='MS', tz='UTC') + date_range = pd.date_range(start=start, end=end, freq="MS", tz="UTC") # If the input DataFrame is empty, create a new one with NaNs for all dates in the range if df.empty: - df = pd.DataFrame(index=date_range, columns=['value']) - df['value'] = None + df = pd.DataFrame(index=date_range, columns=["value"]) + df["value"] = None else: # Reindex to the complete date range, filling missing dates with NaN df = df.reindex(date_range) - df.columns = ['value'] + df.columns = ["value"] return df @@ -109,31 +116,40 @@ def sat_index_service( aggregation_method: AggregationMethod, start_date: datetime, end_date: datetime, - index_type: IndexType + index_type: IndexType, ): # Temporary implementation of GEE Caching strategy current_cache_end_date = datetime( - 2024, 9, 29, tzinfo=timezone.utc) - if start_date < current_cache_end_date and end_date < current_cache_end_date: # current end of cache + 2024, 9, 29, tzinfo=timezone.utc + ) # current end of cache + + # Entire range is within the cache, + # get entire range from cache, process nothing. + if start_date < current_cache_end_date and end_date < current_cache_end_date: cache_start_date = start_date cache_end_date = end_date processing_start_date = None processing_end_date = None + # Partial overlap with the cache, + # get cached part from cache, process the rest until end of range. elif start_date < current_cache_end_date and end_date > current_cache_end_date: cache_start_date = start_date cache_end_date = current_cache_end_date processing_start_date = current_cache_end_date + timedelta(days=1) processing_end_date = end_date + # Entire range is outside the cache, + # get nothing from cache, process entire range. elif start_date > current_cache_end_date: cache_start_date = None cache_end_date = None processing_start_date = start_date processing_end_date = end_date + # Get and process uncached range if processing_start_date: - + print(f'Getting {processing_start_date.date()} to {processing_end_date.date()} from GEE.') masked_images = get_preprocessed_imagery( LocationPolygon[location.value].value, processing_start_date, @@ -143,7 +159,9 @@ def sat_index_service( masked_images, LocationPolygon[location.value].value, index_type ) + # Get cached range if cache_start_date: + print(f'Getting {cache_start_date.date()} to {cache_end_date.date()} from cache.') cached_data_subset = get_cache_subset(cache_start_date, cache_end_date) if processing_start_date and cache_start_date: @@ -152,10 +170,10 @@ def sat_index_service( ndvi_data = cached_data_subset if cache_start_date else NDVI_time_series index_df = initialize_time_series( - ndvi_data, temporal_resolution, aggregation_method) + ndvi_data, temporal_resolution, aggregation_method + ) - filled_df = fill_missing_dates( - index_df, start_date, end_date, temporal_resolution) + filled_df = fill_missing_dates(index_df, start_date, end_date, temporal_resolution) return convert_df_to_list(filled_df) @@ -163,7 +181,9 @@ def sat_index_service( def get_cache_subset(start_date: datetime, end_date: datetime): subset: list[dict] = [] for entry in ndvi_daily_cache: - if entry["timestamp"] >= int(start_date.timestamp()) and entry["timestamp"] <= int(end_date.timestamp()): + if entry["timestamp"] >= int(start_date.timestamp()) and entry[ + "timestamp" + ] <= int(end_date.timestamp()): subset.append(entry) return subset @@ -180,17 +200,17 @@ def convert_df_to_list(df: pd.DataFrame) -> List[Dict[str, Union[int, float, Non """ # Convert the DataFrame index to epoch timestamps and reset index df_reset = df.reset_index() - df_reset['timestamp'] = df_reset['index'].astype(int) // 10**9 - df_reset = df_reset.rename(columns={'value': 'value'}) + df_reset["timestamp"] = df_reset["index"].astype(int) // 10**9 + df_reset = df_reset.rename(columns={"value": "value"}) # Convert to list of dictionaries - result = df_reset[['timestamp', 'value']].to_dict(orient='records') + result = df_reset[["timestamp", "value"]].to_dict(orient="records") # Convert NaN to None (needs to handle empty df as well) for entry in result: - if entry['value'] is None: - entry['value'] = None - elif math.isnan(entry['value']): - entry['value'] = None + if entry["value"] is None: + entry["value"] = None + elif math.isnan(entry["value"]): + entry["value"] = None return result From a5edee1ca12cde8a02bb18e625e32d531d1e53ea Mon Sep 17 00:00:00 2001 From: wherop Date: Fri, 22 Nov 2024 17:24:41 +0100 Subject: [PATCH 20/24] refactor: move NDVI cache to cache directory --- backend/src/{gee => cache}/ndvi_cache.py | 0 backend/src/service.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename backend/src/{gee => cache}/ndvi_cache.py (100%) diff --git a/backend/src/gee/ndvi_cache.py b/backend/src/cache/ndvi_cache.py similarity index 100% rename from backend/src/gee/ndvi_cache.py rename to backend/src/cache/ndvi_cache.py diff --git a/backend/src/service.py b/backend/src/service.py index 78211d4..b9c337c 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -10,7 +10,7 @@ ) from src.gee.image_preprocessing import get_preprocessed_imagery from src.gee.sat_index_info import get_sat_index_info -from src.gee.ndvi_cache import ndvi_daily_cache +from src.cache.ndvi_cache import ndvi_daily_cache from typing import List, Dict, Union import math From b9a52b14b4129b7781493e322b405818c98fa2ef Mon Sep 17 00:00:00 2001 From: wherop Date: Fri, 22 Nov 2024 17:25:49 +0100 Subject: [PATCH 21/24] feat: create MSAVI cache --- backend/src/cache/msavi_cache.py | 544 +++++++++++++++++++++++++++++++ 1 file changed, 544 insertions(+) create mode 100644 backend/src/cache/msavi_cache.py diff --git a/backend/src/cache/msavi_cache.py b/backend/src/cache/msavi_cache.py new file mode 100644 index 0000000..ce310f7 --- /dev/null +++ b/backend/src/cache/msavi_cache.py @@ -0,0 +1,544 @@ +msavi_daily_cache = [ + # Years marked above their section of the data. + + # 2017 + {"timestamp": 1493251200, "value": -4.431000367381644}, + {"timestamp": 1494720000, "value": -5.316995492647151}, + {"timestamp": 1494979200, "value": -7.973086798137311}, + {"timestamp": 1495843200, "value": -7.838449712221872}, + {"timestamp": 1496707200, "value": -1.0467874619591593}, + {"timestamp": 1497571200, "value": -1.9261113425778595}, + {"timestamp": 1498435200, "value": -3.8974326181714947}, + {"timestamp": 1499040000, "value": -4.671271421390005}, + {"timestamp": 1499299200, "value": -3.896947131496574}, + {"timestamp": 1499904000, "value": -1.226862398302503}, + {"timestamp": 1501632000, "value": -1.4971486375794503}, + {"timestamp": 1502755200, "value": -4.060044994911495}, + {"timestamp": 1504483200, "value": -4.268351695704429}, + {"timestamp": 1505088000, "value": -2.786659391004287}, + {"timestamp": 1505952000, "value": -3.4285132064595922}, + {"timestamp": 1509667200, "value": -1.154809081866889}, + {"timestamp": 1511136000, "value": -2.0201818125667486}, + {"timestamp": 1512000000, "value": -2.603414079645819}, + {"timestamp": 1512259200, "value": -1.9874363157053203}, + {"timestamp": 1512864000, "value": -2.041643161438374}, + {"timestamp": 1513123200, "value": -2.849906471581364}, + {"timestamp": 1513555200, "value": -2.158430721914352}, + {"timestamp": 1513728000, "value": -0.22437034596828198}, + {"timestamp": 1513987200, "value": 0.10179884543444519}, + + # 2018 + {"timestamp": 1515888000, "value": -0.0336505695941014}, + {"timestamp": 1516579200, "value": -1.5120684118787913}, + {"timestamp": 1517443200, "value": -2.436240487275879}, + {"timestamp": 1517875200, "value": -2.282794481825409}, + {"timestamp": 1518048000, "value": -2.0870996217045747}, + {"timestamp": 1518480000, "value": -2.2792613736811647}, + {"timestamp": 1518739200, "value": -1.8867529212186391}, + {"timestamp": 1519344000, "value": -1.376529856348907}, + {"timestamp": 1519776000, "value": -0.8174190534865039}, + {"timestamp": 1520035200, "value": -1.2701368134791358}, + {"timestamp": 1521331200, "value": -1.0911406015821747}, + {"timestamp": 1521936000, "value": -0.9154164183275707}, + {"timestamp": 1522627200, "value": -1.227328794032709}, + {"timestamp": 1523059200, "value": -1.661448703694562}, + {"timestamp": 1523232000, "value": -1.6737467481348518}, + {"timestamp": 1523923200, "value": -4.443809296928713}, + {"timestamp": 1524096000, "value": -4.069191039392678}, + {"timestamp": 1524355200, "value": -6.098625321790513}, + {"timestamp": 1524787200, "value": -8.42086121964117}, + {"timestamp": 1524960000, "value": -7.942683528928118}, + {"timestamp": 1525392000, "value": -8.444319825720948}, + {"timestamp": 1525651200, "value": -8.292322086687244}, + {"timestamp": 1525824000, "value": -6.976735281494538}, + {"timestamp": 1526083200, "value": -1.3303846208110004}, + {"timestamp": 1526256000, "value": -7.396700536259861}, + {"timestamp": 1526515200, "value": -0.804238136253652}, + {"timestamp": 1526947200, "value": -5.607996404028049}, + {"timestamp": 1527120000, "value": -4.205717017352484}, + {"timestamp": 1527552000, "value": -4.045806254536262}, + {"timestamp": 1527811200, "value": -3.276410199372174}, + {"timestamp": 1528243200, "value": -4.2712307445347495}, + {"timestamp": 1528416000, "value": -3.585817148450216}, + {"timestamp": 1528675200, "value": -2.885959202345276}, + {"timestamp": 1529107200, "value": -1.8324573025074755}, + {"timestamp": 1529280000, "value": -1.1187289135778689}, + {"timestamp": 1530403200, "value": -2.3347299498431493}, + {"timestamp": 1530576000, "value": -2.03852361485892}, + {"timestamp": 1531008000, "value": -0.8596722673661569}, + {"timestamp": 1531699200, "value": -1.71208585247463}, + {"timestamp": 1531872000, "value": -1.4713061011881976}, + {"timestamp": 1532304000, "value": -2.1801603072734657}, + {"timestamp": 1532563200, "value": -2.275877145002999}, + {"timestamp": 1532736000, "value": -1.9519204082587296}, + {"timestamp": 1532995200, "value": -1.926362792291209}, + {"timestamp": 1533427200, "value": -1.2940295467081178}, + {"timestamp": 1533600000, "value": -1.532616327451566}, + {"timestamp": 1534032000, "value": -1.3274613542145024}, + {"timestamp": 1534896000, "value": -1.103505320481981}, + {"timestamp": 1535155200, "value": -0.8039729082975013}, + {"timestamp": 1535760000, "value": -0.6252516532865182}, + {"timestamp": 1536019200, "value": -0.9021924217913372}, + {"timestamp": 1536192000, "value": -0.8517555348681844}, + {"timestamp": 1536451200, "value": -0.8649430759409419}, + {"timestamp": 1537315200, "value": -0.7074185453523396}, + {"timestamp": 1537747200, "value": -0.10895310410324782}, + {"timestamp": 1538179200, "value": -0.7831497283185018}, + {"timestamp": 1538352000, "value": -0.5634550211236486}, + {"timestamp": 1539216000, "value": -1.0617360823388704}, + {"timestamp": 1539475200, "value": -1.0916332995115228}, + {"timestamp": 1539648000, "value": -0.9795810922759659}, + {"timestamp": 1539907200, "value": -0.9426179492663695}, + {"timestamp": 1540080000, "value": -1.17354596819198}, + {"timestamp": 1540339200, "value": -1.112156161453324}, + {"timestamp": 1540944000, "value": -1.543467480665902}, + {"timestamp": 1541203200, "value": -1.1813709055184496}, + {"timestamp": 1541635200, "value": -0.17950063040098013}, + {"timestamp": 1542499200, "value": -0.42407589913108956}, + {"timestamp": 1543363200, "value": -1.9398449743636732}, + {"timestamp": 1543968000, "value": -1.988094825857856}, + {"timestamp": 1544227200, "value": -0.1434255040739643}, + {"timestamp": 1544400000, "value": -0.6293353122544786}, + {"timestamp": 1545696000, "value": -0.02655733316570224}, + {"timestamp": 1546128000, "value": -0.1690434047924854}, + + # 2019 + {"timestamp": 1546387200, "value": -0.4955474262574897}, + {"timestamp": 1546819200, "value": -0.00858885200538956}, + {"timestamp": 1547424000, "value": -1.8607985135819298}, + {"timestamp": 1548115200, "value": -1.8145645492840798}, + {"timestamp": 1548547200, "value": -0.39740226588526184}, + {"timestamp": 1549584000, "value": -1.7543786047362009}, + {"timestamp": 1549843200, "value": -0.4010353050413589}, + {"timestamp": 1550275200, "value": -2.025613850231601}, + {"timestamp": 1550448000, "value": -1.7696539940370857}, + {"timestamp": 1550707200, "value": -0.6543806812624658}, + {"timestamp": 1551139200, "value": -0.17218658321496605}, + {"timestamp": 1551744000, "value": -0.5444357126730207}, + {"timestamp": 1552435200, "value": -0.9428151830061051}, + {"timestamp": 1552867200, "value": 0.012969494026119709}, + {"timestamp": 1553040000, "value": -1.8665411726202898}, + {"timestamp": 1553299200, "value": -2.945112454358676}, + {"timestamp": 1554163200, "value": -3.6183623961576385}, + {"timestamp": 1554595200, "value": -4.263263121723981}, + {"timestamp": 1554768000, "value": -4.893130625048139}, + {"timestamp": 1555459200, "value": -4.680143129557381}, + {"timestamp": 1555632000, "value": -4.946448300389154}, + {"timestamp": 1555891200, "value": -6.174684742734432}, + {"timestamp": 1556064000, "value": -4.800505980194974}, + {"timestamp": 1557187200, "value": -6.032481750795664}, + {"timestamp": 1557619200, "value": -7.031911033351048}, + {"timestamp": 1558051200, "value": -3.3478156364738694}, + {"timestamp": 1558224000, "value": -5.76397222420251}, + {"timestamp": 1559088000, "value": -5.050031618648295}, + {"timestamp": 1559347200, "value": -5.651993835980702}, + {"timestamp": 1559520000, "value": -5.959873395104083}, + {"timestamp": 1559779200, "value": -5.400815844541562}, + {"timestamp": 1560211200, "value": -4.5017047294797985}, + {"timestamp": 1560384000, "value": -6.585962612258267}, + {"timestamp": 1560816000, "value": -5.884307038399208}, + {"timestamp": 1561075200, "value": -3.9877445482883296}, + {"timestamp": 1561248000, "value": -4.245248500163577}, + {"timestamp": 1561680000, "value": -3.1542461936641364}, + {"timestamp": 1561939200, "value": -1.06466023687381}, + {"timestamp": 1562371200, "value": -0.4053900018141121}, + {"timestamp": 1562803200, "value": -2.5567611930726333}, + {"timestamp": 1562976000, "value": -2.13446089275382}, + {"timestamp": 1563408000, "value": -1.8390466793833884}, + {"timestamp": 1563667200, "value": -1.8217675318689013}, + {"timestamp": 1563840000, "value": -2.4086606895667457}, + {"timestamp": 1564099200, "value": -2.3686107273007964}, + {"timestamp": 1564272000, "value": -2.021466374532932}, + {"timestamp": 1564704000, "value": -0.9367292149389402}, + {"timestamp": 1565395200, "value": -1.4173572500265312}, + {"timestamp": 1565568000, "value": -1.3535227818267588}, + {"timestamp": 1566432000, "value": -2.8182679314990593}, + {"timestamp": 1566691200, "value": -2.781088334966877}, + {"timestamp": 1566864000, "value": -2.7256679100018033}, + {"timestamp": 1567123200, "value": -0.5679139721083297}, + {"timestamp": 1567296000, "value": -1.7220514779979421}, + {"timestamp": 1567555200, "value": -2.13017092493178}, + {"timestamp": 1567728000, "value": -1.7575370766165523}, + {"timestamp": 1568160000, "value": -2.182925898105278}, + {"timestamp": 1568419200, "value": -1.9749729330871977}, + {"timestamp": 1568851200, "value": -2.5585358951393347}, + {"timestamp": 1569024000, "value": -2.2794470184235753}, + {"timestamp": 1569456000, "value": -1.2692098602837003}, + {"timestamp": 1569715200, "value": -0.02457411804370076}, + {"timestamp": 1571011200, "value": -2.845251090546086}, + {"timestamp": 1572307200, "value": -2.7533947572859665}, + {"timestamp": 1572912000, "value": -1.5827990843770674}, + {"timestamp": 1573344000, "value": -4.859530915739711}, + {"timestamp": 1574467200, "value": -1.2617351234374539}, + {"timestamp": 1574899200, "value": -0.44175984767040555}, + {"timestamp": 1575072000, "value": -2.016585892304853}, + {"timestamp": 1575331200, "value": -3.834390865271061}, + {"timestamp": 1575504000, "value": -3.3687185881156245}, + {"timestamp": 1575936000, "value": -3.929150650857071}, + {"timestamp": 1576195200, "value": -3.0320014895357628}, + {"timestamp": 1576368000, "value": -0.5086401792903669}, + {"timestamp": 1576800000, "value": -3.6104612015771678}, + + # 2020 + {"timestamp": 1577923200, "value": -3.737563920629822}, + {"timestamp": 1578355200, "value": -0.005805242125822093}, + {"timestamp": 1578960000, "value": -0.08364172837673736}, + {"timestamp": 1579219200, "value": -2.7852556898975975}, + {"timestamp": 1579392000, "value": -0.29462760063781757}, + {"timestamp": 1579824000, "value": -2.727188408990642}, + {"timestamp": 1580688000, "value": -0.5177023016700376}, + {"timestamp": 1581120000, "value": -2.585271588108896}, + {"timestamp": 1581984000, "value": -1.6812159748399784}, + {"timestamp": 1582243200, "value": -2.328565985486157}, + {"timestamp": 1583107200, "value": -0.4035426022516577}, + {"timestamp": 1583280000, "value": -1.019818398501554}, + {"timestamp": 1583712000, "value": -0.057995498066105544}, + {"timestamp": 1583971200, "value": -2.9039626735576087}, + {"timestamp": 1584144000, "value": -4.0993429139450726}, + {"timestamp": 1584403200, "value": -0.8201701838149547}, + {"timestamp": 1584576000, "value": -1.3515839151004507}, + {"timestamp": 1584835200, "value": -4.503992377268322}, + {"timestamp": 1585008000, "value": -4.050985704258896}, + {"timestamp": 1585267200, "value": -3.401386769327183}, + {"timestamp": 1585699200, "value": -3.88946201342351}, + {"timestamp": 1585872000, "value": -2.599303522753552}, + {"timestamp": 1586131200, "value": -4.061538626802209}, + {"timestamp": 1586304000, "value": -3.4800982529372653}, + {"timestamp": 1586736000, "value": -4.517019347049165}, + {"timestamp": 1586995200, "value": -3.5593581596489003}, + {"timestamp": 1587168000, "value": -4.345296558413699}, + {"timestamp": 1587427200, "value": -4.425319380010606}, + {"timestamp": 1587600000, "value": -4.106659248881034}, + {"timestamp": 1587859200, "value": -3.982874356192161}, + {"timestamp": 1588032000, "value": -3.582948420021519}, + {"timestamp": 1588291200, "value": -4.366621025238526}, + {"timestamp": 1588464000, "value": -1.1206908930180859}, + {"timestamp": 1588723200, "value": -6.4258499510748}, + {"timestamp": 1588896000, "value": -6.459585023122755}, + {"timestamp": 1589328000, "value": -4.16953175698427}, + {"timestamp": 1589587200, "value": -5.240016051472298}, + {"timestamp": 1590192000, "value": -0.12148526515462142}, + {"timestamp": 1590451200, "value": -6.122190070536504}, + {"timestamp": 1590624000, "value": -0.9065447274255518}, + {"timestamp": 1590883200, "value": -5.28926599258797}, + {"timestamp": 1591056000, "value": -4.36288285388885}, + {"timestamp": 1591315200, "value": -2.602488981911402}, + {"timestamp": 1591488000, "value": -1.2077887073754923}, + {"timestamp": 1592179200, "value": -3.6947581575072506}, + {"timestamp": 1592352000, "value": -0.46058505117390525}, + {"timestamp": 1592784000, "value": -3.3853119275818337}, + {"timestamp": 1593043200, "value": -2.0437679998052465}, + {"timestamp": 1593216000, "value": -2.497317971991625}, + {"timestamp": 1594080000, "value": -1.3736509095172529}, + {"timestamp": 1594339200, "value": -0.8396519991635049}, + {"timestamp": 1594512000, "value": -2.263988712031865}, + {"timestamp": 1595203200, "value": -0.7910221105615786}, + {"timestamp": 1596067200, "value": -2.1693851868273404}, + {"timestamp": 1596240000, "value": -1.8475883106196378}, + {"timestamp": 1596499200, "value": -1.4033897173709404}, + {"timestamp": 1596672000, "value": -1.6096636046379615}, + {"timestamp": 1596931200, "value": -1.1288870728545155}, + {"timestamp": 1597104000, "value": -1.3394883303984715}, + {"timestamp": 1597536000, "value": -1.1001062815252551}, + {"timestamp": 1597795200, "value": -0.9341116889843454}, + {"timestamp": 1597968000, "value": -1.0173800101435138}, + {"timestamp": 1598832000, "value": -1.1686268951307206}, + {"timestamp": 1599091200, "value": -1.7605591153673996}, + {"timestamp": 1599264000, "value": -0.07328260928310512}, + {"timestamp": 1599696000, "value": -1.8507526160671577}, + {"timestamp": 1599955200, "value": -2.701098837314166}, + {"timestamp": 1600128000, "value": -3.114659516560067}, + {"timestamp": 1600387200, "value": -3.603330531099459}, + {"timestamp": 1600560000, "value": -3.03120431396832}, + {"timestamp": 1600819200, "value": -2.8100148816153174}, + {"timestamp": 1601251200, "value": -4.266785159003384}, + {"timestamp": 1601424000, "value": -3.9710159407149237}, + {"timestamp": 1601683200, "value": -3.973298389783528}, + {"timestamp": 1602115200, "value": -1.8028489390557498}, + {"timestamp": 1602979200, "value": -0.011541224445218852}, + {"timestamp": 1603584000, "value": -5.767157195924353}, + {"timestamp": 1604448000, "value": -4.134120352990356}, + {"timestamp": 1604707200, "value": -6.151821580539443}, + {"timestamp": 1605139200, "value": -0.24593161180791834}, + {"timestamp": 1605744000, "value": -0.05032974556723841}, + {"timestamp": 1606435200, "value": -1.8668775443509775}, + {"timestamp": 1606608000, "value": -0.6960514239765465}, + {"timestamp": 1606867200, "value": -0.7346994181141604}, + {"timestamp": 1607040000, "value": -0.15411824129496812}, + {"timestamp": 1607299200, "value": -0.005744926622709519}, + {"timestamp": 1607731200, "value": -1.650046096828843}, + {"timestamp": 1608336000, "value": -3.627132506195544}, + {"timestamp": 1608595200, "value": 0.05780431091913084}, + {"timestamp": 1608768000, "value": -0.01295436963111137}, + {"timestamp": 1609200000, "value": -1.628868119675763}, + + # 2021 + {"timestamp": 1609459200, "value": -0.029371912188186414}, + {"timestamp": 1609891200, "value": -1.5927793918024307}, + {"timestamp": 1610323200, "value": -0.1444405529771034}, + {"timestamp": 1611187200, "value": -2.5722410130545685}, + {"timestamp": 1612051200, "value": -0.11870319747996139}, + {"timestamp": 1612915200, "value": -0.0306763259599792}, + {"timestamp": 1613088000, "value": -0.036100496973145706}, + {"timestamp": 1613952000, "value": -1.949104294815591}, + {"timestamp": 1614211200, "value": -1.4042442370222061}, + {"timestamp": 1614643200, "value": -1.992486846839551}, + {"timestamp": 1615248000, "value": -2.0121660431920882}, + {"timestamp": 1615507200, "value": -0.6858985062529075}, + {"timestamp": 1616371200, "value": -2.418532960616541}, + {"timestamp": 1617235200, "value": -0.02583538844696913}, + {"timestamp": 1617408000, "value": -1.2224655508959779}, + {"timestamp": 1617667200, "value": -1.1228865783309585}, + {"timestamp": 1617840000, "value": 0.0012179941656120087}, + {"timestamp": 1618099200, "value": -0.4659063127273096}, + {"timestamp": 1618272000, "value": -0.726307140772426}, + {"timestamp": 1618963200, "value": -4.980860761646362}, + {"timestamp": 1619136000, "value": -3.5989753447964374}, + {"timestamp": 1619568000, "value": -4.856025417132305}, + {"timestamp": 1620000000, "value": -0.03821754336936337}, + {"timestamp": 1620259200, "value": -0.03271299881713246}, + {"timestamp": 1620432000, "value": -4.04198819221969}, + {"timestamp": 1620691200, "value": -6.2085950082003585}, + {"timestamp": 1621123200, "value": -1.4702576218562626}, + {"timestamp": 1621296000, "value": -1.3812850759403357}, + {"timestamp": 1621555200, "value": -5.237347989474782}, + {"timestamp": 1621728000, "value": -4.644603768220883}, + {"timestamp": 1621987200, "value": 0.010969772780466956}, + {"timestamp": 1622419200, "value": -10.550768537059938}, + {"timestamp": 1622592000, "value": -7.904901886618402}, + {"timestamp": 1622851200, "value": -7.0187358243741915}, + {"timestamp": 1623024000, "value": -6.257105372278771}, + {"timestamp": 1623283200, "value": -4.304440700169409}, + {"timestamp": 1623888000, "value": -3.443541233206872}, + {"timestamp": 1624147200, "value": -2.3375278746200805}, + {"timestamp": 1624320000, "value": -0.042969010078800475}, + {"timestamp": 1624579200, "value": -1.8941718610682767}, + {"timestamp": 1624752000, "value": -2.2594890628221043}, + {"timestamp": 1625184000, "value": -1.3874223928738496}, + {"timestamp": 1625443200, "value": -0.060480678460943633}, + {"timestamp": 1625875200, "value": -2.000922879539773}, + {"timestamp": 1626048000, "value": -1.7193515207906735}, + {"timestamp": 1626307200, "value": -2.3044224444762373}, + {"timestamp": 1626480000, "value": -0.45988788421575666}, + {"timestamp": 1626912000, "value": -1.610808717829014}, + {"timestamp": 1627171200, "value": -3.5369894259325854}, + {"timestamp": 1627344000, "value": -3.719390816447317}, + {"timestamp": 1627603200, "value": -2.293601337856621}, + {"timestamp": 1628035200, "value": -4.342401621731726}, + {"timestamp": 1628208000, "value": -3.530646379242294}, + {"timestamp": 1628467200, "value": -4.132581644639337}, + {"timestamp": 1628640000, "value": -2.7759202993197514}, + {"timestamp": 1628899200, "value": -3.3048832544647753}, + {"timestamp": 1629072000, "value": -2.126312674063426}, + {"timestamp": 1629504000, "value": -0.273632740996652}, + {"timestamp": 1629763200, "value": -2.9811844589913696}, + {"timestamp": 1629936000, "value": -2.5354872781498248}, + {"timestamp": 1630195200, "value": -1.0831782798712686}, + {"timestamp": 1630627200, "value": -3.1811683702528297}, + {"timestamp": 1630800000, "value": -1.728952075752586}, + {"timestamp": 1631059200, "value": -3.368706619684609}, + {"timestamp": 1631232000, "value": -3.039886860329177}, + {"timestamp": 1631923200, "value": 0.03415349838777033}, + {"timestamp": 1632096000, "value": -0.006004807411014806}, + {"timestamp": 1632787200, "value": -0.056872452513403185}, + {"timestamp": 1633651200, "value": -5.655762636137656}, + {"timestamp": 1633824000, "value": -5.873610312326366}, + {"timestamp": 1634083200, "value": -4.717324818961415}, + {"timestamp": 1634256000, "value": -0.12303633525621183}, + {"timestamp": 1634947200, "value": -4.038822380161949}, + {"timestamp": 1635120000, "value": -4.149126757949408}, + {"timestamp": 1635379200, "value": -4.757505789998262}, + {"timestamp": 1635552000, "value": -4.186928132688978}, + {"timestamp": 1636243200, "value": -3.0870124192113098}, + {"timestamp": 1637539200, "value": -3.2114665741220922}, + {"timestamp": 1637712000, "value": 0.04337437278697441}, + {"timestamp": 1638144000, "value": 0.014738639934081891}, + {"timestamp": 1638403200, "value": -2.560165354558654}, + {"timestamp": 1638576000, "value": 0.19871610157669053}, + {"timestamp": 1639008000, "value": -0.047256662209476656}, + {"timestamp": 1639267200, "value": -0.050580919294464025}, + {"timestamp": 1639699200, "value": 0.02741377563099516}, + {"timestamp": 1639872000, "value": -0.02470897431490369}, + + # 2022 + {"timestamp": 1641427200, "value": -2.253906560457503}, + {"timestamp": 1641600000, "value": -1.776316065902636}, + {"timestamp": 1642464000, "value": -0.8651157607601431}, + {"timestamp": 1642723200, "value": -0.2038300404410527}, + {"timestamp": 1643328000, "value": -1.2691632873801888}, + {"timestamp": 1643760000, "value": -0.8438458860034452}, + {"timestamp": 1644624000, "value": -1.874021157281405}, + {"timestamp": 1645920000, "value": -0.539159488697499}, + {"timestamp": 1646611200, "value": -1.9039810336070995}, + {"timestamp": 1646784000, "value": -1.7949245326857695}, + {"timestamp": 1647043200, "value": -1.8535684066859253}, + {"timestamp": 1647216000, "value": -1.6675128322975703}, + {"timestamp": 1647648000, "value": -1.5856342663534242}, + {"timestamp": 1647907200, "value": -1.8351703445005467}, + {"timestamp": 1648080000, "value": -1.9443863747741417}, + {"timestamp": 1648339200, "value": -1.8525570064923393}, + {"timestamp": 1648512000, "value": -1.9568586684347689}, + {"timestamp": 1648944000, "value": -1.3721883559710328}, + {"timestamp": 1649635200, "value": -3.257982587049245}, + {"timestamp": 1650067200, "value": -4.048371988020582}, + {"timestamp": 1650240000, "value": -3.9170812117952996}, + {"timestamp": 1650672000, "value": -4.334710308248457}, + {"timestamp": 1650931200, "value": -5.342751714534661}, + {"timestamp": 1651104000, "value": -4.156388985871771}, + {"timestamp": 1651536000, "value": -4.697970837878134}, + {"timestamp": 1651795200, "value": -4.860583606565607}, + {"timestamp": 1651968000, "value": -5.479010375107818}, + {"timestamp": 1652227200, "value": -5.190937377016227}, + {"timestamp": 1652659200, "value": -4.5354788545860325}, + {"timestamp": 1652832000, "value": -5.085450830026249}, + {"timestamp": 1653091200, "value": -3.402619456534638}, + {"timestamp": 1654128000, "value": -5.413656117159736}, + {"timestamp": 1654819200, "value": -3.662764476468461}, + {"timestamp": 1655856000, "value": -2.419774847563414}, + {"timestamp": 1656288000, "value": -1.7368393873596444}, + {"timestamp": 1656720000, "value": -1.5810772686641867}, + {"timestamp": 1656979200, "value": -1.4104082997058258}, + {"timestamp": 1657584000, "value": -1.6854574909736348}, + {"timestamp": 1658016000, "value": -0.7731394915081767}, + {"timestamp": 1658275200, "value": -1.4195104241380918}, + {"timestamp": 1658707200, "value": -1.2655137439698116}, + {"timestamp": 1659571200, "value": -1.1123387618730571}, + {"timestamp": 1659744000, "value": -0.8756535287218514}, + {"timestamp": 1660003200, "value": -0.946651237965853}, + {"timestamp": 1660608000, "value": -0.6432297169112653}, + {"timestamp": 1661299200, "value": -0.5682542976959148}, + {"timestamp": 1661472000, "value": -0.778184881354194}, + {"timestamp": 1662163200, "value": -2.5887382572490525}, + {"timestamp": 1662336000, "value": -3.4051926714742446}, + {"timestamp": 1662768000, "value": -2.790792623026091}, + {"timestamp": 1663459200, "value": -6.860088135583952}, + {"timestamp": 1663632000, "value": -5.192032082406783}, + {"timestamp": 1664064000, "value": -4.165194374985292}, + {"timestamp": 1664496000, "value": -8.909257504857658}, + {"timestamp": 1664755200, "value": -4.136331345998694}, + {"timestamp": 1664928000, "value": -3.249443743821671}, + {"timestamp": 1666483200, "value": -4.6152951292713595}, + {"timestamp": 1666915200, "value": -6.279955471862619}, + {"timestamp": 1667088000, "value": -6.491172170177108}, + {"timestamp": 1667347200, "value": -7.1021373769749}, + {"timestamp": 1668211200, "value": -4.77223593106354}, + {"timestamp": 1668384000, "value": -6.220500157201978}, + {"timestamp": 1669507200, "value": -4.578367402836864}, + {"timestamp": 1671235200, "value": -0.08254701747632309}, + {"timestamp": 1672099200, "value": -1.6004963515406716}, + + # 2023 + {"timestamp": 1672704000, "value": -2.1632787010956784}, + {"timestamp": 1673827200, "value": -2.983477385722785}, + {"timestamp": 1677715200, "value": -1.4165868819117267}, + {"timestamp": 1678579200, "value": -0.9878430407659657}, + {"timestamp": 1679875200, "value": -3.466021499980709}, + {"timestamp": 1680048000, "value": -3.825582990682917}, + {"timestamp": 1680739200, "value": -4.934185080617706}, + {"timestamp": 1682035200, "value": -6.206812129458073}, + {"timestamp": 1683072000, "value": -9.275958556444875}, + {"timestamp": 1683504000, "value": -10.716968297484195}, + {"timestamp": 1683763200, "value": -5.408653793808816}, + {"timestamp": 1683936000, "value": -9.442098368130585}, + {"timestamp": 1684627200, "value": -7.838196738726171}, + {"timestamp": 1685059200, "value": -8.47984955920828}, + {"timestamp": 1685232000, "value": -6.972524075112018}, + {"timestamp": 1685491200, "value": -4.406292021716042}, + {"timestamp": 1685664000, "value": -5.386065534732394}, + {"timestamp": 1685923200, "value": -4.737845392070691}, + {"timestamp": 1686096000, "value": -3.93362223378809}, + {"timestamp": 1687392000, "value": -1.0281388531428395}, + {"timestamp": 1687651200, "value": -2.0277776210674223}, + {"timestamp": 1687824000, "value": -3.1058855995157315}, + {"timestamp": 1688256000, "value": -2.4884785118679584}, + {"timestamp": 1688688000, "value": -3.901995452478836}, + {"timestamp": 1688947200, "value": -2.479749261102836}, + {"timestamp": 1689120000, "value": -2.4953105755389484}, + {"timestamp": 1689379200, "value": -3.083998430925238}, + {"timestamp": 1689552000, "value": -1.5730837328758267}, + {"timestamp": 1689984000, "value": -1.1685138491804112}, + {"timestamp": 1690675200, "value": -2.8959462871988597}, + {"timestamp": 1691539200, "value": -3.9143522850427193}, + {"timestamp": 1691712000, "value": -2.0059515442340894}, + {"timestamp": 1691971200, "value": -4.046766311094322}, + {"timestamp": 1692144000, "value": -5.143252851579925}, + {"timestamp": 1692403200, "value": -3.977662184389684}, + {"timestamp": 1692576000, "value": -2.687668487446206}, + {"timestamp": 1692835200, "value": -4.739718202618833}, + {"timestamp": 1693008000, "value": -3.0932427383310914}, + {"timestamp": 1693440000, "value": -2.3917470541502226}, + {"timestamp": 1693699200, "value": -1.1291876520985438}, + {"timestamp": 1693872000, "value": -4.429118853478426}, + {"timestamp": 1694131200, "value": -3.4127348615565856}, + {"timestamp": 1694304000, "value": -3.2137746258372686}, + {"timestamp": 1694736000, "value": -3.184173382194691}, + {"timestamp": 1695168000, "value": -2.5106827835319963}, + {"timestamp": 1695600000, "value": -0.892445521634119}, + {"timestamp": 1696032000, "value": -1.9478173710739408}, + {"timestamp": 1696723200, "value": -3.338840285518137}, + {"timestamp": 1698192000, "value": -1.4373857288874872}, + {"timestamp": 1699056000, "value": -4.17210574924378}, + {"timestamp": 1699315200, "value": -3.178491343902305}, + {"timestamp": 1700352000, "value": -1.095198175191808}, + {"timestamp": 1700784000, "value": -0.6755551648481929}, + {"timestamp": 1703203200, "value": -2.4320769776043853}, + + # 2024 (until end of September) + {"timestamp": 1704672000, "value": -1.8240732303170757}, + {"timestamp": 1706400000, "value": -2.0359234164407853}, + {"timestamp": 1708128000, "value": -1.1651982353912895}, + {"timestamp": 1708819200, "value": -2.89160848955476}, + {"timestamp": 1709424000, "value": -2.7623708644791067}, + {"timestamp": 1709683200, "value": -2.2604734758474487}, + {"timestamp": 1709856000, "value": -3.0320684120611157}, + {"timestamp": 1710288000, "value": -3.044875564804314}, + {"timestamp": 1710720000, "value": -3.0506387455946324}, + {"timestamp": 1710979200, "value": -3.7108505237985505}, + {"timestamp": 1711411200, "value": -5.132474503968261}, + {"timestamp": 1711843200, "value": -1.6555395941313846}, + {"timestamp": 1712016000, "value": -4.944721667911011}, + {"timestamp": 1713312000, "value": -2.1936541594672163}, + {"timestamp": 1713571200, "value": -4.525621060522565}, + {"timestamp": 1714003200, "value": -5.350975315548725}, + {"timestamp": 1714176000, "value": -2.492177205644747}, + {"timestamp": 1714435200, "value": -10.070425581250989}, + {"timestamp": 1714608000, "value": -8.369960411513166}, + {"timestamp": 1715299200, "value": -8.075975281913461}, + {"timestamp": 1715731200, "value": -7.018195682847631}, + {"timestamp": 1715904000, "value": -5.879113894732184}, + {"timestamp": 1716163200, "value": -1.9426634232290758}, + {"timestamp": 1716595200, "value": -3.4476091230327244}, + {"timestamp": 1717200000, "value": -5.020790501798809}, + {"timestamp": 1717632000, "value": -4.461632656650202}, + {"timestamp": 1717891200, "value": -6.0027612893101185}, + {"timestamp": 1718496000, "value": -3.8380004234155543}, + {"timestamp": 1719187200, "value": -4.311023987624012}, + {"timestamp": 1719360000, "value": -3.5604621195661896}, + {"timestamp": 1719619200, "value": -4.000622873261243}, + {"timestamp": 1720224000, "value": -3.4157165497870023}, + {"timestamp": 1720483200, "value": -2.5417807734102142}, + {"timestamp": 1720656000, "value": -3.2446460227853224}, + {"timestamp": 1720915200, "value": -2.7383587783702614}, + {"timestamp": 1721347200, "value": -3.0862410722785776}, + {"timestamp": 1721520000, "value": -2.914618710471714}, + {"timestamp": 1721779200, "value": -2.0432678736381003}, + {"timestamp": 1722211200, "value": -2.9802316526965424}, + {"timestamp": 1722643200, "value": -3.3619899839479377}, + {"timestamp": 1722816000, "value": -4.030666091594237}, + {"timestamp": 1723075200, "value": -1.1686716051703105}, + {"timestamp": 1723248000, "value": -2.1501821212207193}, + {"timestamp": 1723507200, "value": -4.103490431194776}, + {"timestamp": 1724112000, "value": -2.9218929302120005}, + {"timestamp": 1724371200, "value": -2.74349596609107}, + {"timestamp": 1724803200, "value": -2.2656526045163625}, + {"timestamp": 1724976000, "value": -1.864104950442559}, + {"timestamp": 1725235200, "value": -1.6140829866530924}, + {"timestamp": 1725408000, "value": -1.6863172044651933}, + {"timestamp": 1725667200, "value": -1.6646539826070033}, + {"timestamp": 1726099200, "value": -1.4450195576891183}, + {"timestamp": 1726531200, "value": -1.6790029768346082}, + {"timestamp": 1726704000, "value": -1.7045123289727604}, + {"timestamp": 1726963200, "value": -1.7011517778472642}, + {"timestamp": 1727395200, "value": -1.9010479736367178}, + {"timestamp": 1727568000, "value": -2.048906875218465}, +] From dcf318dfe92dc0b9515f6a8caef6cd834bd3a75d Mon Sep 17 00:00:00 2001 From: wherop Date: Fri, 22 Nov 2024 17:26:43 +0100 Subject: [PATCH 22/24] feat: add a rudamentary caching script (WIP) --- backend/src/gee/caching_script.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 backend/src/gee/caching_script.py diff --git a/backend/src/gee/caching_script.py b/backend/src/gee/caching_script.py new file mode 100644 index 0000000..913b4f7 --- /dev/null +++ b/backend/src/gee/caching_script.py @@ -0,0 +1,30 @@ +import json + +content = [ + {"timestamp": 1493251200, "value": 0.6265267304234295}, + {"timestamp": 1494720000, "value": 0.68603163673333}, + {"timestamp": 1494979200, "value": 0.755257128311451}, +] +var_name = "msavi_daily_cache" +file_name = "../cache/temp_cache.py" +INDENT = " " + +def combine_chache_with_update(): + # update existing cache + return + +def write_year(file, year, results): + file.write(f"{INDENT}# Year {year}\n") + +def write_results_to_cache(results: list, var_name: str, file_name: str): + with open(file_name, "w") as file: + file.write(f"{var_name} = [\n") + for day in results: + file.write(f"{INDENT}{json.dumps(day)},\n") + file.write("]\n") + file.close() + + return + + +# write_results_to_cache(content, var_name, file_name) From 69767bcb9d10d8bde0bf4390105f9f374a73773d Mon Sep 17 00:00:00 2001 From: wherop Date: Fri, 22 Nov 2024 18:03:28 +0100 Subject: [PATCH 23/24] feat: integrate MSAVI cache --- backend/src/service.py | 43 ++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/backend/src/service.py b/backend/src/service.py index b9c337c..7d29a64 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -11,7 +11,9 @@ from src.gee.image_preprocessing import get_preprocessed_imagery from src.gee.sat_index_info import get_sat_index_info from src.cache.ndvi_cache import ndvi_daily_cache +from src.cache.msavi_cache import msavi_daily_cache from typing import List, Dict, Union +from fastapi import HTTPException import math @@ -136,7 +138,9 @@ def sat_index_service( elif start_date < current_cache_end_date and end_date > current_cache_end_date: cache_start_date = start_date cache_end_date = current_cache_end_date - processing_start_date = current_cache_end_date + timedelta(days=1) + processing_start_date = ( + current_cache_end_date + timedelta(days=1) + ) # WARNING: this can cause an empty range request to GEE, TODO: fix empty range case processing_end_date = end_date # Entire range is outside the cache, @@ -149,28 +153,34 @@ def sat_index_service( # Get and process uncached range if processing_start_date: - print(f'Getting {processing_start_date.date()} to {processing_end_date.date()} from GEE.') + print( + f"Getting {processing_start_date.date()} to {processing_end_date.date()} from GEE." + ) masked_images = get_preprocessed_imagery( LocationPolygon[location.value].value, processing_start_date, processing_end_date, ) - NDVI_time_series = get_sat_index_info( + sat_index_time_series = get_sat_index_info( masked_images, LocationPolygon[location.value].value, index_type ) # Get cached range if cache_start_date: - print(f'Getting {cache_start_date.date()} to {cache_end_date.date()} from cache.') - cached_data_subset = get_cache_subset(cache_start_date, cache_end_date) + print( + f"Getting {cache_start_date.date()} to {cache_end_date.date()} from cache." + ) + cached_data_subset = get_cache_subset( + cache_start_date, cache_end_date, index_type + ) if processing_start_date and cache_start_date: - ndvi_data = cached_data_subset + NDVI_time_series + index_data = cached_data_subset + sat_index_time_series else: - ndvi_data = cached_data_subset if cache_start_date else NDVI_time_series + index_data = cached_data_subset if cache_start_date else sat_index_time_series index_df = initialize_time_series( - ndvi_data, temporal_resolution, aggregation_method + index_data, temporal_resolution, aggregation_method ) filled_df = fill_missing_dates(index_df, start_date, end_date, temporal_resolution) @@ -178,9 +188,22 @@ def sat_index_service( return convert_df_to_list(filled_df) -def get_cache_subset(start_date: datetime, end_date: datetime): +def get_cache_subset(start_date: datetime, end_date: datetime, index_type: IndexType): + match index_type: + case IndexType.NDVI: + cache = ndvi_daily_cache + case IndexType.MSAVI: + cache = msavi_daily_cache + case _: + cache = None + + if cache is None: + raise HTTPException( + status_code=404, detail="Cache not found for requested index type." + ) + subset: list[dict] = [] - for entry in ndvi_daily_cache: + for entry in cache: if entry["timestamp"] >= int(start_date.timestamp()) and entry[ "timestamp" ] <= int(end_date.timestamp()): From 21b62889fd2f23d66d3836a8346c8487eabe6203 Mon Sep 17 00:00:00 2001 From: wherop Date: Mon, 25 Nov 2024 16:20:44 +0100 Subject: [PATCH 24/24] fix(index service): handle requests where start or end date equals cache end date (#30) --- backend/src/service.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/src/service.py b/backend/src/service.py index 7d29a64..074c508 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -122,12 +122,12 @@ def sat_index_service( ): # Temporary implementation of GEE Caching strategy current_cache_end_date = datetime( - 2024, 9, 29, tzinfo=timezone.utc + 2024, 9, 29, 23, 59, 59, tzinfo=timezone.utc ) # current end of cache # Entire range is within the cache, # get entire range from cache, process nothing. - if start_date < current_cache_end_date and end_date < current_cache_end_date: + if start_date < current_cache_end_date and end_date <= current_cache_end_date: cache_start_date = start_date cache_end_date = end_date processing_start_date = None @@ -135,12 +135,12 @@ def sat_index_service( # Partial overlap with the cache, # get cached part from cache, process the rest until end of range. - elif start_date < current_cache_end_date and end_date > current_cache_end_date: + elif start_date <= current_cache_end_date and end_date > current_cache_end_date: cache_start_date = start_date cache_end_date = current_cache_end_date processing_start_date = ( - current_cache_end_date + timedelta(days=1) - ) # WARNING: this can cause an empty range request to GEE, TODO: fix empty range case + current_cache_end_date + timedelta(seconds=1) + ) processing_end_date = end_date # Entire range is outside the cache, @@ -151,6 +151,11 @@ def sat_index_service( processing_start_date = start_date processing_end_date = end_date + else: + raise HTTPException( + status_code=422, + detail="Unprocessable input: Input value is not supported." + ) # Get and process uncached range if processing_start_date: print(