diff --git a/backend/src/gee/index.py b/backend/src/gee/index.py index 1834e48..e45001f 100644 --- a/backend/src/gee/index.py +++ b/backend/src/gee/index.py @@ -19,9 +19,12 @@ def get_preprocessed_imagery( image_collection = get_imagery(aoi, start_date, end_date) # Preprocess the imagery + image_collection = image_collection.select( + "SCL", "B8", "B4") # Removing unused bands mosaicked_collection = get_mosaicked_by_date_collection(image_collection) clipped_collection = mosaicked_collection.map(lambda img: img.clip(aoi)) - cloud_masked_collection = clipped_collection.map(lambda img: get_cloud_masked(img)) + cloud_masked_collection = clipped_collection.map( + lambda img: get_cloud_masked(img)) return cloud_masked_collection diff --git a/backend/src/gee/ndvi.py b/backend/src/gee/ndvi.py index 85ccf9a..a571b56 100644 --- a/backend/src/gee/ndvi.py +++ b/backend/src/gee/ndvi.py @@ -53,9 +53,10 @@ def get_ndvi_info( ).getInfo() if len(timestamp_list) != len(index_value_list): - print(f"Timestamps: {len(timestamp_list)}, Values: {len(index_value_list)}") raise ValueError( - "The lists of gee indexing values and timestamps do not have the same size" + "The lists of gee indexing values and timestamps do not have the same size" + + f"Timestamps: {len(timestamp_list)}, Values: {len(index_value_list)}" + ) # Convert the results to a list of dictionaries diff --git a/backend/src/gee/ndvi_cache.py b/backend/src/gee/ndvi_cache.py new file mode 100644 index 0000000..115cfd9 --- /dev/null +++ b/backend/src/gee/ndvi_cache.py @@ -0,0 +1,539 @@ +ndvi_daily_cache = [ + + {'timestamp': 1493251200, 'value': 0.6265267304234295}, + {'timestamp': 1494720000, 'value': 0.68603163673333}, + {'timestamp': 1494979200, 'value': 0.755257128311451}, + {'timestamp': 1495843200, 'value': 0.7354373002096525}, + {'timestamp': 1496707200, 'value': 0.31356286742809775}, + {'timestamp': 1497571200, 'value': 0.4542143670691183}, + {'timestamp': 1498435200, 'value': 0.5919617512091055}, + {'timestamp': 1499040000, 'value': 0.6539697157884732}, + {'timestamp': 1499299200, 'value': 0.6303952412263235}, + {'timestamp': 1499904000, 'value': 0.3453876749102669}, + {'timestamp': 1501632000, 'value': 0.4050406969946647}, + {'timestamp': 1502755200, 'value': 0.6429425803278062}, + {'timestamp': 1504483200, 'value': 0.5918015606195185}, + {'timestamp': 1505088000, 'value': 0.5199806437043044}, + {'timestamp': 1505952000, 'value': 0.5613072154540079}, + {'timestamp': 1509667200, 'value': 0.355040321129086}, + {'timestamp': 1511136000, 'value': 0.4793029710214357}, + {'timestamp': 1512000000, 'value': 0.5144682745732605}, + {'timestamp': 1512259200, 'value': 0.4812850460359476}, + {'timestamp': 1512864000, 'value': 0.4915769963854998}, + {'timestamp': 1513123200, 'value': 0.5612609179087754}, + {'timestamp': 1513555200, 'value': 0.49397625433694664}, + {'timestamp': 1513728000, 'value': 0.10078945539648299}, + {'timestamp': 1513987200, 'value': -0.05380305031374222}, + # until here its 2017 + {'timestamp': 1515888000, 'value': 0.01634385135231174}, + {'timestamp': 1516579200, 'value': 0.4084963960575644}, + {'timestamp': 1517443200, 'value': 0.5275309839482423}, + {'timestamp': 1517875200, 'value': 0.506521617561721}, + {'timestamp': 1518048000, 'value': 0.490625119426628}, + {'timestamp': 1518480000, 'value': 0.5111775938710303}, + {'timestamp': 1518739200, 'value': 0.46794661547132277}, + {'timestamp': 1519344000, 'value': 0.36935062396906804}, + {'timestamp': 1519776000, 'value': 0.27336033950917427}, + {'timestamp': 1520035200, 'value': 0.37743127832902273}, + {'timestamp': 1521331200, 'value': 0.342227538494118}, + {'timestamp': 1521936000, 'value': 0.3081052442616005}, + {'timestamp': 1522627200, 'value': 0.37092985707956994}, + {'timestamp': 1523059200, 'value': 0.4297057750906333}, + {'timestamp': 1523232000, 'value': 0.4387548022615023}, + {'timestamp': 1523923200, 'value': 0.65378630233386}, + {'timestamp': 1524096000, 'value': 0.633789637457917}, + {'timestamp': 1524355200, 'value': 0.7060110186976779}, + {'timestamp': 1524787200, 'value': 0.7624711921453309}, + {'timestamp': 1524960000, 'value': 0.7497715661757918}, + {'timestamp': 1525392000, 'value': 0.7596553124834161}, + {'timestamp': 1525651200, 'value': 0.754118311745244}, + {'timestamp': 1525824000, 'value': 0.7232205427443583}, + {'timestamp': 1526083200, 'value': 0.3822415269246948}, + {'timestamp': 1526256000, 'value': 0.7340398528120343}, + {'timestamp': 1526515200, 'value': 0.28521040360598626}, + {'timestamp': 1526947200, 'value': 0.6952578447035102}, + {'timestamp': 1527120000, 'value': 0.6232964708384111}, + {'timestamp': 1527552000, 'value': 0.6300091691019815}, + {'timestamp': 1527811200, 'value': 0.5837729773361593}, + {'timestamp': 1528243200, 'value': 0.6409030517490615}, + {'timestamp': 1528416000, 'value': 0.6041067426351739}, + {'timestamp': 1528675200, 'value': 0.5428701270313011}, + {'timestamp': 1529107200, 'value': 0.44278144287635474}, + {'timestamp': 1529280000, 'value': 0.34814687536377137}, + {'timestamp': 1530403200, 'value': 0.5099651844736147}, + {'timestamp': 1530576000, 'value': 0.4806309790189844}, + {'timestamp': 1531008000, 'value': 0.29236687187783805}, + {'timestamp': 1531699200, 'value': 0.4351651719304671}, + {'timestamp': 1531872000, 'value': 0.40396922025585447}, + {'timestamp': 1532304000, 'value': 0.4929734694779009}, + {'timestamp': 1532563200, 'value': 0.5049480674886351}, + {'timestamp': 1532736000, 'value': 0.47175783767312296}, + {'timestamp': 1532995200, 'value': 0.4686040013961713}, + {'timestamp': 1533427200, 'value': 0.3689835971360573}, + {'timestamp': 1533600000, 'value': 0.41796636527900427}, + {'timestamp': 1534032000, 'value': 0.38439341812607486}, + {'timestamp': 1534896000, 'value': 0.3416124871522577}, + {'timestamp': 1535155200, 'value': 0.2764639043940805}, + {'timestamp': 1535760000, 'value': 0.2294651988350261}, + {'timestamp': 1536019200, 'value': 0.2994392537612457}, + {'timestamp': 1536192000, 'value': 0.2836145495281138}, + {'timestamp': 1536451200, 'value': 0.2877444495163961}, + {'timestamp': 1537315200, 'value': 0.2518901279728607}, + {'timestamp': 1537747200, 'value': 0.04412068554646227}, + {'timestamp': 1538179200, 'value': 0.27060413178935944}, + {'timestamp': 1538352000, 'value': 0.218526678880176}, + {'timestamp': 1539216000, 'value': 0.3358831303480651}, + {'timestamp': 1539475200, 'value': 0.34343979370280736}, + {'timestamp': 1539648000, 'value': 0.32026366381651145}, + {'timestamp': 1539907200, 'value': 0.31340923136862847}, + {'timestamp': 1540080000, 'value': 0.36016472321904647}, + {'timestamp': 1540339200, 'value': 0.3460083515305792}, + {'timestamp': 1540944000, 'value': 0.42180934711053264}, + {'timestamp': 1541203200, 'value': 0.3606669028141891}, + {'timestamp': 1541635200, 'value': 0.03708648541774034}, + {'timestamp': 1542499200, 'value': 0.17338581915345114}, + {'timestamp': 1543363200, 'value': 0.4738111853498088}, + {'timestamp': 1543968000, 'value': 0.47951016892420906}, + {'timestamp': 1544227200, 'value': 0.0663400635450533}, + {'timestamp': 1544400000, 'value': 0.22558175917054904}, + {'timestamp': 1545696000, 'value': 0.01207277210298325}, + {'timestamp': 1546128000, 'value': 0.06949516092806259}, + # until here its 2018 + + {'timestamp': 1546387200, 'value': 0.19575113537918123}, + {'timestamp': 1546819200, 'value': 0.004088486747965585}, + {'timestamp': 1547424000, 'value': 0.4628066318766027}, + {'timestamp': 1548115200, 'value': 0.4601170296239082}, + {'timestamp': 1548547200, 'value': 0.16502884449147368}, + {'timestamp': 1549584000, 'value': 0.45229330852409055}, + {'timestamp': 1549843200, 'value': 0.15426414088875784}, + {'timestamp': 1550275200, 'value': 0.4844492269864073}, + {'timestamp': 1550448000, 'value': 0.45415724658453716}, + {'timestamp': 1550707200, 'value': 0.23666395258440662}, + {'timestamp': 1551139200, 'value': 0.07907500390519201}, + {'timestamp': 1551744000, 'value': 0.1974367818867876}, + {'timestamp': 1552435200, 'value': 0.2763185157667595}, + {'timestamp': 1552867200, 'value': -0.0071800199836781695}, + {'timestamp': 1553040000, 'value': 0.44023582421288454}, + {'timestamp': 1553299200, 'value': 0.5616058583472479}, + {'timestamp': 1554163200, 'value': 0.6088354782316263}, + {'timestamp': 1554595200, 'value': 0.6399512199914456}, + {'timestamp': 1554768000, 'value': 0.6646672015472888}, + {'timestamp': 1555459200, 'value': 0.6485335011250782}, + {'timestamp': 1555632000, 'value': 0.6660331815652318}, + {'timestamp': 1555891200, 'value': 0.6974715796915939}, + {'timestamp': 1556064000, 'value': 0.6591090636161403}, + {'timestamp': 1557187200, 'value': 0.6921844436335354}, + {'timestamp': 1557619200, 'value': 0.7237710469613807}, + {'timestamp': 1558051200, 'value': 0.5622858292688305}, + {'timestamp': 1558224000, 'value': 0.6935684926748218}, + {'timestamp': 1559088000, 'value': 0.6245298968235982}, + {'timestamp': 1559347200, 'value': 0.6883854192631761}, + {'timestamp': 1559520000, 'value': 0.7016563575969763}, + {'timestamp': 1559779200, 'value': 0.6850383691684236}, + {'timestamp': 1560211200, 'value': 0.656470990596835}, + {'timestamp': 1560384000, 'value': 0.7238889987682}, + {'timestamp': 1560816000, 'value': 0.7035307391094375}, + {'timestamp': 1561075200, 'value': 0.6196358167172958}, + {'timestamp': 1561248000, 'value': 0.5980658765203083}, + {'timestamp': 1561680000, 'value': 0.5682400437486497}, + {'timestamp': 1561939200, 'value': 0.3400334709746356}, + {'timestamp': 1562371200, 'value': 0.16810500318259972}, + {'timestamp': 1562803200, 'value': 0.5256886782823283}, + {'timestamp': 1562976000, 'value': 0.4828430203411382}, + {'timestamp': 1563408000, 'value': 0.4477864993285167}, + {'timestamp': 1563667200, 'value': 0.43142048885631473}, + {'timestamp': 1563840000, 'value': 0.5123785941775288}, + {'timestamp': 1564099200, 'value': 0.5118334691193546}, + {'timestamp': 1564272000, 'value': 0.47658669387040065}, + {'timestamp': 1564704000, 'value': 0.2993281732463428}, + {'timestamp': 1565395200, 'value': 0.3997237032477171}, + {'timestamp': 1565568000, 'value': 0.3734611545776927}, + {'timestamp': 1566432000, 'value': 0.553340821019502}, + {'timestamp': 1566691200, 'value': 0.5513021771738195}, + {'timestamp': 1566864000, 'value': 0.5463825185642255}, + {'timestamp': 1567123200, 'value': 0.21662176920867515}, + {'timestamp': 1567296000, 'value': 0.43273218662635077}, + {'timestamp': 1567555200, 'value': 0.48772592943947113}, + {'timestamp': 1567728000, 'value': 0.4215565424979785}, + {'timestamp': 1568160000, 'value': 0.49199222075663573}, + {'timestamp': 1568419200, 'value': 0.4546686358405888}, + {'timestamp': 1568851200, 'value': 0.5316299742986982}, + {'timestamp': 1569024000, 'value': 0.5050756284663493}, + {'timestamp': 1569456000, 'value': 0.3714272473547276}, + {'timestamp': 1569715200, 'value': 0.012086493540667431}, + {'timestamp': 1571011200, 'value': 0.5601561861315829}, + {'timestamp': 1572307200, 'value': 0.5356334635640078}, + {'timestamp': 1572912000, 'value': 0.4391747111811623}, + {'timestamp': 1573344000, 'value': 0.6659406925942615}, + {'timestamp': 1574467200, 'value': 0.32388295893529057}, + {'timestamp': 1574899200, 'value': 0.17902818025326842}, + {'timestamp': 1575072000, 'value': 0.4713729520715623}, + {'timestamp': 1575331200, 'value': 0.6132103540678879}, + {'timestamp': 1575504000, 'value': 0.5894704386046352}, + {'timestamp': 1575936000, 'value': 0.6217669074338891}, + {'timestamp': 1576195200, 'value': 0.5687294579081753}, + {'timestamp': 1576368000, 'value': 0.18916815180865942}, + {'timestamp': 1576800000, 'value': 0.6037218544209375}, + # until here its 2019 + {'timestamp': 1577923200, 'value': 0.6106607485321933}, + {'timestamp': 1578355200, 'value': 0.002772249651503641}, + {'timestamp': 1578960000, 'value': 0.039889901477437366}, + {'timestamp': 1579219200, 'value': 0.5498426602606883}, + {'timestamp': 1579392000, 'value': 0.12463008162965945}, + {'timestamp': 1579824000, 'value': 0.5452118568383161}, + {'timestamp': 1580688000, 'value': 0.19285440472844556}, + {'timestamp': 1581120000, 'value': 0.5349233397200529}, + {'timestamp': 1581984000, 'value': 0.41061245567924765}, + {'timestamp': 1582243200, 'value': 0.4987324301884347}, + {'timestamp': 1583107200, 'value': 0.16397724497593927}, + {'timestamp': 1583280000, 'value': 0.32556816604560035}, + {'timestamp': 1583712000, 'value': 0.028160691939547327}, + {'timestamp': 1583971200, 'value': 0.40284770443025025}, + {'timestamp': 1584144000, 'value': 0.6349186440600055}, + {'timestamp': 1584403200, 'value': 0.2791129158697314}, + {'timestamp': 1584576000, 'value': 0.3762696064585376}, + {'timestamp': 1584835200, 'value': 0.6511059563604233}, + {'timestamp': 1585008000, 'value': 0.6329986751370975}, + {'timestamp': 1585267200, 'value': 0.5987580526634962}, + {'timestamp': 1585699200, 'value': 0.6227551970900924}, + {'timestamp': 1585872000, 'value': 0.5347724961569185}, + {'timestamp': 1586131200, 'value': 0.6327020615368077}, + {'timestamp': 1586304000, 'value': 0.6006515508260071}, + {'timestamp': 1586736000, 'value': 0.6522948086132051}, + {'timestamp': 1586995200, 'value': 0.6110782070702317}, + {'timestamp': 1587168000, 'value': 0.6417717497420365}, + {'timestamp': 1587427200, 'value': 0.6438999314473854}, + {'timestamp': 1587600000, 'value': 0.6296070325555527}, + {'timestamp': 1587859200, 'value': 0.6139715295880748}, + {'timestamp': 1588032000, 'value': 0.6016991503602547}, + {'timestamp': 1588291200, 'value': 0.6048452268959833}, + {'timestamp': 1588464000, 'value': 0.3548655861699142}, + {'timestamp': 1588723200, 'value': 0.7106839685170496}, + {'timestamp': 1588896000, 'value': 0.7112474894558717}, + {'timestamp': 1589328000, 'value': 0.6296398878166072}, + {'timestamp': 1589587200, 'value': 0.6739577862539349}, + {'timestamp': 1590192000, 'value': 0.0571304518695734}, + {'timestamp': 1590451200, 'value': 0.7060073637909001}, + {'timestamp': 1590624000, 'value': 0.2973821568775205}, + {'timestamp': 1590883200, 'value': 0.6854767595194105}, + {'timestamp': 1591056000, 'value': 0.6485029921918724}, + {'timestamp': 1591315200, 'value': 0.49870885670099085}, + {'timestamp': 1591488000, 'value': 0.3763445984837111}, + {'timestamp': 1592179200, 'value': 0.6149271317935514}, + {'timestamp': 1592352000, 'value': 0.1414903367375361}, + {'timestamp': 1592784000, 'value': 0.5948858659053022}, + {'timestamp': 1593043200, 'value': 0.4498251464639131}, + {'timestamp': 1593216000, 'value': 0.5272039951120531}, + {'timestamp': 1594080000, 'value': 0.3837704256571959}, + {'timestamp': 1594339200, 'value': 0.2911646173299671}, + {'timestamp': 1594512000, 'value': 0.4977832283157321}, + {'timestamp': 1595203200, 'value': 0.20726488571115734}, + {'timestamp': 1596067200, 'value': 0.49307978331855673}, + {'timestamp': 1596240000, 'value': 0.4566579362561879}, + {'timestamp': 1596499200, 'value': 0.391521811571442}, + {'timestamp': 1596672000, 'value': 0.42554546950592553}, + {'timestamp': 1596931200, 'value': 0.35022409279627237}, + {'timestamp': 1597104000, 'value': 0.38468834195220786}, + {'timestamp': 1597536000, 'value': 0.3412284738505061}, + {'timestamp': 1597795200, 'value': 0.30528961677949606}, + {'timestamp': 1597968000, 'value': 0.32196833124223506}, + {'timestamp': 1598832000, 'value': 0.35366390955095417}, + {'timestamp': 1599091200, 'value': 0.4343526544785653}, + {'timestamp': 1599264000, 'value': 0.03448406973182179}, + {'timestamp': 1599696000, 'value': 0.37114051174253926}, + {'timestamp': 1599955200, 'value': 0.5749910861856561}, + {'timestamp': 1600128000, 'value': 0.584619444298503}, + {'timestamp': 1600387200, 'value': 0.6151645893185002}, + {'timestamp': 1600560000, 'value': 0.5793651023317429}, + {'timestamp': 1600819200, 'value': 0.5599830871951561}, + {'timestamp': 1601251200, 'value': 0.6473475164629133}, + {'timestamp': 1601424000, 'value': 0.632010548709421}, + {'timestamp': 1601683200, 'value': 0.6410922055636433}, + {'timestamp': 1602115200, 'value': 0.4740622153388786}, + {'timestamp': 1602979200, 'value': 0.005711123344876248}, + {'timestamp': 1603584000, 'value': 0.6994891137287609}, + {'timestamp': 1604448000, 'value': 0.608658248994832}, + {'timestamp': 1604707200, 'value': 0.7115847640768033}, + {'timestamp': 1605139200, 'value': 0.08949784970353686}, + {'timestamp': 1605744000, 'value': 0.022849772434432095}, + {'timestamp': 1606435200, 'value': 0.4188422043235015}, + {'timestamp': 1606608000, 'value': 0.2500683303648727}, + {'timestamp': 1606867200, 'value': 0.2604494519978076}, + {'timestamp': 1607040000, 'value': 0.07083324936938587}, + {'timestamp': 1607299200, 'value': 0.002813341091232573}, + {'timestamp': 1607731200, 'value': 0.4187637973213316}, + {'timestamp': 1608336000, 'value': 0.612775111399876}, + {'timestamp': 1608595200, 'value': -0.029898428378950404}, + {'timestamp': 1608768000, 'value': 0.004315493270424968}, + {'timestamp': 1609200000, 'value': 0.4312730098931475}, + # until here its 2020 + {'timestamp': 1609459200, 'value': 0.01435058501894068}, + {'timestamp': 1609891200, 'value': 0.2923432869827817}, + {'timestamp': 1610323200, 'value': 0.0668237159232399}, + {'timestamp': 1611187200, 'value': 0.5369455034572239}, + {'timestamp': 1612051200, 'value': 0.055436857855142414}, + {'timestamp': 1612915200, 'value': 0.014911965149072238}, + {'timestamp': 1613088000, 'value': 0.01704268959229717}, + {'timestamp': 1613952000, 'value': 0.47666126930181013}, + {'timestamp': 1614211200, 'value': 0.40249965224608275}, + {'timestamp': 1614643200, 'value': 0.48217023910839407}, + {'timestamp': 1615248000, 'value': 0.4857652322292789}, + {'timestamp': 1615507200, 'value': 0.23827403158690943}, + {'timestamp': 1616371200, 'value': 0.5281767438543209}, + {'timestamp': 1617235200, 'value': 0.012463302374166159}, + {'timestamp': 1617408000, 'value': 0.36797845579231414}, + {'timestamp': 1617667200, 'value': 0.3413998686870444}, + {'timestamp': 1617840000, 'value': -0.0007593096602570172}, + {'timestamp': 1618099200, 'value': 0.1588694278504701}, + {'timestamp': 1618272000, 'value': 0.2530503929402777}, + {'timestamp': 1618963200, 'value': 0.6782696743923541}, + {'timestamp': 1619136000, 'value': 0.5994564865783631}, + {'timestamp': 1619568000, 'value': 0.6737351778513451}, + {'timestamp': 1620000000, 'value': 0.016924522307793323}, + {'timestamp': 1620259200, 'value': 0.01606474366095763}, + {'timestamp': 1620432000, 'value': 0.6330257332134035}, + {'timestamp': 1620691200, 'value': 0.717899089620151}, + {'timestamp': 1621123200, 'value': 0.42380173968839757}, + {'timestamp': 1621296000, 'value': 0.4033266298501044}, + {'timestamp': 1621555200, 'value': 0.6920621748795662}, + {'timestamp': 1621728000, 'value': 0.6260287142711787}, + {'timestamp': 1621987200, 'value': -0.005978637506810127}, + {'timestamp': 1622419200, 'value': 0.7904180101910425}, + {'timestamp': 1622592000, 'value': 0.7507581742359998}, + {'timestamp': 1622851200, 'value': 0.7340596869750035}, + {'timestamp': 1623024000, 'value': 0.713082625860254}, + {'timestamp': 1623283200, 'value': 0.6162225833638466}, + {'timestamp': 1623888000, 'value': 0.5970207971386315}, + {'timestamp': 1624147200, 'value': 0.5150093669774768}, + {'timestamp': 1624320000, 'value': 0.021000450337780315}, + {'timestamp': 1624579200, 'value': 0.45922488065462064}, + {'timestamp': 1624752000, 'value': 0.4967584306180959}, + {'timestamp': 1625184000, 'value': 0.3931810201670696}, + {'timestamp': 1625443200, 'value': 0.029347269193163553}, + {'timestamp': 1625875200, 'value': 0.4619241476221313}, + {'timestamp': 1626048000, 'value': 0.4226732311911146}, + {'timestamp': 1626307200, 'value': 0.49927513744860536}, + {'timestamp': 1626480000, 'value': 0.18278688072834576}, + {'timestamp': 1626912000, 'value': 0.41382215157983376}, + {'timestamp': 1627171200, 'value': 0.6010501555679375}, + {'timestamp': 1627344000, 'value': 0.6117434219873504}, + {'timestamp': 1627603200, 'value': 0.5208266892285692}, + {'timestamp': 1628035200, 'value': 0.6441416594696726}, + {'timestamp': 1628208000, 'value': 0.6034584964961202}, + {'timestamp': 1628467200, 'value': 0.64590786606526}, + {'timestamp': 1628640000, 'value': 0.4772097732538933}, + {'timestamp': 1628899200, 'value': 0.5880408265167602}, + {'timestamp': 1629072000, 'value': 0.44680911441258525}, + {'timestamp': 1629504000, 'value': 0.109105974373757}, + {'timestamp': 1629763200, 'value': 0.5395000130664906}, + {'timestamp': 1629936000, 'value': 0.458826030047935}, + {'timestamp': 1630195200, 'value': 0.28350180200114466}, + {'timestamp': 1630627200, 'value': 0.5688902152406968}, + {'timestamp': 1630800000, 'value': 0.43670135868371135}, + {'timestamp': 1631059200, 'value': 0.5948213293638251}, + {'timestamp': 1631232000, 'value': 0.554547020677893}, + {'timestamp': 1631923200, 'value': -0.017812937577812448}, + {'timestamp': 1632096000, 'value': 0.0019416418307588795}, + {'timestamp': 1632787200, 'value': 0.027521948760183122}, + {'timestamp': 1633651200, 'value': 0.6992170331018968}, + {'timestamp': 1633824000, 'value': 0.7061244648094013}, + {'timestamp': 1634083200, 'value': 0.6225944600006217}, + {'timestamp': 1634256000, 'value': 0.05714529819266201}, + {'timestamp': 1634947200, 'value': 0.6253835953395109}, + {'timestamp': 1635120000, 'value': 0.646177337901081}, + {'timestamp': 1635379200, 'value': 0.6667752468768585}, + {'timestamp': 1635552000, 'value': 0.6417093892688839}, + {'timestamp': 1636243200, 'value': 0.5549119008730556}, + {'timestamp': 1637539200, 'value': 0.5842417213194347}, + {'timestamp': 1637712000, 'value': -0.02299490551774671}, + {'timestamp': 1638144000, 'value': -0.00758547438802522}, + {'timestamp': 1638403200, 'value': 0.5143914299255691}, + {'timestamp': 1638576000, 'value': -0.1109406938627911}, + {'timestamp': 1639008000, 'value': 0.023029429420381983}, + {'timestamp': 1639267200, 'value': 0.024638896217661042}, + {'timestamp': 1639699200, 'value': -0.014363757133942142}, + {'timestamp': 1639872000, 'value': 0.011914837591533221}, + # until here its 2021 + {'timestamp': 1641427200, 'value': 0.5101549880831526}, + {'timestamp': 1641600000, 'value': 0.3887980581208711}, + {'timestamp': 1642464000, 'value': 0.2908277073649076}, + {'timestamp': 1642723200, 'value': 0.09096975011191563}, + {'timestamp': 1643328000, 'value': 0.369768095625778}, + {'timestamp': 1643760000, 'value': 0.2962413837894405}, + {'timestamp': 1644624000, 'value': 0.46836335847631205}, + {'timestamp': 1645920000, 'value': 0.2082614694292724}, + {'timestamp': 1646611200, 'value': 0.47022668892142844}, + {'timestamp': 1646784000, 'value': 0.45634158198743174}, + {'timestamp': 1647043200, 'value': 0.4641304764728491}, + {'timestamp': 1647216000, 'value': 0.439508319453624}, + {'timestamp': 1647648000, 'value': 0.4221169001419796}, + {'timestamp': 1647907200, 'value': 0.4425966960328659}, + {'timestamp': 1648080000, 'value': 0.4753782294251874}, + {'timestamp': 1648339200, 'value': 0.45627407595897207}, + {'timestamp': 1648512000, 'value': 0.47646076351947864}, + {'timestamp': 1648944000, 'value': 0.38671477286937855}, + {'timestamp': 1649635200, 'value': 0.58796790004507}, + {'timestamp': 1650067200, 'value': 0.6347346101398396}, + {'timestamp': 1650240000, 'value': 0.6263818349781571}, + {'timestamp': 1650672000, 'value': 0.6495226867754919}, + {'timestamp': 1650931200, 'value': 0.6867098674822344}, + {'timestamp': 1651104000, 'value': 0.6295148354799598}, + {'timestamp': 1651536000, 'value': 0.6663085005558228}, + {'timestamp': 1651795200, 'value': 0.6612043296879485}, + {'timestamp': 1651968000, 'value': 0.6902022479410631}, + {'timestamp': 1652227200, 'value': 0.6758318769248401}, + {'timestamp': 1652659200, 'value': 0.6560463825331692}, + {'timestamp': 1652832000, 'value': 0.6743604469746679}, + {'timestamp': 1653091200, 'value': 0.5917613969845943}, + {'timestamp': 1654128000, 'value': 0.6859302960341542}, + {'timestamp': 1654819200, 'value': 0.6118822200279516}, + {'timestamp': 1655856000, 'value': 0.5172830428976952}, + {'timestamp': 1656288000, 'value': 0.43986954061211364}, + {'timestamp': 1656547200, 'value': 0.34190041012449773}, + {'timestamp': 1656720000, 'value': 0.4193678789443398}, + {'timestamp': 1656979200, 'value': 0.393259289566778}, + {'timestamp': 1657584000, 'value': 0.4309890930798414}, + {'timestamp': 1658016000, 'value': 0.2777647567881633}, + {'timestamp': 1658275200, 'value': 0.39715504435734067}, + {'timestamp': 1658707200, 'value': 0.37016542504876004}, + {'timestamp': 1659571200, 'value': 0.34231296403384054}, + {'timestamp': 1659744000, 'value': 0.28926778249008184}, + {'timestamp': 1660003200, 'value': 0.30904499302081206}, + {'timestamp': 1660608000, 'value': 0.23752333515678647}, + {'timestamp': 1661299200, 'value': 0.21692575013363097}, + {'timestamp': 1661472000, 'value': 0.26788093919798717}, + {'timestamp': 1662163200, 'value': 0.5217756707981661}, + {'timestamp': 1662336000, 'value': 0.5977810346275365}, + {'timestamp': 1662768000, 'value': 0.5379871186171694}, + {'timestamp': 1663459200, 'value': 0.7349933166750889}, + {'timestamp': 1663632000, 'value': 0.673641456907865}, + {'timestamp': 1664064000, 'value': 0.6215492199142049}, + {'timestamp': 1664496000, 'value': 0.772380642343471}, + {'timestamp': 1664755200, 'value': 0.6334922152602114}, + {'timestamp': 1664928000, 'value': 0.6156581292973645}, + {'timestamp': 1666483200, 'value': 0.6692634931805789}, + {'timestamp': 1666915200, 'value': 0.7228286861507973}, + {'timestamp': 1667088000, 'value': 0.726920807653625}, + {'timestamp': 1667347200, 'value': 0.7427448875178663}, + {'timestamp': 1668211200, 'value': 0.6738504720290368}, + {'timestamp': 1668384000, 'value': 0.721222057771743}, + {'timestamp': 1669507200, 'value': 0.666140043780319}, + {'timestamp': 1671235200, 'value': 0.039560165834786186}, + {'timestamp': 1672099200, 'value': 0.4181342412037926}, + + # until here its 2022 + {'timestamp': 1672704000, 'value': 0.49311103866901945}, + {'timestamp': 1673827200, 'value': 0.5721801522171637}, + {'timestamp': 1677715200, 'value': 0.40309570282911744}, + {'timestamp': 1678579200, 'value': 0.3285946798542066}, + {'timestamp': 1679875200, 'value': 0.604913363250996}, + {'timestamp': 1680048000, 'value': 0.6241211192214813}, + {'timestamp': 1680739200, 'value': 0.6770332524516463}, + {'timestamp': 1682035200, 'value': 0.7195395171743766}, + {'timestamp': 1683072000, 'value': 0.776642542110257}, + {'timestamp': 1683504000, 'value': 0.7946412386885652}, + {'timestamp': 1683763200, 'value': 0.6962946310451835}, + {'timestamp': 1683936000, 'value': 0.7803666870020902}, + {'timestamp': 1684627200, 'value': 0.744074654951325}, + {'timestamp': 1685059200, 'value': 0.7603736326247179}, + {'timestamp': 1685232000, 'value': 0.7307521424847342}, + {'timestamp': 1685491200, 'value': 0.6595660036736165}, + {'timestamp': 1685664000, 'value': 0.6844185268873992}, + {'timestamp': 1685923200, 'value': 0.6609223570577207}, + {'timestamp': 1686096000, 'value': 0.6243770761357701}, + {'timestamp': 1687392000, 'value': 0.33966505122465546}, + {'timestamp': 1687651200, 'value': 0.46991814590987957}, + {'timestamp': 1687824000, 'value': 0.5637975455700079}, + {'timestamp': 1688256000, 'value': 0.5214708303888645}, + {'timestamp': 1688688000, 'value': 0.6235262779374806}, + {'timestamp': 1688947200, 'value': 0.5294325381096169}, + {'timestamp': 1689120000, 'value': 0.5378739374319399}, + {'timestamp': 1689379200, 'value': 0.5774819523508761}, + {'timestamp': 1689552000, 'value': 0.4300296294988784}, + {'timestamp': 1689984000, 'value': 0.3540605442689726}, + {'timestamp': 1690675200, 'value': 0.5559975830254852}, + {'timestamp': 1691539200, 'value': 0.6153487749947343}, + {'timestamp': 1691712000, 'value': 0.4640778921573764}, + {'timestamp': 1691971200, 'value': 0.6340361324676178}, + {'timestamp': 1692144000, 'value': 0.6643802566638307}, + {'timestamp': 1692403200, 'value': 0.6067618892145015}, + {'timestamp': 1692576000, 'value': 0.4995997577630159}, + {'timestamp': 1692835200, 'value': 0.6413726523764097}, + {'timestamp': 1693008000, 'value': 0.5552765524029254}, + {'timestamp': 1693440000, 'value': 0.5019676160726875}, + {'timestamp': 1693699200, 'value': 0.3575762499142059}, + {'timestamp': 1693872000, 'value': 0.636932829286457}, + {'timestamp': 1694131200, 'value': 0.5860366775305434}, + {'timestamp': 1694304000, 'value': 0.5678766403737106}, + {'timestamp': 1694736000, 'value': 0.5551531160988741}, + {'timestamp': 1695168000, 'value': 0.5267156551730596}, + {'timestamp': 1695600000, 'value': 0.3034994712244936}, + {'timestamp': 1696032000, 'value': 0.4584922258377994}, + {'timestamp': 1696723200, 'value': 0.5904255552176838}, + {'timestamp': 1698192000, 'value': 0.41143405956899415}, + {'timestamp': 1699056000, 'value': 0.6294013230687814}, + {'timestamp': 1699315200, 'value': 0.5625790498918815}, + {'timestamp': 1700352000, 'value': 0.3344593256089601}, + {'timestamp': 1700784000, 'value': 0.22997949110107285}, + {'timestamp': 1703203200, 'value': 0.5129322858835728}, + # until here its 2023 + {'timestamp': 1704672000, 'value': 0.4537940461130249}, + {'timestamp': 1706400000, 'value': 0.4841096797858553}, + {'timestamp': 1708128000, 'value': 0.3511160651300334}, + {'timestamp': 1708819200, 'value': 0.562864984877086}, + {'timestamp': 1709424000, 'value': 0.5528860685305712}, + {'timestamp': 1709683200, 'value': 0.4956625869394157}, + {'timestamp': 1709856000, 'value': 0.5740671639202845}, + {'timestamp': 1710288000, 'value': 0.5754883333185951}, + {'timestamp': 1710720000, 'value': 0.5781645983396947}, + {'timestamp': 1710979200, 'value': 0.6237791569348801}, + {'timestamp': 1711411200, 'value': 0.6805453663367135}, + {'timestamp': 1711843200, 'value': 0.44476014092304844}, + {'timestamp': 1712016000, 'value': 0.7037068616309425}, + {'timestamp': 1713312000, 'value': 0.44905630802780866}, + {'timestamp': 1713571200, 'value': 0.6595994526976465}, + {'timestamp': 1714003200, 'value': 0.6618991807578869}, + {'timestamp': 1714176000, 'value': 0.5086040253726309}, + {'timestamp': 1714435200, 'value': 0.7871618291679429}, + {'timestamp': 1714608000, 'value': 0.7642049393963221}, + {'timestamp': 1715299200, 'value': 0.7507461516420122}, + {'timestamp': 1715731200, 'value': 0.729646566522368}, + {'timestamp': 1715904000, 'value': 0.7007678963716821}, + {'timestamp': 1716163200, 'value': 0.4666880552899499}, + {'timestamp': 1716595200, 'value': 0.5894210184248679}, + {'timestamp': 1717200000, 'value': 0.6822153554153109}, + {'timestamp': 1717632000, 'value': 0.6532572845376999}, + {'timestamp': 1717891200, 'value': 0.70896413011386}, + {'timestamp': 1718496000, 'value': 0.603215338515721}, + {'timestamp': 1719187200, 'value': 0.637128334417188}, + {'timestamp': 1719360000, 'value': 0.6068163184736923}, + {'timestamp': 1719619200, 'value': 0.6288699313084715}, + {'timestamp': 1720224000, 'value': 0.5924314172946463}, + {'timestamp': 1720483200, 'value': 0.5156037447155328}, + {'timestamp': 1720656000, 'value': 0.5877750112506149}, + {'timestamp': 1720915200, 'value': 0.5446529073721998}, + {'timestamp': 1721347200, 'value': 0.5742237606531847}, + {'timestamp': 1721520000, 'value': 0.5641866773489524}, + {'timestamp': 1721779200, 'value': 0.4648095090560619}, + {'timestamp': 1722211200, 'value': 0.5643601252638971}, + {'timestamp': 1722643200, 'value': 0.5854776683005347}, + {'timestamp': 1722816000, 'value': 0.6287237036738268}, + {'timestamp': 1723075200, 'value': 0.36412344373600897}, + {'timestamp': 1723248000, 'value': 0.49056984190497194}, + {'timestamp': 1723507200, 'value': 0.6288169228142412}, + {'timestamp': 1724112000, 'value': 0.5628947382513431}, + {'timestamp': 1724371200, 'value': 0.5533013108657124}, + {'timestamp': 1724803200, 'value': 0.5050495959276382}, + {'timestamp': 1724976000, 'value': 0.4609939427470864}, + {'timestamp': 1725235200, 'value': 0.4467849685147322}, + {'timestamp': 1725408000, 'value': 0.4353219027209249}, + {'timestamp': 1725667200, 'value': 0.42904769396575165}, + {'timestamp': 1726099200, 'value': 0.4045242606090997}, + {'timestamp': 1726531200, 'value': 0.4328800438447044}, + {'timestamp': 1726704000, 'value': 0.43806171404605726}, + {'timestamp': 1726963200, 'value': 0.43907114056947844}, + {'timestamp': 1727395200, 'value': 0.46624384621001974}, + {'timestamp': 1727568000, 'value': 0.46813784900297967} + # Sep 29 2024 until here +] diff --git a/backend/src/routes/ndvi_router.py b/backend/src/routes/ndvi_router.py index e60e9ec..0e60ffc 100644 --- a/backend/src/routes/ndvi_router.py +++ b/backend/src/routes/ndvi_router.py @@ -1,8 +1,7 @@ -from datetime import datetime +from datetime import datetime, timezone from fastapi import APIRouter, Query from fastapi.responses import JSONResponse - from src.constants import ( AggregationMethod, LocationName, @@ -13,7 +12,7 @@ from src.utils.temporal import get_optimistic_rounding from src.validation.models import NDVIResponse from src.validation.utils import ( - validate_timestamp_in_range, + validate_timestamp_in_range_of_S2_imagery, validate_timestamp_start_date_before_end_date, ) @@ -22,20 +21,22 @@ @ndvi_router.get("/ndvi", response_model=NDVIResponse) async def get_temperature_data( - startDate: int = Query(..., description="Start date as UNIX timestamp in seconds"), - endDate: int = Query(..., description="End date as UNIX timestamp in seconds"), + startDate: int = Query(..., + description="Start date as UNIX timestamp in seconds"), + endDate: int = Query(..., + description="End date as UNIX timestamp in seconds"), location: LocationName = Query(..., description="Location name"), temporalResolution: TemporalResolution = Query( ..., description="Temporal resolution" ), - aggregation: AggregationMethod = Query(..., description="Aggregation method"), + aggregation: AggregationMethod = Query(..., + description="Aggregation method"), ): - validate_timestamp_in_range(startDate) - validate_timestamp_in_range(endDate) - validate_timestamp_start_date_before_end_date(startDate, endDate) - start_date_dt = datetime.utcfromtimestamp(startDate) - end_date_dt = datetime.utcfromtimestamp(endDate) + validate_timestamp_start_date_before_end_date(startDate, endDate) + validate_timestamp_in_range_of_S2_imagery(startDate, endDate) + start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc) + end_date_dt = datetime.fromtimestamp(endDate, tz=timezone.utc) rounded_start_date, rounded_end_date = get_optimistic_rounding( start_date_dt, end_date_dt, temporalResolution @@ -61,5 +62,4 @@ async def get_temperature_data( "data": data, } - print(response) return JSONResponse(content=response) diff --git a/backend/src/service.py b/backend/src/service.py index 8c4871b..21e4ee1 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -1,77 +1,106 @@ -from datetime import datetime +from datetime import datetime, timezone, timedelta import pandas as pd from src.constants import AggregationMethod, LocationPolygon, TemporalResolution from src.gee.index import get_preprocessed_imagery from src.gee.ndvi import get_ndvi_info +from src.gee.ndvi_cache import ndvi_daily_cache +from typing import List, Dict, Union +import math -def aggregate_time_series( - ndvi_info: list[dict], - temporal_resolution, - aggregation_method, - start_date: datetime, - end_date: datetime, -): - # Generate a complete date range based on temporal resolution - if temporal_resolution == "DAILY": - date_range = pd.date_range(start=start_date, end=end_date, freq="D", tz="UTC") - elif temporal_resolution == "MONTHLY": - date_range = pd.date_range(start=start_date, end=end_date, freq="MS", tz="UTC") - else: - raise ValueError("Unsupported temporal resolution") - - # Create a DataFrame with the full date range, initially filled with None - df = pd.DataFrame(index=date_range) - df.index.name = "timestamp" - df["value"] = None - - # Convert ndvi_info to a DataFrame and set the index - if temporal_resolution == "MONTHLY": - for record in ndvi_info: - record["timestamp"] = record["timestamp"].replace(day=1) - info_df = pd.DataFrame(ndvi_info) - info_df["timestamp"] = pd.to_datetime(info_df["timestamp"], unit="s", utc=True) - - # Align info_df to the temporal resolution - if temporal_resolution == "DAILY": - info_df["timestamp"] = info_df["timestamp"].dt.floor("D") - elif temporal_resolution == "MONTHLY": - info_df["timestamp"] = info_df["timestamp"].dt.to_period("M").dt.to_timestamp() - - info_df.set_index("timestamp", inplace=True) - - # Update the full DataFrame with actual NDVI values - df.loc[info_df.index, "value"] = info_df["value"] - - # Resample the DataFrame based on the temporal resolution - resampled_df = ( - df.resample("D") if temporal_resolution == "DAILY" else df.resample("M") - ) - - # Aggregate the resampled DataFrame based on the aggregation method, ignoring None values - if aggregation_method == "MEAN": - aggregated_df = resampled_df.mean() - elif aggregation_method == "MIN": - aggregated_df = resampled_df.min() - elif aggregation_method == "MAX": - aggregated_df = resampled_df.max() - elif aggregation_method == "MEDIAN": - aggregated_df = resampled_df.median() +def initialize_time_series( + time_series: List[Dict[str, Union[int, float]]], + temporal_resolution: TemporalResolution, + aggregation_method: AggregationMethod +) -> pd.DataFrame: + """ + Initializes a pandas DataFrame from a time series and applies temporal resolution and aggregation. + + Parameters: + time_series (List[Dict[str, Union[int, float]]]): List of dicts with 'timestamp' and 'value'. + temporal_resolution (TemporalResolution): Temporal resolution, either DAILY or MONTHLY. + aggregation_method (AggregationMethod): Aggregation method to use if resolution is MONTHLY. + + Returns: + pd.DataFrame: The resulting DataFrame with applied resolution and aggregation. + """ + # Check if the time_series is empty + if not time_series: + # Return an empty DataFrame with a datetime index and 'value' column in UTC + if temporal_resolution == TemporalResolution.MONTHLY: + empty_index = pd.date_range( + start="1970-01-01", periods=0, freq='MS', tz='UTC') + else: + empty_index = pd.date_range( + start="1970-01-01", periods=0, freq='D', tz='UTC') + + return pd.DataFrame(index=empty_index, columns=['value']) + + # Convert timestamps to datetime in UTC and create DataFrame + df = pd.DataFrame(time_series) + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s', utc=True) + df.set_index('timestamp', inplace=True) + + # Resample based on temporal resolution and apply aggregation if needed + if temporal_resolution == TemporalResolution.MONTHLY: + if aggregation_method == AggregationMethod.MEAN: + df = df.resample('MS').mean() + elif aggregation_method == AggregationMethod.MEDIAN: + df = df.resample('MS').median() + elif aggregation_method == AggregationMethod.MAX: + df = df.resample('MS').max() + elif aggregation_method == AggregationMethod.MIN: + df = df.resample('MS').min() + # If DAILY, do nothing as time series is already in daily format + return df + + +def fill_missing_dates( + df: pd.DataFrame, + start: datetime, + end: datetime, + temporal_resolution: TemporalResolution +) -> pd.DataFrame: + """ + Fills missing entries in the time series, adding NaN for missing days or months. + + Parameters: + df (pd.DataFrame): Input DataFrame with timestamps as index. + start (datetime): Start datetime for filling. + end (datetime): End datetime for filling. + temporal_resolution (TemporalResolution): Temporal resolution, either DAILY or MONTHLY. + + Returns: + pd.DataFrame: DataFrame with missing dates or months filled with NaN values. + """ + # Ensure start and end are in UTC + if start.tzinfo is None: + start = start.replace(tzinfo=timezone.utc) else: - raise ValueError("Unsupported aggregation method") - - # Replace NaNs with None for final output consistency - aggregated_df = aggregated_df.where(pd.notnull(aggregated_df), None) + start = start.astimezone(timezone.utc) - # Convert the aggregated DataFrame back to a list of dicts with ISO format timestamps - aggregated_info = [ - {"timestamp": record["timestamp"].isoformat(), "value": record["value"]} - for record in aggregated_df.reset_index().to_dict(orient="records") - ] + if end.tzinfo is None: + end = end.replace(tzinfo=timezone.utc) + else: + end = end.astimezone(timezone.utc) + + # Generate the complete date range based on the temporal resolution + if temporal_resolution == TemporalResolution.DAILY: + date_range = pd.date_range(start=start, end=end, freq='D', tz='UTC') + elif temporal_resolution == TemporalResolution.MONTHLY: + date_range = pd.date_range(start=start, end=end, freq='MS', tz='UTC') + # If the input DataFrame is empty, create a new one with NaNs for all dates in the range + if df.empty: + df = pd.DataFrame(index=date_range, columns=['value']) + df['value'] = None + else: + # Reindex to the complete date range, filling missing dates with NaN + df = df.reindex(date_range) - return aggregated_info + df.columns = ['value'] + return df def ndvi_service( @@ -81,15 +110,86 @@ def ndvi_service( start_date: datetime, end_date: datetime, ): - masked_images = get_preprocessed_imagery( - LocationPolygon[location.value].value, - start_date, - end_date, - ) - NDVI_time_series = get_ndvi_info( - masked_images, LocationPolygon[location.value].value - ) - aggregated_data_dict = aggregate_time_series( - NDVI_time_series, temporal_resolution, aggregation_method, start_date, end_date - ) - return aggregated_data_dict + # Temporary implementation of GEE Caching strategy + current_cache_end_date = datetime( + 2024, 9, 29, tzinfo=timezone.utc) + if start_date < current_cache_end_date and end_date < current_cache_end_date: # current end of cache + cache_start_date = start_date + cache_end_date = end_date + processing_start_date = None + processing_end_date = None + + elif start_date < current_cache_end_date and end_date > current_cache_end_date: + cache_start_date = start_date + cache_end_date = current_cache_end_date + processing_start_date = current_cache_end_date + timedelta(days=1) + processing_end_date = end_date + + elif start_date > current_cache_end_date: + cache_start_date = None + cache_end_date = None + processing_start_date = start_date + processing_end_date = end_date + + if processing_start_date: + + masked_images = get_preprocessed_imagery( + LocationPolygon[location.value].value, + processing_start_date, + processing_end_date, + ) + NDVI_time_series = get_ndvi_info( + masked_images, LocationPolygon[location.value].value + ) + + if cache_start_date: + cached_data_subset = get_cache_subset(cache_start_date, cache_end_date) + + if processing_start_date and cache_start_date: + ndvi_data = cached_data_subset + NDVI_time_series + else: + ndvi_data = cached_data_subset if cache_start_date else NDVI_time_series + + index_df = initialize_time_series( + ndvi_data, temporal_resolution, aggregation_method) + + filled_df = fill_missing_dates( + index_df, start_date, end_date, temporal_resolution) + + return convert_df_to_list(filled_df) + + +def get_cache_subset(start_date: datetime, end_date: datetime): + subset: list[dict] = [] + for entry in ndvi_daily_cache: + if entry["timestamp"] >= int(start_date.timestamp()) and entry["timestamp"] <= int(end_date.timestamp()): + subset.append(entry) + return subset + + +def convert_df_to_list(df: pd.DataFrame) -> List[Dict[str, Union[int, float, None]]]: + """ + Converts a DataFrame with a datetime index back to a list of dictionaries in the original format. + + Parameters: + df (pd.DataFrame): Input DataFrame with datetime index and a 'value' column. + + Returns: + List[Dict[str, Union[int, float, None]]]: List of dictionaries with 'timestamp' in epoch format and 'value'. + """ + # Convert the DataFrame index to epoch timestamps and reset index + df_reset = df.reset_index() + df_reset['timestamp'] = df_reset['index'].astype(int) // 10**9 + df_reset = df_reset.rename(columns={'value': 'value'}) + + # Convert to list of dictionaries + result = df_reset[['timestamp', 'value']].to_dict(orient='records') + + # Convert NaN to None (needs to handle empty df as well) + for entry in result: + if entry['value'] is None: + entry['value'] = None + elif math.isnan(entry['value']): + entry['value'] = None + + return result diff --git a/backend/src/utils/temporal.py b/backend/src/utils/temporal.py index 5eb73cf..90bc123 100644 --- a/backend/src/utils/temporal.py +++ b/backend/src/utils/temporal.py @@ -1,3 +1,4 @@ +from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta, timezone from src.constants import TemporalResolution @@ -7,46 +8,30 @@ def get_optimistic_rounding( start_date: datetime, end_date: datetime, temporal_resolution: TemporalResolution ) -> (datetime, datetime): """Get optimistic rounding for the start date based on the temporal resolution.""" - start_date = start_date.astimezone(timezone.utc) - end_date = end_date.astimezone(timezone.utc) + if temporal_resolution == TemporalResolution.DAILY: - start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0) - end_date = end_date.replace(hour=23, minute=59, second=59, microsecond=999999) + start_date = first_second_of_day(start_date) + end_date = last_second_of_day(end_date) elif temporal_resolution == TemporalResolution.MONTHLY: - start_date = get_start_of_day(start_date) - end_date = last_time_of_month(end_date) - # Set end_date to the last day of the month at 23:59:59.999999 - next_month = end_date.replace(day=1, month=end_date.month % 12 + 1) - end_date = next_month - timedelta(seconds=1) - else: - raise ValueError(f"Temporal resolution {temporal_resolution} not supported.") - print(start_date, end_date) - return start_date, end_date - + start_date = first_second_of_month(start_date) + end_date = last_second_of_month(end_date) -def last_time_of_month(dt: datetime) -> datetime: - # Move to the beginning of the next month - if dt.month == 12: - next_month = datetime(dt.year + 1, 1, 1) - else: - next_month = datetime(dt.year, dt.month + 1, 1) + return start_date, end_date - # Subtract one microsecond to get the last moment of the given month - last_time = next_month - timedelta(microseconds=1) - return last_time +def first_second_of_day(dt: datetime) -> datetime: + return dt.replace(hour=0, minute=0, second=0, microsecond=0) -def get_start_of_day(dt: datetime) -> datetime: - return dt.replace(hour=0, minute=0, second=0, microsecond=0) +def last_second_of_day(dt: datetime) -> datetime: + return dt.replace(hour=23, minute=59, second=59, microsecond=0) -def get_month_start(unix_timestamp: int) -> int: - # Convert Unix timestamp to a datetime object in UTC - dt = datetime.fromtimestamp(unix_timestamp, tz=timezone.utc) +def first_second_of_month(dt: datetime) -> datetime: + return dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - # Create a new datetime object for the beginning of the month - month_start = datetime(dt.year, dt.month, 1, tzinfo=timezone.utc) - # Convert back to Unix timestamp - return int(month_start.timestamp()) +def last_second_of_month(dt: datetime) -> datetime: + next_month = dt.replace(day=1, hour=0, minute=0, + second=0, microsecond=0) + relativedelta(months=1) + return (next_month - timedelta(seconds=1)).replace(tzinfo=timezone.utc) diff --git a/backend/src/validation/utils.py b/backend/src/validation/utils.py index 72d3488..d08708e 100644 --- a/backend/src/validation/utils.py +++ b/backend/src/validation/utils.py @@ -1,4 +1,5 @@ from fastapi import HTTPException +import time # Timestamp @@ -6,11 +7,22 @@ def validate_timestamp_in_range(timestamp): min_timestamp = 1388530801 # 01/01/2014 max_timestamp = 2208985201 # 01/01/2040 if not (min_timestamp <= timestamp <= max_timestamp): - raise HTTPException(status_code=400, detail=f"Timestamp must be between {min_timestamp} and {max_timestamp}") + raise HTTPException( + status_code=400, detail=f"Timestamp must be between {min_timestamp} and {max_timestamp}") return timestamp +def validate_timestamp_in_range_of_S2_imagery(start_timestamp, end_timestamp): + min_timestamp = 1491004800 # 01/04/2017 + max_timestamp = int(time.time()) # Current timestamp + if not (min_timestamp <= start_timestamp <= max_timestamp and min_timestamp <= end_timestamp <= max_timestamp): + raise HTTPException( + status_code=400, detail=f"Timestamp must be between {min_timestamp} and now ({max_timestamp})") + return + + def validate_timestamp_start_date_before_end_date(startDate, endDate): if endDate <= startDate: - raise HTTPException(status_code=400, detail="endDate must be after startDate") - return endDate \ No newline at end of file + raise HTTPException( + status_code=400, detail="endDate must be after startDate") + return endDate