From 40364a2bbf26b7182d2afea25088e273e71df227 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Wed, 19 Feb 2025 15:46:58 +0100 Subject: [PATCH 1/6] Update API dates use timespan --- src/main.jl | 85 ++++++++++++++++++++++++------------------------ test/testdata.jl | 4 ++- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/main.jl b/src/main.jl index 1d21782..5fb952e 100644 --- a/src/main.jl +++ b/src/main.jl @@ -11,12 +11,13 @@ const argparsesettings = ArgParseSettings() help = "Polarisation that should be stacked" default = "VH" - "--years", "--year", "-y" - help = "Year in which the RQA Trend should be detected. - We take a buffer of six month before and after the year to end up with two years of data." - default = [2018, 2019, 2020, 2021, 2022, 2023] - nargs = '+' - arg_type = Int + "--start-date" + help = "Start date of the time series to analyze in ISO 8601 format YYYY-MM-DD" + arg_type = String + + "--end-date" + help = "End date of the time series to analyze in ISO 8601 format YYYY-MM-DD" + arg_type = String "--orbit", "-o" help = "One of: Orbit number, 'A' for ascending, 'D' for descending, '*' for all orbits" @@ -58,13 +59,14 @@ end function main(; tiles::Vector{String}, - continent::String, + continent::String, indir::String, outdir="out.zarr", - years=[2018, 2019, 2020, 2021, 2022, 2023], - polarisation="VH", - orbit="*", - threshold=3.0, + start_date::Date, + end_date::Date, + polarisation="VH", + orbit="*", + threshold=3.0, folders=["V01R01", "V0M2R4", "V1M0R1", "V1M1R1", "V1M1R2"] ) if isdir(indir) && isempty(indir) @@ -89,42 +91,39 @@ function main(; relorbits = unique([split(basename(x), "_")[5][2:end] for x in allfilenames]) @show relorbits for relorbit in relorbits - for y in years - - filenames = allfilenames[findall(contains("$(relorbit)_E"), allfilenames)] - @time cube = gdalcube(filenames) - - path = joinpath(YAXDefaults.workdir[], "$(tilefolder)_rqatrend_$(polarisation)_$(relorbit)_thresh_$(threshold)_year_$(y)") - @show path - ispath(path * ".done") && continue - ispath(path * "_zerotimesteps.done") && continue + filenames = allfilenames[findall(contains("$(relorbit)_E"), allfilenames)] + @time cube = gdalcube(filenames) + + path = joinpath(YAXDefaults.workdir[], "$(tilefolder)_rqatrend_$(polarisation)_$(relorbit)_thresh_$(threshold)") + @show path + ispath(path * ".done") && continue + ispath(path * "_zerotimesteps.done") && continue + + tcube = cube[Time=start_date .. end_date] + @show size(cube) + @show size(tcube) + if size(tcube, 3) == 0 + touch(path * "_zerotimesteps.done") + continue + end + try + @time rqatrend(tcube; thresh=threshold, outpath=path * ".zarr", overwrite=true) + catch e - tcube = cube[Time=Date(y - 1, 7, 1) .. Date(y + 1, 7, 1)] - @show size(cube) - @show size(tcube) - if size(tcube, 3) == 0 - touch(path * "_zerotimesteps.done") + if hasproperty(e, :captured) && e.captured.ex isa ArchGDAL.GDAL.GDALError + println("Found GDALError:") + println(e.captured.ex.msg) continue + else + rethrow(e) end - try - @time rqatrend(tcube; thresh=threshold, outpath=path * ".zarr", overwrite=true) - catch e - - if hasproperty(e, :captured) && e.captured.ex isa ArchGDAL.GDAL.GDALError - println("Found GDALError:") - println(e.captured.ex.msg) - continue - else - rethrow(e) - end - end - #=@everywhere begin - fname = "$(VERSION)_$(getpid())_$(time_ns()).heapsnapshot" - Profile.take_heap_snapshot(fname;streaming=true) - end - =# - touch(path * ".done") end + #=@everywhere begin + fname = "$(VERSION)_$(getpid())_$(time_ns()).heapsnapshot" + Profile.take_heap_snapshot(fname;streaming=true) + end + =# + touch(path * ".done") end end end \ No newline at end of file diff --git a/test/testdata.jl b/test/testdata.jl index c203934..868fd6a 100644 --- a/test/testdata.jl +++ b/test/testdata.jl @@ -5,7 +5,7 @@ ensure_artifact_installed("rqatestdata", "Artifacts.toml") testdatapath = joinpath(artifact"rqatestdata", "RQADeforestationTestData-2.0") - testdir = "tmp/testdata" + testdir = "tmp/testdata" rm(testdir, recursive=true, force=true) mkpath(testdir) outdir = "$testdir/out.zarr" @@ -16,6 +16,8 @@ copy!(ARGS, [ "--tile", "E051N018T3", "--continent", "EU", + "--start-date", "2021-01-01", + "--end-date", "2021-12-31", "--in-dir", indir, "--out-dir", outdir, ]) From 9853697a2e03eaa5889abbea5d92f471366be3e0 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Wed, 19 Feb 2025 16:11:00 +0100 Subject: [PATCH 2/6] Fix dest types --- src/main.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main.jl b/src/main.jl index 5fb952e..377c417 100644 --- a/src/main.jl +++ b/src/main.jl @@ -13,11 +13,15 @@ const argparsesettings = ArgParseSettings() "--start-date" help = "Start date of the time series to analyze in ISO 8601 format YYYY-MM-DD" - arg_type = String + required = true + arg_type = Date + dest_name = "start_date" "--end-date" help = "End date of the time series to analyze in ISO 8601 format YYYY-MM-DD" - arg_type = String + required = true + arg_type = Date + dest_name = "end_date" "--orbit", "-o" help = "One of: Orbit number, 'A' for ascending, 'D' for descending, '*' for all orbits" From c10f65558ab016b9e601cd1a8ee5381a12753097 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Wed, 19 Feb 2025 16:22:58 +0100 Subject: [PATCH 3/6] Add ArgParse.parse_item for type Date --- src/main.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.jl b/src/main.jl index 377c417..d8017a0 100644 --- a/src/main.jl +++ b/src/main.jl @@ -2,6 +2,9 @@ using ArgParse using YAXArrays: YAXDefaults const argparsesettings = ArgParseSettings() + +ArgParse.parse_item(::Type{Date}, x::AbstractString) = Date(x) + @add_arg_table! argparsesettings begin "--threshold", "-t" help = "Threshold for the recurrence matrix computation" From 0c034da915fff34efb28fa6adca694c4f7bc0126 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Thu, 20 Feb 2025 11:34:24 +0100 Subject: [PATCH 4/6] Add warning for sub year periods see #78 --- src/main.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main.jl b/src/main.jl index d8017a0..91a2dcc 100644 --- a/src/main.jl +++ b/src/main.jl @@ -85,6 +85,11 @@ function main(; mkdir(outdir) @info "Write output to $outdir" end + + if end_date - start_date < Dates.Day(365) + @warn "Selected time series is less than a year. This will introduce seasonal bias." + end + YAXDefaults.workdir[] = outdir corruptedfiles = "corrupted_tiles.txt" From 2303981b5ff0982422d6e34fc7caeffd415c7b52 Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Thu, 20 Feb 2025 11:55:40 +0100 Subject: [PATCH 5/6] Improve seasonal warning --- src/main.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.jl b/src/main.jl index 91a2dcc..00667e2 100644 --- a/src/main.jl +++ b/src/main.jl @@ -86,8 +86,8 @@ function main(; @info "Write output to $outdir" end - if end_date - start_date < Dates.Day(365) - @warn "Selected time series is less than a year. This will introduce seasonal bias." + if Dates.Day(start_date) != Dates.Day(end_date) || Dates.Month(start_date) != Dates.Month(end_date) + @warn "Selected time series does not include a multiple of whole years. This might introduce seasonal bias." end YAXDefaults.workdir[] = outdir From d26bacaaa9c0c84a5521a63eabad24108849553f Mon Sep 17 00:00:00 2001 From: Daniel Loos Date: Mon, 24 Feb 2025 11:02:10 +0100 Subject: [PATCH 6/6] Apply suggestion Co-authored-by: @felixcremer --- src/main.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.jl b/src/main.jl index 00667e2..d8ff5b1 100644 --- a/src/main.jl +++ b/src/main.jl @@ -86,7 +86,7 @@ function main(; @info "Write output to $outdir" end - if Dates.Day(start_date) != Dates.Day(end_date) || Dates.Month(start_date) != Dates.Month(end_date) + if monthday(start_date) != monthday(end_date) @warn "Selected time series does not include a multiple of whole years. This might introduce seasonal bias." end