diff --git a/Project.toml b/Project.toml index c5c5f7b..aa36d38 100644 --- a/Project.toml +++ b/Project.toml @@ -39,10 +39,8 @@ YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" [compat] -AllocCheck = "0.2.0" ArchGDAL = "0.10" ArgParse = "1" -BenchmarkTools = "1.6.0" CairoMakie = "0.12" ConstructionBase = "1" DimensionalData = "0.29" @@ -61,7 +59,6 @@ LoggingExtras = "1" Missings = "1" NetCDF = "0.12" Proj = "1" -Random = "1.10.0, 1.11.0" Rasters = "0.12,0.13" RecurrenceAnalysis = "2" Statistics = "1" @@ -72,11 +69,3 @@ YAXArrays = "0.5" Zarr = "0.9" julia = "1.10" -[extras] -AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" -BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[targets] -test = ["Test", "AllocCheck", "BenchmarkTools", "Random"] diff --git a/src/RQADeforestation.jl b/src/RQADeforestation.jl index 806ab43..353f81e 100644 --- a/src/RQADeforestation.jl +++ b/src/RQADeforestation.jl @@ -16,10 +16,10 @@ include("analysis.jl") # TODO what is still needed from analysis now that rqatr include("timestats.jl") -function main(tiles = ["E048N018T3"]; pol="VH", orbit="*", thresh=3.0) +function main(tiles=["E048N018T3"]; pol="VH", orbit="*", thresh=3.0) indir = "/eodc/products/eodc.eu/S1_CSAR_IWGRDH/SIG0/" continent = "EU" - folders = ["V01R01","V0M2R4", "V1M0R1", "V1M1R1", "V1M1R2"] + folders = ["V01R01", "V0M2R4", "V1M0R1", "V1M1R1", "V1M1R2"] corruptedfiles = "corrupted_tiles.txt" # TODO save the corrupt files to a txt for investigation for tilefolder in tiles @@ -32,35 +32,35 @@ function main(tiles = ["E048N018T3"]; pol="VH", orbit="*", thresh=3.0) relorbits = unique([split(basename(x), "_")[5][2:end] for x in allfilenames]) @show relorbits for relorbit in relorbits - for y in [2018,2019,2020,2021,2022, 2023] + for y in [2018, 2019, 2020, 2021, 2022, 2023] - filenames = allfilenames[findall(contains("$(relorbit)_E"), allfilenames)] - @time cube = gdalcube(filenames) + filenames = allfilenames[findall(contains("$(relorbit)_E"), allfilenames)] + @time cube = gdalcube(filenames) path = joinpath(YAXDefaults.workdir[], "$(tilefolder)_rqatrend_$(pol)_$(relorbit)_thresh_$(thresh)_year_$(y)") @show path - ispath(path*".done") && continue - ispath(path*"_zerotimesteps.done") && continue + ispath(path * ".done") && continue + ispath(path * "_zerotimesteps.done") && continue - tcube = cube[Time=Date(y-1, 7,1)..Date(y+1,7,1)] + tcube = cube[Time=Date(y - 1, 7, 1) .. Date(y + 1, 7, 1)] @show size(cube) @show size(tcube) if size(tcube, Ti) == 0 - touch(path*"_zerotimesteps.done") + touch(path * "_zerotimesteps.done") continue end - try - @time rqatrend(tcube; thresh, outpath=path * ".zarr", overwrite=true) - catch e - - if e.captured.ex isa ArchGDAL.GDAL.GDALError - println("Found GDALError:") - println(e.captured.ex.msg) - continue - else - rethrow(e) + try + @time rqatrend(tcube; thresh, outpath=path * ".zarr", overwrite=true) + catch e + + if e.captured.ex isa ArchGDAL.GDAL.GDALError + println("Found GDALError:") + println(e.captured.ex.msg) + continue + else + rethrow(e) + end end - end #=@everywhere begin fname = "$(VERSION)_$(getpid())_$(time_ns()).heapsnapshot" Profile.take_heap_snapshot(fname;streaming=true) diff --git a/test/Artifacts.toml b/test/Artifacts.toml new file mode 100644 index 0000000..fff4e89 --- /dev/null +++ b/test/Artifacts.toml @@ -0,0 +1,6 @@ +[rqatestdata] +git-tree-sha1 = "ba3717febf44a19ce75ac4df625f8b0281077745" + + [[rqatestdata.download]] + url = "https://github.com/meggart/RQADeforestationTestData/archive/refs/tags/v1.0.tar.gz" + sha256 = "2db9f0d215446a10588e2f3b9117e7f6c7a23ba3f369aa168714b6ba777fcfe9" \ No newline at end of file diff --git a/test/Project.toml b/test/Project.toml new file mode 100644 index 0000000..80f1b74 --- /dev/null +++ b/test/Project.toml @@ -0,0 +1,10 @@ +[deps] +AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" diff --git a/test/runtests.jl b/test/runtests.jl index 0e4110a..862e37c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,10 +2,20 @@ using RQADeforestation using Test import AllocCheck import Random +import Pkg: Artifacts.@artifact_str, ensure_artifact_installed +using DimensionalData +using YAXArrays +using Dates +using Random +using Statistics + Random.seed!(1234) +ensure_artifact_installed("rqatestdata", "Artifacts.toml") +testdatapath = joinpath(artifact"rqatestdata", "RQADeforestationTestData-1.0") + @testset "RQADeforestation.jl" begin - # Write your tests here. + @test isfile(joinpath(testdatapath, "V01R01", "EQUI7_EU020M", "E051N018T3", "SIG0_20210818T051717__VH_D095_E051N018T3_EU020M_V01R01_S1BIWGRDH.tif")) x = 1:0.01:30 y = sin.(x) + 0.1x + rand(length(x)) @@ -14,12 +24,25 @@ Random.seed!(1234) @test isempty(AllocCheck.check_allocs(RQADeforestation.rqatrend_impl, Tuple{Vector{Float64}})) - y2 = similar(y, Union{Float64, Missing}) + y2 = similar(y, Union{Float64,Missing}) copy!(y2, y) - y2[[1,4,10,20,33,65]] .= missing + y2[[1, 4, 10, 20, 33, 65]] .= missing @test isapprox(RQADeforestation.rqatrend_impl(y2; thresh=0.5), -0.11069045524336744) @test isempty(AllocCheck.check_allocs(RQADeforestation.rqatrend_impl, Tuple{Vector{Union{Float64,Missing}}})) + mock_axes = ( + Ti(Date("2022-01-01"):Day(1):Date("2022-01-30")), + X(range(1, 10, length=10)), + Y(range(1, 5, length=15)), + ) + mock_data = rand(30, 10, 15) + mock_props = Dict() + mock_cube = YAXArray(mock_axes, mock_data, mock_props) + + mock_trend = rqatrend(mock_cube; thresh=0.5) + @test mock_trend.axes == (mock_cube.X, mock_cube.Y) + diff = abs(mean(mock_trend)) + @test diff < 0.2 end