From c709cee6f250043f35c646e61b83263e2b4bb7e4 Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 15:22:40 +0100 Subject: [PATCH 1/5] Remove timestats --- Project.toml | 10 ++--- src/RQADeforestation.jl | 1 - src/main.jl | 98 +++++++++-------------------------------- 3 files changed, 24 insertions(+), 85 deletions(-) diff --git a/Project.toml b/Project.toml index fe1a238..48ec735 100644 --- a/Project.toml +++ b/Project.toml @@ -4,7 +4,6 @@ authors = ["Felix Cremer , Daniel Loos Date: Wed, 29 Oct 2025 15:35:19 +0100 Subject: [PATCH 2/5] Remove PyramidScheme --- Project.toml | 1 - src/RQADeforestation.jl | 1 - 2 files changed, 2 deletions(-) diff --git a/Project.toml b/Project.toml index 48ec735..4f683b0 100644 --- a/Project.toml +++ b/Project.toml @@ -31,7 +31,6 @@ Missings = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Proj = "c94c279d-25a6-4763-9509-64d165bea63e" -PyramidScheme = "ec211b67-1c2c-4319-878f-eaee078ee145" Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" RecurrenceAnalysis = "639c3291-70d9-5ea2-8c5b-839eba1ee399" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" diff --git a/src/RQADeforestation.jl b/src/RQADeforestation.jl index 8e024c1..f2d4664 100644 --- a/src/RQADeforestation.jl +++ b/src/RQADeforestation.jl @@ -8,7 +8,6 @@ using Zarr using Distributed: myid using NetCDF using TestItems -using PyramidScheme: PyramidScheme using DimensionalData: DimensionalData as DD export gdalcube, rqatrend From 15308fec112c9988a77aad88c09526ca32eab57d Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 16:32:29 +0100 Subject: [PATCH 3/5] Use LazyArtifacts --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index b413803..968e1c9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,3 +1,4 @@ +using LazyArtifacts using RQADeforestation # doctests do not run as testitem as of now, hence it is included here From bffc5cd16d9e210cd04d4ad4050657357ec705fe Mon Sep 17 00:00:00 2001 From: Felix Cremer Date: Mon, 10 Nov 2025 13:35:31 +0100 Subject: [PATCH 4/5] Add LazyArtifacts import to testdata.jl --- test/testdata.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/testdata.jl b/test/testdata.jl index 2b548a1..078f78a 100644 --- a/test/testdata.jl +++ b/test/testdata.jl @@ -1,6 +1,7 @@ @testitem "testdata main" begin import Pkg: Artifacts.@artifact_str + using LazyArtifacts testdatapath = artifact"rqatestdata/RQADeforestationTestData-2.0" testdir = tempname() @@ -55,4 +56,4 @@ end copy!(ARGS, OLD_ARGS) @test outdir |> readdir |> length > 1 -end \ No newline at end of file +end From ea42ef4d4b945bdc2cad81b35822e86bde9e2032 Mon Sep 17 00:00:00 2001 From: danlooo Date: Thu, 13 Nov 2025 10:59:38 +0100 Subject: [PATCH 5/5] Fix ambiguity --- src/cluster.jl | 87 +++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/src/cluster.jl b/src/cluster.jl index 9ada46f..a36a8fd 100644 --- a/src/cluster.jl +++ b/src/cluster.jl @@ -7,30 +7,32 @@ using FillArrays #maskfolder = "data/forestcompressed" function meanvote(orbits, significance_thresh=-1.28) - s,n = 0.0,0 + s, n = 0.0, 0 for i in eachindex(orbits) if orbits[i] != 0 && !isnan(orbits[i]) s += orbits[i] n += 1 end end - m = s/n - m < significance_thresh ? 1 : 0 + m = s / n + return m < significance_thresh ? 1 : 0 end -function filtersmallcomps!(xout,xin_unfiltered,forestmask,comborbits,connsize;dims=:,threaded=false) - xin = broadcast(xin_unfiltered,forestmask) do x,m - ismissing(m) ? zero(x) : x*m +function filtersmallcomps!( + xout, xin_unfiltered, forestmask, comborbits, connsize; dims=:, threaded=false +) + xin = broadcast(xin_unfiltered, forestmask) do x, m + ismissing(m) ? zero(x) : x * m end - x = similar(Array{Float64},(axes(xin,1),axes(xin,2),Base.OneTo(1))) - for j in axes(x,2), i in axes(x,1) - x[i,j,1] = comborbits(view(xin,i,j,:)) + x = similar(Array{Float64}, (axes(xin, 1), axes(xin, 2), Base.OneTo(1))) + for j in axes(x, 2), i in axes(x, 1) + x[i, j, 1] = comborbits(view(xin, i, j, :)) end lc = label_components(x) c = counter(lc) for ix in eachindex(xout) v = lc[ix] - if v==0 || c[v] < connsize + if v == 0 || c[v] < connsize xout[ix] = 0 else xout[ix] = 1 @@ -38,47 +40,66 @@ function filtersmallcomps!(xout,xin_unfiltered,forestmask,comborbits,connsize;di end end -function postprocess(a,target_array::YAXArray,forestmask::YAXArray, orbitcombine=meanvote;minsize=30,max_cache=5e8) - nx,ny,nz = size(a) - windowsx = DAE.MovingWindow(1 - minsize,1,2*minsize + 1,nx,(1,nx)) - windowsy = DAE.MovingWindow(1 - minsize,1,2*minsize + 1,ny,(1,ny)) +function postprocess( + a::YAXArray, + target_array::YAXArray, + forestmask::YAXArray, + orbitcombine=meanvote; + minsize=30, + max_cache=5e8, +) + nx, ny, nz = size(a) + windowsx = DAE.MovingWindow(1 - minsize, 1, 2 * minsize + 1, nx, (1, nx)) + windowsy = DAE.MovingWindow(1 - minsize, 1, 2 * minsize + 1, ny, (1, ny)) windowsz = [1:nz] - inar1 = DAE.InputArray(a.data,windows=(windowsx,windowsy,windowsz)); - inar2 = DAE.InputArray(forestmask.data, windows=(windowsx,windowsy)); - inars = (inar1,inar2) - outchunks = (target_array.chunks.chunks...,DAE.RegularChunks(1,0,1)) - outars = DAE.create_outwindows((nx,ny,1),chunks=outchunks); - uf = DAE.create_userfunction(filtersmallcomps!,UInt8,is_blockfunction=true,is_mutating=true,args=(orbitcombine,minsize)) - op = DAE.GMDWop(inars,(outars,),uf) - plan = DAE.optimize_loopranges(op,max_cache) - runner=DAE.LocalRunner(op,plan,(reshape(target_array.data,(nx,ny,1)),)) + inar1 = DAE.InputArray(a.data; windows=(windowsx, windowsy, windowsz)) + inar2 = DAE.InputArray(forestmask.data; windows=(windowsx, windowsy)) + inars = (inar1, inar2) + outchunks = (target_array.chunks.chunks..., DAE.RegularChunks(1, 0, 1)) + outars = DAE.create_outwindows((nx, ny, 1); chunks=outchunks) + uf = DAE.create_userfunction( + filtersmallcomps!, + UInt8; + is_blockfunction=true, + is_mutating=true, + args=(orbitcombine, minsize), + ) + op = DAE.GMDWop(inars, (outars,), uf) + plan = DAE.optimize_loopranges(op, max_cache) + runner = DAE.LocalRunner(op, plan, (reshape(target_array.data, (nx, ny, 1)),)) run(runner) - target_array + return target_array end function postprocess(tile::AbstractString, indir, outpath, maskfolder) if isdir(outpath) - return + return nothing end forpath = only(glob("*$tile*", maskfolder)) @show forpath - allfiles = readdir(indir, join=true) + allfiles = readdir(indir; join=true) @show allfiles - orbitfiles = filter(x->occursin(tile,string(x)), allfiles) + orbitfiles = filter(x -> occursin(tile, string(x)), allfiles) @show orbitfiles - orbits = filter(x->occursin(".zarr", string(x)), orbitfiles) + orbits = filter(x -> occursin(".zarr", string(x)), orbitfiles) @show orbits - orbitname = map(o->split(basename(o),'_')[4],orbits) + orbitname = map(o -> split(basename(o), '_')[4], orbits) d = DD.format(Dim{:orbit}(orbitname)) - files = DD.DimArray(orbits,d) + files = DD.DimArray(orbits, d) ds = open_mfdataset(string.(files)) nx, ny = size(ds.layer) - outds_skeleton = Dataset(;defo=YAXArray((ds.X,ds.Y),Fill(UInt8(0),nx,ny),chunks=DAE.GridChunks((nx,ny),(256,256)))) - dsout = savedataset(outds_skeleton,path=outpath,skeleton=true,overwrite=true) + outds_skeleton = Dataset(; + defo=YAXArray( + (ds.X, ds.Y), + Fill(UInt8(0), nx, ny); + chunks=DAE.GridChunks((nx, ny), (256, 256)), + ), + ) + dsout = savedataset(outds_skeleton; path=outpath, skeleton=true, overwrite=true) forest = Cube(forpath) #masked = map(*, ds.layer, setchunks(forest,DiskArrays.eachchunk(ds.layer.chunks))) - @time postprocess(ds.layer,dsout.defo, forest) + @time postprocess(ds.layer, dsout.defo, forest) @time PyramidScheme.buildpyramids(outpath) end