From 510c70047da425cc68acd798ac42ed72e2f821c6 Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 13:36:19 +0100 Subject: [PATCH 1/6] Remove missing file --- src/RQADeforestation.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/RQADeforestation.jl b/src/RQADeforestation.jl index 0e81a71..8e024c1 100644 --- a/src/RQADeforestation.jl +++ b/src/RQADeforestation.jl @@ -18,7 +18,6 @@ include("auxil.jl") include("rqatrend.jl") include("analysis.jl") # TODO what is still needed from analysis now that rqatrend is in its own file? include("cluster.jl") -include("timestats.jl") include("main.jl") end \ No newline at end of file From 7a1e8e5c796a088290aea180257c640e88447f95 Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 13:39:10 +0100 Subject: [PATCH 2/6] Replace self hosted runner with ubuntu --- .github/workflows/CI.yml | 2 +- .github/workflows/TagBot.yml | 2 +- .github/workflows/build.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 11604e5..ee79cf8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -26,7 +26,7 @@ jobs: - "1" - "lts" os: - - self-hosted + - ubuntu-latest arch: - x64 env: diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index 430a918..0cd3114 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -23,7 +23,7 @@ permissions: jobs: TagBot: if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' - runs-on: [self-hosted, ubuntu-latest] + runs-on: ubuntu-latest steps: - uses: JuliaRegistries/TagBot@v1 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d73e58..6e5908c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: jobs: build: - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v5 From d27c2acfb7c00cde8dca21913d75e33946108a4a Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 13:40:45 +0100 Subject: [PATCH 3/6] Use ubuntu runner for documenter --- .github/workflows/Documenter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index b45dd96..d6105ce 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -29,7 +29,7 @@ concurrency: jobs: # Build job build: - runs-on: self-hosted + runs-on: ubuntu-latest env: JULIA_NUM_THREADS: auto steps: From 5410a41200c052a73458f576bd828ee4c7f5808b Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 13:51:49 +0100 Subject: [PATCH 4/6] Update dep versions --- Project.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index 5b65689..27ad995 100644 --- a/Project.toml +++ b/Project.toml @@ -47,19 +47,19 @@ YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c" Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99" [compat] -AWSS3 = "0.10.4" AllocCheck = "0.2.2" Aqua = "0.8.13" ArchGDAL = "0.10" ArgParse = "1" +AWSS3 = "0.10.4" BenchmarkTools = "1.6.0" ConstructionBase = "1" DataStructures = "0.18.22" Dates = "1.10" DimensionalData = "0.29" DiskArrayEngine = "0.1, 0.2" -DiskArrayTools = "0.1" DiskArrays = "0.4" +DiskArrayTools = "0.1" Distances = "0.10" Distributed = "1.10" Distributions = "0.25" @@ -71,8 +71,9 @@ GDAL = "1" GeoFormatTypes = "0.4" Glob = "1" ImageMorphology = "0.4.5" +julia = "1.11" KML = "0.2" -LazyArtifacts = "1.11.0" +LazyArtifacts = "1" Libdl = "1.10" LinearAlgebra = "1.10" LoggingExtras = "1" @@ -81,6 +82,7 @@ Missings = "1" NetCDF = "0.12" Pkg = "1.10" Proj = "1" +PyramidScheme = "0.1" PythonCall = "0.9.25" Random = "1.10" Rasters = "0.12,0.13, 0.14" @@ -97,7 +99,6 @@ UnicodePlots = "3" YAXArrayBase = "0.6, 0.7" YAXArrays = "0.5, 0.6, 0.7" Zarr = "0.9" -julia = "1.11" [extras] AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" From e4745c6e00029b43e2842221b39ae93cebd77031 Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 14:05:00 +0100 Subject: [PATCH 5/6] Resolve Ambiguity postprocess --- src/cluster.jl | 87 +++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/src/cluster.jl b/src/cluster.jl index 9ada46f..a36a8fd 100644 --- a/src/cluster.jl +++ b/src/cluster.jl @@ -7,30 +7,32 @@ using FillArrays #maskfolder = "data/forestcompressed" function meanvote(orbits, significance_thresh=-1.28) - s,n = 0.0,0 + s, n = 0.0, 0 for i in eachindex(orbits) if orbits[i] != 0 && !isnan(orbits[i]) s += orbits[i] n += 1 end end - m = s/n - m < significance_thresh ? 1 : 0 + m = s / n + return m < significance_thresh ? 1 : 0 end -function filtersmallcomps!(xout,xin_unfiltered,forestmask,comborbits,connsize;dims=:,threaded=false) - xin = broadcast(xin_unfiltered,forestmask) do x,m - ismissing(m) ? zero(x) : x*m +function filtersmallcomps!( + xout, xin_unfiltered, forestmask, comborbits, connsize; dims=:, threaded=false +) + xin = broadcast(xin_unfiltered, forestmask) do x, m + ismissing(m) ? zero(x) : x * m end - x = similar(Array{Float64},(axes(xin,1),axes(xin,2),Base.OneTo(1))) - for j in axes(x,2), i in axes(x,1) - x[i,j,1] = comborbits(view(xin,i,j,:)) + x = similar(Array{Float64}, (axes(xin, 1), axes(xin, 2), Base.OneTo(1))) + for j in axes(x, 2), i in axes(x, 1) + x[i, j, 1] = comborbits(view(xin, i, j, :)) end lc = label_components(x) c = counter(lc) for ix in eachindex(xout) v = lc[ix] - if v==0 || c[v] < connsize + if v == 0 || c[v] < connsize xout[ix] = 0 else xout[ix] = 1 @@ -38,47 +40,66 @@ function filtersmallcomps!(xout,xin_unfiltered,forestmask,comborbits,connsize;di end end -function postprocess(a,target_array::YAXArray,forestmask::YAXArray, orbitcombine=meanvote;minsize=30,max_cache=5e8) - nx,ny,nz = size(a) - windowsx = DAE.MovingWindow(1 - minsize,1,2*minsize + 1,nx,(1,nx)) - windowsy = DAE.MovingWindow(1 - minsize,1,2*minsize + 1,ny,(1,ny)) +function postprocess( + a::YAXArray, + target_array::YAXArray, + forestmask::YAXArray, + orbitcombine=meanvote; + minsize=30, + max_cache=5e8, +) + nx, ny, nz = size(a) + windowsx = DAE.MovingWindow(1 - minsize, 1, 2 * minsize + 1, nx, (1, nx)) + windowsy = DAE.MovingWindow(1 - minsize, 1, 2 * minsize + 1, ny, (1, ny)) windowsz = [1:nz] - inar1 = DAE.InputArray(a.data,windows=(windowsx,windowsy,windowsz)); - inar2 = DAE.InputArray(forestmask.data, windows=(windowsx,windowsy)); - inars = (inar1,inar2) - outchunks = (target_array.chunks.chunks...,DAE.RegularChunks(1,0,1)) - outars = DAE.create_outwindows((nx,ny,1),chunks=outchunks); - uf = DAE.create_userfunction(filtersmallcomps!,UInt8,is_blockfunction=true,is_mutating=true,args=(orbitcombine,minsize)) - op = DAE.GMDWop(inars,(outars,),uf) - plan = DAE.optimize_loopranges(op,max_cache) - runner=DAE.LocalRunner(op,plan,(reshape(target_array.data,(nx,ny,1)),)) + inar1 = DAE.InputArray(a.data; windows=(windowsx, windowsy, windowsz)) + inar2 = DAE.InputArray(forestmask.data; windows=(windowsx, windowsy)) + inars = (inar1, inar2) + outchunks = (target_array.chunks.chunks..., DAE.RegularChunks(1, 0, 1)) + outars = DAE.create_outwindows((nx, ny, 1); chunks=outchunks) + uf = DAE.create_userfunction( + filtersmallcomps!, + UInt8; + is_blockfunction=true, + is_mutating=true, + args=(orbitcombine, minsize), + ) + op = DAE.GMDWop(inars, (outars,), uf) + plan = DAE.optimize_loopranges(op, max_cache) + runner = DAE.LocalRunner(op, plan, (reshape(target_array.data, (nx, ny, 1)),)) run(runner) - target_array + return target_array end function postprocess(tile::AbstractString, indir, outpath, maskfolder) if isdir(outpath) - return + return nothing end forpath = only(glob("*$tile*", maskfolder)) @show forpath - allfiles = readdir(indir, join=true) + allfiles = readdir(indir; join=true) @show allfiles - orbitfiles = filter(x->occursin(tile,string(x)), allfiles) + orbitfiles = filter(x -> occursin(tile, string(x)), allfiles) @show orbitfiles - orbits = filter(x->occursin(".zarr", string(x)), orbitfiles) + orbits = filter(x -> occursin(".zarr", string(x)), orbitfiles) @show orbits - orbitname = map(o->split(basename(o),'_')[4],orbits) + orbitname = map(o -> split(basename(o), '_')[4], orbits) d = DD.format(Dim{:orbit}(orbitname)) - files = DD.DimArray(orbits,d) + files = DD.DimArray(orbits, d) ds = open_mfdataset(string.(files)) nx, ny = size(ds.layer) - outds_skeleton = Dataset(;defo=YAXArray((ds.X,ds.Y),Fill(UInt8(0),nx,ny),chunks=DAE.GridChunks((nx,ny),(256,256)))) - dsout = savedataset(outds_skeleton,path=outpath,skeleton=true,overwrite=true) + outds_skeleton = Dataset(; + defo=YAXArray( + (ds.X, ds.Y), + Fill(UInt8(0), nx, ny); + chunks=DAE.GridChunks((nx, ny), (256, 256)), + ), + ) + dsout = savedataset(outds_skeleton; path=outpath, skeleton=true, overwrite=true) forest = Cube(forpath) #masked = map(*, ds.layer, setchunks(forest,DiskArrays.eachchunk(ds.layer.chunks))) - @time postprocess(ds.layer,dsout.defo, forest) + @time postprocess(ds.layer, dsout.defo, forest) @time PyramidScheme.buildpyramids(outpath) end From 8412a11678dab15ba6bd38927b3da042ac29983c Mon Sep 17 00:00:00 2001 From: danlooo Date: Wed, 29 Oct 2025 14:14:12 +0100 Subject: [PATCH 6/6] Set test julia version --- .github/workflows/CI.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ee79cf8..42756c3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -24,7 +24,6 @@ jobs: matrix: version: - "1" - - "lts" os: - ubuntu-latest arch: