Fix missing file timestats and S3 config

.github/workflows/CI.yml

@@ -24,9 +24,8 @@ jobs:
       matrix:
         version:
           - "1"
-          - "lts"
         os:
-          - self-hosted
+          - ubuntu-latest
         arch:
           - x64
     env:

.github/workflows/Documenter.yml

@@ -29,7 +29,7 @@ concurrency:
 jobs:
   # Build job
   build:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     env:
       JULIA_NUM_THREADS: auto
     steps:

.github/workflows/TagBot.yml

@@ -23,7 +23,7 @@ permissions:
 jobs:
   TagBot:
     if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
-    runs-on: [self-hosted, ubuntu-latest]
+    runs-on: ubuntu-latest
     steps:
       - uses: JuliaRegistries/TagBot@v1
         with:

.github/workflows/build.yml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
 jobs:
   build:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5

Project.toml

@@ -47,19 +47,19 @@ YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
 Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"
 
 [compat]
-AWSS3 = "0.10.4"
 AllocCheck = "0.2.2"
 Aqua = "0.8.13"
 ArchGDAL = "0.10"
 ArgParse = "1"
+AWSS3 = "0.10.4"
 BenchmarkTools = "1.6.0"
 ConstructionBase = "1"
 DataStructures = "0.18.22"
 Dates = "1.10"
 DimensionalData = "0.29"
 DiskArrayEngine = "0.1, 0.2"
-DiskArrayTools = "0.1"
 DiskArrays = "0.4"
+DiskArrayTools = "0.1"
 Distances = "0.10"
 Distributed = "1.10"
 Distributions = "0.25"
@@ -71,8 +71,9 @@ GDAL = "1"
 GeoFormatTypes = "0.4"
 Glob = "1"
 ImageMorphology = "0.4.5"
+julia = "1.11"
 KML = "0.2"
-LazyArtifacts = "1.11.0"
+LazyArtifacts = "1"
 Libdl = "1.10"
 LinearAlgebra = "1.10"
 LoggingExtras = "1"
@@ -81,6 +82,7 @@ Missings = "1"
 NetCDF = "0.12"
 Pkg = "1.10"
 Proj = "1"
+PyramidScheme = "0.1"
 PythonCall = "0.9.25"
 Random = "1.10"
 Rasters = "0.12,0.13, 0.14"
@@ -97,7 +99,6 @@ UnicodePlots = "3"
 YAXArrayBase = "0.6, 0.7"
 YAXArrays = "0.5, 0.6, 0.7"
 Zarr = "0.9"
-julia = "1.11"
 
 [extras]
 AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"

src/RQADeforestation.jl

@@ -18,7 +18,6 @@ include("auxil.jl")
 include("rqatrend.jl")
 include("analysis.jl")  # TODO what is still needed from analysis now that rqatrend is in its own file?
 include("cluster.jl")
-include("timestats.jl")
 include("main.jl")
 
 end

src/cluster.jl

@@ -7,78 +7,99 @@ using FillArrays
 #maskfolder = "data/forestcompressed"
 
 function meanvote(orbits, significance_thresh=-1.28)
-    s,n = 0.0,0
+    s, n = 0.0, 0
     for i in eachindex(orbits)
         if orbits[i] != 0 && !isnan(orbits[i])
             s += orbits[i]
             n += 1
         end
     end
-    m = s/n
-    m < significance_thresh ? 1 : 0
+    m = s / n
+    return m < significance_thresh ? 1 : 0
 end
 
-function filtersmallcomps!(xout,xin_unfiltered,forestmask,comborbits,connsize;dims=:,threaded=false)
-    xin = broadcast(xin_unfiltered,forestmask) do x,m
-        ismissing(m) ? zero(x) : x*m
+function filtersmallcomps!(
+    xout, xin_unfiltered, forestmask, comborbits, connsize; dims=:, threaded=false
+)
+    xin = broadcast(xin_unfiltered, forestmask) do x, m
+        ismissing(m) ? zero(x) : x * m
     end
-    x = similar(Array{Float64},(axes(xin,1),axes(xin,2),Base.OneTo(1)))
-    for j in axes(x,2), i in axes(x,1)
-        x[i,j,1] = comborbits(view(xin,i,j,:))
+    x = similar(Array{Float64}, (axes(xin, 1), axes(xin, 2), Base.OneTo(1)))
+    for j in axes(x, 2), i in axes(x, 1)
+        x[i, j, 1] = comborbits(view(xin, i, j, :))
     end
     lc = label_components(x)
     c = counter(lc)
     for ix in eachindex(xout)
         v = lc[ix]
-        if v==0 || c[v] < connsize
+        if v == 0 || c[v] < connsize
             xout[ix] = 0
         else
             xout[ix] = 1
         end
     end
 end
 
-function postprocess(a,target_array::YAXArray,forestmask::YAXArray, orbitcombine=meanvote;minsize=30,max_cache=5e8)
-    nx,ny,nz = size(a)
-    windowsx = DAE.MovingWindow(1 - minsize,1,2*minsize + 1,nx,(1,nx))
-    windowsy = DAE.MovingWindow(1 - minsize,1,2*minsize + 1,ny,(1,ny))
+function postprocess(
+    a::YAXArray,
+    target_array::YAXArray,
+    forestmask::YAXArray,
+    orbitcombine=meanvote;
+    minsize=30,
+    max_cache=5e8,
+)
+    nx, ny, nz = size(a)
+    windowsx = DAE.MovingWindow(1 - minsize, 1, 2 * minsize + 1, nx, (1, nx))
+    windowsy = DAE.MovingWindow(1 - minsize, 1, 2 * minsize + 1, ny, (1, ny))
     windowsz = [1:nz]
-    inar1 = DAE.InputArray(a.data,windows=(windowsx,windowsy,windowsz));
-    inar2 = DAE.InputArray(forestmask.data, windows=(windowsx,windowsy));
-    inars = (inar1,inar2)
-    outchunks = (target_array.chunks.chunks...,DAE.RegularChunks(1,0,1))
-    outars = DAE.create_outwindows((nx,ny,1),chunks=outchunks);
-    uf = DAE.create_userfunction(filtersmallcomps!,UInt8,is_blockfunction=true,is_mutating=true,args=(orbitcombine,minsize))
-    op = DAE.GMDWop(inars,(outars,),uf)
-    plan = DAE.optimize_loopranges(op,max_cache)
-    runner=DAE.LocalRunner(op,plan,(reshape(target_array.data,(nx,ny,1)),))
+    inar1 = DAE.InputArray(a.data; windows=(windowsx, windowsy, windowsz))
+    inar2 = DAE.InputArray(forestmask.data; windows=(windowsx, windowsy))
+    inars = (inar1, inar2)
+    outchunks = (target_array.chunks.chunks..., DAE.RegularChunks(1, 0, 1))
+    outars = DAE.create_outwindows((nx, ny, 1); chunks=outchunks)
+    uf = DAE.create_userfunction(
+        filtersmallcomps!,
+        UInt8;
+        is_blockfunction=true,
+        is_mutating=true,
+        args=(orbitcombine, minsize),
+    )
+    op = DAE.GMDWop(inars, (outars,), uf)
+    plan = DAE.optimize_loopranges(op, max_cache)
+    runner = DAE.LocalRunner(op, plan, (reshape(target_array.data, (nx, ny, 1)),))
     run(runner)
-    target_array
+    return target_array
 end
 
 function postprocess(tile::AbstractString, indir, outpath, maskfolder)
     if isdir(outpath)
-        return
+        return nothing
     end
 
     forpath = only(glob("*$tile*", maskfolder))
     @show forpath
-    allfiles = readdir(indir, join=true)
+    allfiles = readdir(indir; join=true)
     @show allfiles
-    orbitfiles = filter(x->occursin(tile,string(x)), allfiles)
+    orbitfiles = filter(x -> occursin(tile, string(x)), allfiles)
     @show orbitfiles
-    orbits = filter(x->occursin(".zarr", string(x)), orbitfiles)
+    orbits = filter(x -> occursin(".zarr", string(x)), orbitfiles)
     @show orbits
-    orbitname = map(o->split(basename(o),'_')[4],orbits)
+    orbitname = map(o -> split(basename(o), '_')[4], orbits)
     d = DD.format(Dim{:orbit}(orbitname))
-    files = DD.DimArray(orbits,d)
+    files = DD.DimArray(orbits, d)
     ds = open_mfdataset(string.(files))
     nx, ny = size(ds.layer)
-    outds_skeleton = Dataset(;defo=YAXArray((ds.X,ds.Y),Fill(UInt8(0),nx,ny),chunks=DAE.GridChunks((nx,ny),(256,256))))
-    dsout = savedataset(outds_skeleton,path=outpath,skeleton=true,overwrite=true)
+    outds_skeleton = Dataset(;
+        defo=YAXArray(
+            (ds.X, ds.Y),
+            Fill(UInt8(0), nx, ny);
+            chunks=DAE.GridChunks((nx, ny), (256, 256)),
+        ),
+    )
+    dsout = savedataset(outds_skeleton; path=outpath, skeleton=true, overwrite=true)
     forest = Cube(forpath)
     #masked = map(*, ds.layer, setchunks(forest,DiskArrays.eachchunk(ds.layer.chunks)))
-    @time postprocess(ds.layer,dsout.defo, forest)
+    @time postprocess(ds.layer, dsout.defo, forest)
     @time PyramidScheme.buildpyramids(outpath)
 end