Remove timestats

Author: danlooo

Project.toml

@@ -4,7 +4,6 @@ authors = ["Felix Cremer <[email protected]>, Daniel Loos <dloos@bgc-
 version = "1.0.0-DEV"
 
 [deps]
-AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
 ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
@@ -25,9 +24,9 @@ GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f"
 Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
 ImageMorphology = "787d08f9-d448-5407-9aad-5290dd7ab264"
 KML = "1284bf3a-1e3d-4f4e-a7a9-b9d235a28f35"
+LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
-Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20"
 Missings = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
 NetCDF = "30363a11-5582-574a-97bb-aa9a979735b9"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -46,7 +45,6 @@ YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
 Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"
 
 [compat]
-AWSS3 = "0.10.4"
 AllocCheck = "0.2.2"
 Aqua = "0.8.13"
 ArchGDAL = "0.10"
@@ -71,10 +69,10 @@ GeoFormatTypes = "0.4"
 Glob = "1"
 ImageMorphology = "0.4.5"
 KML = "0.2"
+LazyArtifacts = "1.11.0"
 Libdl = "1.10"
 LinearAlgebra = "1.10"
 LoggingExtras = "1"
-Minio = "0.2.2"
 Missings = "1"
 NetCDF = "0.12"
 Pkg = "1.10"
@@ -93,7 +91,7 @@ TestItems = "1.0"
 TimeseriesSurrogates = "2"
 UnicodePlots = "3"
 YAXArrayBase = "0.6, 0.7"
-YAXArrays = "0.5, 0.6"
+YAXArrays = "0.5, 0.6, 0.7"
 Zarr = "0.9"
 julia = "1.11"
 
@@ -102,13 +100,13 @@ AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 StaticTools = "86c06d3c-3f03-46de-9781-57580aa96d0a"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
-Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [targets]
 test = ["Test", "TestItemRunner", "Pkg", "Random", "AllocCheck", "BenchmarkTools", "Aqua", "Documenter", "StaticTools", "PythonCall", "Libdl"]

src/RQADeforestation.jl

@@ -18,7 +18,6 @@ include("auxil.jl")
 include("rqatrend.jl")
 include("analysis.jl")  # TODO what is still needed from analysis now that rqatrend is in its own file?
 include("cluster.jl")
-include("timestats.jl")
 include("main.jl")
 
 end

src/main.jl

@@ -1,16 +1,7 @@
 using ArgParse
 using YAXArrays: YAXDefaults
-using ArchGDAL: ArchGDAL
-using PyramidScheme
-using AWSS3: global_aws_config, S3Path
-using FilePathsBase: exists
 
 
-#using YAXArrays, Zarr
-using Minio: MinioConfig
-
-global_aws_config(MinioConfig("http://s3.fairsendd.eodchosting.eu",region="us-east-1"))
-
 const argparsesettings = ArgParseSettings()
 
 ArgParse.parse_item(::Type{Date}, x::AbstractString) = Date(x)
@@ -78,121 +69,72 @@ function main(;
     tiles::Vector{String},
     continent::String,
     indir::String,
-    outstore=Zarr.S3Store("europe-forest-change"),
-    outdir="results",
-    tempfolder = S3Path(outstore.bucket, "intermediates/"),
+    outdir="out.zarr",
     start_date::Date,
     end_date::Date,
     polarisation="VH",
     orbit="D",
     threshold=3.0,
     folders=["V1M0R1", "V1M1R1", "V1M1R2"],
-    stack=:dae,
-    postprocess=true,
-    forestdir="data/forest20m_new",
-    delete_intermediate=false
+    stack=:dae
 )
-#global_aws_config(MinioConfig("http://s3.fairsendd.eodchosting.eu",region="us-east-1",username="ufew8gJku5hRY7VD6jbEjRi8VnvDfeEv",password="dqZdzWCLB7a9gTshL29AnQWGqL3krwnS"))
     in(orbit, ["A", "D"]) || error("Orbit needs to be either A or D")
     if isdir(indir) && isempty(indir)
         error("Input directory $indir must not be empty")
     end
-    
-    if isdir(tempfolder)
+    if isdir(outdir)
         @warn "Resume from existing output directory"
     else
-        mkdir(tempfolder, recursive=true)
+        mkdir(outdir)
         @info "Write output to $outdir"
     end
+
     if monthday(start_date) != monthday(end_date)
         @warn "Selected time series does not include a multiple of whole years. This might introduce seasonal bias."
     end
 
-    YAXDefaults.workdir[] = tempfolder
-    @show typeof(tempfolder)
+    YAXDefaults.workdir[] = outdir
 
-    corruptedfiles = open("corrupted_tiles.txt", "w")
+    corruptedfiles = "corrupted_tiles.txt"
     # TODO save the corrupt files to a txt for investigation
     for tilefolder in tiles
-        @show tilefolder
-        outpath = joinpath(outdir, "postprocess_$tilefolder.zarr/")
-        @show outpath
-        if outpath in Zarr.subdirs(outstore, outdir)
-            println("Skip already processed tile $tilefolder")
-            continue
-        end
-        sub = first(folders)
-        #@show glob("$(sub)/*$(continent)*20M/$(tilefolder)*/*$(polarisation)_$(orbit)*.tif", indir)
-        filenamelist = [glob("$(sub)/*$(continent)*20M/$(tilefolder)*/*$(polarisation)_$(orbit)*.tif", indir) for sub in folders]
+        filenamelist = [glob("$(sub)/*$(continent)*20M/$(tilefolder)/*$(polarisation)_$(orbit)*.tif", indir) for sub in folders]
         allfilenames = collect(Iterators.flatten(filenamelist))
-        #@show allfilenames
+
         relorbits = unique([split(basename(x), "_")[5][2:end] for x in allfilenames])
         @show relorbits
 
         for relorbit in relorbits
-            path = S3Path(joinpath(YAXDefaults.workdir[], "$(tilefolder)_rqatrend_$(polarisation)_$(orbit)$(relorbit)_thresh_$(threshold)"))
-            #s3path = "s3://"*joinpath(outstore.bucket, path)
-            @show path
-            exists(path * ".done") && continue
-            exists(path * "_zerotimesteps.done") && continue
             filenames = allfilenames[findall(contains("$(relorbit)_E"), allfilenames)]
             @time cube = gdalcube(filenames, stack)
-            
 
+            path = joinpath(YAXDefaults.workdir[], "$(tilefolder)_rqatrend_$(polarisation)_$(orbit)$(relorbit)_thresh_$(threshold)")
+            @show path
+            ispath(path * ".done") && continue
+            ispath(path * "_zerotimesteps.done") && continue
 
             tcube = cube[Time=start_date .. end_date]
             @show size(cube)
             @show size(tcube)
             if size(tcube, 3) == 0
-                touch(S3Path(path * "_zerotimesteps.done"))
+                touch(path * "_zerotimesteps.done")
                 continue
             end
             try
-                orbitoutpath = string(path * ".zarr/")
-                # This is only necessary because overwrite=true doesn't work on S3 based Zarr files in YAXArrays
-                # See https://github.com/JuliaDataCubes/YAXArrays.jl/issues/511
-                if exists(S3Path(orbitoutpath))
-                    println("Deleting path $orbitoutpath")    
-                    rm(S3Path(orbitoutpath), recursive=true)
-                end
-                @show orbitoutpath
-                # This seems to ignore the overwrite keyword when the outpath point to S3.
-                @time rqatrend(tcube; thresh=threshold, outpath=orbitoutpath, overwrite=true)
-                if delete_intermediate == false
-                    PyramidScheme.buildpyramids(orbitoutpath)
-                    Zarr.consolidate_metadata(orbitoutpath)
-                end
+                outpath = path * ".zarr"
+                @time rqatrend(tcube; thresh=threshold, outpath=outpath, overwrite=true)
+                Zarr.consolidate_metadata(outpath)
             catch e
 
                 if hasproperty(e, :captured) && e.captured.ex isa ArchGDAL.GDAL.GDALError
-                    println(corruptedfiles, "Found GDALError:")
-                    println(corruptedfiles, e.captured.ex.msg)
+                    println("Found GDALError:")
+                    println(e.captured.ex.msg)
                     continue
                 else
                     rethrow(e)
                 end
             end
-            donepath = path * ".done"
-            @show donepath
-            touch(S3Path(path * ".done"))
-        end
-        if postprocess
-            @show outpath
-            DD.STRICT_BROADCAST_CHECKS[] = false
-
-            RQADeforestation.postprocess(tilefolder, tempfolder, outpath, forestdir)
-            Zarr.consolidate_metadata(outpath)
-            DD.STRICT_BROADCAST_CHECKS[] = true
-            #base = basename(outpath)
-            #@show base
-            #command = `aws --endpoint-url http://s3.fairsendd.eodchosting.eu s3 cp --recursive $outpath s3://europe-forest-change/$base`
-
-            #run(command)
+            touch(path * ".done")
         end
-        if delete_intermediate
-            rm(tempfolder, force=true, recursive=true)
-        end
-
-
     end
 end