[builder] save/load, export/import

Author: stefanmaierhofer

src/Aardvark.Geometry.Quadtree/Builder.fs

@@ -3,6 +3,9 @@
 open Aardvark.Base
 open System
 open System.Collections.Generic
+open Serialization
+
+#nowarn "1337"
 
 module Builder =
 
@@ -77,7 +80,7 @@ module Builder =
                 if debugOutput then
                     printfn "[DEBUG] CREATED QNode with split limit = %d" qnode.SplitLimitExponent
 
-                let xs = qnode |> InMemoryNode |> Query.All Query.Config.Default |> Seq.map (fun x -> x.GetSamples<V4f>(Defs.HeightsBilinear4f)) |> Array.ofSeq |> Array.collect id
+                //let xs = qnode |> InMemoryNode |> Query.All Query.Config.Default |> Seq.map (fun x -> x.GetSamples<V4f>(Defs.HeightsBilinear4f)) |> Array.ofSeq |> Array.collect id
                 //for x in xs do printfn "%A" x
                 qnode |> InMemoryNode
 
@@ -108,6 +111,8 @@ module Builder =
                     )
 
                 let hasMask = subNodes |> Array.exists (fun n -> n.HasMask)
+                if debugOutput && hasMask then
+                    printfn "[DEBUG] has mask %A" rootCell
                 let result = { Id = Guid.NewGuid(); ExactBoundingBox = ebb; Cell = rootCell; SplitLimitExponent = BuildConfig.Default.SplitLimitPowerOfTwo; HasMask = hasMask; SubNodes = subNodes }
                 result |> InMemoryInner
 
@@ -118,7 +123,6 @@ module Builder =
         let sampleExponent = (patches |> Array.distinctBy (fun x -> x.SampleExponent) |> Array.exactlyOne).SampleExponent
         build' sampleExponent rootCell patches
 
-
 /// Creates a quadtree from many small patches.
 type Builder () =
 
@@ -127,6 +131,12 @@ type Builder () =
     let mutable isEmpty = true
     let mutable layerCount = 0
 
+    static let ensureDir path =
+        let path = System.IO.Path.GetFullPath(path)
+        let dir = System.IO.DirectoryInfo(path)
+        if not dir.Exists then dir.Create()
+        path
+
     /// Add patch.
     member this.Add (patch : LayerSet) : unit = lock l (fun () ->
 
@@ -150,6 +160,12 @@ type Builder () =
         list.Add(patch)
         )
 
+    /// Add patch.
+    member this.Add (patch : LayerSet option) : unit =
+        match patch with
+        | None -> ()
+        | Some patch -> this.Add(patch)
+
     /// Add patch.
     member this.Add (patch : ILayer) : unit =
         this.Add(LayerSet([| patch |]))
@@ -158,6 +174,18 @@ type Builder () =
     member this.Add (patch : QNode) : unit =
         this.Add(patch.LayerSet)
 
+    /// Add all leaf nodes of given quadtree as patches.
+    member this.Add (root : QNodeRef) : unit =
+        root |> Quadtree.EnumerateLeafNodesInMemory |> this.AddRange
+
+    /// Add multiple patches.
+    member this.AddRange (patches : seq<QNode>) : unit =
+        for patch in patches do this.Add(patch)
+
+    /// Add multiple patches.
+    member this.AddRange (patches : seq<LayerSet>) : unit =
+        for patch in patches do this.Add(patch)
+
     /// Build a quadtree from all the patches that have been added to this builder.
     member this.Build () : QNodeRef option =
 
@@ -173,7 +201,11 @@ type Builder () =
             | Some state -> Quadtree.Merge Dominance.SecondDominates state item |> Some
             ) 
             None // initial state
-        
+      
+    /// Enumerate all patches.
+    member this.GetPatches () : seq<LayerSet> =
+        patches |> Seq.map (fun kv -> kv.Value) |> Seq.collect id
+
     member this.Print () : unit =
 
         printfn "Builder("
@@ -186,4 +218,57 @@ type Builder () =
         printfn "  global root cell = %A" (Cell2d bbGlobal)
 
         printfn "  )"
-        ()
+        ()
+
+    /// Save builder. Returns id of saved builder.
+    member this.Save (options : SerializationOptions) : Guid =
+        Defs.init ()
+        let patches = this.GetPatches() |> Array.ofSeq
+        Serialization.SavePatches options patches
+
+    /// Load builder with given id.
+    static member Load (options : SerializationOptions) (id : Guid) : Builder option =
+        Defs.init ()
+        match Serialization.LoadPatches options id with
+        | None -> None
+        | Some patches ->
+            let builder = Builder()
+            builder.AddRange(patches)
+            Some builder
+
+    /// Export builder to directory.
+    member this.Export (dir : string) : Guid =
+
+        let storeDir = ensureDir dir
+
+        let now = DateTimeOffset.Now
+        let fileNameKey = sprintf("builder.%04d%02d%02d%02d%02d%02d.%d.key.txt") now.Year now.Month now.Day now.Hour now.Minute now.Second now.Ticks
+
+        use store = new Uncodium.SimpleStore.SimpleDiskStore(System.IO.Path.Combine(storeDir, "store.dur"))
+        let options = SerializationOptions.SimpleStore(store)
+        let key = this.Save(options)
+
+        System.IO.File.WriteAllText(System.IO.Path.Combine(storeDir, fileNameKey), key.ToString())
+
+        key
+
+    /// Imports builder with given id from directory.
+    static member Import (dir : string, id : Guid) : Builder option =
+
+        let storeFileName =
+            if System.IO.File.Exists(dir) then
+                dir
+            else
+                if System.IO.Directory.Exists(dir) then
+                    System.IO.Path.Combine(dir, "store.dur")
+                else
+                    sprintf "Unknown path %s. Error 094f64b9-ba1e-4d9d-a5e1-bc471665f332." dir |> failwith
+
+        use store = new Uncodium.SimpleStore.SimpleDiskStore(storeFileName)
+        let options = SerializationOptions.SimpleStore(store)
+        match Serialization.LoadPatches options id with
+        | None -> None
+        | Some patches ->
+            let builder = Builder()
+            builder.AddRange(patches)
+            Some builder

src/Aardvark.Geometry.Quadtree/Defs.fs

@@ -111,7 +111,9 @@ module Defs =
     let VolumesBilinear4d      = def "9aadc102-68b7-4e6f-ba22-bad1440e1bcf" "Quadtree.VolumesBilinear4d" "Quadtree. Volume value (height difference) per sample as bilinear params. volume(x,y) = A + B*x + C*y+ D*x*y, where x,y in range [-sample.Size/2, +sample.Size/2], (x=0,y=0) corresponds to center of sample (A), and A=v.X, B=v.Y, C=v.Z, D=v.W. V4d[]." Durable.Aardvark.V4dArray
     let VolumesBilinear4dLayer = def "fd547ab1-88e9-4fc2-9c7d-a67545bfd3d1" "Quadtree.VolumesBilinear4d.Layer" "Quadtree. VolumesBilinearParams4d layer. DurableMapAligned16." Durable.Primitives.DurableMapAligned16
 
-    let Mask1b                 = def "6f33738c-eddc-411e-932f-326249d95f46" "Aardvark.Geometry.Quadtree.ByteMask" "UInt8[]." Durable.Primitives.UInt8Array
+    let Mask1b                 = def "6f33738c-eddc-411e-932f-326249d95f46" "Quadtree.ByteMask" "UInt8[]." Durable.Primitives.UInt8Array
+
+    let PatchIds               = def "3505d69b-79bf-449f-8740-13a0eda219b1" "Quadtree.PatchIds" "Guid[]." Durable.Primitives.GuidArray
 
     let private def2layer = Map.ofList [
         (Heights1f         , Heights1fLayer         )

src/Aardvark.Geometry.Quadtree/Serialization.fs

@@ -262,7 +262,7 @@ module Serialization =
     (*
         QNode
 
-        durable definition c74fad23-1211-4073-94e5-54b778e0d295
+        durable definition c74fad23-1211-4073-94e5-54b778e0d295 (Quadtree.NodeLeaf)
     *)
 
     let private encodeQNode (options : SerializationOptions) (n : QNode) : byte[] =
@@ -381,7 +381,63 @@ module Serialization =
             InMemoryInner { Id = id; ExactBoundingBox = ebb; Cell = cell; SplitLimitExponent = sle; HasMask = hasMask; SubNodes = ns }
         | None    -> 
             InMemoryNode(QNode(id, ebb, cell, sle, layerSet))
-                    
+            
+            
+    (*
+        LayerSet
+
+        durable definition c39a978b-00f5-485f-b0b3-d2cf9599016b (Quadtree.Layers)
+    *)
+
+    let private encodeLayerSet (options : SerializationOptions) (x : LayerSet) : byte[] =
+
+        let map = List<KeyValuePair<Durable.Def, obj>>()
+            
+        // layers
+        for layer in x.Layers do
+            let layerDef = Defs.GetLayerFromDef layer.Def
+            let dm = layer.Materialize().ToDurableMap ()
+            map.Add(kvp layerDef dm)
+        
+        DurableCodec.Serialize(Defs.Layers, map)
+
+    let private decodeLayerSet (options: SerializationOptions) (def : Durable.Def) (o : obj) =
+        
+        let map = o :?> ImmutableDictionary<Durable.Def, obj>
+        
+        let layers : ILayer[] =  
+            map 
+            |> Seq.choose (fun kv ->
+                match Defs.TryGetDefFromLayer kv.Key with
+                | Some def -> 
+                    let m = kv.Value :?> ImmutableDictionary<Durable.Def, obj>
+                    Layer.FromDurableMap def m |> Some
+                | None -> None
+                )
+            |> Seq.toArray
+
+        let layerSet = LayerSet(layers)
+
+        invariant (layers.Length > 0) "71249698-2b8f-4423-80d6-43122ac27f9c"
+
+        layerSet
+
+
+    (*
+        Patch IDs
+
+        durable definition 3505d69b-79bf-449f-8740-13a0eda219b1 (Quadtree.PatchIds)
+    *)
+
+    let private encodePatchIds (options : SerializationOptions) (ids : seq<Guid>) : byte[] =
+        DurableCodec.Serialize(Defs.PatchIds, ids |> Array.ofSeq)
+
+    let private decodePatchIds (options: SerializationOptions) (def : Durable.Def) (o : obj) : Guid[] =
+        
+        invariant (def = Defs.PatchIds) "c68e7236-21b4-4919-ba14-afa75e69d0cc"
+
+        let ids = o :?> Guid[]
+        ids
 
     (*
         save/encode
@@ -407,9 +463,25 @@ module Serialization =
         | InMemoryMerge n       -> saveBuffer n.Id (fun () -> encodeQMergeNode options n) [| n.First; n.Second |]
         | LinkedNode n          -> saveBuffer n.Id (fun () -> encodeQLinkedNode options n) [| n.Target |]
 
+    let SaveLayerSet (options: SerializationOptions) (layerSet : LayerSet) : Guid =
+        
+        let id = Guid.NewGuid()
+        let buffer = encodeLayerSet options layerSet
+        options.Save id buffer
+        id
+
+    let SavePatches (options: SerializationOptions) (patches : seq<LayerSet>) : Guid =
+        
+        let buffer = patches |> Seq.map(SaveLayerSet options) |> encodePatchIds options
+
+        let id = Guid.NewGuid()
+        options.Save id buffer
+
+        id
+
 
     (*
-        load
+        load/decode
     *)
 
     let private decoders : Map<Guid, Decoder> = Map.ofList <| [
@@ -433,6 +505,44 @@ module Serialization =
             let struct (def, o) = DurableCodec.Deserialize(buffer)
             decode options def o
 
+    let LoadLayerSet (options: SerializationOptions) (id : Guid) : LayerSet option =
+
+        match options.TryLoad id with
+        | None -> None
+        | Some buffer ->
+            let struct (def, o) = DurableCodec.Deserialize(buffer)
+            let map = o :?> ImmutableDictionary<Durable.Def, obj>
+            let layers = map |> Seq.map (fun kv -> Layer.FromDurableMap kv.Key map)
+            for kv in map do
+                let bar = Layer.FromDurableMap kv.Key map
+                ()
+
+            None
+
+    let LoadPatches (options: SerializationOptions) (id : Guid) : LayerSet[] option =
+
+        match options.TryLoad id with
+        | None -> None
+        | Some buffer ->
+            let struct (def, o) = DurableCodec.Deserialize(buffer)
+            let patchIds = decodePatchIds options def o
+
+            let layerSets = 
+                patchIds |> Array.map(fun patchId ->
+                    match options.TryLoad patchId with
+
+                    | None ->
+                        sprintf "Patch (id=%A) not found. Error 033031e0-03a5-4b3f-bf3c-88b7f4215373." patchId
+                        |> failwith
+
+                    | Some buffer ->
+                        let struct (def, o) = DurableCodec.Deserialize(buffer)
+                        let layerSet = decodeLayerSet options def o
+                        layerSet
+                    )
+
+            Some layerSets
+
     /// Enumerates node ids of given quadtree.
     let rec EnumerateKeys (outOfCore : bool) (qtree : QNodeRef) : Guid seq = seq {
         let recurse = EnumerateKeys outOfCore