V3 json

.gitignore

@@ -26,3 +26,6 @@ release.properties
 /here-naksha-lib-log
 /vault/
 !/vault/README.md
+/json_data/*
+!/json_data/
+!/json_data/README.md

buildSrc/src/main/kotlin/Descriptions.kt

@@ -19,6 +19,7 @@ private val Descriptions = mapOf(
     //"here-naksha-lib-heapcache" to "TBD",
     "here-naksha-lib-hub" to "TBD",
     "here-naksha-lib-jbon" to "Naksha library, adding support to encode and decode JBON (Java Binary Object Notation).",
+    "here-naksha-lib-json" to "Naksha library, adding support to encode and decode JSON.",
     "here-naksha-lib-model" to "Naksha library, adding the Storage-Abstraction-Layer of Naksha, this is the base of all Naksha storage operations. It defines interfaces, helper classes, abstract base classes, and more, needed to use storage implementations or assisting in making new storage implementations.",
     "here-naksha-lib-psql" to "Naksha library, implementation of the Naksha Storage-Abstraction-Layer.",
     "here-naksha-lib-view" to "Naksha library, adding capabilities to combine multiple storages, maps, collections into a single virtual view.",

gradle.properties

@@ -16,7 +16,7 @@ mavenPassword=YourPassword
 #   here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/LibPsql.kt (adminVersion property)
 #     Warning: Only update LibPsql version, if there is a change in SQL functions!
 #              The reason is, that this version is encoded in the database, and when updated, forced an upgrade!
-version=3.0.0-beta.29
+version=3.0.0-beta.30
 
 org.gradle.jvmargs=-Xmx12g
 kotlin.code.style=official

gradle/libs.versions.toml

@@ -10,9 +10,15 @@ log4j = "2.25.1"
 junit = "5.9.2"
 test_containers = "1.20.6"
 kotlin = "2.1.20"
+fastdouble = "2.0.1"
+jmh = "1.37"
+gson = "2.13.2" # https://github.com/google/gson
+jsonio = "4.61.0" # https://github.com/jdereg/json-io
+fastjson = "2.0.31" # https://github.com/alibaba/fastjson
+simdjson = "0.4.0" # https://github.com/simdjson/simdjson-java
 
 [libraries]
-jetbrains_annotations = { module = "org.jetbrains:annotations", version = "24.0.1" }
+jetbrains_annotations = { module = "org.jetbrains:annotations", version = "26.0.2-1" }
 
 vertx_core = { module = "io.vertx:vertx-core", version.ref = "vertx" }
 vertx_config = { module = "io.vertx:vertx-config", version.ref = "vertx" }
@@ -102,6 +108,15 @@ lz4_java = "org.lz4:lz4-java:1.8.0"
 
 picocli = { module = "info.picocli:picocli", version = "4.7.7" }
 
+fastdouble = { module = "ch.randelshofer:fastdoubleparser", version.ref = "fastdouble" }
+
+jmh_core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" }
+jmh_ann = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" }
+gson = { module = "com.google.code.gson:gson", version.ref = "gson" }
+jsonio = { module = "com.cedarsoftware:json-io", version.ref = "jsonio" }
+fastjson = { module = "com.alibaba:fastjson", version.ref="fastjson" }
+simdjson = { module = "org.simdjson:simdjson-java", version.ref = "simdjson" }
+
 [plugins]
 kotlin_multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" }
 kotlin_js_plain_objects = { id = "org.jetbrains.kotlin.plugin.js-plain-objects", version.ref = "kotlin" }
@@ -111,6 +126,7 @@ spotless = { id = "spotless", version = "6.22.0" }
 vanniktechMavenPublish = { id = "com.vanniktech.maven.publish", version = "0.31.0" }
 #vanniktechMavenPublish = { id = "com.vanniktech.maven.publish", version = "0.1.0-SNAPSHOT" }
 axion_release = { id = "pl.allegro.tech.build.axion-release", version = "1.20.1"}
+#jmh = { id = "me.champeau.jmh", version = "0.7.2"}
 
 [bundles]
 vertx = [
@@ -139,3 +155,4 @@ testing = [
 ]
 jackson = ["jackson_core", "jackson_core_annotations", "jackson_core_databind", "jackson_core_dataformat", "jackson_kotlin"]
 logging = ["slf4j_api", "jcl_slf4j", "log4j_core", "log4j_api", "log4j_jcl", "log4j_slf4j"]
+#perftest = ["jmh_core", "jmh_ann", "gson"]

here-naksha-app-service/src/jvmMain/resources/swagger/openapi.yaml

@@ -10,7 +10,7 @@ servers:
 info:
   title: "Naskha Hub-API"
   description: "Naksha Hub-API is a REST API to provide simple access to geo data."
-  version: "3.0.0-beta.25"
+  version: "3.0.0-beta.30"
 
 security:
   - AccessToken: [ ]

here-naksha-cli/src/jvmTest/java/com/here/naksha/cli/copy/CopyCliTest.java

@@ -262,7 +262,7 @@ void shouldFailWithInvalidSrcNakshaStorage() {
         TestCommandLine.CommandResult result = commandLine.execute(testCase.args());
 
         // Then
-        testCase.assertMatches(result);
+        assertEquals(1, result.exitCode());
     }
 
     @Test
@@ -288,7 +288,7 @@ void shouldFailWithInvalidTargetNakshaStorage() {
         TestCommandLine.CommandResult result = commandLine.execute(testCase.args());
 
         // Then
-        testCase.assertMatches(result);
+        assertEquals(1, result.exitCode());
     }
 
     @Test

here-naksha-lib-base/BENCHMARK.md

@@ -0,0 +1,37 @@
+# Benchmark
+We test including GC, to see the impact of GC to the parser performance. The test does:
+
+- Load all files into memory, considering the requested memory-quota between JSON bytes and HEAP.
+- Then parse the first file 10 times to warmup.
+- Force a garbage collection, measure how much memory is now used (information only).
+- Start timer
+- Parse all files, loop 10 times, keep the result of the last round.
+  - Note: We parse file 1, 2, 3, ... 10, then restart the loop, so not 10 times the first, 10 times the second, ...!
+- End timer
+- Clear all cached JSON bytes (set references to `null`).
+- Force a garbage collection, measure how much memory is now used.
+  - This should measure how much memory is kept on heap by each parser.
+
+This shows us what impact garbage collection has to parser performance, and it measures roughly how much memory the files eventually consume at HEAP, after GC done its job, which is not unimportant either.
+
+## Prepare the test
+You need around 51 GiB of free disk space. Go to [json_data/README.md](../json_data/README.md) and follow the instructions to download the JSON test data.
+
+## -XX:+UseG1GC -Xmx12g -Xms12g ; json_data/ {parser} 10 18.0
+- `naksha: 5.90 seconds, json-size: 825.97 MiB, heap-usage: 1925.90 MiB, 140.09 MiB/second, source-to-heap ratio: 2.33`
+- `jackson: 7.07 seconds, json-size: 825.97 MiB, heap-usage: 4806.29 MiB, 116.83 MiB/second, source-to-heap ratio: 5.82`
+- `naksha_jackson: 8.05 seconds, json-size: 825.97 MiB, heap-usage: 5224.46 MiB, 102.59 MiB/second, source-to-heap ratio: 6.33`
+- `fastjson: 10.09 seconds, json-size: 825.97 MiB, heap-usage: 4932.15 MiB, 81.84 MiB/second, source-to-heap ratio: 5.97`
+- `gson: 17.57 seconds, json-size: 825.97 MiB, heap-usage: 6791.26 MiB, 47.00 MiB/second, source-to-heap ratio: 8.22`
+- `jsonio: 22.06 seconds, json-size: 825.97 MiB, heap-usage: 5143.81 MiB, 37.43 MiB/second, source-to-heap ratio: 6.23`
+
+Overview:
+
+| Parser         | Time   | Json-Size     | Heap-Usage   | Parser rate  | JSON : HEAP |
+|----------------|--------|---------------|--------------|--------------|-------------|
+| naksha         | 5.90   | 825.97 MiB    | 1925.90 MiB  | 140.09 MiB/s | 2.33        |
+| jackson        | 7.07   | 825.97 MiB    | 4806.29 MiB  | 116.83 MiB/s | 5.82        |
+| naksha_jackson | 8.05   | 825.97 MiB    | 5224.46 MiB  | 102.59 MiB/s | 6.33        |
+| fastjson       | 10.09  | 825.97 MiB    | 4932.15 MiB  | 81.84 MiB/s  | 5.97        |
+| gson           | 17.57  | 825.97 MiB    | 6791.26 MiB  | 47.00 MiB/s  | 8.22        |
+| jsonio         | 22.06  | 825.97 MiB    | 5143.81 MiB  | 37.43 MiB/s  | 6.23        |

here-naksha-lib-base/build.gradle.kts

@@ -1,10 +1,11 @@
-import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi
 import org.jetbrains.kotlin.gradle.dsl.JsModuleKind
 import org.jetbrains.kotlin.gradle.dsl.JsSourceMapEmbedMode
 import org.jetbrains.kotlin.gradle.dsl.JsSourceMapNamesPolicy
 
 plugins {
     alias(libs.plugins.kotlin.multiplatform)
+    // we use this, ones we're back to java, currently it breaks the multi-platform build!
+    // alias(libs.plugins.jmh)
 }
 
 description = gatherDescription()
@@ -33,9 +34,17 @@ kotlin {
             dependencies {
                 implementation(kotlin("stdlib-jdk8"))
                 implementation(kotlin("reflect"))
-                api(libs.kotlinx.datetime)
+                implementation(libs.fastdouble)
+                implementation(libs.jetbrains.annotations)
+                api(libs.kotlinx.datetime.get().toString()) {
+                   exclude(group = "org.jetbrains.annotations")
+                }
                 api(libs.lz4.java)
                 implementation(libs.jackson.kotlin)
+                implementation(libs.gson)
+                implementation(libs.jsonio)
+                implementation(libs.fastjson)
+                // implementation(libs.simdjson) // Ones we have Java 25 !
                 // https://mvnrepository.com/artifact/org.slf4j
                 api(libs.slf4j.api)
                 implementation(libs.slf4j.console)
@@ -86,6 +95,10 @@ tasks {
     getByName<ProcessResources>("jvmProcessResources") {
         dependsOn("jsNodeProductionLibraryDistribution" ) // "jsBrowserDistribution"
     }
+//    getByName<JavaCompile>("jvmCompile") {
+//        options.annotationProcessorPath = configurations
+//        options.jm jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.36'
+//    }
     getByName<Jar>("jvmJar") { dependsOn("jvmProcessResources") }
     // Test
     getByName<ProcessResources>("jvmTestProcessResources") { dependsOn("jvmProcessResources") }

here-naksha-lib-base/src/commonMain/kotlin/naksha/base/Platform.kt

@@ -477,6 +477,14 @@ expect class Platform {
          */
         fun fromJSON(json: String): Any?
 
+        /**
+         * Deserialize the given JSON.
+         * @param json The JSON UTF-8 encoded byte array to parse.
+         * @return The parsed JSON.
+         * @see [FromJsonOptions.DEFAULT]
+         */
+        fun fromJSON(utf8: ByteArray): Any?
+
         /**
          * Deserialize the given JSON.
          * @param json the JSON string to parse.

here-naksha-lib-base/src/jsMain/kotlin/naksha/base/Platform.kt

@@ -831,5 +831,9 @@ return obj;
         init {
             initialize()
         }
+
+        actual fun fromJSON(utf8: ByteArray): Any? {
+            TODO("Not yet implemented")
+        }
     }
 }