diff --git a/benchmark/README.md b/benchmark/README.md index 5735471..4caaf31 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,6 +1,10 @@ -# JSON5 vs JSON Performance Benchmark +# JSON5 Performance Benchmark - Three-way Comparison -This module provides benchmarking tools to compare the performance of JSON5 serialization/deserialization against standard JSON using kotlinx.serialization. +This module provides benchmarking tools to compare the performance of JSON5 serialization/deserialization across three different implementations: + +1. **JSON5** (this project) - Uses kotlinx.serialization +2. **JSON** (kotlinx.serialization) - Standard JSON baseline +3. **External-JSON5** (at.syntaxerror.json5:2.1.0) - External JSON5 library ## Running the Benchmark @@ -27,9 +31,9 @@ The benchmark generates two types of reports: ### Summary Report - File: `benchmark_summary_YYYY-MM-DD_HH-mm-ss.txt` -- Contains human-readable performance comparisons -- Shows which format is faster and by how much -- Includes overall statistics +- Contains human-readable performance comparisons across all three libraries +- Shows relative performance comparisons between each pair of libraries +- Includes overall statistics and rankings ## Test Data Types @@ -59,39 +63,37 @@ To run the benchmark module tests: ## Sample Results -Based on typical runs, JSON standard library generally performs 2-6x faster than JSON5 for both serialization and deserialization, with the performance gap being larger for more complex data structures. +Based on typical runs across the three libraries: + +- **JSON** (kotlinx.serialization) is consistently the fastest +- **External-JSON5** performs better than this project's JSON5 implementation +- **JSON5** (this project) offers kotlinx.serialization integration but with slower performance Example output: ``` -SimplePerson Serialization: JSON5=0.027ms, JSON=0.013ms -ComplexPerson Serialization: JSON5=0.083ms, JSON=0.015ms -Company Serialization: JSON5=0.200ms, JSON=0.032ms +SimplePerson Serialization: JSON5=0.028ms, JSON=0.010ms, External-JSON5=0.011ms +ComplexPerson Serialization: JSON5=0.073ms, JSON=0.018ms, External-JSON5=0.018ms +Company Serialization: JSON5=0.163ms, JSON=0.031ms, External-JSON5=0.054ms ``` ### Benchmark Result Snapshot -| Case | Type | JSON5 Avg (ms) | JSON Avg (ms) | Speedup (JSON) | -| ------------------- | --------------- | -------------- | ------------- | -------------- | -| SimplePerson | Serialization | 0.056 | 0.020 | 2.77× | -| SimplePerson | Deserialization | 0.064 | 0.022 | 2.93× | -| ComplexPerson | Serialization | 0.089 | 0.019 | 4.59× | -| ComplexPerson | Deserialization | 0.113 | 0.030 | 3.76× | -| Company | Serialization | 0.226 | 0.059 | 3.81× | -| Company | Deserialization | 0.254 | 0.090 | 2.83× | -| NumberTypes | Serialization | 0.032 | 0.003 | 9.43× | -| NumberTypes | Deserialization | 0.021 | 0.003 | 6.60× | -| CollectionTypes | Serialization | 0.067 | 0.009 | 7.41× | -| CollectionTypes | Deserialization | 0.059 | 0.025 | 2.37× | -| SimplePersonList100 | Serialization | 0.153 | 0.042 | 3.64× | -| SimplePersonList100 | Deserialization | 0.234 | 0.039 | 5.99× | -| ComplexPersonList50 | Serialization | 0.388 | 0.059 | 6.59× | -| ComplexPersonList50 | Deserialization | 0.452 | 0.089 | 5.09× | - -**Overall Average Time** - -* JSON5: **0.158 ms** -* JSON: **0.036 ms** -* 🔥 Overall: **KotlinX JSON is 4.33× faster than JSON5** - - -benchmark bar chart +| Case | Type | JSON5 (ms) | JSON (ms) | External-JSON5 (ms) | Performance Ranking | +| ------------------- | --------------- | ---------- | --------- | ------------------- | ------------------- | +| SimplePerson | Serialization | 0.028 | 0.010 | 0.011 | JSON > Ext-JSON5 > JSON5 | +| SimplePerson | Deserialization | 0.049 | 0.014 | 0.017 | JSON > Ext-JSON5 > JSON5 | +| ComplexPerson | Serialization | 0.073 | 0.018 | 0.018 | JSON ≈ Ext-JSON5 > JSON5 | +| ComplexPerson | Deserialization | 0.101 | 0.021 | 0.020 | Ext-JSON5 > JSON > JSON5 | +| Company | Serialization | 0.163 | 0.031 | 0.054 | JSON > Ext-JSON5 > JSON5 | +| Company | Deserialization | 0.200 | 0.081 | 0.117 | JSON > Ext-JSON5 > JSON5 | + +**Overall Performance Comparison:** +- **JSON** is **3.90×** faster than **JSON5** and **2.75×** faster than **External-JSON5** +- **External-JSON5** is **1.42×** faster than **JSON5** + +## Key Insights + +- **kotlinx.serialization JSON** remains the performance leader +- **External JSON5 library** provides a good balance of JSON5 features with reasonable performance +- **This project's JSON5** offers seamless kotlinx.serialization integration but at a performance cost +- Choose based on your priorities: performance (JSON), JSON5 features with good performance (External-JSON5), or kotlinx.serialization integration (this project) diff --git a/benchmark/build.gradle.kts b/benchmark/build.gradle.kts index e94d037..7629c76 100644 --- a/benchmark/build.gradle.kts +++ b/benchmark/build.gradle.kts @@ -11,6 +11,9 @@ dependencies { // kotlinx.serialization for JSON comparison implementation(libs.bundles.kotlinxEcosystem) + + // External JSON5 library for comparison + implementation("at.syntaxerror:json5:2.1.0") // Test dependencies testImplementation(kotlin("test")) diff --git a/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/BenchmarkRunner.kt b/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/BenchmarkRunner.kt index 11a573b..d5cb289 100644 --- a/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/BenchmarkRunner.kt +++ b/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/BenchmarkRunner.kt @@ -117,16 +117,35 @@ object BenchmarkRunner { val json5Result = resultList.find { it.format == "JSON5" } val jsonResult = resultList.find { it.format == "JSON" } + val externalJson5Result = resultList.find { it.format == "External-JSON5" } - if (json5Result != null && jsonResult != null) { - appendLine(" JSON5: ${String.format("%.3f", json5Result.averageTimeMillis)} ms avg") - appendLine(" JSON: ${String.format("%.3f", jsonResult.averageTimeMillis)} ms avg") + if (json5Result != null && jsonResult != null && externalJson5Result != null) { + appendLine(" JSON5: ${String.format("%.3f", json5Result.averageTimeMillis)} ms avg") + appendLine(" JSON: ${String.format("%.3f", jsonResult.averageTimeMillis)} ms avg") + appendLine(" External-JSON5: ${String.format("%.3f", externalJson5Result.averageTimeMillis)} ms avg") - val speedup = json5Result.averageTimeMillis / jsonResult.averageTimeMillis - if (speedup > 1.0) { - appendLine(" → JSON is ${String.format("%.2f", speedup)}x faster than JSON5") + // Compare all libraries + val json5VsJson = json5Result.averageTimeMillis / jsonResult.averageTimeMillis + val externalVsJson = externalJson5Result.averageTimeMillis / jsonResult.averageTimeMillis + val json5VsExternal = json5Result.averageTimeMillis / externalJson5Result.averageTimeMillis + + appendLine(" Comparisons:") + if (json5VsJson > 1.0) { + appendLine(" → JSON is ${String.format("%.2f", json5VsJson)}x faster than JSON5") + } else { + appendLine(" → JSON5 is ${String.format("%.2f", 1.0 / json5VsJson)}x faster than JSON") + } + + if (externalVsJson > 1.0) { + appendLine(" → JSON is ${String.format("%.2f", externalVsJson)}x faster than External-JSON5") + } else { + appendLine(" → External-JSON5 is ${String.format("%.2f", 1.0 / externalVsJson)}x faster than JSON") + } + + if (json5VsExternal > 1.0) { + appendLine(" → External-JSON5 is ${String.format("%.2f", json5VsExternal)}x faster than JSON5") } else { - appendLine(" → JSON5 is ${String.format("%.2f", 1.0 / speedup)}x faster than JSON") + appendLine(" → JSON5 is ${String.format("%.2f", 1.0 / json5VsExternal)}x faster than External-JSON5") } } appendLine() @@ -136,18 +155,37 @@ object BenchmarkRunner { appendLine("Overall Statistics:") val json5Results = results.filter { it.format == "JSON5" } val jsonResults = results.filter { it.format == "JSON" } + val externalJson5Results = results.filter { it.format == "External-JSON5" } val avgJson5Time = json5Results.map { it.averageTimeMillis }.average() val avgJsonTime = jsonResults.map { it.averageTimeMillis }.average() + val avgExternalJson5Time = externalJson5Results.map { it.averageTimeMillis }.average() - appendLine("Average JSON5 time: ${String.format("%.3f", avgJson5Time)} ms") - appendLine("Average JSON time: ${String.format("%.3f", avgJsonTime)} ms") + appendLine("Average JSON5 time: ${String.format("%.3f", avgJson5Time)} ms") + appendLine("Average JSON time: ${String.format("%.3f", avgJsonTime)} ms") + appendLine("Average External-JSON5 time: ${String.format("%.3f", avgExternalJson5Time)} ms") + + val json5VsJson = avgJson5Time / avgJsonTime + val externalVsJson = avgExternalJson5Time / avgJsonTime + val json5VsExternal = avgJson5Time / avgExternalJson5Time + + appendLine("\nOverall Comparisons:") + if (json5VsJson > 1.0) { + appendLine(" → JSON is ${String.format("%.2f", json5VsJson)}x faster than JSON5") + } else { + appendLine(" → JSON5 is ${String.format("%.2f", 1.0 / json5VsJson)}x faster than JSON") + } + + if (externalVsJson > 1.0) { + appendLine(" → JSON is ${String.format("%.2f", externalVsJson)}x faster than External-JSON5") + } else { + appendLine(" → External-JSON5 is ${String.format("%.2f", 1.0 / externalVsJson)}x faster than JSON") + } - val overallSpeedup = avgJson5Time / avgJsonTime - if (overallSpeedup > 1.0) { - appendLine("Overall: JSON is ${String.format("%.2f", overallSpeedup)}x faster than JSON5") + if (json5VsExternal > 1.0) { + appendLine(" → External-JSON5 is ${String.format("%.2f", json5VsExternal)}x faster than JSON5") } else { - appendLine("Overall: JSON5 is ${String.format("%.2f", 1.0 / overallSpeedup)}x faster than JSON") + appendLine(" → JSON5 is ${String.format("%.2f", 1.0 / json5VsExternal)}x faster than External-JSON5") } } @@ -161,9 +199,10 @@ object BenchmarkRunner { val (dataType, operation) = key.split("_") val json5Result = resultList.find { it.format == "JSON5" } val jsonResult = resultList.find { it.format == "JSON" } + val externalJson5Result = resultList.find { it.format == "External-JSON5" } - if (json5Result != null && jsonResult != null) { - println("$dataType $operation: JSON5=${String.format("%.3f", json5Result.averageTimeMillis)}ms, JSON=${String.format("%.3f", jsonResult.averageTimeMillis)}ms") + if (json5Result != null && jsonResult != null && externalJson5Result != null) { + println("$dataType $operation: JSON5=${String.format("%.3f", json5Result.averageTimeMillis)}ms, JSON=${String.format("%.3f", jsonResult.averageTimeMillis)}ms, External-JSON5=${String.format("%.3f", externalJson5Result.averageTimeMillis)}ms") } } } diff --git a/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/SerializationBenchmark.kt b/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/SerializationBenchmark.kt index 9e325d0..d732b4d 100644 --- a/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/SerializationBenchmark.kt +++ b/benchmark/src/main/kotlin/dev/hossain/json5kt/benchmark/SerializationBenchmark.kt @@ -6,6 +6,7 @@ import kotlinx.serialization.encodeToString import kotlinx.serialization.decodeFromString import kotlinx.serialization.serializer import kotlin.system.measureNanoTime +import at.syntaxerror.json5.* /** * Benchmark class for comparing JSON5 and standard JSON performance. @@ -38,6 +39,35 @@ class SerializationBenchmark { } } + /** + * Converts a kotlinx.serialization object to a plain object that can be used with the external JSON5 library. + * This is done by serializing to JSON with kotlinx.serialization and then parsing with external JSON5 library. + */ + private fun convertToPlainObject(data: T, serializer: kotlinx.serialization.SerializationStrategy): Any? { + val jsonString = json.encodeToString(serializer, data) + val parser = JSONParser(jsonString) + return parser.nextValue() + } + + /** + * Converts a JSON5 string to a plain object using the external JSON5 library. + */ + private fun parseWithExternalLibrary(json5String: String): Any? { + val parser = JSONParser(json5String) + return parser.nextValue() + } + + /** + * Converts a plain object to JSON5 string using the external JSON5 library. + */ + private fun stringifyWithExternalLibrary(obj: Any?): String { + return when (obj) { + is JSONObject -> JSONStringify.toString(obj, 0) + is JSONArray -> JSONStringify.toString(obj, 0) + else -> obj?.toString() ?: "null" + } + } + /** * Runs serialization benchmark for the given data and serializer. */ @@ -89,6 +119,27 @@ class SerializationBenchmark { ) ) + // Benchmark external JSON5 library serialization + val externalJson5Object = convertToPlainObject(data, serializer) + val externalJson5SerializationTime = measureNanoTime { + repeat(iterations) { + stringifyWithExternalLibrary(externalJson5Object) + } + } + + results.add( + BenchmarkResult( + operation = "Serialization", + dataType = dataTypeName, + format = "External-JSON5", + iterations = iterations, + totalTimeNanos = externalJson5SerializationTime, + averageTimeNanos = externalJson5SerializationTime / iterations, + averageTimeMicros = (externalJson5SerializationTime / iterations) / 1000.0, + averageTimeMillis = (externalJson5SerializationTime / iterations) / 1_000_000.0 + ) + ) + return results } @@ -144,6 +195,26 @@ class SerializationBenchmark { ) ) + // Benchmark external JSON5 library deserialization + val externalJson5DeserializationTime = measureNanoTime { + repeat(iterations) { + parseWithExternalLibrary(json5String) + } + } + + results.add( + BenchmarkResult( + operation = "Deserialization", + dataType = dataTypeName, + format = "External-JSON5", + iterations = iterations, + totalTimeNanos = externalJson5DeserializationTime, + averageTimeNanos = externalJson5DeserializationTime / iterations, + averageTimeMicros = (externalJson5DeserializationTime / iterations) / 1000.0, + averageTimeMillis = (externalJson5DeserializationTime / iterations) / 1_000_000.0 + ) + ) + return results } diff --git a/benchmark/src/test/kotlin/dev/hossain/json5kt/benchmark/BenchmarkTest.kt b/benchmark/src/test/kotlin/dev/hossain/json5kt/benchmark/BenchmarkTest.kt index fc10343..f23481e 100644 --- a/benchmark/src/test/kotlin/dev/hossain/json5kt/benchmark/BenchmarkTest.kt +++ b/benchmark/src/test/kotlin/dev/hossain/json5kt/benchmark/BenchmarkTest.kt @@ -45,18 +45,22 @@ class BenchmarkTest { assertNotNull(results) assertTrue(results.isNotEmpty()) - // Should have 4 results: JSON5 and JSON for both serialization and deserialization - assertEquals(4, results.size) + // Should have 6 results: JSON5, JSON, and External-JSON5 for both serialization and deserialization + assertEquals(6, results.size) val json5SerializationResult = results.find { it.format == "JSON5" && it.operation == "Serialization" } val jsonSerializationResult = results.find { it.format == "JSON" && it.operation == "Serialization" } + val externalJson5SerializationResult = results.find { it.format == "External-JSON5" && it.operation == "Serialization" } val json5DeserializationResult = results.find { it.format == "JSON5" && it.operation == "Deserialization" } val jsonDeserializationResult = results.find { it.format == "JSON" && it.operation == "Deserialization" } + val externalJson5DeserializationResult = results.find { it.format == "External-JSON5" && it.operation == "Deserialization" } assertNotNull(json5SerializationResult) assertNotNull(jsonSerializationResult) + assertNotNull(externalJson5SerializationResult) assertNotNull(json5DeserializationResult) assertNotNull(jsonDeserializationResult) + assertNotNull(externalJson5DeserializationResult) // Verify basic properties assertEquals("TestPerson", json5SerializationResult!!.dataType) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index eb49a8f..52b5936 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -6,7 +6,7 @@ plugins { } kotlin { - jvmToolchain(21) // Changed from 23 to 21 + jvmToolchain(17) // Use Java 17 to match available JVM } dependencies {