Skip to content

Commit 6bdff5b

Browse files
authored
Merge pull request #331 from gnieh/performances/jsonpatch-diff
Improve JsonPatch diff performances
2 parents c2e8c82 + 97aee07 commit 6bdff5b

File tree

11 files changed

+164
-107
lines changed

11 files changed

+164
-107
lines changed

.github/workflows/ci.yml

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -94,11 +94,11 @@ jobs:
9494

9595
- name: Make target directories
9696
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
97-
run: mkdir -p circe/jvm/target testkit/native/target target testkit/js/target .js/target core/.native/target playJson/jvm/target sprayJson/target core/.js/target circe/js/target core/.jvm/target .jvm/target .native/target circe/native/target playJson/js/target testkit/jvm/target project/target
97+
run: mkdir -p circe/jvm/target testkit/native/target target testkit/js/target .js/target core/.native/target playJson/jvm/target benchmarks/.jvm/target sprayJson/target core/.js/target circe/js/target core/.jvm/target .jvm/target .native/target circe/native/target playJson/js/target testkit/jvm/target project/target
9898

9999
- name: Compress target directories
100100
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
101-
run: tar cf targets.tar circe/jvm/target testkit/native/target target testkit/js/target .js/target core/.native/target playJson/jvm/target sprayJson/target core/.js/target circe/js/target core/.jvm/target .jvm/target .native/target circe/native/target playJson/js/target testkit/jvm/target project/target
101+
run: tar cf targets.tar circe/jvm/target testkit/native/target target testkit/js/target .js/target core/.native/target playJson/jvm/target benchmarks/.jvm/target sprayJson/target core/.js/target circe/js/target core/.jvm/target .jvm/target .native/target circe/native/target playJson/js/target testkit/jvm/target project/target
102102

103103
- name: Upload target directories
104104
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
@@ -181,32 +181,32 @@ jobs:
181181
tar xf targets.tar
182182
rm targets.tar
183183
184-
- name: Download target directories (2.13.9, rootJS)
184+
- name: Download target directories (2.13.10, rootJS)
185185
uses: actions/download-artifact@v3
186186
with:
187-
name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.9-rootJS
187+
name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.10-rootJS
188188

189-
- name: Inflate target directories (2.13.9, rootJS)
189+
- name: Inflate target directories (2.13.10, rootJS)
190190
run: |
191191
tar xf targets.tar
192192
rm targets.tar
193193
194-
- name: Download target directories (2.13.9, rootJVM)
194+
- name: Download target directories (2.13.10, rootJVM)
195195
uses: actions/download-artifact@v3
196196
with:
197-
name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.9-rootJVM
197+
name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.10-rootJVM
198198

199-
- name: Inflate target directories (2.13.9, rootJVM)
199+
- name: Inflate target directories (2.13.10, rootJVM)
200200
run: |
201201
tar xf targets.tar
202202
rm targets.tar
203203
204-
- name: Download target directories (2.13.9, rootNative)
204+
- name: Download target directories (2.13.10, rootNative)
205205
uses: actions/download-artifact@v3
206206
with:
207-
name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.9-rootNative
207+
name: target-${{ matrix.os }}-${{ matrix.java }}-2.13.10-rootNative
208208

209-
- name: Inflate target directories (2.13.9, rootNative)
209+
- name: Inflate target directories (2.13.10, rootNative)
210210
run: |
211211
tar xf targets.tar
212212
rm targets.tar

.gitignore

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,9 @@ syntax: glob
2020
.idea
2121
*.iml
2222

23-
#bsp
24-
.bsp
25-
2623
# building
2724
target
25+
out
2826
build
2927
null
3028
tmp*
@@ -51,3 +49,15 @@ build.log
5149
#ensime
5250
.ensime*
5351
ensime.sbt
52+
53+
# website
54+
site/content/api
55+
site/content/documentation/
56+
site/output
57+
58+
.metals/
59+
.bloop/
60+
.bsp/
61+
metals.sbt
62+
63+
.vscode/settings.json

.scalafmt.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version = "3.7.2"
1+
version = "3.7.1"
22
maxColumn = 120
33
danglingParentheses.preset = false
44
align.preset = some
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package diffson
2+
3+
import org.openjdk.jmh.annotations.State
4+
import org.openjdk.jmh.annotations.Scope
5+
import org.openjdk.jmh.annotations.BenchmarkMode
6+
import org.openjdk.jmh.annotations.Mode
7+
import org.openjdk.jmh.annotations.Fork
8+
import org.openjdk.jmh.annotations.Warmup
9+
import org.openjdk.jmh.annotations.Measurement
10+
11+
import diffson.circe._
12+
import diffson.jsonpatch.lcsdiff._
13+
import diffson.lcs._
14+
15+
import io.circe.syntax._
16+
import io.circe.Json
17+
import org.openjdk.jmh.annotations.Benchmark
18+
19+
@BenchmarkMode(Array(Mode.Throughput))
20+
@State(Scope.Benchmark)
21+
@Fork(value = 1)
22+
@Warmup(iterations = 3, time = 2)
23+
@Measurement(iterations = 5, time = 2)
24+
class PatienceBenchmarks {
25+
26+
implicit val lcs = new Patience[Json]
27+
28+
private def createJson(depth: Int, arrayStep: Int) =
29+
List
30+
.range(depth, 0, -1)
31+
.foldLeft(Json.obj("array" := List.range(0, 1000, arrayStep).map(n => Json.obj("n" := n, "other" := "common")))) {
32+
(acc, idx) =>
33+
Json.obj(s"key$idx" := acc, "other" := arrayStep)
34+
}
35+
36+
def array(size: Int, step: Int) =
37+
Json.obj("array" := List.range(0, size, step))
38+
39+
val deep1 =
40+
createJson(100, 1)
41+
42+
val deep2 =
43+
createJson(100, 2)
44+
45+
val array1 =
46+
array(1000, 2)
47+
48+
val array2 =
49+
array(1000, 1)
50+
51+
@Benchmark
52+
def diffArray() =
53+
diff(array1, array2)
54+
55+
@Benchmark
56+
def diffDeep() =
57+
diff(deep1, deep2)
58+
}

build.sbt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,16 @@ lazy val circe = crossProject(JSPlatform, JVMPlatform, NativePlatform)
9090
)
9191
)
9292
.dependsOn(core, testkit % Test)
93+
94+
lazy val benchmarks = crossProject(JVMPlatform)
95+
.crossType(CrossType.Pure)
96+
.in(file("benchmarks"))
97+
.enablePlugins(NoPublishPlugin, JmhPlugin)
98+
.settings(commonSettings: _*)
99+
.settings(
100+
name := "diffson-benchmarks",
101+
libraryDependencies ++= Seq(
102+
"io.circe" %% "circe-literal" % circeVersion
103+
)
104+
)
105+
.dependsOn(circe)

core/src/main/scala/diffson/jsonpatch/JsonDiff.scala

Lines changed: 43 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -20,73 +20,45 @@ package jsonpatch
2020
import lcs._
2121
import jsonpointer._
2222

23-
import cats.implicits._
23+
import cats.syntax.all._
24+
import cats.data.Chain
25+
import cats.Eval
2426

2527
import scala.annotation.tailrec
2628

2729
class JsonDiff[Json](diffArray: Boolean, rememberOld: Boolean)(implicit J: Jsony[Json], Lcs: Lcs[Json])
2830
extends Diff[Json, JsonPatch[Json]] {
2931
def diff(json1: Json, json2: Json): JsonPatch[Json] =
30-
JsonPatch(diff(json1, json2, Pointer.Root))
32+
JsonPatch(diff(json1, json2, Pointer.Root).value.toList)
3133

32-
private def diff(json1: Json, json2: Json, pointer: Pointer): List[Operation[Json]] =
33-
if (json1 === json2)
34-
// if they are equal, this one is easy...
35-
Nil
36-
else
37-
(json1, json2) match {
38-
case (JsObject(fields1), JsObject(fields2)) => fieldsDiff(fields1.toList, fields2.toList, pointer)
39-
case (JsArray(arr1), JsArray(arr2)) if diffArray => arraysDiff(arr1.toList, arr2.toList, pointer)
40-
case (_, _) => List(Replace(pointer, json2, if (rememberOld) Some(json1) else None))
41-
}
34+
private def diff(json1: Json, json2: Json, pointer: Pointer): Eval[Chain[Operation[Json]]] =
35+
(json1, json2) match {
36+
case (JsObject(fields1), JsObject(fields2)) => fieldsDiff(fields1.toList, fields2, pointer)
37+
case (JsArray(arr1), JsArray(arr2)) if diffArray => arraysDiff(arr1.toList, arr2.toList, pointer)
38+
case _ if json1 === json2 =>
39+
// if they are equal, this one is easy...
40+
Eval.now(Chain.empty)
41+
case _ => Eval.now(Chain.one(Replace(pointer, json2, if (rememberOld) Some(json1) else None)))
42+
}
4243

4344
private def fieldsDiff(fields1: List[(String, Json)],
44-
fields2: List[(String, Json)],
45-
path: Pointer): List[Operation[Json]] = {
46-
// sort fields by name in both objects
47-
val sorted1 = fields1.sortBy(_._1)
48-
val sorted2 = fields2.sortBy(_._1)
49-
@tailrec
50-
def associate(fields1: List[(String, Json)],
51-
fields2: List[(String, Json)],
52-
acc: List[(Option[(String, Json)], Option[(String, Json)])])
53-
: List[(Option[(String, Json)], Option[(String, Json)])] = (fields1, fields2) match {
54-
case (f1 :: t1, f2 :: t2) if f1._1 == f2._1 =>
55-
// same name, associate both
56-
associate(t1, t2, (Some(f1), Some(f2)) :: acc)
57-
case (f1 :: t1, f2 :: _) if f1._1 < f2._1 =>
58-
// the first field is not present in the second object
59-
associate(t1, fields2, (Some(f1), None) :: acc)
60-
case (_ :: _, f2 :: t2) =>
61-
// the second field is not present in the first object
62-
associate(fields1, t2, (None, Some(f2)) :: acc)
63-
case (_, Nil) =>
64-
fields1.map(Some(_) -> None) ::: acc
65-
case (Nil, _) =>
66-
fields2.map(None -> Some(_)) ::: acc
45+
fields2: Map[String, Json],
46+
path: Pointer): Eval[Chain[Operation[Json]]] =
47+
fields1 match {
48+
case (fld, value1) :: fields1 =>
49+
fields2.get(fld) match {
50+
case Some(value2) =>
51+
fieldsDiff(fields1, fields2 - fld, path).flatMap(d => diff(value1, value2, path / fld).map(_ ++ d))
52+
case None =>
53+
// field is not in the second object, delete it
54+
fieldsDiff(fields1, fields2, path).map(
55+
_.prepend(Remove(path / fld, if (rememberOld) Some(value1) else None)))
56+
}
57+
case Nil =>
58+
Eval.now(Chain.fromSeq(fields2.toList).map { case (fld, value) => Add(path / fld, value) })
6759
}
68-
@tailrec
69-
def fields(fs: List[(Option[(String, Json)], Option[(String, Json)])],
70-
acc: List[Operation[Json]]): List[Operation[Json]] = fs match {
71-
case (Some(f1), Some(f2)) :: tl if f1 == f2 =>
72-
// all right, nothing changed
73-
fields(tl, acc)
74-
case (Some(f1), Some(f2)) :: tl =>
75-
// same field name, different values
76-
fields(tl, diff(f1._2, f2._2, path / f1._1) ::: acc)
77-
case (Some(f1), None) :: tl =>
78-
// the field was deleted
79-
fields(tl, Remove[Json](path / f1._1, if (rememberOld) Some(f1._2) else None) :: acc)
80-
case (None, Some(f2)) :: tl =>
81-
// the field was added
82-
fields(tl, Add(path / f2._1, f2._2) :: acc)
83-
case _ =>
84-
acc
85-
}
86-
fields(associate(sorted1, sorted2, Nil), Nil)
87-
}
8860

89-
private def arraysDiff(arr1: List[Json], arr2: List[Json], path: Pointer): List[Operation[Json]] = {
61+
private def arraysDiff(arr1: List[Json], arr2: List[Json], path: Pointer): Eval[Chain[Operation[Json]]] = {
9062
// get the longest common subsequence in the array
9163
val lcs = Lcs.lcs(arr1, arr2)
9264

@@ -104,27 +76,27 @@ class JsonDiff[Json](diffArray: Boolean, rememberOld: Boolean)(implicit J: Jsony
10476

10577
// add a bunch of values to an array starting at the specified index
10678
@tailrec
107-
def add(arr: List[Json], idx: Int, acc: List[Operation[Json]]): List[Operation[Json]] = arr match {
108-
case v :: tl => add(tl, idx + 1, Add(path / idx, v) :: acc)
109-
case Nil => acc.reverse
79+
def add(arr: List[Json], idx: Int, acc: Chain[Operation[Json]]): Chain[Operation[Json]] = arr match {
80+
case v :: tl => add(tl, idx + 1, acc.append(Add(path / idx, v)))
81+
case Nil => acc
11082
}
11183

11284
// remove a bunch of array elements starting by the last one in the range
113-
def remove(from: Int, until: Int, shift: Int, arr: List[Json]): List[Operation[Json]] =
114-
(for (idx <- until to from by -1)
115-
yield Remove[Json](path / idx, if (rememberOld) Some(arr(idx - shift)) else None)).toList
85+
def remove(from: Int, until: Int, shift: Int, arr: List[Json]): Chain[Operation[Json]] =
86+
Chain.fromSeq(
87+
for (idx <- until to from by -1)
88+
yield Remove[Json](path / idx, if (rememberOld) Some(arr(idx - shift)) else None))
11689

11790
// now iterate over the first array to computes what was added, what was removed and what was modified
118-
@tailrec
11991
def loop(
12092
arr1: List[Json], // the first array
12193
arr2: List[Json], // the second array
12294
idx1: Int, // current index in the first array
12395
shift1: Int, // current index shift in the first array (due to elements being add or removed)
12496
idx2: Int, // current index in the second array
12597
lcs: List[(Int, Int)], // the list of remaining matching indices
126-
acc: List[Operation[Json]] // the already accumulated result
127-
): List[Operation[Json]] = (arr1, arr2) match {
98+
acc: Chain[Operation[Json]] // the already accumulated result
99+
): Eval[Chain[Operation[Json]]] = (arr1, arr2) match {
128100
case (_ :: tl1, _) if isCommon1(idx1, lcs) =>
129101
// all values in arr2 were added until the index of common value
130102
val until = lcs.head._2
@@ -134,7 +106,7 @@ class JsonDiff[Json](diffArray: Boolean, rememberOld: Boolean)(implicit J: Jsony
134106
shift1 + until - idx2,
135107
until + 1,
136108
lcs.tail,
137-
add(arr2.take(until - idx2), idx1 + shift1, Nil) reverse_::: acc)
109+
acc ++ add(arr2.take(until - idx2), idx1 + shift1, Chain.empty))
138110
case (_, _ :: tl2) if isCommon2(idx2, lcs) =>
139111
// all values in arr1 were removed until the index of common value
140112
val until = lcs.head._1
@@ -144,18 +116,18 @@ class JsonDiff[Json](diffArray: Boolean, rememberOld: Boolean)(implicit J: Jsony
144116
shift1 - (until - idx1),
145117
idx2 + 1,
146118
lcs.tail,
147-
remove(idx1 + shift1, until - 1 + shift1, idx1 + shift1, arr1) reverse_::: acc)
119+
acc ++ remove(idx1 + shift1, until - 1 + shift1, idx1 + shift1, arr1))
148120
case (v1 :: tl1, v2 :: tl2) =>
149121
// values are different, recursively compute the diff of these values
150-
loop(tl1, tl2, idx1 + 1, shift1, idx2 + 1, lcs, diff(v1, v2, path / (idx1 + shift1)) reverse_::: acc)
122+
diff(v1, v2, path / (idx1 + shift1)).flatMap(d => loop(tl1, tl2, idx1 + 1, shift1, idx2 + 1, lcs, acc ++ d))
151123
case (_, Nil) =>
152124
// all subsequent values in arr1 were removed
153-
remove(idx1 + shift1, idx1 + arr1.size - 1 + shift1, idx1 + shift1, arr1) reverse_::: acc
125+
Eval.now(acc ++ remove(idx1 + shift1, idx1 + arr1.size - 1 + shift1, idx1 + shift1, arr1))
154126
case (Nil, _) =>
155127
// all subsequent value in arr2 were added
156-
arr2.map(Add(path / "-", _)) reverse_::: acc
128+
Eval.now(acc ++ Chain.fromSeq(arr2.map(Add(path / "-", _))))
157129
}
158130

159-
loop(arr1, arr2, 0, 0, 0, lcs, Nil).reverse
131+
loop(arr1, arr2, 0, 0, 0, lcs, Chain.empty)
160132
}
161133
}

core/src/main/scala/diffson/jsonpatch/package.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ package object jsonpatch {
2323
object lcsdiff {
2424
object remembering {
2525
implicit def JsonDiffDiff[Json: Jsony: Lcs]: Diff[Json, JsonPatch[Json]] =
26-
new JsonDiff[Json](true, true)
26+
new JsonDiff[Json](true, true)(implicitly, implicitly[Lcs[Json]].savedHashes)
2727
}
2828
implicit def JsonDiffDiff[Json: Jsony: Lcs]: Diff[Json, JsonPatch[Json]] =
29-
new JsonDiff[Json](true, false)
29+
new JsonDiff[Json](true, false)(implicitly, implicitly[Lcs[Json]].savedHashes)
3030
}
3131

3232
object simplediff {

core/src/main/scala/diffson/lcs/Patience.scala

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import cats.Eq
2020
import cats.implicits._
2121

2222
import scala.annotation.tailrec
23-
import scala.collection.SortedMap
2423
import scala.collection.immutable.TreeMap
2524
import scala.collection.compat._
2625

@@ -43,17 +42,17 @@ class Patience[T: Eq](withFallback: Boolean = true) extends Lcs[T] {
4342
/** Returns occurrences that appear only once in the list, associated with their index */
4443
private def uniques(l: List[T]): Map[T, Int] = {
4544
@tailrec
46-
def loop(l: List[Occurrence], acc: Map[T, Int]): Map[T, Int] = l match {
47-
case (value, idx) :: tl =>
45+
def loop(l: List[T], idx: Int, acc: Map[T, Int]): Map[T, Int] = l match {
46+
case value :: tl =>
4847
if (acc.contains(value))
4948
// not unique, remove it from the accumulator and go further
50-
loop(tl, acc - value)
49+
loop(tl, idx + 1, acc - value)
5150
else
52-
loop(tl, acc + (value -> idx))
51+
loop(tl, idx + 1, acc.updated(value, idx))
5352
case Nil =>
5453
acc
5554
}
56-
loop(l.zipWithIndex, Map.empty)
55+
loop(l, 0, Map.empty)
5756
}
5857

5958
/** Takes all occurences from the first sequence and order them as in the second sequence if it is present */

project/plugins.sbt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ addSbtPlugin("org.typelevel" % "sbt-typelevel" % "0.4.18")
22
addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.13.0")
33
addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.10")
44
addSbtPlugin("org.portable-scala" % "sbt-scala-native-crossproject" % "1.2.0")
5+
addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.4")

0 commit comments

Comments
 (0)