Compare test results using json diff (#743)

Use json diff to compare results of pipeline tests. This avoids failures caused by
different format of equal numbers.
After this change two json documents are equal if their values are, independently
of their format.

Diff returned is not a unified diff anymore, it makes reference now to the different
fields following a json-like format.

Author: jsoriano

go.mod

@@ -16,13 +16,12 @@ require (
 	github.com/fatih/color v1.13.0
 	github.com/go-git/go-billy/v5 v5.3.1
 	github.com/go-git/go-git/v5 v5.4.2
-	github.com/google/go-cmp v0.5.7
 	github.com/google/go-github/v32 v32.1.0
 	github.com/google/go-querystring v1.1.0
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
-	github.com/kylelemons/godebug v1.1.0
 	github.com/magefile/mage v1.13.0
 	github.com/mholt/archiver/v3 v3.5.1
+	github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.4.0
@@ -75,6 +74,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/btree v1.0.1 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.2.0 // indirect

go.sum

@@ -783,8 +783,6 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -902,6 +900,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
+github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
 github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=

internal/testrunner/runners/pipeline/runner_test.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,283 +43,6 @@ func TestStripEmptyTestResults(t *testing.T) {
 	require.Equal(t, actual.events[3], json.RawMessage(thirdTestResult))
 }
 
-var diffUliteTests = []struct {
-	name string
-	a, b string
-	u    int
-	want string
-}{
-	{
-		name: "no diff",
-		u:    3,
-		a: `a
-b
-c
-d
-`,
-		b: `a
-b
-c
-d
-`,
-		want: "",
-	},
-	{
-		name: "first line",
-		u:    3,
-		a: `a change
-b
-c
-d
-e
-`,
-		b: `a
-b
-c
-d
-e
-`,
-		want: `--- want
-+++ got
-@@ -1 +1 @@
-- a change
-+ a
-  b
-  c
-  d`,
-	},
-	{
-		name: "last line",
-		u:    3,
-		a: `a
-b
-c
-d
-e change
-`,
-		b: `a
-b
-c
-d
-e
-`,
-		want: `--- want
-+++ got
-@@ -2 +2 @@
-  b
-  c
-  d
-- e change
-+ e
-  `,
-	},
-	{
-		name: "middle",
-		u:    3,
-		a: `a
-b
-c
-d change
-e
-f
-g
-h
-`,
-		b: `a
-b
-c
-d
-e
-f
-g
-h
-`,
-		want: `--- want
-+++ got
-@@ -1 +1 @@
-  a
-  b
-  c
-- d change
-+ d
-  e
-  f
-  g`,
-	},
-	{
-		name: "close pair",
-		u:    3,
-		a: `a
-b
-c
-d change
-e
-f
-g
-h
-i
-j
-k
-l
-m
-`,
-		b: `a
-b
-c
-d
-e
-f
-g
-h
-i change
-j
-k
-l
-m
-`,
-		want: `--- want
-+++ got
-@@ -1 +1 @@
-  a
-  b
-  c
-- d change
-+ d
-  e
-  f
-  g
-  h
-- i
-+ i change
-  j
-  k
-  l`,
-	},
-	{
-		name: "far pair",
-		u:    3,
-		a: `a
-b
-c change
-d
-e
-f
-g
-h
-i
-j
-k
-l
-m
-`,
-		b: `a
-b
-c
-d
-e
-f
-g
-h
-i
-j
-k
-l change
-m
-`,
-		want: `--- want
-+++ got
-@@ -1 +1 @@
-  a
-  b
-- c change
-+ c
-  d
-  e
-  f
-@@ -9 +9 @@
-  i
-  j
-  k
-- l
-+ l change
-  m
-  `,
-	},
-	{
-		name: "far pair addition",
-		u:    3,
-		a: `a
-b
-c change
-d
-e
-f
-g
-a
-b
-c
-d
-e
-f
-g
-h
-i
-j
-k
-l
-m
-`,
-		b: `a
-b
-c
-d
-e
-f
-g
-h
-i
-j
-k
-l change
-m
-`,
-		want: `--- want
-+++ got
-@@ -1 +1 @@
-  a
-  b
-- c change
-- d
-- e
-- f
-- g
-- a
-- b
-  c
-  d
-  e
-@@ -16 +9 @@
-  i
-  j
-  k
-- l
-+ l change
-  m
-  `,
-	},
-}
-
-func TestDiffUlite(t *testing.T) {
-	for _, test := range diffUliteTests {
-		t.Run(test.name, func(t *testing.T) {
-			got := diffUlite(test.a, test.b, test.u)
-			if got != test.want {
-				t.Errorf("unexpected result\n%s", cmp.Diff(got, test.want))
-			}
-		})
-	}
-}
-
 var jsonUnmarshalUsingNumberTests = []struct {
 	name string
 	msg  string