Skip to content

Commit 11421c3

Browse files
jwierzbojwierzbo
authored
Disallow unknown fields feature (TG-212) (#366)
* Disallow unknown fields feature (TG-212) * PR review fixes * Disable concurrency test in active failover mode
1 parent ed6c2a9 commit 11421c3

20 files changed

+227
-55
lines changed

‎.travis.yml‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,14 @@ language: go
3030
env:
3131
- TEST_SUITE=run-unit-tests GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch ALWAYS=1
3232
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6
33-
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest ALWAYS=1
3433
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.7
35-
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:3.8.0-nightly
36-
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:3.9.0-nightly
37-
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest
34+
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.8
35+
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest ALWAYS=1
36+
- TEST_SUITE=run-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest TEST_DISALLOW_UNKNOWN_FIELDS=false
3837
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.6
39-
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest ALWAYS=1
4038
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.7
41-
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:3.8.0-nightly
42-
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:3.9.0-nightly
39+
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb:3.8
40+
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb:latest ALWAYS=1
4341
- TEST_SUITE=run-v2-tests-single GOIMAGE=gcr.io/gcr-for-testing/golang:1.16.6-stretch STARTER=gcr.io/gcr-for-testing/arangodb/arangodb-starter:latest ALPINE_IMAGE=gcr.io/gcr-for-testing/alpine:3.4 ARANGODB=gcr.io/gcr-for-testing/arangodb/arangodb-preview:latest
4442

4543
script:

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,7 @@ __test_go_test:
382382
-e TEST_DEBUG='$(TEST_DEBUG)' \
383383
-e TEST_ENABLE_SHUTDOWN=$(TEST_ENABLE_SHUTDOWN) \
384384
-e TEST_REQUEST_LOG=$(TEST_REQUEST_LOG) \
385+
-e TEST_DISALLOW_UNKNOWN_FIELDS=$(TEST_DISALLOW_UNKNOWN_FIELDS) \
385386
-e GODEBUG=tls13=1 \
386387
-e CGO_ENABLED=$(CGO_ENABLED) \
387388
-w /usr/code/ \

‎http/connection_wrapper.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package http
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"reflect"
9+
"strings"
10+
11+
"github.com/arangodb/go-driver"
12+
"github.com/arangodb/go-velocypack"
13+
)
14+
15+
type connectionDebugWrapper struct {
16+
driver.Connection
17+
ct driver.ContentType
18+
}
19+
20+
func NewConnectionDebugWrapper(conn driver.Connection, ct driver.ContentType) driver.Connection {
21+
return &connectionDebugWrapper{conn, ct}
22+
}
23+
24+
func (c *connectionDebugWrapper) Do(ctx context.Context, req driver.Request) (driver.Response, error) {
25+
if c.ct == driver.ContentTypeJSON {
26+
resp, err := c.Connection.Do(ctx, req)
27+
if err != nil {
28+
return resp, err
29+
}
30+
31+
httpResponse, ok := resp.(*httpJSONResponse)
32+
if !ok {
33+
panic("can not cast response to the httpJSONResponse type!")
34+
}
35+
36+
return &responseDebugWrapper{httpResponse}, err
37+
38+
}
39+
return c.Connection.Do(ctx, req)
40+
}
41+
42+
func (c *connectionDebugWrapper) Unmarshal(data driver.RawObject, result interface{}) error {
43+
ct := c.ct
44+
if ct == driver.ContentTypeVelocypack && len(data) >= 2 {
45+
// Poor mans auto detection of json
46+
l := len(data)
47+
if (data[0] == '{' && data[l-1] == '}') || (data[0] == '[' && data[l-1] == ']') {
48+
ct = driver.ContentTypeJSON
49+
}
50+
}
51+
switch ct {
52+
case driver.ContentTypeJSON:
53+
decoder := json.NewDecoder(strings.NewReader(string(data)))
54+
decoder.DisallowUnknownFields()
55+
56+
if err := decoder.Decode(result); err != nil {
57+
return driver.WithStack(err)
58+
}
59+
60+
if err := json.Unmarshal(data, result); err != nil {
61+
fmt.Printf("Struct: %s \n", reflect.TypeOf(result).String())
62+
fmt.Printf("Response: %s \n\n", string(data))
63+
return driver.WithStack(errors.New(fmt.Sprintf("Struct: %s, Error: %s", reflect.TypeOf(result).String(), err.Error())))
64+
}
65+
case driver.ContentTypeVelocypack:
66+
if err := velocypack.Unmarshal(velocypack.Slice(data), result); err != nil {
67+
return driver.WithStack(err)
68+
}
69+
default:
70+
return driver.WithStack(fmt.Errorf("unsupported content type %d", int(c.ct)))
71+
}
72+
return nil
73+
}
74+
75+
type responseDebugWrapper struct {
76+
*httpJSONResponse
77+
}
78+
79+
func (r *responseDebugWrapper) ParseBody(field string, result interface{}) error {
80+
if field == "" {
81+
decoder := json.NewDecoder(strings.NewReader(string(r.httpJSONResponse.rawResponse)))
82+
decoder.DisallowUnknownFields()
83+
84+
if err := decoder.Decode(result); err != nil {
85+
fmt.Printf("Struct: %s \n", reflect.TypeOf(result).String())
86+
fmt.Printf("Response: %s \n\n", string(r.httpJSONResponse.rawResponse))
87+
return driver.WithStack(errors.New(fmt.Sprintf("Struct: %s, Error: %s", reflect.TypeOf(result).String(), err.Error())))
88+
}
89+
}
90+
return r.httpJSONResponse.ParseBody(field, result)
91+
}

‎test/client_test.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,15 @@ func createConnectionFromEnvWitLog(t testEnv, logger zerolog.Logger) driver.Conn
160160

161161
// createConnectionFromEnv initializes a Connection from information specified in environment variables.
162162
func createConnectionFromEnv(t testEnv) driver.Connection {
163+
disallowUnknownFields := os.Getenv("TEST_DISALLOW_UNKNOWN_FIELDS")
164+
if disallowUnknownFields == "true" {
165+
return createConnection(t, true)
166+
}
167+
return createConnection(t, false)
168+
}
169+
170+
// createConnection initializes a Connection from information specified in environment variables.
171+
func createConnection(t testEnv, disallowUnknownFields bool) driver.Connection {
163172
connSpec := os.Getenv("TEST_CONNECTION")
164173
connVer := os.Getenv("TEST_CVERSION")
165174
switch connSpec {
@@ -184,6 +193,9 @@ func createConnectionFromEnv(t testEnv) driver.Connection {
184193
if err != nil {
185194
t.Fatalf("Failed to create new vst connection: %s", describe(err))
186195
}
196+
if disallowUnknownFields {
197+
return http.NewConnectionDebugWrapper(conn, driver.ContentTypeVelocypack)
198+
}
187199
return conn
188200

189201
case "http", "":
@@ -196,6 +208,9 @@ func createConnectionFromEnv(t testEnv) driver.Connection {
196208
if err != nil {
197209
t.Fatalf("Failed to create new http connection: %s", describe(err))
198210
}
211+
if disallowUnknownFields {
212+
return http.NewConnectionDebugWrapper(conn, config.ContentType)
213+
}
199214
return conn
200215

201216
default:
@@ -206,6 +221,15 @@ func createConnectionFromEnv(t testEnv) driver.Connection {
206221

207222
// createClientFromEnv initializes a Client from information specified in environment variables.
208223
func createClientFromEnv(t testEnv, waitUntilReady bool) driver.Client {
224+
disallowUnknownFields := os.Getenv("TEST_DISALLOW_UNKNOWN_FIELDS")
225+
if disallowUnknownFields == "true" {
226+
return createClient(t, waitUntilReady, true)
227+
}
228+
return createClient(t, waitUntilReady, false)
229+
}
230+
231+
// createClient initializes a Client from information specified in environment variables.
232+
func createClient(t testEnv, waitUntilReady bool, disallowUnknownFields bool) driver.Client {
209233
runPProfServerOnce.Do(func() {
210234
if os.Getenv("TEST_PPROF") != "" {
211235
go func() {
@@ -217,7 +241,7 @@ func createClientFromEnv(t testEnv, waitUntilReady bool) driver.Client {
217241
}
218242
})
219243

220-
conn := createConnectionFromEnv(t)
244+
conn := createConnection(t, disallowUnknownFields)
221245
if os.Getenv("TEST_REQUEST_LOG") != "" {
222246
conn = WrapLogger(t, conn)
223247
}

‎test/collection_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,8 @@ func TestCollectionName(t *testing.T) {
376376

377377
// TestCollectionTruncate creates a collection, adds some documents and truncates it.
378378
func TestCollectionTruncate(t *testing.T) {
379-
c := createClientFromEnv(t, true)
379+
// don't use disallowUnknownFields in this test - we have here custom structs defined
380+
c := createClient(t, true, false)
380381
db := ensureDatabase(nil, c, "collection_test", nil, t)
381382
name := "test_collection_truncate"
382383
col, err := db.CreateCollection(nil, name, nil)

‎test/concurrency_test.go‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,14 @@ func TestConcurrentCreateSmallDocuments(t *testing.T) {
4040
if testing.Short() {
4141
t.Skip("Skip on short tests")
4242
}
43-
c := createClientFromEnv(t, true)
43+
44+
// Disable those tests for active failover
45+
if getTestMode() == testModeResilientSingle {
46+
t.Skip("Disabled in active failover mode")
47+
}
48+
49+
// don't use disallowUnknownFields in this test - we have here custom structs defined
50+
c := createClient(t, true, false)
4451

4552
version, err := c.Version(nil)
4653
if err != nil {
@@ -124,7 +131,14 @@ func TestConcurrentCreateBigDocuments(t *testing.T) {
124131
if testing.Short() {
125132
t.Skip("Skip on short tests")
126133
}
127-
c := createClientFromEnv(t, true)
134+
135+
// Disable those tests for active failover
136+
if getTestMode() == testModeResilientSingle {
137+
t.Skip("Disabled in active failover mode")
138+
}
139+
140+
// don't use disallowUnknownFields in this test - we have here custom structs defined
141+
c := createClient(t, true, false)
128142

129143
version, err := c.Version(nil)
130144
if err != nil {

‎test/cursor_test.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,8 @@ func TestCreateCursorWithMaxRuntime(t *testing.T) {
8585
// TestCreateCursor creates several cursors.
8686
func TestCreateCursor(t *testing.T) {
8787
ctx := context.Background()
88-
c := createClientFromEnv(t, true)
88+
// don't use disallowUnknownFields in this test - we have here custom structs defined
89+
c := createClient(t, true, false)
8990
db := ensureDatabase(ctx, c, "cursor_test", nil, t)
9091

9192
// Create data set
@@ -327,7 +328,8 @@ func TestCreateCursorReturnNull(t *testing.T) {
327328
func TestCreateStreamCursor(t *testing.T) {
328329
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
329330
defer cancel()
330-
c := createClientFromEnv(t, true)
331+
// don't use disallowUnknownFields in this test - we have here custom structs defined
332+
c := createClient(t, true, false)
331333

332334
version, err := c.Version(nil)
333335
if err != nil {

‎test/document_create_test.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ func createDocument(ctx context.Context, col driver.Collection, document interfa
4141

4242
// TestCreateDocument creates a document and then checks that it exists.
4343
func TestCreateDocument(t *testing.T) {
44-
c := createClientFromEnv(t, true)
44+
// don't use disallowUnknownFields in this test - we have here custom structs defined
45+
c := createClient(t, true, false)
4546
db := ensureDatabase(nil, c, "document_test", nil, t)
4647
col := ensureCollection(nil, db, "document_test", nil, t)
4748
doc := UserDoc{
@@ -70,7 +71,8 @@ func TestCreateDocument(t *testing.T) {
7071

7172
// TestCreateDocumentWithKey creates a document with given key and then checks that it exists.
7273
func TestCreateDocumentWithKey(t *testing.T) {
73-
c := createClientFromEnv(t, true)
74+
// don't use disallowUnknownFields in this test - we have here custom structs defined
75+
c := createClient(t, true, false)
7476
db := ensureDatabase(nil, c, "document_test", nil, t)
7577
col := ensureCollection(nil, db, "document_withKey_test", nil, t)
7678
doc := UserDocWithKey{
@@ -104,7 +106,8 @@ func TestCreateDocumentWithKey(t *testing.T) {
104106
// TestCreateDocumentReturnNew creates a document and checks the document returned in in ReturnNew.
105107
func TestCreateDocumentReturnNew(t *testing.T) {
106108
ctx := context.Background()
107-
c := createClientFromEnv(t, true)
109+
// don't use disallowUnknownFields in this test - we have here custom structs defined
110+
c := createClient(t, true, false)
108111
db := ensureDatabase(ctx, c, "document_test", nil, t)
109112
col := ensureCollection(ctx, db, "document_test", nil, t)
110113
doc := UserDoc{
@@ -160,7 +163,8 @@ func TestCreateDocumentNil(t *testing.T) {
160163
// TestCreateDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled,
161164
// and then checks that it exists.
162165
func TestCreateDocumentInWaitForSyncCollection(t *testing.T) {
163-
c := createClientFromEnv(t, true)
166+
// don't use disallowUnknownFields in this test - we have here custom structs defined
167+
c := createClient(t, true, false)
164168
db := ensureDatabase(nil, c, "document_test", nil, t)
165169
col := ensureCollection(nil, db, "TestCreateDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{
166170
WaitForSync: true,

‎test/document_read_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ import (
3131

3232
// TestReadDocumentWithIfMatch creates a document and reads it with a non-matching revision.
3333
func TestReadDocumentWithIfMatch(t *testing.T) {
34-
c := createClientFromEnv(t, true)
34+
// don't use disallowUnknownFields in this test - we have here custom structs defined
35+
c := createClient(t, true, false)
3536
db := ensureDatabase(nil, c, "document_read_test", nil, t)
3637
col := ensureCollection(nil, db, "document_read_test", nil, t)
3738
doc := UserDoc{
@@ -50,7 +51,7 @@ func TestReadDocumentWithIfMatch(t *testing.T) {
5051
if err != nil {
5152
t.Fatalf("Failed to read document: %s", describe(err))
5253
}
53-
if meta2 != meta {
54+
if meta2.Key != meta.Key || meta2.Rev != meta.Rev || meta2.ID != meta.ID {
5455
t.Error("Read wrong meta data.")
5556
}
5657

‎test/document_remove_test.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ func TestRemoveDocument(t *testing.T) {
6363
// TestRemoveDocumentReturnOld creates a document, removes it checks the ReturnOld value.
6464
func TestRemoveDocumentReturnOld(t *testing.T) {
6565
ctx := context.Background()
66-
c := createClientFromEnv(t, true)
66+
// don't use disallowUnknownFields in this test - we have here custom structs defined
67+
c := createClient(t, true, false)
6768
db := ensureDatabase(ctx, c, "document_test", nil, t)
6869
col := ensureCollection(ctx, db, "document_test", nil, t)
6970
doc := UserDoc{
@@ -174,7 +175,8 @@ func TestRemoveDocumentKeyEmpty(t *testing.T) {
174175
// removes it and then checks the removal has succeeded.
175176
func TestRemoveDocumentInWaitForSyncCollection(t *testing.T) {
176177
ctx := context.Background()
177-
c := createClientFromEnv(t, true)
178+
// don't use disallowUnknownFields in this test - we have here custom structs defined
179+
c := createClient(t, true, false)
178180
db := ensureDatabase(ctx, c, "document_test", nil, t)
179181
col := ensureCollection(ctx, db, "TestRemoveDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{
180182
WaitForSync: true,

0 commit comments

Comments
 (0)