diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37bc6d3..7697949 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,26 +1,22 @@ -on: [push] +on: + push: + branches: '*' + pull_request: + branches: '*' + name: Test jobs: test: - strategy: - matrix: - go-version: [1.16.x] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + name: Unit Tests + runs-on: ubuntu-latest steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - name: Checkout code - uses: actions/checkout@v2 - - name: Install dependencies - run: | - go get -u honnef.co/go/tools/cmd/staticcheck@latest - go get -u golang.org/x/tools/cmd/goimports - - name: Run staticcheck - run: staticcheck ./... - - name: Check code formatting - run: test -z $(goimports -l .) - - name: Run Test - run: go test ./... + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '1.20.1' + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Unit tests + run: go test ./... -race \ No newline at end of file diff --git a/.gitignore b/.gitignore index 66fd13c..fd2129c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ *.dll *.so *.dylib +.idea +.idea/* # Test binary, built with `go test -c` *.test @@ -13,3 +15,7 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +# specific files +jlib/timeparse/outputdata.json +jlib/timeparse/outputdata_lite.json \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b40568..2376b63 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to blues/jsonata-go +# Contributing to blues/jsonata-go [xiatechs fork] We love pull requests from everyone. By participating in this project, you agree to abide by the Blues Inc [code of conduct]. @@ -18,7 +18,7 @@ clean up inconsistent whitespace ) * by closing [issues][] * by reviewing patches -[issues]: https://github.com/blues/jsonata-go/issues +[issues]: https://github.com/xiatechs/jsonata-go/issues ## Submitting an Issue @@ -55,7 +55,7 @@ clean up inconsistent whitespace ) * If you don't know how to add tests, please put in a PR and leave a comment asking for help. We love helping! -[repo]: https://github.com/blues/jsonata-go/tree/master +[repo]: https://github.com/xiatechs/jsonata-go/tree/master [fork]: https://help.github.com/articles/fork-a-repo/ [branch]: https://help.github.com/articles/creating-and-deleting-branches-within-your-repository/ diff --git a/README.md b/README.md index aaa8abf..c60d015 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,17 @@ It currently has feature parity with jsonata-js 1.5.4. As well as a most of the ## Install - go get github.com/blues/jsonata-go + go get github.com/xiatechs/jsonata-go ## Usage ```Go import ( - "encoding/json" + "github.com/goccy/go-json" "fmt" "log" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) const jsonString = ` @@ -56,7 +56,7 @@ func main() { ## JSONata Server A locally hosted version of [JSONata Exerciser](http://try.jsonata.org/) -for testing is [available here](https://github.com/blues/jsonata-go/jsonata-server). +for testing is [available here](https://github.com/xiatechs/jsonata-go/jsonata-server). ## JSONata tests A CLI tool for running jsonata-go against the [JSONata test suite](https://github.com/jsonata-js/jsonata/tree/master/test/test-suite) is [available here](./jsonata-test). diff --git a/callable.go b/callable.go index ec08501..87700e6 100644 --- a/callable.go +++ b/callable.go @@ -5,18 +5,20 @@ package jsonata import ( - "encoding/json" "fmt" + "github.com/goccy/go-json" "reflect" "regexp" "strings" + "sync" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type callableName struct { + mu sync.Mutex name string } @@ -25,6 +27,8 @@ func (n callableName) Name() string { } func (n *callableName) SetName(s string) { + n.mu.Lock() + defer n.mu.Unlock() n.name = s } @@ -76,6 +80,7 @@ func newGoCallableParam(typ reflect.Type) goCallableParam { // A goCallable represents a built-in or third party Go function. // It implements the Callable interface. type goCallable struct { + mu sync.Mutex callableName callableMarshaler fn reflect.Value @@ -205,6 +210,8 @@ func makeGoCallableParams(typ reflect.Type) []goCallableParam { } func (c *goCallable) SetContext(context reflect.Value) { + c.mu.Lock() + defer c.mu.Unlock() c.context = context } @@ -682,7 +689,7 @@ func (f *transformationCallable) Call(argv []reflect.Value) (reflect.Value, erro obj, err := f.clone(argv[0]) if err != nil { - return undefined, newEvalError(ErrClone, nil, nil) + return undefined, newEvalError(ErrClone, nil, nil, 0) } if obj == undefined { @@ -739,7 +746,7 @@ func (f *transformationCallable) updateEntries(item reflect.Value) error { } if !jtypes.IsMap(updates) { - return newEvalError(ErrIllegalUpdate, f.updates, nil) + return newEvalError(ErrIllegalUpdate, f.updates, nil, 0) } for _, key := range updates.MapKeys() { @@ -759,7 +766,7 @@ func (f *transformationCallable) deleteEntries(item reflect.Value) error { deletes = arrayify(deletes) if !jtypes.IsArrayOf(deletes, jtypes.IsString) { - return newEvalError(ErrIllegalDelete, f.deletes, nil) + return newEvalError(ErrIllegalDelete, f.deletes, nil, 0) } for i := 0; i < deletes.Len(); i++ { diff --git a/callable_test.go b/callable_test.go index 75062bc..f07493b 100644 --- a/callable_test.go +++ b/callable_test.go @@ -11,10 +11,13 @@ import ( "regexp" "sort" "strings" + "sync" "testing" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/stretchr/testify/assert" + + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) var ( @@ -2328,8 +2331,8 @@ func testTransformationCallable(t *testing.T, tests []transformationCallableTest } } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("transform %d: expected error %v, got %v", i+1, test.Error, err) + if err != nil && test.Error != nil { + assert.EqualError(t, err, test.Error.Error()) } } } @@ -2371,6 +2374,7 @@ func TestRegexCallable(t *testing.T) { "groups": []string{}, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2379,6 +2383,7 @@ func TestRegexCallable(t *testing.T) { groups: []string{}, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ad", @@ -2387,6 +2392,7 @@ func TestRegexCallable(t *testing.T) { groups: []string{}, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ab", @@ -2395,6 +2401,7 @@ func TestRegexCallable(t *testing.T) { groups: []string{}, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", @@ -2425,6 +2432,7 @@ func TestRegexCallable(t *testing.T) { }, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2435,6 +2443,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ad", @@ -2445,6 +2454,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ab", @@ -2455,6 +2465,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", @@ -2491,6 +2502,7 @@ func TestRegexCallable(t *testing.T) { }, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2502,6 +2514,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ad", @@ -2513,6 +2526,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ab", @@ -2524,6 +2538,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..af2978b --- /dev/null +++ b/config/config.go @@ -0,0 +1,7 @@ +package config + +var defaultDivisionPrecision int32 = 8 + +func GetDivisionPrecision() int32 { + return defaultDivisionPrecision +} diff --git a/env.go b/env.go index bbbefc5..c2847da 100644 --- a/env.go +++ b/env.go @@ -11,9 +11,11 @@ import ( "strings" "unicode/utf8" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jlib/join" + "github.com/xiatechs/jsonata-go/jlib/timeparse" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type environment struct { @@ -67,6 +69,91 @@ var ( var baseEnv = initBaseEnv(map[string]Extension{ + /* + EXTENDED START + */ + "objmerge": { + Func: jlib.ObjMerge, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + + "sjoin": { + Func: jlib.SimpleJoin, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + + "eval": { + Func: RunEval, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "unescape": { + Func: jlib.Unescape, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "hashmd5": { + Func: jlib.HashMD5, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "hash256": { + Func: jlib.Hash256, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "dateTimeDim": { + Func: timeparse.TimeDateDimensions, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "dateTimeDimLite": { + Func: timeparse.TimeDateDimensionsLite, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "timeSince": { + Func: timeparse.Since, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "objectsToDocument": { + Func: jlib.ObjectsToDocument, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + + "oneToManyJoin": { + Func: join.OneToManyJoin, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + + "accumulatingSlice": { + Func: jlib.FoldArray, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + + "renameKeys": { + Func: jlib.RenameKeys, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + + /* + EXTENDED END + */ + // String functions "string": { @@ -442,7 +529,6 @@ func undefinedHandlerAppend(argv []reflect.Value) bool { // Context handlers func contextHandlerSubstring(argv []reflect.Value) bool { - // If substring() is called with one or two numeric arguments, // use the evaluation context as the first argument. switch len(argv) { diff --git a/error.go b/error.go index f5d383c..3020e5c 100644 --- a/error.go +++ b/error.go @@ -9,7 +9,7 @@ import ( "fmt" "regexp" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // ErrUndefined is returned by the evaluation methods when @@ -50,39 +50,41 @@ const ( ) var errmsgs = map[ErrType]string{ - ErrNonIntegerLHS: `left side of the "{{value}}" operator must evaluate to an integer`, - ErrNonIntegerRHS: `right side of the "{{value}}" operator must evaluate to an integer`, - ErrNonNumberLHS: `left side of the "{{value}}" operator must evaluate to a number`, - ErrNonNumberRHS: `right side of the "{{value}}" operator must evaluate to a number`, - ErrNonComparableLHS: `left side of the "{{value}}" operator must evaluate to a number or string`, - ErrNonComparableRHS: `right side of the "{{value}}" operator must evaluate to a number or string`, - ErrTypeMismatch: `both sides of the "{{value}}" operator must have the same type`, - ErrNonCallable: `cannot call non-function {{token}}`, - ErrNonCallableApply: `cannot use function application with non-function {{token}}`, - ErrNonCallablePartial: `cannot partially apply non-function {{token}}`, - ErrNumberInf: `result of the "{{value}}" operator is out of range`, - ErrNumberNaN: `result of the "{{value}}" operator is not a valid number`, - ErrMaxRangeItems: `range operator has too many items`, - ErrIllegalKey: `object key {{token}} does not evaluate to a string`, - ErrDuplicateKey: `multiple object keys evaluate to the value "{{value}}"`, - ErrClone: `object transformation: cannot make a copy of the object`, - ErrIllegalUpdate: `the insert/update clause of an object transformation must evaluate to an object`, - ErrIllegalDelete: `the delete clause of an object transformation must evaluate to an array of strings`, - ErrNonSortable: `expressions in a sort term must evaluate to strings or numbers`, - ErrSortMismatch: `expressions in a sort term must have the same type`, + ErrNonIntegerLHS: `left side of the "{{value}}" operator must evaluate to an integer, position:{{position}}, arguments: {{arguments}}`, + ErrNonIntegerRHS: `right side of the "{{value}}" operator must evaluate to an integer, position:{{position}}, arguments: {{arguments}}`, + ErrNonNumberLHS: `left side of the "{{value}}" operator must evaluate to a number, position:{{position}}, arguments: {{arguments}}`, + ErrNonNumberRHS: `right side of the "{{value}}" operator must evaluate to a number, position:{{position}}, arguments: {{arguments}}`, + ErrNonComparableLHS: `left side of the "{{value}}" operator must evaluate to a number or string, position:{{position}}, arguments: {{arguments}}`, + ErrNonComparableRHS: `right side of the "{{value}}" operator must evaluate to a number or string, position:{{position}}, arguments: {{arguments}}`, + ErrTypeMismatch: `both sides of the "{{value}}" operator must have the same type, position:{{position}}, arguments: {{arguments}}`, + ErrNonCallable: `cannot call non-function {{token}}, position:{{position}}, arguments: {{arguments}}`, + ErrNonCallableApply: `cannot use function application with non-function {{token}}, position:{{position}}, arguments: {{arguments}}`, + ErrNonCallablePartial: `cannot partially apply non-function {{token}}, position:{{position}}, arguments: {{arguments}}`, + ErrNumberInf: `result of the "{{value}}" operator is out of range, position:{{position}}, arguments: {{arguments}}`, + ErrNumberNaN: `result of the "{{value}}" operator is not a valid number, position:{{position}}, arguments: {{arguments}}`, + ErrMaxRangeItems: `range operator has too many items, position:{{position}}, arguments: {{arguments}}`, + ErrIllegalKey: `object key {{token}} does not evaluate to a string, position:{{position}}, arguments: {{arguments}}`, + ErrDuplicateKey: `multiple object keys evaluate to the value "{{value}}", position:{{position}}, arguments: {{arguments}}`, + ErrClone: `object transformation: cannot make a copy of the object, position:{{position}}, arguments: {{arguments}}`, + ErrIllegalUpdate: `the insert/update clause of an object transformation must evaluate to an object, position:{{position}}, arguments: {{arguments}}`, + ErrIllegalDelete: `the delete clause of an object transformation must evaluate to an array of strings, position:{{position}}, arguments: {{arguments}}`, + ErrNonSortable: `expressions in a sort term must evaluate to strings or numbers, position:{{position}}, arguments: {{arguments}}`, + ErrSortMismatch: `expressions in a sort term must have the same type, position:{{position}}, arguments: {{arguments}}`, } -var reErrMsg = regexp.MustCompile("{{(token|value)}}") +var reErrMsg = regexp.MustCompile("{{(token|value|position|arguments)}}") // An EvalError represents an error during evaluation of a // JSONata expression. type EvalError struct { - Type ErrType - Token string - Value string + Type ErrType + Token string + Value string + Pos int + Arguments string } -func newEvalError(typ ErrType, token interface{}, value interface{}) *EvalError { +func newEvalError(typ ErrType, token interface{}, value interface{}, pos int) *EvalError { stringify := func(v interface{}) string { switch v := v.(type) { @@ -99,6 +101,7 @@ func newEvalError(typ ErrType, token interface{}, value interface{}) *EvalError Type: typ, Token: stringify(token), Value: stringify(value), + Pos: pos, } } @@ -115,6 +118,10 @@ func (e EvalError) Error() string { return e.Token case "{{value}}": return e.Value + case "{{arguments}}": + return e.Arguments + case "{{position}}": + return fmt.Sprintf("%v", e.Pos) default: return match } @@ -146,8 +153,10 @@ func (e ArgCountError) Error() string { // expression contains a function call with the wrong argument // type. type ArgTypeError struct { - Func string - Which int + Func string + Which int + Pos int + Arguments string } func newArgTypeError(f jtypes.Callable, which int) *ArgTypeError { @@ -158,5 +167,5 @@ func newArgTypeError(f jtypes.Callable, which int) *ArgTypeError { } func (e ArgTypeError) Error() string { - return fmt.Sprintf("argument %d of function %q does not match function signature", e.Which, e.Func) + return fmt.Sprintf("argument %d of function %q does not match function signature, position: %v, arguments: %v", e.Which, e.Func, e.Pos, e.Arguments) } diff --git a/errrors_test.go b/errrors_test.go new file mode 100644 index 0000000..8b23059 --- /dev/null +++ b/errrors_test.go @@ -0,0 +1,191 @@ +package jsonata + +import ( + "github.com/goccy/go-json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrors(t *testing.T) { + // your JSON data + data1 := `{ + "employees": [ + { + "firstName": "John", + "lastName": "Doe", + "department": "Sales", + "salary": 50000, + "joining_date": "2020-05-12", + "details": { + "address": "123 Main St", + "city": "New York", + "state": "NY" + } + }, + { + "firstName": "Anna", + "lastName": "Smith", + "department": "Marketing", + "salary": 60000, + "joining_date": "2019-07-01", + "details": { + "address": "456 Market St", + "city": "San Francisco", + "state": "CA" + } + }, + { + "firstName": "Peter", + "lastName": "Jones", + "department": "Sales", + "salary": 70000, + "joining_date": "2021-01-20", + "details": { + "address": "789 Broad St", + "city": "Los Angeles", + "state": "CA" + } + } + ] +}` + var data interface{} + + // Decode JSON. + err := json.Unmarshal([]byte(data1), &data) + assert.NoError(t, err) + t.Run("wrong arithmetic errors", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.firstName + 5") + + // Evaluate. + _, err := e.Eval(data) + assert.Error(t, err, "left side of the \"value:+, position: 20\" operator must evaluate to a number") + }) + t.Run("Cannot call non-function token:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.details.state.$address()") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "cannot call non-function $address, position:25, arguments: ") + + }) + t.Run("Trying to get the maximum of a string field:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$max(employees.firstName)") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "cannot call max on an array with non-number types, position: 1, arguments: number:0 value:[John Anna Peter] ") + + }) + t.Run("Invalid Function Call on a non-array field:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.department.$count()") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "function \"count\" takes 1 argument(s), got 0, position: 22, arguments: ") + + }) + t.Run("Cannot use wildcard on non-object type:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.*.salary") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "no results found") + }) + t.Run("Indexing on non-array type:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.firstName[1]") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "no results found") + + }) + t.Run("Use of an undefined variable:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$undefinedVariable") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "no results found") + + }) + t.Run("Use of an undefined function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$undefinedFunction()") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "cannot call non-function $undefinedFunction, position:1, arguments: ") + + }) + t.Run("Comparison of incompatible types:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.firstName > employees.salary") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "left side of the \">\" operator must evaluate to a number or string, position:0, arguments: ") + + }) + t.Run("Use of an invalid JSONata operator:", func(t *testing.T) { + + // Create expression. + _, err := Compile("employees ! employees") + assert.EqualError(t, err, "syntax error: '', position: 10") + }) + t.Run("Incorrect use of the reduce function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$reduce(employees.firstName, function($acc, $val) { $acc + $val })") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "left side of the \"+\" operator must evaluate to a number, position:57, arguments: ") + + }) + t.Run("Incorrect use of the map function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$map(employees, function($employee) { $employee.firstName + 5 })") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "left side of the \"+\" operator must evaluate to a number, position:58") + + }) + t.Run("Incorrect use of the filter function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$filter(employees, function($employee) { $employee.salary.$uppercase() })") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "argument 1 of function \"uppercase\" does not match function signature, position: 1, arguments: number") + + }) + t.Run("Incorrect use of the join function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$join(employees.firstName, 5)") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "argument 2 of function \"join\" does not match function signature, position: 1, arguments: number:0 value:[John Anna Peter] number:1 value:5 ") + + }) +} diff --git a/eval.go b/eval.go index a66b561..6141596 100644 --- a/eval.go +++ b/eval.go @@ -9,10 +9,13 @@ import ( "math" "reflect" "sort" + "sync" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/shopspring/decimal" + "github.com/xiatechs/jsonata-go/config" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) var undefined reflect.Value @@ -202,9 +205,6 @@ func evalPath(node *jparse.PathNode, data reflect.Value, env *environment) (refl return undefined, err } - if jtypes.IsArray(output) && jtypes.Resolve(output).Len() == 0 { - return undefined, nil - } } if node.KeepArrays { @@ -312,7 +312,7 @@ func evalNegation(node *jparse.NegationNode, data reflect.Value, env *environmen n, ok := jtypes.AsNumber(rhs) if !ok { - return undefined, newEvalError(ErrNonNumberRHS, node.RHS, "-") + return undefined, newEvalError(ErrNonNumberRHS, node.RHS, "-", node.Pos()) } return reflect.ValueOf(-n), nil @@ -353,11 +353,11 @@ func evalRange(node *jparse.RangeNode, data reflect.Value, env *environment) (re // If either side is not an integer, return an error. if lhsOK && !lhsInteger { - return undefined, newEvalError(ErrNonIntegerLHS, node.LHS, "..") + return undefined, newEvalError(ErrNonIntegerLHS, node.LHS, "..", 0) } if rhsOK && !rhsInteger { - return undefined, newEvalError(ErrNonIntegerRHS, node.RHS, "..") + return undefined, newEvalError(ErrNonIntegerRHS, node.RHS, "..", 0) } // If either side is undefined or the left side is greater @@ -370,7 +370,7 @@ func evalRange(node *jparse.RangeNode, data reflect.Value, env *environment) (re // Check for integer overflow or an array size that exceeds // our upper bound. if size < 0 || size > maxRangeItems { - return undefined, newEvalError(ErrMaxRangeItems, "..", nil) + return undefined, newEvalError(ErrMaxRangeItems, "..", nil, 0) } results := reflect.MakeSlice(typeInterfaceSlice, size, size) @@ -475,7 +475,7 @@ func groupItemsByKey(obj *jparse.ObjectNode, items reflect.Value, env *environme key := s.Value if _, ok := results[key]; ok { - return nil, newEvalError(ErrDuplicateKey, keyNode, key) + return nil, newEvalError(ErrDuplicateKey, keyNode, key, 0) } results[key] = keyIndexes{ @@ -493,7 +493,7 @@ func groupItemsByKey(obj *jparse.ObjectNode, items reflect.Value, env *environme key, ok := jtypes.AsString(v) if !ok { - return nil, newEvalError(ErrIllegalKey, keyNode, nil) + return nil, newEvalError(ErrIllegalKey, keyNode, nil, 0) } idx, ok := results[key] @@ -506,7 +506,7 @@ func groupItemsByKey(obj *jparse.ObjectNode, items reflect.Value, env *environme } if idx.pair != i { - return nil, newEvalError(ErrDuplicateKey, keyNode, key) + return nil, newEvalError(ErrDuplicateKey, keyNode, key, 0) } idx.items = append(idx.items, j) @@ -715,20 +715,20 @@ func buildSortInfo(items reflect.Value, terms []jparse.SortTerm, env *environmen switch { case jtypes.IsNumber(v): if isStringTerm[j] { - return nil, newEvalError(ErrSortMismatch, term.Expr, nil) + return nil, newEvalError(ErrSortMismatch, term.Expr, nil, 0) } values[j] = v isNumberTerm[j] = true case jtypes.IsString(v): if isNumberTerm[j] { - return nil, newEvalError(ErrSortMismatch, term.Expr, nil) + return nil, newEvalError(ErrSortMismatch, term.Expr, nil, 0) } values[j] = v isStringTerm[j] = true default: - return nil, newEvalError(ErrNonSortable, term.Expr, nil) + return nil, newEvalError(ErrNonSortable, term.Expr, nil, 0) } } @@ -829,6 +829,7 @@ func evalTypedLambda(node *jparse.TypedLambdaNode, data reflect.Value, env *envi func evalObjectTransformation(node *jparse.ObjectTransformationNode, data reflect.Value, env *environment) (reflect.Value, error) { f := &transformationCallable{ callableName: callableName{ + sync.Mutex{}, "transform", }, pattern: node.Pattern, @@ -848,7 +849,7 @@ func evalPartial(node *jparse.PartialNode, data reflect.Value, env *environment) fn, ok := jtypes.AsCallable(v) if !ok { - return undefined, newEvalError(ErrNonCallablePartial, node.Func, nil) + return undefined, newEvalError(ErrNonCallablePartial, node.Func, nil, 0) } f := &partialCallable{ @@ -880,7 +881,7 @@ func evalFunctionCall(node *jparse.FunctionCallNode, data reflect.Value, env *en fn, ok := jtypes.AsCallable(v) if !ok { - return undefined, newEvalError(ErrNonCallable, node.Func, nil) + return undefined, newEvalError(ErrNonCallable, node.Func, reflect.ValueOf(data), node.Func.Pos()) } if setter, ok := fn.(nameSetter); ok { @@ -903,8 +904,32 @@ func evalFunctionCall(node *jparse.FunctionCallNode, data reflect.Value, env *en argv[i] = v } + res, err := fn.Call(argv) + if err != nil { + return res, updateError(err, node, transformArgsToString(argv)) + } + return res, nil +} + +func updateError(err error, node *jparse.FunctionCallNode, stringArgs string) error { + newErr, ok := err.(*ArgTypeError) + if ok { + newErr.Pos = node.Func.Pos() + newErr.Arguments = stringArgs + return newErr + } - return fn.Call(argv) + return fmt.Errorf("%v, position: %v, arguments: %v", err, node.Func.Pos(), stringArgs) +} + +func transformArgsToString(argv []reflect.Value) string { + argvString := "" + for i, value := range argv { + if value.IsValid() && value.CanInterface() { + argvString += fmt.Sprintf("number:%v value:%v ", i, value.Interface()) + } + } + return argvString } func evalFunctionApplication(node *jparse.FunctionApplicationNode, data reflect.Value, env *environment) (reflect.Value, error) { @@ -931,7 +956,7 @@ func evalFunctionApplication(node *jparse.FunctionApplicationNode, data reflect. // Check that the right hand side is callable. f2, ok := jtypes.AsCallable(rhs) if !ok { - return undefined, newEvalError(ErrNonCallableApply, node.RHS, "~>") + return undefined, newEvalError(ErrNonCallableApply, node.RHS, "~>", 0) } // If the left hand side is not callable, call the right @@ -977,12 +1002,13 @@ func evalNumericOperator(node *jparse.NumericOperatorNode, data reflect.Value, e } // Return an error if either side is not a number. + if lhsOK && !lhsNumber { - return undefined, newEvalError(ErrNonNumberLHS, node.LHS, node.Type) + return undefined, newEvalError(ErrNonNumberLHS, node.LHS, node.Type, node.Pos()) } if rhsOK && !rhsNumber { - return undefined, newEvalError(ErrNonNumberRHS, node.RHS, node.Type) + return undefined, newEvalError(ErrNonNumberRHS, node.RHS, node.Type, 0) } // Return undefined if either side is undefined. @@ -990,29 +1016,34 @@ func evalNumericOperator(node *jparse.NumericOperatorNode, data reflect.Value, e return undefined, nil } + lhsDecimal, rhsDecimal := decimal.NewFromFloat(lhs), decimal.NewFromFloat(rhs) + var x float64 switch node.Type { case jparse.NumericAdd: - x = lhs + rhs + x = lhsDecimal.Add(rhsDecimal).RoundCeil(config.GetDivisionPrecision()).InexactFloat64() case jparse.NumericSubtract: - x = lhs - rhs + x = lhsDecimal.Sub(rhsDecimal).RoundCeil(config.GetDivisionPrecision()).InexactFloat64() case jparse.NumericMultiply: - x = lhs * rhs + x = lhsDecimal.Mul(rhsDecimal).Truncate(config.GetDivisionPrecision()).InexactFloat64() case jparse.NumericDivide: x = lhs / rhs + if !math.IsInf(x, 0) && !math.IsNaN(x) { + x = lhsDecimal.Div(rhsDecimal).RoundCeil(config.GetDivisionPrecision()).InexactFloat64() + } case jparse.NumericModulo: - x = math.Mod(lhs, rhs) + x = lhsDecimal.Mod(rhsDecimal).Truncate(config.GetDivisionPrecision()).InexactFloat64() default: panicf("unrecognised numeric operator %q", node.Type) } if math.IsInf(x, 0) { - return undefined, newEvalError(ErrNumberInf, nil, node.Type) + return undefined, newEvalError(ErrNumberInf, nil, node.Type, 0) } if math.IsNaN(x) { - return undefined, newEvalError(ErrNumberNaN, nil, node.Type) + return undefined, newEvalError(ErrNumberNaN, nil, node.Type, 0) } return reflect.ValueOf(x), nil @@ -1047,16 +1078,16 @@ func evalComparisonOperator(node *jparse.ComparisonOperatorNode, data reflect.Va // left side type does not equal right side type. if needComparableTypes(node.Type) { if lhs != undefined && !lhsNumber && !lhsString { - return undefined, newEvalError(ErrNonComparableLHS, node.LHS, node.Type) + return undefined, newEvalError(ErrNonComparableLHS, node.LHS, node.Type, 0) } if rhs != undefined && !rhsNumber && !rhsString { - return undefined, newEvalError(ErrNonComparableRHS, node.RHS, node.Type) + return undefined, newEvalError(ErrNonComparableRHS, node.RHS, node.Type, 0) } if lhs != undefined && rhs != undefined && (lhsNumber != rhsNumber || lhsString != rhsString) { - return undefined, newEvalError(ErrTypeMismatch, nil, node.Type) + return undefined, newEvalError(ErrTypeMismatch, nil, node.Type, 0) } } diff --git a/eval_test.go b/eval_test.go index 80c9378..4c1b4df 100644 --- a/eval_test.go +++ b/eval_test.go @@ -11,9 +11,11 @@ import ( "strings" "testing" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/stretchr/testify/assert" + + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type evalTestCase struct { @@ -4297,7 +4299,6 @@ func testEvalTestCases(t *testing.T, tests []evalTestCase) { } v, err := eval(test.Input, reflect.ValueOf(test.Data), env) - var output interface{} if v.IsValid() && v.CanInterface() { output = v.Interface() @@ -4309,11 +4310,12 @@ func testEvalTestCases(t *testing.T, tests []evalTestCase) { } if !equal(output, test.Output) { - t.Errorf("%s: Expected %v, got %v", test.Input, test.Output, output) + t.Errorf("%s: Expected: %v, got: %v", test.Input, test.Output, output) } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("%s: Expected error %v, got %v", test.Input, test.Error, err) + if err != nil && test.Error != nil { + assert.EqualError(t, err, test.Error.Error()) } + } } diff --git a/example_eval_test.go b/example_eval_test.go index fd35592..65eb0b4 100644 --- a/example_eval_test.go +++ b/example_eval_test.go @@ -5,11 +5,11 @@ package jsonata_test import ( - "encoding/json" "fmt" + "github.com/goccy/go-json" "log" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) const jsonString = ` diff --git a/example_exts_test.go b/example_exts_test.go index 2c02e18..7a1bd05 100644 --- a/example_exts_test.go +++ b/example_exts_test.go @@ -9,7 +9,7 @@ import ( "log" "strings" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) // diff --git a/extendedTestFiles/arrayIndexingComplex/input.json b/extendedTestFiles/arrayIndexingComplex/input.json new file mode 100644 index 0000000..4902504 --- /dev/null +++ b/extendedTestFiles/arrayIndexingComplex/input.json @@ -0,0 +1,6 @@ +[ + {"Code":"root.list[0].items[0].value","Val":"Item0-0"}, + {"Code":"root.list[0].items[1].value","Value":"Item0-1"}, + {"Code":"root.list[1].items[0].value","Val":"Item1-0"}, + {"Code":"root.list[2].info[2].details[1].desc","Value":"DeepNestedValue"} +] diff --git a/extendedTestFiles/arrayIndexingComplex/input.jsonata b/extendedTestFiles/arrayIndexingComplex/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/arrayIndexingComplex/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/arrayIndexingComplex/output.json b/extendedTestFiles/arrayIndexingComplex/output.json new file mode 100644 index 0000000..c96aca4 --- /dev/null +++ b/extendedTestFiles/arrayIndexingComplex/output.json @@ -0,0 +1,29 @@ +{ + "root": { + "list": [ + { + "items": [ + {"value":"Item0-0"}, + {"value":"Item0-1"} + ] + }, + { + "items": [ + {"value":"Item1-0"} + ] + }, + { + "info": [ + {}, + {}, + { + "details": [ + {}, + {"desc": "DeepNestedValue"} + ] + } + ] + } + ] + } +} diff --git a/extendedTestFiles/arrayIndexingSimple/input.json b/extendedTestFiles/arrayIndexingSimple/input.json new file mode 100644 index 0000000..256c35a --- /dev/null +++ b/extendedTestFiles/arrayIndexingSimple/input.json @@ -0,0 +1,4 @@ +[ + {"Code": "employees[0].name", "Val": "Alice"}, + {"Code": "employees[1].name", "Value": "Bob"} +] \ No newline at end of file diff --git a/extendedTestFiles/arrayIndexingSimple/input.jsonata b/extendedTestFiles/arrayIndexingSimple/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/arrayIndexingSimple/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/arrayIndexingSimple/output.json b/extendedTestFiles/arrayIndexingSimple/output.json new file mode 100644 index 0000000..028c5fe --- /dev/null +++ b/extendedTestFiles/arrayIndexingSimple/output.json @@ -0,0 +1,10 @@ +{ + "employees": [ + { + "name": "Alice" + }, + { + "name": "Bob" + } + ] +} \ No newline at end of file diff --git a/extendedTestFiles/noValNoValue/input.json b/extendedTestFiles/noValNoValue/input.json new file mode 100644 index 0000000..abd3291 --- /dev/null +++ b/extendedTestFiles/noValNoValue/input.json @@ -0,0 +1,4 @@ +[ + {"Code":"topKey","OtherField":"none"}, + {"Code":"anotherKey"} +] diff --git a/extendedTestFiles/noValNoValue/input.jsonata b/extendedTestFiles/noValNoValue/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/noValNoValue/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/noValNoValue/output.json b/extendedTestFiles/noValNoValue/output.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/extendedTestFiles/noValNoValue/output.json @@ -0,0 +1 @@ +{} diff --git a/extendedTestFiles/readme.md b/extendedTestFiles/readme.md new file mode 100644 index 0000000..9eb13bc --- /dev/null +++ b/extendedTestFiles/readme.md @@ -0,0 +1,13 @@ +## how? + +First, create a folder with a name i.e "exampleFunc" + +Second, add three files: input.json, input.jsonata and output.json + +Third, make sure the three files match context below: + +``` +input.json - this is the test file you wish to assert against. +input.jsonata - this is the jsonata that will be applied to the input json +output.json - this is what you expect the output to be +``` \ No newline at end of file diff --git a/extendedTestFiles/valAndValueAtArrayIndex/input.json b/extendedTestFiles/valAndValueAtArrayIndex/input.json new file mode 100644 index 0000000..3f8aba0 --- /dev/null +++ b/extendedTestFiles/valAndValueAtArrayIndex/input.json @@ -0,0 +1,5 @@ +[ + {"Code":"data[2].entry","Val":"ThirdEntryVal","Value":"ThirdEntryValue"}, + {"Code":"data[1].entry","Value":"SecondEntryValue"}, + {"Code":"data[0].entry","Val":"FirstEntryVal"} +] diff --git a/extendedTestFiles/valAndValueAtArrayIndex/input.jsonata b/extendedTestFiles/valAndValueAtArrayIndex/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/valAndValueAtArrayIndex/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/valAndValueAtArrayIndex/output.json b/extendedTestFiles/valAndValueAtArrayIndex/output.json new file mode 100644 index 0000000..10f612d --- /dev/null +++ b/extendedTestFiles/valAndValueAtArrayIndex/output.json @@ -0,0 +1,7 @@ +{ + "data": [ + { "entry": "FirstEntryVal" }, + { "entry": "SecondEntryValue" }, + { "entry": "ThirdEntryVal" } + ] +} diff --git a/extendedTestFiles/valIsNullUseValue/input.json b/extendedTestFiles/valIsNullUseValue/input.json new file mode 100644 index 0000000..f86bba5 --- /dev/null +++ b/extendedTestFiles/valIsNullUseValue/input.json @@ -0,0 +1,3 @@ +[ + {"Code": "testKey", "Val": null, "Value": "RealValue"} +] \ No newline at end of file diff --git a/extendedTestFiles/valIsNullUseValue/input.jsonata b/extendedTestFiles/valIsNullUseValue/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/valIsNullUseValue/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/valIsNullUseValue/output.json b/extendedTestFiles/valIsNullUseValue/output.json new file mode 100644 index 0000000..37e89be --- /dev/null +++ b/extendedTestFiles/valIsNullUseValue/output.json @@ -0,0 +1,3 @@ +{ + "testKey": "RealValue" +} \ No newline at end of file diff --git a/extendedTestFiles/valPriority/input.json b/extendedTestFiles/valPriority/input.json new file mode 100644 index 0000000..fde9a14 --- /dev/null +++ b/extendedTestFiles/valPriority/input.json @@ -0,0 +1,4 @@ +[ + {"Code": "person.name", "Val": "Alice", "Value": "ShouldNotUse"}, + {"Code": "person.age", "Val": 30} +] \ No newline at end of file diff --git a/extendedTestFiles/valPriority/input.jsonata b/extendedTestFiles/valPriority/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/valPriority/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/valPriority/output.json b/extendedTestFiles/valPriority/output.json new file mode 100644 index 0000000..67420c7 --- /dev/null +++ b/extendedTestFiles/valPriority/output.json @@ -0,0 +1,6 @@ +{ + "person": { + "name": "Alice", + "age": 30 + } +} \ No newline at end of file diff --git a/extended_test.go b/extended_test.go new file mode 100644 index 0000000..4194f80 --- /dev/null +++ b/extended_test.go @@ -0,0 +1,71 @@ +package jsonata + +import ( + "log" + "os" + "path/filepath" + "testing" + + "github.com/goccy/go-json" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testCasesPath = "extendedTestFiles" + +const ( + expectedInputFile = "input.json" + expectedOutputFile = "output.json" + expectedInputJsonata = "input.jsonata" +) + +func TestChassis(t *testing.T) { + entries, err := os.ReadDir(testCasesPath) + if err != nil { + log.Fatalf("Failed to read directory: %v", err) + } + + for _, entry := range entries { + if entry.IsDir() { + testCase := entry.Name() + + testCasePath := filepath.Join(testCasesPath, testCase) + + t.Run(testCasePath, func(t *testing.T) { + runTest(t, + filepath.Join(testCasePath, expectedInputFile), + filepath.Join(testCasePath, expectedOutputFile), + filepath.Join(testCasePath, expectedInputJsonata), + ) + }) + } + } +} + +func runTest(t *testing.T, inputfile, outputfile, jsonatafile string) { + inputBytes, err := os.ReadFile(inputfile) + require.NoError(t, err) + + outputBytes, err := os.ReadFile(outputfile) + require.NoError(t, err) + + jsonataBytes, err := os.ReadFile(jsonatafile) + require.NoError(t, err) + + expr, err := Compile(string(jsonataBytes)) + require.NoError(t, err) + + var input, output interface{} + + err = json.Unmarshal(inputBytes, &input) + require.NoError(t, err) + + result, err := expr.Eval(input) + require.NoError(t, err) + + err = json.Unmarshal(outputBytes, &output) + require.NoError(t, err) + + assert.Equal(t, result, output) +} diff --git a/go.mod b/go.mod index 6a0e2e5..fbee8af 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,16 @@ -module github.com/blues/jsonata-go +module github.com/xiatechs/jsonata-go -go 1.16 +go 1.20 + +require ( + github.com/goccy/go-json v0.10.2 + github.com/ncruces/go-strftime v0.1.9 + github.com/shopspring/decimal v1.3.1 + github.com/stretchr/testify v1.8.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index e69de29..7deb495 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,21 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jlib/aggregate.go b/jlib/aggregate.go index 735ff82..475711b 100644 --- a/jlib/aggregate.go +++ b/jlib/aggregate.go @@ -8,7 +8,9 @@ import ( "fmt" "reflect" - "github.com/blues/jsonata-go/jtypes" + "github.com/shopspring/decimal" + "github.com/xiatechs/jsonata-go/config" + "github.com/xiatechs/jsonata-go/jtypes" ) // Sum returns the total of an array of numbers. If the array is @@ -24,17 +26,17 @@ func Sum(v reflect.Value) (float64, error) { v = jtypes.Resolve(v) - var sum float64 + var sum decimal.Decimal for i := 0; i < v.Len(); i++ { n, ok := jtypes.AsNumber(v.Index(i)) if !ok { return 0, fmt.Errorf("cannot call sum on an array with non-number types") } - sum += n + sum = sum.Add(decimal.NewFromFloat(n)) } - return sum, nil + return sum.RoundCeil(config.GetDivisionPrecision()).InexactFloat64(), nil } // Max returns the largest value in an array of numbers. If the @@ -115,15 +117,15 @@ func Average(v reflect.Value) (float64, error) { return 0, jtypes.ErrUndefined } - var sum float64 + var sum decimal.Decimal for i := 0; i < v.Len(); i++ { n, ok := jtypes.AsNumber(v.Index(i)) if !ok { return 0, fmt.Errorf("cannot call average on an array with non-number types") } - sum += n + sum = sum.Add(decimal.NewFromFloat(n)) } - return sum / float64(v.Len()), nil + return sum.Div(decimal.NewFromInt(int64(v.Len()))).RoundCeil(config.GetDivisionPrecision()).InexactFloat64(), nil } diff --git a/jlib/array.go b/jlib/array.go index f16f711..690bc70 100644 --- a/jlib/array.go +++ b/jlib/array.go @@ -10,7 +10,7 @@ import ( "reflect" "sort" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // Count (golint) @@ -45,14 +45,14 @@ func Distinct(v reflect.Value) interface{} { for i := 0; i < items.Len(); i++ { item := jtypes.Resolve(items.Index(i)) - if jtypes.IsMap(item) { - // We can't hash a map, so convert it to a + if jtypes.IsMap(item) || jtypes.IsArray(item) { + // We can't hash a map or array, so convert it to a // string that is hashable - mapItem := fmt.Sprint(item.Interface()) - if _, ok := visited[mapItem]; ok { + unhashableItem := fmt.Sprint(item.Interface()) + if _, ok := visited[unhashableItem]; ok { continue } - visited[mapItem] = struct{}{} + visited[unhashableItem] = struct{}{} distinctValues = reflect.Append(distinctValues, item) continue diff --git a/jlib/boolean.go b/jlib/boolean.go index b4325f4..067e2f9 100644 --- a/jlib/boolean.go +++ b/jlib/boolean.go @@ -7,7 +7,7 @@ package jlib import ( "reflect" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // Boolean (golint) diff --git a/jlib/date.go b/jlib/date.go index 70a9a2c..bcc47d1 100644 --- a/jlib/date.go +++ b/jlib/date.go @@ -8,26 +8,37 @@ import ( "fmt" "regexp" "strconv" + "strings" "time" + "unicode" - "github.com/blues/jsonata-go/jlib/jxpath" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib/jxpath" + "github.com/xiatechs/jsonata-go/jtypes" ) // 2006-01-02T15:04:05.000Z07:00 const defaultFormatTimeLayout = "[Y]-[M01]-[D01]T[H01]:[m]:[s].[f001][Z01:01t]" +const ( + amSuffix = "am" + pmSuffix = "pm" + MST = "07:00" +) + var defaultParseTimeLayouts = []string{ "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z01:01t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z0100t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s]", + "[Y0001]-[M01]-[D01]", "[Y]-[M01]-[D01]", + "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]", + "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]", + "[H01]", "[Y]", } // FromMillis (golint) func FromMillis(ms int64, picture jtypes.OptionalString, tz jtypes.OptionalString) (string, error) { - t := msToTime(ms).UTC() if tz.String != "" { @@ -94,27 +105,43 @@ func parseTimeZone(tz string) (*time.Location, error) { // ToMillis (golint) func ToMillis(s string, picture jtypes.OptionalString, tz jtypes.OptionalString) (int64, error) { + var err error + var t time.Time + layouts := defaultParseTimeLayouts if picture.String != "" { layouts = []string{picture.String} } // TODO: How are timezones used for parsing? - for _, l := range layouts { - if t, err := parseTime(s, l); err == nil { + if t, err = parseTime(s, l); err == nil { return timeToMS(t), nil } } - return 0, fmt.Errorf("could not parse time %q", s) + return 0, err } var reMinus7 = regexp.MustCompile("-(0*7)") +const ( + timeDateOnly = "2006-01-02" + timeDateTime = "2006-01-02 15:04:05" +) + func parseTime(s string, picture string) (time.Time, error) { // Go's reference time: Mon Jan 2 15:04:05 MST 2006 - refTime := time.Date(2006, time.January, 2, 15, 4, 5, 0, time.FixedZone("MST", -7*60*60)) + refTime := time.Date( + 2006, + time.January, + 2, + 15, + 4, + 5, + 0, + time.FixedZone("MST", -7*60*60), + ) layout, err := jxpath.FormatTime(refTime, picture) if err != nil { @@ -124,14 +151,76 @@ func parseTime(s string, picture string) (time.Time, error) { // Replace -07:00 with Z07:00 layout = reMinus7.ReplaceAllString(layout, "Z$1") - t, err := time.Parse(layout, s) + var formattedTime = s + switch layout { + case timeDateOnly: + if len(formattedTime) > len(timeDateOnly) { + formattedTime = formattedTime[:len(timeDateOnly)] + } + case time.RFC3339: + // If the layout contains a time zone but the date string doesn't, lets remove it. + // Otherwise, if the layout contains a timezone and the time string doesn't add a default + // The default is currently MST which is GMT -7. + if !strings.Contains(formattedTime, "Z") { + layout = layout[:len(timeDateTime)] + } else { + formattedTimeWithTimeZone := strings.Split(formattedTime, "Z") + if len(formattedTimeWithTimeZone) == 2 { + formattedTime += MST + } + } + } + + // Occasionally date time strings contain a T in the string and the layout doesn't, if that's the + // case, lets remove it. + if strings.Contains(formattedTime, "T") && !strings.Contains(layout, "T") { + formattedTime = strings.ReplaceAll(formattedTime, "T", "") + } else if !strings.Contains(formattedTime, "T") && strings.Contains(layout, "T") { + layout = strings.ReplaceAll(layout, "T", "") + } + + sanitisedLayout := strings.ToLower(stripSpaces(layout)) + sanitisedDateTime := strings.ToLower(stripSpaces(formattedTime)) + + sanitisedLayout = addSuffixIfNotExists(sanitisedLayout, sanitisedDateTime) + sanitisedDateTime = addSuffixIfNotExists(sanitisedDateTime, sanitisedLayout) + + t, err := time.Parse(sanitisedLayout, sanitisedDateTime) if err != nil { - return time.Time{}, fmt.Errorf("could not parse time %q", s) + return time.Time{}, fmt.Errorf( + "could not parse time %q due to inconsistency in layout and date time string, date %s layout %s", + s, + sanitisedDateTime, + sanitisedLayout, + ) } return t, nil } +// It isn't consistent that both the date time string and format have a PM/AM suffix. If we find the suffix +// on one of the strings, add it to the other. Sometimes we can have conflicting suffixes for example the layout +// is always in PM 2006-01-0215:04:05pm but the actual date time string could be AM 2023-01-3110:44:59am. +// If this is the case, just ignore it as the time will parse correctly. +func addSuffixIfNotExists(s string, target string) string { + if strings.HasSuffix(target, amSuffix) && !strings.HasSuffix(s, amSuffix) && !strings.HasSuffix(s, pmSuffix) { + return s + amSuffix + } + + return s +} + +func stripSpaces(str string) string { + return strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + // if the character is a space, drop it + return -1 + } + // else keep it in the string + return r + }, str) +} + func msToTime(ms int64) time.Time { return time.Unix(ms/1000, (ms%1000)*int64(time.Millisecond)) } diff --git a/jlib/date_test.go b/jlib/date_test.go index 7c42c62..f956dab 100644 --- a/jlib/date_test.go +++ b/jlib/date_test.go @@ -9,12 +9,11 @@ import ( "testing" "time" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) func TestFromMillis(t *testing.T) { - date := time.Date(2018, time.September, 30, 15, 58, 5, int(762*time.Millisecond), time.UTC) input := date.UnixNano() / int64(time.Millisecond) @@ -28,10 +27,10 @@ func TestFromMillis(t *testing.T) { Picture: "[Y0001]-[M01]-[D01]", Output: "2018-09-30", }, - /*{ + { Picture: "[[[Y0001]-[M01]-[D01]]]", Output: "[2018-09-30]", - },*/ + }, { Picture: "[M]-[D]-[Y]", Output: "9-30-2018", @@ -117,3 +116,102 @@ func TestFromMillis(t *testing.T) { } } } + +func TestToMillis(t *testing.T) { + var picture jtypes.OptionalString + var tz jtypes.OptionalString + + t.Run("2023-01-31T10:44:59.800 is truncated to [Y0001]-[M01]-[D01]", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("2023-01-31T10:44:59.800 can be parsed", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("Whitespace is trimmed to ensure layout and time string match", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-3110:44:59", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("Milliseconds are ignored from the date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01][H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-3110:44:59.100", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("T is removed from date time string if it doesn't appear in the layout", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("T is removed from layout string if it doesn't appear in the date time", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59.800", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("No picture is passed to the to millis function", func(t *testing.T) { + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:47:06.260", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("Picture contains timezone (using RFC3339 format) but no timezone provided in date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01][Z]")) + _, err := jlib.ToMillis("2023-01-31T10:47:06.260", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("[P] placeholder within date format & date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]")) + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59 AM", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("AM present on date time string but not in the layout", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59 AM", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) +} diff --git a/jlib/fold.go b/jlib/fold.go new file mode 100644 index 0000000..112824c --- /dev/null +++ b/jlib/fold.go @@ -0,0 +1,19 @@ +package jlib + +import "errors" + +func FoldArray(input interface{}) ([][]interface{}, error) { + inputSlice, ok := input.([]interface{}) + if !ok { + return nil, errors.New("input for $foldarray was not an []interface type") + } + + result := make([][]interface{}, len(inputSlice)) + + for i := range inputSlice { + result[i] = make([]interface{}, i+1) + copy(result[i], inputSlice[:i+1]) + } + + return result, nil +} diff --git a/jlib/fold_test.go b/jlib/fold_test.go new file mode 100644 index 0000000..ab236c5 --- /dev/null +++ b/jlib/fold_test.go @@ -0,0 +1,40 @@ +package jlib + +import ( + "github.com/goccy/go-json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFold(t *testing.T) { + t.Run("fold a []interface", func(t *testing.T) { + input := []interface{}{ + map[string]interface{}{"Amount": 8}, + map[string]interface{}{"Amount": -1}, + map[string]interface{}{"Amount": 0}, + map[string]interface{}{"Amount": -3}, + map[string]interface{}{"Amount": -4}, + } + + output, err := FoldArray(input) + assert.NoError(t, err) + + outputJSON, err := json.Marshal(output) + assert.NoError(t, err) + + assert.Equal(t, string(outputJSON), `[[{"Amount":8}],[{"Amount":8},{"Amount":-1}],[{"Amount":8},{"Amount":-1},{"Amount":0}],[{"Amount":8},{"Amount":-1},{"Amount":0},{"Amount":-3}],[{"Amount":8},{"Amount":-1},{"Amount":0},{"Amount":-3},{"Amount":-4}]]`) + }) + + t.Run("fold - not an []interface{}", func(t *testing.T) { + input := "testing" + + output, err := FoldArray(input) + assert.Error(t, err) + + outputJSON, err := json.Marshal(output) + assert.NoError(t, err) + + assert.Equal(t, string(outputJSON), `null`) + }) +} diff --git a/jlib/hash.go b/jlib/hash.go new file mode 100644 index 0000000..712dcdb --- /dev/null +++ b/jlib/hash.go @@ -0,0 +1,31 @@ +package jlib + +import ( + "crypto/md5" + "crypto/sha256" + "encoding/hex" +) + +// Hash a input string into a md5 string +// for deduplication purposes +func HashMD5(input string) string { + hash := md5.Sum([]byte(input)) + + hashedString := hex.EncodeToString(hash[:]) + + return hashedString +} + +// Hash a input string into a sha256 string +// for deduplication purposes +func Hash256(input string) string { + hasher := sha256.New() + + hasher.Write([]byte(input)) + + hashedBytes := hasher.Sum(nil) + + hashedString := hex.EncodeToString(hashedBytes) + + return hashedString +} diff --git a/jlib/hof.go b/jlib/hof.go index 6452a3b..b7b07c4 100644 --- a/jlib/hof.go +++ b/jlib/hof.go @@ -8,7 +8,7 @@ import ( "fmt" "reflect" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // Map (golint) diff --git a/jlib/jlib.go b/jlib/jlib.go index 644a044..9147665 100644 --- a/jlib/jlib.go +++ b/jlib/jlib.go @@ -11,7 +11,7 @@ import ( "reflect" "time" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) func init() { @@ -57,7 +57,17 @@ func (s StringCallable) toInterface() interface{} { // TypeOf implements the jsonata $type function that returns the data type of // the argument func TypeOf(x interface{}) (string, error) { + if fmt.Sprintf("%v", x) == "" { + return "null", nil + } + v := reflect.ValueOf(x) + + switch x.(type) { + case *interface{}: + return "null", nil + } + if jtypes.IsCallable(v) { return "function", nil } @@ -77,11 +87,7 @@ func TypeOf(x interface{}) (string, error) { return "object", nil } - switch x.(type) { - case *interface{}: - return "null", nil - } - xType := reflect.TypeOf(x).String() - return "", fmt.Errorf("unknown type %s", xType) + + return xType, nil } diff --git a/jlib/join/join.go b/jlib/join/join.go new file mode 100644 index 0000000..ad1f048 --- /dev/null +++ b/jlib/join/join.go @@ -0,0 +1,255 @@ +package join + +import ( + "errors" + "fmt" + "reflect" +) + +// OneToManyJoin performs a join operation between two slices of maps/structs based on specified keys. +// It supports different types of joins: left, right, inner, and full. +func OneToManyJoin(leftArr, rightArr interface{}, leftKey, rightKey, rightArrayName, joinType string) (interface{}, error) { + // Convert input to slices of interfaces + trueLeftArr, ok := leftArr.([]interface{}) + if !ok { + return nil, errors.New("left input must be an array of Objects") + } + + trueRightArr, ok := rightArr.([]interface{}) + if !ok { + return nil, errors.New("right input must be an array of Objects") + } + + // Maps for tracking processed items + alreadyProcessed := make(map[string]bool) + rightProcessed := make(map[string]bool) + + // Create a map for faster lookup of rightArr elements based on the key + rightMap := make(map[string][]interface{}) + for _, item := range trueRightArr { + itemMap, ok := item.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + rightMap[strVal] = append(rightMap[strVal], item) + } + } + } + + // Slice to store the merged results + var result []map[string]interface{} + leftMatched := make(map[string]interface{}) + + // Iterate through the left array and perform the join + for _, leftItem := range trueLeftArr { + itemMap, ok := leftItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[leftKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the right items to join + rightItems := rightMap[strVal] + + // Perform the join based on the join type + if joinType == "left" || joinType == "full" || (joinType == "inner" && len(rightItems) > 0) { + mergedItem := mergeItems(leftItem, rightItems, rightArrayName) + result = append(result, mergedItem) + } + + // Mark items as processed + leftMatched[strVal] = leftItem + alreadyProcessed[strVal] = true + } + } + } + + // Add items from the right array for right or full join + if joinType == "right" || joinType == "full" { + for _, rightItem := range trueRightArr { + itemMap, ok := rightItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the left item to merge with + var leftItemToMerge interface{} + if leftMatch, ok := leftMatched[strVal]; ok { + leftItemToMerge = leftMatch + } else { + leftItemToMerge = map[string]interface{}{rightKey: itemKey} + } + + // Handle right and full join separately to avoid duplication + if joinType == "right" && !rightProcessed[strVal] { + mergedItem := mergeItems(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } else if joinType == "full" && !rightProcessed[strVal] && !alreadyProcessed[strVal] { + mergedItem := mergeItems(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } + } + } + } + } + + return result, nil +} + +func mergeItems(leftItem interface{}, rightItems []interface{}, rightArrayName string) map[string]interface{} { + mergedItem := make(map[string]interface{}) + + // Check if leftItem is a map or a struct and merge accordingly + leftVal := reflect.ValueOf(leftItem) + if leftVal.Kind() == reflect.Map { + // Merge fields from the map + for _, key := range leftVal.MapKeys() { + mergedItem[key.String()] = leftVal.MapIndex(key).Interface() + } + } else { + // Merge fields from the struct + leftType := leftVal.Type() + for i := 0; i < leftVal.NumField(); i++ { + fieldName := leftType.Field(i).Name + fieldValue := leftVal.Field(i).Interface() + mergedItem[fieldName] = fieldValue + } + } + + // If there are matching items in the right array, add them under the specified name + if len(rightItems) > 0 { + mergedItem[rightArrayName] = rightItems + } + + return mergedItem +} + +// OneToManyJoin2 performs a join operation between two slices of maps/structs based on specified keys. +// It supports different types of joins: left, right, inner, and full. +func OneToManyJoin2(leftArr, rightArr interface{}, leftKey, rightKey, rightArrayName, joinType string) (interface{}, error) { + // Convert input to slices of interfaces + trueLeftArr, ok := leftArr.([]interface{}) + if !ok { + return nil, errors.New("left input must be an array of Objects") + } + + trueRightArr, ok := rightArr.([]interface{}) + if !ok { + return nil, errors.New("right input must be an array of Objects") + } + + // Maps for tracking processed items + alreadyProcessed := make(map[string]bool) + rightProcessed := make(map[string]bool) + + // Create a map for faster lookup of rightArr elements based on the key + rightMap := make(map[string][]interface{}) + for _, item := range trueRightArr { + itemMap, ok := item.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + rightMap[strVal] = append(rightMap[strVal], item) + } + } + } + + // Slice to store the merged results + var result []map[string]interface{} + leftMatched := make(map[string]interface{}) + + // Iterate through the left array and perform the join + for _, leftItem := range trueLeftArr { + itemMap, ok := leftItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[leftKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the right items to join + rightItems := rightMap[strVal] + + // Perform the join based on the join type + if joinType == "left" || joinType == "full" || (joinType == "inner" && len(rightItems) > 0) { + mergedItem := mergeItemsNew(leftItem, rightItems, rightArrayName) + result = append(result, mergedItem) + } + + // Mark items as processed + leftMatched[strVal] = leftItem + alreadyProcessed[strVal] = true + } + } + } + + // Add items from the right array for right or full join + if joinType == "right" || joinType == "full" { + for _, rightItem := range trueRightArr { + itemMap, ok := rightItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the left item to merge with + var leftItemToMerge interface{} + if leftMatch, ok := leftMatched[strVal]; ok { + leftItemToMerge = leftMatch + } else { + leftItemToMerge = map[string]interface{}{rightKey: itemKey} + } + + // Handle right and full join separately to avoid duplication + if joinType == "right" && !rightProcessed[strVal] { + mergedItem := mergeItemsNew(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } else if joinType == "full" && !rightProcessed[strVal] && !alreadyProcessed[strVal] { + mergedItem := mergeItemsNew(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } + } + } + } + } + + return result, nil +} + +// use reflect sparingly to avoid performance issues +func mergeStruct(item interface{}) map[string]interface{} { + mergedItem := make(map[string]interface{}) + val := reflect.ValueOf(item) + + for i := 0; i < val.NumField(); i++ { + fieldName := val.Type().Field(i).Name + fieldValue := val.Field(i).Interface() + mergedItem[fieldName] = fieldValue + } + + return mergedItem +} + +func mergeItemsNew(leftItem interface{}, rightItems []interface{}, rightArrayName string) map[string]interface{} { + mergedItem := make(map[string]interface{}) + + switch left := leftItem.(type) { + case map[string]interface{}: + for key, value := range left { + mergedItem[key] = value + } + case nil: + // skip + default: + structFields := mergeStruct(left) + for key, value := range structFields { + mergedItem[key] = value + } + } + + if len(rightItems) > 0 { + mergedItem[rightArrayName] = rightItems + } + + return mergedItem +} diff --git a/jlib/join/join_bench_test.go b/jlib/join/join_bench_test.go new file mode 100644 index 0000000..fbb36f1 --- /dev/null +++ b/jlib/join/join_bench_test.go @@ -0,0 +1,273 @@ +package join + +import ( + "github.com/goccy/go-json" + "log" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func BenchmarkOldJoin(t *testing.B) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.B) { + for i := 0; i < t.N; i++ { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + } + }) + } + + log.Println("done") + time.Sleep(2 * time.Second) +} + +func BenchmarkNewJoin(t *testing.B) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.B) { + for i := 0; i < t.N; i++ { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin2(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + } + }) + } +} diff --git a/jlib/join/join_test.go b/jlib/join/join_test.go new file mode 100644 index 0000000..6a9a953 --- /dev/null +++ b/jlib/join/join_test.go @@ -0,0 +1,265 @@ +package join + +import ( + "github.com/goccy/go-json" + "log" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOneToManyJoin(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} + +func TestOneToManyJoinNew(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin2(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} diff --git a/jlib/jxpath/formatdate_test.go b/jlib/jxpath/formatdate_test.go index 0ceaa9d..c673266 100644 --- a/jlib/jxpath/formatdate_test.go +++ b/jlib/jxpath/formatdate_test.go @@ -11,7 +11,6 @@ import ( ) func TestFormatYear(t *testing.T) { - input := time.Date(2018, time.April, 1, 12, 0, 0, 0, time.UTC) data := []struct { @@ -24,6 +23,10 @@ func TestFormatYear(t *testing.T) { Picture: "[Y]", Output: "2018", }, + { + Picture: "[Y0001] [M01] [D01]", + Output: "2018 04 01", + }, { Picture: "[Y1]", Output: "2018", @@ -81,8 +84,22 @@ func TestFormatYear(t *testing.T) { } } -func TestFormatTimezone(t *testing.T) { +func TestFormatYearAndTimezone(t *testing.T) { + location, _ := time.LoadLocation("Europe/Rome") + input := time.Date(2018, time.April, 1, 12, 0, 0, 0, location) + + picture := "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]" + got, err := FormatTime(input, picture) + if err != nil { + t.Errorf("unable to format time %+v", err) + } + if got != "2018-04-01 12:00:00 pm" { + t.Errorf("got %s expected %s", got, "2018-04-01 12:00:00 pm") + } +} + +func TestFormatTimezone(t *testing.T) { const minutes = 60 const hours = 60 * minutes diff --git a/jlib/new.go b/jlib/new.go new file mode 100644 index 0000000..e38b200 --- /dev/null +++ b/jlib/new.go @@ -0,0 +1,426 @@ +package jlib + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/goccy/go-json" +) + +// Unescape an escaped json string into JSON (once) +func Unescape(input string) (interface{}, error) { + var output interface{} + + err := json.Unmarshal([]byte(input), &output) + if err != nil { + return output, fmt.Errorf("unescape json unmarshal error: %v", err) + } + + return output, nil +} + +func getVal(input interface{}) string { + return fmt.Sprintf("%v", input) +} + +const ( + arrDelimiter = "|" + keyDelimiter = "¬" +) + +// SimpleJoin - a multi-key multi-level full OR join - very simple and useful in certain circumstances +func SimpleJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { + if !(v.IsValid() && v.CanInterface() && v2.IsValid() && v2.CanInterface()) { + return nil, nil + } + + i1, ok := v.Interface().([]interface{}) + if !ok { + return nil, fmt.Errorf("both objects must be slice of objects") + } + + i2, ok := v2.Interface().([]interface{}) + if !ok { + return nil, fmt.Errorf("both objects must be slice of objects") + } + + field1Arr := strings.Split(field1, arrDelimiter) // todo: only works as an OR atm + + field2Arr := strings.Split(field2, arrDelimiter) + + if len(field1Arr) != len(field2Arr) { + return nil, fmt.Errorf("field arrays must be same length") + } + + relationMap := make(map[string]*relation) + + for index := range field1Arr { + addItems(relationMap, i1, i2, field1Arr[index], field2Arr[index]) + } + + output := make([]interface{}, 0) + + for index := range relationMap { + output = append(output, relationMap[index].generateItem()) + } + + return output, nil +} + +type relation struct { + object map[string]interface{} + related []interface{} +} + +func newRelation(input map[string]interface{}) *relation { + return &relation{ + object: input, + related: make([]interface{}, 0), + } +} + +func (r *relation) generateItem() map[string]interface{} { + newitem := make(map[string]interface{}) + + for key := range r.object { + newitem[key] = r.object[key] + + for index := range r.related { + if val, ok := r.related[index].(map[string]interface{}); ok { + for key := range val { + newitem[key] = val[key] + } + } + } + + } + + return newitem +} + +func addItems(relationMap map[string]*relation, i1, i2 []interface{}, field1, field2 string) { + for a := range i1 { + item1, ok := i1[a].(map[string]interface{}) + if !ok { + continue + } + + key := fmt.Sprintf("%v", item1) + + if _, ok := relationMap[key]; !ok { + relationMap[key] = newRelation(item1) + } + + rel := relationMap[key] + + f1 := getMapStringValue(strings.Split(field1, keyDelimiter), 0, item1) + if f1 == nil { + continue + } + + for b := range i2 { + f2 := getMapStringValue(strings.Split(field2, keyDelimiter), 0, i2[b]) + if f2 == nil { + continue + } + + if f1 == f2 { + rel.related = append(rel.related, i2[b]) + } + } + + relationMap[key] = rel + } +} + +func outsideRange(fieldArr []string, index int) bool { + return index > len(fieldArr)-1 +} + +func getMapStringValue(fieldArr []string, index int, item interface{}) interface{} { + if outsideRange(fieldArr, index) { + return nil + } + + if obj, ok := item.(map[string]interface{}); ok { + for key := range obj { + if key == fieldArr[index] { + if len(fieldArr)-1 == index { + return obj[key] + } else { + index++ + new := getMapStringValue(fieldArr, index, obj[key]) + if new != nil { + return new + } + } + } + } + } + + return getArrayValue(fieldArr, index, item) +} + +func getArrayValue(fieldArr []string, index int, item interface{}) interface{} { + if outsideRange(fieldArr, index) { + return nil + } + + if obj, ok := item.([]interface{}); ok { + for value := range obj { + a := fmt.Sprintf("%v", fieldArr[index]) + b := fmt.Sprintf("%v", obj[value]) + if a == b { + if len(fieldArr)-1 == index { + return item + } else { + index++ + new := getMapStringValue(fieldArr, index, obj) + if new != nil { + return new + } + } + } + } + } + + return getSingleValue(fieldArr, index, item) +} + +func getSingleValue(fieldArr []string, index int, item interface{}) interface{} { + if outsideRange(fieldArr, index) { + return nil + } + + a := fmt.Sprintf("%v", fieldArr[index]) + b := fmt.Sprintf("%v", item) + if a == b { + if len(fieldArr)-1 == index { + return item + } else { + index++ + new := getMapStringValue(fieldArr, index, item) + if new != nil { + return new + } + } + } + + return nil +} + +// ObjMerge - merge two map[string]interface{} objects together - if they have unique keys +func ObjMerge(i1, i2 interface{}) interface{} { + output := make(map[string]interface{}) + + merge1, ok1 := i1.(map[string]interface{}) + merge2, ok2 := i2.(map[string]interface{}) + if !ok1 || !ok2 { + return output + } + + for key := range merge1 { + output[key] = merge1[key] + } + + for key := range merge2 { + output[key] = merge2[key] + } + + return output +} + +// setValue handles setting values in a nested structure including array indices +func setValue(obj map[string]interface{}, path string, value interface{}) { + parts := strings.Split(path, ".") + current := obj + + for i := 0; i < len(parts)-1; i++ { + part := parts[i] + + // Check if this part contains an array index + arrayIndex := -1 + if idx := strings.Index(part, "["); idx != -1 { + // Extract the array index + if end := strings.Index(part, "]"); end != -1 { + indexStr := part[idx+1 : end] + if index, err := strconv.Atoi(indexStr); err == nil { + arrayIndex = index + part = part[:idx] // Remove the array notation from the part + } + } + } + + // Handle array index if present + if arrayIndex != -1 { + // Ensure the current part exists and is an array + arr, exists := current[part].([]interface{}) + if !exists { + arr = make([]interface{}, 0) + current[part] = arr + } + + // Extend array if needed + for len(arr) <= arrayIndex { + arr = append(arr, make(map[string]interface{})) + } + current[part] = arr // Important: update the array in the map + + // Get or create map at array index + if arr[arrayIndex] == nil { + arr[arrayIndex] = make(map[string]interface{}) + } + + current = arr[arrayIndex].(map[string]interface{}) + } else { + // Normal object property + next, exists := current[part].(map[string]interface{}) + if !exists { + next = make(map[string]interface{}) + current[part] = next + } + current = next + } + } + + // Handle the final part + lastPart := parts[len(parts)-1] + if idx := strings.Index(lastPart, "["); idx != -1 { + // Handle array index in the final part + if end := strings.Index(lastPart, "]"); end != -1 { + indexStr := lastPart[idx+1 : end] + if index, err := strconv.Atoi(indexStr); err == nil { + part := lastPart[:idx] + arr, exists := current[part].([]interface{}) + if !exists { + arr = make([]interface{}, 0) + } + // Extend array if needed + for len(arr) <= index { + arr = append(arr, nil) + } + arr[index] = value + current[part] = arr // Important: update the array in the map + return + } + } + } + // Set value for non-array final part + current[lastPart] = value +} + +// objectsToDocument converts an array of Items to a nested map according to the Code paths. +func ObjectsToDocument(input interface{}) (interface{}, error) { + trueInput, ok := input.([]interface{}) + if !ok { + return nil, errors.New("$objectsToDocument input must be an array of objects") + } + + output := make(map[string]interface{}) + for _, itemToInterface := range trueInput { + item, ok := itemToInterface.(map[string]interface{}) + if !ok { + return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Val/Value fields") + } + + code, ok := item["Code"].(string) + if code == "" || !ok { + return nil, errors.New("$objectsToDocument input must contain a 'Code' field that is non-empty string") + } + + var value interface{} + if val, exists := item["Val"]; exists && val != nil { + // Use Val only if it's not nil + value = val + } else if val, exists := item["Value"]; exists && val != nil { + // Use Value if Val doesn't exist or was nil + value = val + } + + if value != nil { + setValue(output, code, value) + } + } + + return output, nil +} + +// TransformRule defines a transformation rule with a search substring and a new name. +type TransformRule struct { + SearchSubstring string + NewName string +} + +// RenameKeys applies a series of transformations to the keys in a JSON-compatible data structure. +// 'data' is the original data where keys need to be transformed. +// 'rulesInterface' is expected to be a slice of interface{}, where each element is a slice containing two strings: +// the substring to search for in the keys, and the new name to replace the key with. +func RenameKeys(data interface{}, rulesInterface interface{}) (interface{}, error) { + // Attempt to assert rulesInterface as a slice of interface{} + rulesRaw, ok := rulesInterface.([]interface{}) + if !ok { + return nil, fmt.Errorf("rules must be a slice of interface{}") + } + + // Process each rule, converting it into a TransformRule + var rules []TransformRule + for _, r := range rulesRaw { + rule, ok := r.([]interface{}) + if !ok || len(rule) != 2 { + return nil, fmt.Errorf("each rule must be an array of two strings") + } + + searchSubstring, ok1 := rule[0].(string) + newName, ok2 := rule[1].(string) + if !ok1 || !ok2 { + return nil, fmt.Errorf("each rule must be an array of two strings") + } + + rules = append(rules, TransformRule{SearchSubstring: searchSubstring, NewName: newName}) + } + + // Marshal the original data into JSON + jsonData, err := json.Marshal(data) + if err != nil { + return nil, err + } + + // Unmarshal the JSON into a map for easy manipulation + var mapData map[string]interface{} + err = json.Unmarshal(jsonData, &mapData) + if err != nil { + return nil, err + } + + // Create a new map to store the modified data + newData := make(map[string]interface{}) + for key, value := range mapData { + newKey := key // Default to the original key + // Apply transformation rules + for _, rule := range rules { + if strings.Contains(key, rule.SearchSubstring) { + newKey = rule.NewName // Update the key if rule matches + break + } + } + newData[newKey] = value // Store the value with the new key + } + + // Re-marshal to JSON to maintain the same data type as the input + resultJSON, err := json.Marshal(newData) + if err != nil { + return nil, err + } + + // Unmarshal the JSON back into an interface{} for the return value + var result interface{} + err = json.Unmarshal(resultJSON, &result) + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/jlib/new_test.go b/jlib/new_test.go new file mode 100644 index 0000000..8eee88b --- /dev/null +++ b/jlib/new_test.go @@ -0,0 +1,284 @@ +package jlib + +import ( + "reflect" + "testing" + + "github.com/goccy/go-json" + + "github.com/stretchr/testify/assert" +) + +func TestSJoin(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + expectedOutput string + }{ + { + description: "simple join", + object1: `[{"test": { + "id": 1, + "age": 5 + }}]`, + object2: `[{"test": { + "id": 1, + "name": "Tim" + }}]`, + joinStr1: "id", + joinStr2: "id", + expectedOutput: "[{\"test\":{\"age\":5,\"id\":1}}]", + }, + { + description: "nested join", + object1: `[ + { + "age": 5, + "id": 1 + } + ]`, + object2: `[ + { + "test": { + "id": 1, + "name": "Tim" + } + } + ]`, + joinStr1: "id", + joinStr2: "test¬id", + expectedOutput: "[{\"age\":5,\"id\":1,\"test\":{\"id\":1,\"name\":\"Tim\"}}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + i1 := reflect.ValueOf(o1) + i2 := reflect.ValueOf(o2) + + output, err := SimpleJoin(i1, i2, tt.joinStr1, tt.joinStr2) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} + +func TestRenameKeys(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + expectedOutput string + }{ + { + description: "Rename Keys", + object1: `{ + "itemLineId": "1", + "unitPrice": 104.5, + "percentageDiscountValue": 5, + "discountedLinePrice": 104.5, + "name": "LD Wrong Price", + "discountAmount": 5.5, + "discountType": "AMOUNT", + "discountReasonCode": "9901", + "discountReasonName": "LD Wrong Price" + }`, + object2: `[["ReasonCode","reasonCode"],["ReasonName","reasonName"],["DiscountValue","value"],["Amount","amount"],["Type","type"],["LinePrice","linePrice"]]`, + expectedOutput: "{\"amount\":5.5,\"itemLineId\":\"1\",\"linePrice\":104.5,\"name\":\"LD Wrong Price\",\"reasonCode\":\"9901\",\"reasonName\":\"LD Wrong Price\",\"type\":\"AMOUNT\",\"unitPrice\":104.5,\"value\":5}", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := RenameKeys(o1, o2) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} + +func TestSetValue_ArrayIndexing(t *testing.T) { + tests := []struct { + description string + input map[string]interface{} + code string + value interface{} + expectedOutput map[string]interface{} + }{ + { + description: "Set simple value at array index", + input: map[string]interface{}{}, + code: "employees[0].name", + value: "Alice", + expectedOutput: map[string]interface{}{ + "employees": []interface{}{ + map[string]interface{}{"name": "Alice"}, + }, + }, + }, + { + description: "Extend array to meet required index", + input: map[string]interface{}{}, + code: "employees[2].name", + value: "Charlie", + expectedOutput: map[string]interface{}{ + "employees": []interface{}{ + map[string]interface{}{}, + map[string]interface{}{}, + map[string]interface{}{"name": "Charlie"}, + }, + }, + }, + { + description: "Nested arrays and objects", + input: map[string]interface{}{}, + code: "company.departments[1].staff[0].role", + value: "Manager", + expectedOutput: map[string]interface{}{ + "company": map[string]interface{}{ + "departments": []interface{}{ + map[string]interface{}{}, + map[string]interface{}{ + "staff": []interface{}{ + map[string]interface{}{"role": "Manager"}, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.description, func(t *testing.T) { + setValue(tt.input, tt.code, tt.value) + assert.Equal(t, tt.expectedOutput, tt.input) + }) + } +} + +func TestObjectsToDocument_ValPriority(t *testing.T) { + tests := []struct { + description string + input string + expectedOutput string + }{ + { + description: "Use Val when present", + input: `[ + {"Code":"person.name","Val":"Alice","Value":"ShouldNotUse"}, + {"Code":"person.age","Val":30} + ]`, + expectedOutput: `{"person":{"age":30,"name":"Alice"}}`, + }, + { + description: "Use Value when Val not present", + input: `[ + {"Code":"person.name","Value":"Bob"}, + {"Code":"person.age","Val":25} + ]`, + // "person.name" should come from Value since Val is not present + expectedOutput: `{"person":{"age":25,"name":"Bob"}}`, + }, + { + description: "Array indexing in Code with Val", + input: `[ + {"Code":"people[0].name","Val":"Carol"}, + {"Code":"people[1].name","Value":"Dave"} + ]`, + expectedOutput: `{"people":[{"name":"Carol"},{"name":"Dave"}]}`, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.description, func(t *testing.T) { + var inputData interface{} + err := json.Unmarshal([]byte(tt.input), &inputData) + assert.NoError(t, err) + + output, err := ObjectsToDocument(inputData) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} + +func TestObjectsToDocument_ComplexArrayPaths(t *testing.T) { + tests := []struct { + description string + input string + expectedOutput string + }{ + { + description: "Set multiple nested array values", + input: `[ + {"Code":"root.list[0].items[0].value","Val":"Item0-0"}, + {"Code":"root.list[0].items[1].value","Value":"Item0-1"}, + {"Code":"root.list[1].items[0].value","Val":"Item1-0"} + ]`, + expectedOutput: `{"root":{"list":[{"items":[{"value":"Item0-0"},{"value":"Item0-1"}]},{"items":[{"value":"Item1-0"}]}]}}`, + }, + { + description: "No Val or Value", + input: `[ + {"Code":"topKey","Whatever":"none"}, + {"Code":"anotherKey","Val":null} + ]`, + // No Val or Value means keys are not set + expectedOutput: `{}`, + }, + { + description: "Val is nil, use Value", + input: `[ + {"Code":"testKey","Val":null,"Value":"RealValue"} + ]`, + expectedOutput: `{"testKey":"RealValue"}`, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.description, func(t *testing.T) { + var inputData interface{} + err := json.Unmarshal([]byte(tt.input), &inputData) + assert.NoError(t, err) + + output, err := ObjectsToDocument(inputData) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} diff --git a/jlib/number.go b/jlib/number.go index 8cb0e46..ebc9282 100644 --- a/jlib/number.go +++ b/jlib/number.go @@ -13,7 +13,7 @@ import ( "strconv" "strings" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) var reNumber = regexp.MustCompile(`^-?(([0-9]+))(\.[0-9]+)?([Ee][-+]?[0-9]+)?$`) @@ -116,7 +116,7 @@ func Random() float64 { // It does this by converting back and forth to strings to // avoid floating point rounding errors, e.g. // -// 4.525 * math.Pow10(2) returns 452.50000000000006 +// 4.525 * math.Pow10(2) returns 452.50000000000006 func multByPow10(x float64, n int) float64 { if n == 0 || math.IsNaN(x) || math.IsInf(x, 0) { return x diff --git a/jlib/number_test.go b/jlib/number_test.go index 51b422f..478855f 100644 --- a/jlib/number_test.go +++ b/jlib/number_test.go @@ -8,8 +8,8 @@ import ( "fmt" "testing" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) func TestRound(t *testing.T) { diff --git a/jlib/object.go b/jlib/object.go index ca43f8d..36284ab 100644 --- a/jlib/object.go +++ b/jlib/object.go @@ -8,7 +8,7 @@ import ( "fmt" "reflect" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // typeInterfaceMap is the reflect.Type for map[string]interface{}. diff --git a/jlib/object_test.go b/jlib/object_test.go index e2e8dc7..3e97ece 100644 --- a/jlib/object_test.go +++ b/jlib/object_test.go @@ -12,8 +12,8 @@ import ( "strings" "testing" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) type eachTest struct { diff --git a/jlib/string.go b/jlib/string.go index cd49c6f..014acf8 100644 --- a/jlib/string.go +++ b/jlib/string.go @@ -7,8 +7,8 @@ package jlib import ( "bytes" "encoding/base64" - "encoding/json" "fmt" + "github.com/goccy/go-json" "math" "net/url" "reflect" @@ -17,8 +17,8 @@ import ( "strings" "unicode/utf8" - "github.com/blues/jsonata-go/jlib/jxpath" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib/jxpath" + "github.com/xiatechs/jsonata-go/jtypes" ) // String converts a JSONata value to a string. Values that are @@ -231,9 +231,9 @@ func Join(values reflect.Value, separator jtypes.OptionalString) (string, error) // regular expression in the source string. Each object in the // array has the following fields: // -// match - the substring matched by the regex -// index - the starting offset of this match -// groups - any captured groups for this match +// match - the substring matched by the regex +// index - the starting offset of this match +// groups - any captured groups for this match // // The optional third argument specifies the maximum number // of matches to return. By default, Match returns all matches. diff --git a/jlib/string_test.go b/jlib/string_test.go index 21a91c6..5fcb544 100644 --- a/jlib/string_test.go +++ b/jlib/string_test.go @@ -13,8 +13,8 @@ import ( "testing" "unicode/utf8" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) var typereplaceCallable = reflect.TypeOf((*replaceCallable)(nil)).Elem() diff --git a/jlib/timeparse/testdata.json b/jlib/timeparse/testdata.json new file mode 100644 index 0000000..f1327eb --- /dev/null +++ b/jlib/timeparse/testdata.json @@ -0,0 +1,422 @@ +[ + { + "testDesc": "test id 62", + "input_srcTs": "2023-08-05T23:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023217, + "DateId": "Dates_20230805", + "DateKey": 20230805, + "DateTimeKey": 20230805232341454, + "HourId": "Hours_2023080523", + "HourKey": 2023080523, + "Millis": 1691274221454, + "RawValue": "2023-08-05T23:23:41.454Z", + "UTC": "2023-08-05T22:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 22, + "Local": "2023-08-05T23:23:41.454+01:00", + "DateLocal": "2023-08-05", + "HourLocal": 23 + } + }, + { + "testDesc": "test id 61", + "input_srcTs": "2023-08-05T23:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "UTC", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-05T23:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "test id 54", + "input_srcTs": "2024-10-26T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202410, + "YearWeek": 202443, + "YearIsoWeek": 202443, + "YearDay": 2024300, + "DateId": "Dates_20241026", + "DateKey": 20241026, + "DateTimeKey": 20241026113456000, + "HourId": "Hours_2024102611", + "HourKey": 2024102611, + "Millis": 1729942496000, + "RawValue": "2024-10-26T12:34:56Z", + "UTC": "2024-10-26T11:34:56.000Z", + "DateUTC": "2024-10-26", + "HourUTC": 11, + "Local": "2024-10-26T11:34:56.000+00:00", + "DateLocal": "2024-10-26", + "HourLocal": 11 + } + }, + { + "testDesc": "test id 53", + "input_srcTs": "2024-10-26T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202410, + "YearWeek": 202443, + "YearIsoWeek": 202443, + "YearDay": 2024300, + "DateId": "Dates_20241026", + "DateKey": 20241026, + "DateTimeKey": 20241026113456000, + "HourId": "Hours_2024102611", + "HourKey": 2024102611, + "Millis": 1729942496000, + "RawValue": "2024-10-26T12:34:56Z", + "UTC": "2024-10-26T11:34:56.000Z", + "DateUTC": "2024-10-26", + "HourUTC": 11, + "Local": "2024-10-26T11:34:56.000+00:00", + "DateLocal": "2024-10-26", + "HourLocal": 11 + } + }, + { + "testDesc": "test id 52", + "input_srcTs": "2024-03-30T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202403, + "YearWeek": 202413, + "YearIsoWeek": 202413, + "YearDay": 2024090, + "DateId": "Dates_20240330", + "DateKey": 20240330, + "DateTimeKey": 20240330123456000, + "HourId": "Hours_2024033012", + "HourKey": 2024033012, + "Millis": 1711802096000, + "RawValue": "2024-03-30T12:34:56Z", + "UTC": "2024-03-30T12:34:56.000Z", + "DateUTC": "2024-03-30", + "HourUTC": 12, + "Local": "2024-03-30T12:34:56.000+00:00", + "DateLocal": "2024-03-30", + "HourLocal": 12 + } + }, + { + "testDesc": "test id 51", + "input_srcTs": "2024-03-31T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202403, + "YearWeek": 202413, + "YearIsoWeek": 202413, + "YearDay": 2024091, + "DateId": "Dates_20240331", + "DateKey": 20240331, + "DateTimeKey": 20240331113456000, + "HourId": "Hours_2024033111", + "HourKey": 2024033111, + "Millis": 1711884896000, + "RawValue": "2024-03-31T12:34:56Z", + "UTC": "2024-03-31T11:34:56.000Z", + "DateUTC": "2024-03-31", + "HourUTC": 11, + "Local": "2024-03-31T11:34:56.000+00:00", + "DateLocal": "2024-03-31", + "HourLocal": 11 + } + }, + { + "testDesc": "/* isoweek 53 - yearweek 0 test */", + "input_srcTs": "2021-01-03T12:13:14Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202101, + "YearWeek": 202100, + "YearIsoWeek": 202053, + "YearDay": 2021003, + "DateId": "Dates_20210103", + "DateKey": 20210103, + "DateTimeKey": 20210103121314000, + "HourId": "Hours_2021010312", + "HourKey": 2021010312, + "Millis": 1609675994000, + "RawValue": "2021-01-03T12:13:14Z", + "UTC": "2021-01-03T12:13:14.000Z", + "DateUTC": "2021-01-03", + "HourUTC": 12, + "Local": "2021-01-03T12:13:14.000+00:00", + "DateLocal": "2021-01-03", + "HourLocal": 12 + } + }, + { + "testDesc": "/* isoweek 52 - yearweek 0 test */", + "input_srcTs": "2023-01-01T12:13:14Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202301, + "YearWeek": 202300, + "YearIsoWeek": 202252, + "YearDay": 2023001, + "DateId": "Dates_20230101", + "DateKey": 20230101, + "DateTimeKey": 20230101121314000, + "HourId": "Hours_2023010112", + "HourKey": 2023010112, + "Millis": 1672575194000, + "RawValue": "2023-01-01T12:13:14Z", + "UTC": "2023-01-01T12:13:14.000Z", + "DateUTC": "2023-01-01", + "HourUTC": 12, + "Local": "2023-01-01T12:13:14.000+00:00", + "DateLocal": "2023-01-01", + "HourLocal": 12 + } + }, + { + "testDesc": "/* source has explicit input UTC, output UTC - this will convert XF official formatting */", + "input_srcTs": "2023-08-06T00:23:41Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "UTC", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691281421000, + "RawValue": "2023-08-06T00:23:41Z", + "UTC": "2023-08-06T00:23:41.000Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T00:23:41.000+00:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source explicit UTC, Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "UTC", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806012341454, + "HourId": "Hours_2023080601", + "HourKey": 2023080601, + "Millis": 1691281421454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-06T00:23:41.454Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T01:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 1 + } + }, + { + "testDesc": "/* source explicit src TZ, UTC = 2023-08-05T23:23:41.454 (NOTE: Day before!), Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454+01:00", + "input_srcFormat": "2006-01-02T15:04:05.999999999Z07:00", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454+01:00", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source impliciti TZ, bad use of a 'Z' whici is meant to meant UTC - same output as above as equiv */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* utc to america/new_york */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "America/New_York", + "DateDim": { + "TimeZone": "America/New_York", + "TimeZoneOffset": "-04:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023217, + "DateId": "Dates_20230805", + "DateKey": 20230805, + "DateTimeKey": 20230805192341454, + "HourId": "Hours_2023080519", + "HourKey": 2023080519, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-05T19:23:41.454-04:00", + "DateLocal": "2023-08-05", + "HourLocal": 19 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z'*/", + "input_srcTs": "2023-08-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z' - january*/", + "input_srcTs": "2023-01-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202301, + "YearWeek": 202301, + "YearIsoWeek": 202301, + "YearDay": 2023006, + "DateId": "Dates_20230106", + "DateKey": 20230106, + "DateTimeKey": 20230106002341454, + "HourId": "Hours_2023010600", + "HourKey": 2023010600, + "Millis": 1672964621454, + "RawValue": "2023-01-06T00:23:41.454", + "UTC": "2023-01-06T00:23:41.454Z", + "DateUTC": "2023-01-06", + "HourUTC": 0, + "Local": "2023-01-06T00:23:41.454+00:00", + "DateLocal": "2023-01-06", + "HourLocal": 0 + } + } + ] \ No newline at end of file diff --git a/jlib/timeparse/testdata_lite.json b/jlib/timeparse/testdata_lite.json new file mode 100644 index 0000000..c239026 --- /dev/null +++ b/jlib/timeparse/testdata_lite.json @@ -0,0 +1,198 @@ +[ + { + "testDesc": "/* source has explicit input UTC, output UTC - this will convert XF official formatting */", + "input_srcTs": "2023-08-06T00:23:41Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "UTC", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691281421000, + "RawValue": "2023-08-06T00:23:41Z", + "UTC": "2023-08-06T00:23:41.000Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T00:23:41.000+00:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source explicit UTC, Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "UTC", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806012341454, + "HourId": "Hours_2023080601", + "HourKey": 2023080601, + "Millis": 1691281421454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-06T00:23:41.454Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T01:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 1 + } + }, + { + "testDesc": "/* source explicit src TZ, UTC = 2023-08-05T23:23:41.454 (NOTE: Day before!), Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454+01:00", + "input_srcFormat": "2006-01-02T15:04:05.999999999Z07:00", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454+01:00", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source impliciti TZ, bad use of a 'Z' whici is meant to meant UTC - same output as above as equiv */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* utc to america/new_york */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "America/New_York", + "DateDim": { + "TimeZone": "America/New_York", + "TimeZoneOffset": "-04:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023217, + "DateId": "Dates_20230805", + "DateKey": 20230805, + "DateTimeKey": 20230805192341454, + "HourId": "Hours_2023080519", + "HourKey": 2023080519, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-05T19:23:41.454-04:00", + "DateLocal": "2023-08-05", + "HourLocal": 19 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z'*/", + "input_srcTs": "2023-08-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z' - january*/", + "input_srcTs": "2023-01-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202301, + "YearWeek": 202301, + "YearIsoWeek": 202301, + "YearDay": 2023006, + "DateId": "Dates_20230106", + "DateKey": 20230106, + "DateTimeKey": 20230106002341454, + "HourId": "Hours_2023010600", + "HourKey": 2023010600, + "Millis": 1672964621454, + "RawValue": "2023-01-06T00:23:41.454", + "UTC": "2023-01-06T00:23:41.454Z", + "DateUTC": "2023-01-06", + "HourUTC": 0, + "Local": "2023-01-06T00:23:41.454+00:00", + "DateLocal": "2023-01-06", + "HourLocal": 0 + } + } + ] \ No newline at end of file diff --git a/jlib/timeparse/timeparse.go b/jlib/timeparse/timeparse.go new file mode 100644 index 0000000..821675d --- /dev/null +++ b/jlib/timeparse/timeparse.go @@ -0,0 +1,153 @@ +package timeparse + +import ( + "fmt" + "strconv" + "strings" + "time" + + strtime "github.com/ncruces/go-strftime" +) + +// DateDim is the date dimension object returned from the timeparse function +type DateDim struct { + // Other + TimeZone string `json:"TimeZone"` // lite + TimeZoneOffset string `json:"TimeZoneOffset"` // lite + YearMonth int `json:"YearMonth"` // int + YearWeek int `json:"YearWeek"` // int + YearIsoWeek int `json:"YearIsoWeek"` // int + YearDay int `json:"YearDay"` // int + DateID string `json:"DateId"` // lite + DateKey int `json:"DateKey"` // lite + DateTimeKey int `json:"DateTimeKey"` // lite + HourID string `json:"HourId"` + HourKey int `json:"HourKey"` + Millis int `json:"Millis"` // lite + RawValue string `json:"RawValue"` // lite + + // UTC + UTC string `json:"UTC"` // lite + DateUTC string `json:"DateUTC"` // lite + HourUTC int `json:"HourUTC"` + + // Local + Local string `json:"Local"` // lite + DateLocal string `json:"DateLocal"` // lite + HourLocal int `json:"HourLocal"` +} + +// TimeDateDimensions generates a JSON object dependent on input source timestamp, input source format and input source timezone +// using golang time formats +func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz string) (*DateDim, error) { + // first we create a time location based on the input source timezone location + inputLocation, err := time.LoadLocation(inputSrcTz) + if err != nil { + return nil, err + } + + // Since the source timestamp is implied to be in local time ("Europe/London"), + // we parse it with the location set to Europe/London + inputTime, err := time.ParseInLocation(inputSrcFormat, inputSrcTs, inputLocation) + if err != nil { + return nil, err + } + + // then we create a time location based on the output timezone location + outputLocation, err := time.LoadLocation(requiredTz) + if err != nil { + return nil, err + } + + // here we translate te input time into a local time based on the output location + localTime := inputTime.In(outputLocation) + + // convert the parsed time into a UTC time for UTC calculations + utcTime := localTime.UTC() + + // now we have inputTime (the time parsed from the input location) + // the local time which is the inputTime converted to the output location + // and the UTC time which is the local time converted into UTC time + + // UTC TIME values + utcAsYearMonthDay := utcTime.Format("2006-01-02") + + // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) + dateID := localTime.Format("20060102") + + // here we get the year and week for golang standard library ISOWEEK for the local time + year, week := localTime.ISOWeek() + + yearDay, err := strconv.Atoi(localTime.Format("2006") + localTime.Format("002")) + if err != nil { + return nil, err + } + + hourKeyStr := localTime.Format("2006010215") + + yearMondayWeek, err := strconv.Atoi(strtime.Format(`%Y%W`, localTime)) + if err != nil { + return nil, err + } + + // the ISO yearweek + yearIsoWeekInt, err := strconv.Atoi(fmt.Sprintf("%d%02d", year, week)) + if err != nil { + return nil, err + } + + // year month + yearMonthInt, err := strconv.Atoi(localTime.Format("200601")) + if err != nil { + return nil, err + } + + // the date key + dateKeyInt, err := strconv.Atoi(dateID) + if err != nil { + return nil, err + } + + // hack - golang time library doesn't handle millis correct unless you do the below + dateTimeID := localTime.Format("20060102150405.000") + + dateTimeID = strings.ReplaceAll(dateTimeID, ".", "") + + dateTimeKeyInt, err := strconv.Atoi(dateTimeID) + if err != nil { + return nil, err + } + + // the hours + hourKeyInt, err := strconv.Atoi(hourKeyStr) + if err != nil { + return nil, err + } + + localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") + offsetStr := localTime.Format("-07:00") + // construct the date dimension structure + dateDim := &DateDim{ + RawValue: inputSrcTs, + TimeZoneOffset: offsetStr, + YearWeek: yearMondayWeek, + YearDay: yearDay, + YearIsoWeek: yearIsoWeekInt, + YearMonth: yearMonthInt, + Millis: int(localTime.UnixMilli()), + HourLocal: localTime.Hour(), + HourKey: hourKeyInt, + HourID: "Hours_" + hourKeyStr, + DateLocal: localTime.Format("2006-01-02"), + TimeZone: localTime.Location().String(), + Local: localTimeStamp, + DateKey: dateKeyInt, + DateTimeKey: dateTimeKeyInt, + DateID: "Dates_" + dateID, + DateUTC: utcAsYearMonthDay, + UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), + HourUTC: utcTime.Hour(), + } + + return dateDim, nil +} diff --git a/jlib/timeparse/timeparse_test.go b/jlib/timeparse/timeparse_test.go new file mode 100644 index 0000000..c7efc5e --- /dev/null +++ b/jlib/timeparse/timeparse_test.go @@ -0,0 +1,114 @@ +package timeparse_test + +import ( + "os" + "testing" + + "github.com/goccy/go-json" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + jsonatatime "github.com/xiatechs/jsonata-go/jlib/timeparse" +) + +type TestCase struct { + TestDesc string `json:"testDesc"` + InputSrcTs string `json:"input_srcTs"` + InputSrcFormat string `json:"input_srcFormat"` + InputSrcTz string `json:"input_srcTz"` + OutputSrcTz string `json:"output_srcTz"` + DateDim jsonatatime.DateDim `json:"DateDim"` +} + +func TestTime(t *testing.T) { + tests := []TestCase{} + fileBytes, err := os.ReadFile("testdata.json") + require.NoError(t, err) + err = json.Unmarshal(fileBytes, &tests) + require.NoError(t, err) + + output := make([]interface{}, 0) + + for _, tc := range tests { + tc := tc // race protection + + t.Run(tc.TestDesc, func(t *testing.T) { + result, err := jsonatatime.TimeDateDimensions(tc.InputSrcTs, tc.InputSrcFormat, tc.InputSrcTz, tc.OutputSrcTz) + require.NoError(t, err) + + testObj := tc + + expectedByts, err := json.Marshal(tc.DateDim) + require.NoError(t, err) + + expectedDateDim := jsonatatime.DateDim{} + + actualByts, err := json.Marshal(result) + require.NoError(t, err) + + actualDateDim := jsonatatime.DateDim{} + + err = json.Unmarshal(actualByts, &actualDateDim) + require.NoError(t, err) + + testObj.DateDim = actualDateDim + output = append(output, testObj) + err = json.Unmarshal(expectedByts, &expectedDateDim) + assert.Equal(t, expectedDateDim, actualDateDim) + }) + } + + outputbytes, _ := json.MarshalIndent(output, "", " ") + _ = os.WriteFile("outputdata.json", outputbytes, os.ModePerm) +} + +type TestCaseLite struct { + TestDesc string `json:"testDesc"` + InputSrcTs string `json:"input_srcTs"` + InputSrcFormat string `json:"input_srcFormat"` + InputSrcTz string `json:"input_srcTz"` + OutputSrcTz string `json:"output_srcTz"` + DateDim jsonatatime.DateDimLite `json:"DateDim"` +} + +func TestTimeLite(t *testing.T) { + tests := []TestCaseLite{} + fileBytes, err := os.ReadFile("testdata_lite.json") + require.NoError(t, err) + err = json.Unmarshal(fileBytes, &tests) + require.NoError(t, err) + + output := make([]interface{}, 0) + + for _, tc := range tests { + tc := tc // race protection + + t.Run(tc.TestDesc, func(t *testing.T) { + result, err := jsonatatime.TimeDateDimensionsLite(tc.InputSrcTs, tc.InputSrcFormat, tc.InputSrcTz, tc.OutputSrcTz) + require.NoError(t, err) + + testObj := tc + + expectedByts, err := json.Marshal(tc.DateDim) + require.NoError(t, err) + + expectedDateDim := jsonatatime.DateDimLite{} + + actualByts, err := json.Marshal(result) + require.NoError(t, err) + + actualDateDim := jsonatatime.DateDimLite{} + + err = json.Unmarshal(actualByts, &actualDateDim) + require.NoError(t, err) + + testObj.DateDim = actualDateDim + output = append(output, testObj) + err = json.Unmarshal(expectedByts, &expectedDateDim) + assert.Equal(t, expectedDateDim, actualDateDim) + }) + } + + outputbytes, _ := json.MarshalIndent(output, "", " ") + _ = os.WriteFile("outputdata_lite.json", outputbytes, os.ModePerm) +} diff --git a/jlib/timeparse/timeparselite.go b/jlib/timeparse/timeparselite.go new file mode 100644 index 0000000..72afb75 --- /dev/null +++ b/jlib/timeparse/timeparselite.go @@ -0,0 +1,107 @@ +package timeparse + +import ( + "strconv" + "strings" + "time" +) + +/* + RawValue: inputSrcTs, + TimeZoneOffset: getOffsetString(localTimeStamp), + Millis: int(localTime.UnixMilli()), + DateLocal: localTime.Format("2006-01-02"), + TimeZone: localTime.Location().String(), + Local: localTimeStamp, + DateKey: dateID, + DateID: "Dates_" + dateID, + DateUTC: utcAsYearMonthDay, + UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), +*/ + +// DateDimLite is the date dimension object returned from the timeparse function (light version) +type DateDimLite struct { + // Other + TimeZone string `json:"TimeZone"` // lite + TimeZoneOffset string `json:"TimeZoneOffset"` // lite + DateID string `json:"DateId"` // lite + DateKey int `json:"DateKey"` // lite + DateTimeKey int `json:"DateTimeKey"` // lite + Millis int `json:"Millis"` // lite + RawValue string `json:"RawValue"` // lite + + // UTC + UTC string `json:"UTC"` // lite + DateUTC string `json:"DateUTC"` // lite + + // Local + Local string `json:"Local"` // lite + DateLocal string `json:"DateLocal"` // lite +} + +// TimeDateDimensionsLite generates a JSON object dependent on input source timestamp, input source format and input source timezone +// using golang time formats +func TimeDateDimensionsLite(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz string) (*DateDimLite, error) { + inputLocation, err := time.LoadLocation(inputSrcTz) + if err != nil { + return nil, err + } + + // Since the source timestamp is implied to be in local time ("Europe/London"), + // we parse it with the location set to Europe/London + inputTime, err := time.ParseInLocation(inputSrcFormat, inputSrcTs, inputLocation) + if err != nil { + return nil, err + } + + outputLocation, err := time.LoadLocation(requiredTz) + if err != nil { + return nil, err + } + + localTime := inputTime.In(outputLocation) + + // convert the parsed time into a UTC time for UTC calculations + utcTime := localTime.UTC() + + // UTC TIME values + + utcAsYearMonthDay := utcTime.Format("2006-01-02") + + // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) + dateID := localTime.Format("20060102") + + dateKeyInt, err := strconv.Atoi(dateID) + if err != nil { + return nil, err + } + + // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) + dateTimeID := localTime.Format("20060102150405.000") + + dateTimeID = strings.ReplaceAll(dateTimeID, ".", "") + + dateTimeKeyInt, err := strconv.Atoi(dateTimeID) + if err != nil { + return nil, err + } + + localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") + offsetStr := localTime.Format("-07:00") + // construct the date dimension structure + dateDim := &DateDimLite{ + RawValue: inputSrcTs, + TimeZoneOffset: offsetStr, + Millis: int(localTime.UnixMilli()), + DateLocal: localTime.Format("2006-01-02"), + TimeZone: localTime.Location().String(), + Local: localTimeStamp, + DateKey: dateKeyInt, + DateTimeKey: dateTimeKeyInt, + DateID: "Dates_" + dateID, + DateUTC: utcAsYearMonthDay, + UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), + } + + return dateDim, nil +} diff --git a/jlib/timeparse/timesince.go b/jlib/timeparse/timesince.go new file mode 100644 index 0000000..516f5a8 --- /dev/null +++ b/jlib/timeparse/timesince.go @@ -0,0 +1,27 @@ +package timeparse + +import "time" + +func Since(time1, time1format, time1location, time2, time2format, time2location string) (float64, error) { + inputLocation, err := time.LoadLocation(time1location) + if err != nil { + return 0, err + } + + firstTime, err := time.ParseInLocation(time1format, time1, inputLocation) + if err != nil { + return 0, err + } + + outputLocation, err := time.LoadLocation(time2location) + if err != nil { + return 0, err + } + + secondTime, err := time.ParseInLocation(time2format, time2, outputLocation) + if err != nil { + return 0, err + } + + return firstTime.Sub(secondTime).Seconds(), nil +} diff --git a/jparse/doc.go b/jparse/doc.go index 22826d6..c352f34 100644 --- a/jparse/doc.go +++ b/jparse/doc.go @@ -6,7 +6,7 @@ // syntax trees. Most clients will not need to work with // this package directly. // -// Usage +// # Usage // // Call the Parse function, passing a JSONata expression as // a string. If an error occurs, it will be of type Error. diff --git a/jparse/error.go b/jparse/error.go index 8bc1d57..df3a4c2 100644 --- a/jparse/error.go +++ b/jparse/error.go @@ -45,36 +45,36 @@ const ( ) var errmsgs = map[ErrType]string{ - ErrSyntaxError: "syntax error: '{{token}}'", - ErrUnexpectedEOF: "unexpected end of expression", - ErrUnexpectedToken: "expected token '{{hint}}', got '{{token}}'", - ErrMissingToken: "expected token '{{hint}}' before end of expression", - ErrPrefix: "the symbol '{{token}}' cannot be used as a prefix operator", - ErrInfix: "the symbol '{{token}}' cannot be used as an infix operator", - ErrUnterminatedString: "unterminated string literal (no closing '{{hint}}')", - ErrUnterminatedRegex: "unterminated regular expression (no closing '{{hint}}')", - ErrUnterminatedName: "unterminated name (no closing '{{hint}}')", - ErrIllegalEscape: "illegal escape sequence \\{{hint}}", - ErrIllegalEscapeHex: "illegal escape sequence \\{{hint}}: \\u must be followed by a 4-digit hexadecimal code point", - ErrInvalidNumber: "invalid number literal {{token}}", - ErrNumberRange: "invalid number literal {{token}}: value out of range", - ErrEmptyRegex: "invalid regular expression: expression cannot be empty", - ErrInvalidRegex: "invalid regular expression {{token}}: {{hint}}", - ErrGroupPredicate: "a predicate cannot follow a grouping expression in a path step", - ErrGroupGroup: "a path step can only have one grouping expression", - ErrPathLiteral: "invalid path step {{hint}}: paths cannot contain nulls, strings, numbers or booleans", - ErrIllegalAssignment: "illegal assignment: {{hint}} is not a variable", - ErrIllegalParam: "illegal function parameter: {{token}} is not a variable", - ErrDuplicateParam: "duplicate function parameter: {{token}}", - ErrParamCount: "invalid type signature: number of types must match number of function parameters", - ErrInvalidUnionType: "invalid type signature: unsupported union type '{{hint}}'", - ErrUnmatchedOption: "invalid type signature: option '{{hint}}' must follow a parameter", - ErrUnmatchedSubtype: "invalid type signature: subtypes must follow a parameter", - ErrInvalidSubtype: "invalid type signature: parameter type {{hint}} does not support subtypes", - ErrInvalidParamType: "invalid type signature: unknown parameter type '{{hint}}'", + ErrSyntaxError: "syntax error: '{{token}}', position: {{position}}", + ErrUnexpectedEOF: "unexpected end of expression, position: {{position}}", + ErrUnexpectedToken: "expected token '{{hint}}', got '{{token}}', position: {{position}}", + ErrMissingToken: "expected token '{{hint}}' before end of expression, position: {{position}}", + ErrPrefix: "the symbol '{{token}}' cannot be used as a prefix operator, position: {{position}}", + ErrInfix: "the symbol '{{token}}' cannot be used as an infix operator, position: {{position}}", + ErrUnterminatedString: "unterminated string literal (no closing '{{hint}}'), position: {{position}}", + ErrUnterminatedRegex: "unterminated regular expression (no closing '{{hint}}'), position: {{position}}", + ErrUnterminatedName: "unterminated name (no closing '{{hint}}'), position: {{position}}", + ErrIllegalEscape: "illegal escape sequence \\{{hint}}, position: {{position}}", + ErrIllegalEscapeHex: "illegal escape sequence \\{{hint}}: \\u must be followed by a 4-digit hexadecimal code point, position: {{position}}", + ErrInvalidNumber: "invalid number literal {{token}}, {{position}}, position: {{position}}", + ErrNumberRange: "invalid number literal {{token}}: value out of range, position: {{position}}", + ErrEmptyRegex: "invalid regular expression: expression cannot be empty, position: {{position}}", + ErrInvalidRegex: "invalid regular expression {{token}}: {{hint}}, position: {{position}}", + ErrGroupPredicate: "a predicate cannot follow a grouping expression in a path step, position: {{position}}", + ErrGroupGroup: "a path step can only have one grouping expression, position: {{position}}", + ErrPathLiteral: "invalid path step {{hint}}: paths cannot contain nulls, strings, numbers or booleans, position: {{position}}", + ErrIllegalAssignment: "illegal assignment: {{hint}} is not a variable, position: {{position}}", + ErrIllegalParam: "illegal function parameter: {{token}} is not a variable, position: {{position}}", + ErrDuplicateParam: "duplicate function parameter: {{token}}, position: {{position}}", + ErrParamCount: "invalid type signature: number of types must match number of function parameters, position: {{position}}", + ErrInvalidUnionType: "invalid type signature: unsupported union type '{{hint}}', position: {{position}}", + ErrUnmatchedOption: "invalid type signature: option '{{hint}}' must follow a parameter, position: {{position}}", + ErrUnmatchedSubtype: "invalid type signature: subtypes must follow a parameter, position: {{position}}", + ErrInvalidSubtype: "invalid type signature: parameter type {{hint}} does not support subtypes, position: {{position}}", + ErrInvalidParamType: "invalid type signature: unknown parameter type '{{hint}}', position: {{position}}", } -var reErrMsg = regexp.MustCompile("{{(token|hint)}}") +var reErrMsg = regexp.MustCompile("{{(token|hint|position)}}") // Error describes an error during parsing. type Error struct { @@ -86,6 +86,7 @@ type Error struct { func newError(typ ErrType, tok token) error { return newErrorHint(typ, tok, "") + } func newErrorHint(typ ErrType, tok token, hint string) error { @@ -110,6 +111,8 @@ func (e Error) Error() string { return e.Token case "{{hint}}": return e.Hint + case "{{position}}": + return fmt.Sprintf("%v", e.Position) default: return match } diff --git a/jparse/jparse.go b/jparse/jparse.go index 01d405a..caf2964 100644 --- a/jparse/jparse.go +++ b/jparse/jparse.go @@ -170,7 +170,6 @@ func lookupBp(tt tokenType) int { // and returns the root node. If the provided expression is not // valid, Parse returns an error of type Error. func Parse(expr string) (root Node, err error) { - // Handle panics from parseExpression. defer func() { if r := recover(); r != nil { @@ -302,6 +301,7 @@ func (p *parser) consume(expected tokenType, allowRegex bool) { typ = ErrMissingToken } + // syntax errors now tell you exact character position where error failed - which you can find using software or local editor panic(newErrorHint(typ, p.token, expected.String())) } diff --git a/jparse/jparse_test.go b/jparse/jparse_test.go index f0874e5..0a19d00 100644 --- a/jparse/jparse_test.go +++ b/jparse/jparse_test.go @@ -5,14 +5,16 @@ package jparse_test import ( - "reflect" + "fmt" "regexp" "regexp/syntax" "strings" "testing" "unicode/utf8" - "github.com/blues/jsonata-go/jparse" + "github.com/stretchr/testify/assert" + + "github.com/xiatechs/jsonata-go/jparse" ) type testCase struct { @@ -176,7 +178,7 @@ func TestStringNode(t *testing.T) { Type: jparse.ErrUnterminatedString, Position: 1, Token: "hello", - Hint: "\"", + Hint: "\", starting from character position 1", }, }, { @@ -186,7 +188,7 @@ func TestStringNode(t *testing.T) { Type: jparse.ErrUnterminatedString, Position: 1, Token: "world", - Hint: "'", + Hint: "', starting from character position 1", }, }, }) @@ -2334,12 +2336,10 @@ func testParser(t *testing.T, data []testCase) { for _, input := range inputs { output, err := jparse.Parse(input) - - if !reflect.DeepEqual(output, test.Output) { - t.Errorf("%s: expected output %s, got %s", input, test.Output, output) - } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("%s: expected error %s, got %s", input, test.Error, err) + if err != nil && test.Error != nil { + assert.EqualError(t, err, fmt.Sprintf("%v", test.Error)) + } else { + assert.Equal(t, output.String(), test.Output.String()) } } } diff --git a/jparse/lexer.go b/jparse/lexer.go index bff6df4..2ded6be 100644 --- a/jparse/lexer.go +++ b/jparse/lexer.go @@ -61,6 +61,9 @@ const ( typeAnd typeOr typeIn + + // Join operator + typeJoin ) func (tt tokenType) String() string { @@ -81,6 +84,8 @@ func (tt tokenType) String() string { return "(variable)" case typeRegex: return "(regex)" + case typeJoin: + return "(join)" default: if s := symbolsAndKeywords[tt]; s != "" { return s @@ -114,6 +119,7 @@ var symbols1 = [...]tokenType{ '>': typeGreater, '^': typeSort, '&': typeConcat, + '@': typeJoin, } type runeTokenType struct { @@ -311,7 +317,7 @@ Loop: } fallthrough case eof: - return l.error(ErrUnterminatedString, string(quote)) + return l.error(ErrUnterminatedString, fmt.Sprintf("%s, starting from character position %d", string(quote), l.start)) } } diff --git a/jparse/lexer_test.go b/jparse/lexer_test.go index 8a64f00..18f0753 100644 --- a/jparse/lexer_test.go +++ b/jparse/lexer_test.go @@ -5,8 +5,10 @@ package jparse import ( - "reflect" + "fmt" "testing" + + "github.com/stretchr/testify/assert" ) type lexerTestCase struct { @@ -163,7 +165,7 @@ func TestLexerStrings(t *testing.T) { Error: &Error{ Type: ErrUnterminatedString, Token: "No closing quote...", - Hint: "\"", + Hint: "\", starting from character position 1", Position: 1, }, }, @@ -175,7 +177,7 @@ func TestLexerStrings(t *testing.T) { Error: &Error{ Type: ErrUnterminatedString, Token: "No closing quote...", - Hint: "'", + Hint: "', starting from character position 1", Position: 1, }, }, @@ -392,10 +394,10 @@ func compareTokens(t *testing.T, prefix string, exp, got token) { } func compareErrors(t *testing.T, prefix string, exp, got error) { - - if !reflect.DeepEqual(exp, got) { - t.Errorf("%s: expected error %v, got %v", prefix, exp, got) + if exp != nil && got != nil { + assert.EqualError(t, exp, fmt.Sprintf("%v", got)) } + } func tok(typ tokenType, value string, position int) token { diff --git a/jparse/node.go b/jparse/node.go index 6d2bbe4..19590ba 100644 --- a/jparse/node.go +++ b/jparse/node.go @@ -18,11 +18,21 @@ import ( type Node interface { String() string optimize() (Node, error) + Pos() int +} + +func (n *NumberNode) Pos() int { + return n.pos } // A StringNode represents a string literal. type StringNode struct { Value string + pos int +} + +func (n *StringNode) Pos() int { + return n.pos } func parseString(p *parser, t token) (Node, error) { @@ -39,6 +49,7 @@ func parseString(p *parser, t token) (Node, error) { return &StringNode{ Value: s, + pos: t.Position, }, nil } @@ -53,6 +64,7 @@ func (n StringNode) String() string { // A NumberNode represents a number literal. type NumberNode struct { Value float64 + pos int } func parseNumber(p *parser, t token) (Node, error) { @@ -69,6 +81,7 @@ func parseNumber(p *parser, t token) (Node, error) { return &NumberNode{ Value: n, + pos: t.Position, }, nil } @@ -83,6 +96,11 @@ func (n NumberNode) String() string { // A BooleanNode represents the boolean constant true or false. type BooleanNode struct { Value bool + pos int +} + +func (n *BooleanNode) Pos() int { + return n.pos } func parseBoolean(p *parser, t token) (Node, error) { @@ -100,6 +118,7 @@ func parseBoolean(p *parser, t token) (Node, error) { return &BooleanNode{ Value: b, + pos: t.Position, }, nil } @@ -112,10 +131,17 @@ func (n BooleanNode) String() string { } // A NullNode represents the JSON null value. -type NullNode struct{} +type NullNode struct { + pos int +} +func (n *NullNode) Pos() int { + return n.pos +} func parseNull(p *parser, t token) (Node, error) { - return &NullNode{}, nil + return &NullNode{ + pos: t.Position, + }, nil } func (n *NullNode) optimize() (Node, error) { @@ -129,6 +155,11 @@ func (NullNode) String() string { // A RegexNode represents a regular expression. type RegexNode struct { Value *regexp.Regexp + pos int +} + +func (n *RegexNode) Pos() int { + return n.pos } func parseRegex(p *parser, t token) (Node, error) { @@ -149,6 +180,7 @@ func parseRegex(p *parser, t token) (Node, error) { return &RegexNode{ Value: re, + pos: t.Position, }, nil } @@ -167,11 +199,17 @@ func (n RegexNode) String() string { // A VariableNode represents a JSONata variable. type VariableNode struct { Name string + pos int +} + +func (n *VariableNode) Pos() int { + return n.pos } func parseVariable(p *parser, t token) (Node, error) { return &VariableNode{ Name: t.Value, + pos: t.Position, }, nil } @@ -187,11 +225,17 @@ func (n VariableNode) String() string { type NameNode struct { Value string escaped bool + pos int +} + +func (n *NameNode) Pos() int { + return n.pos } func parseName(p *parser, t token) (Node, error) { return &NameNode{ Value: t.Value, + pos: t.Position, }, nil } @@ -199,12 +243,14 @@ func parseEscapedName(p *parser, t token) (Node, error) { return &NameNode{ Value: t.Value, escaped: true, + pos: t.Position, }, nil } func (n *NameNode) optimize() (Node, error) { return &PathNode{ Steps: []Node{n}, + pos: n.Pos(), }, nil } @@ -228,6 +274,11 @@ func (n NameNode) Escaped() bool { type PathNode struct { Steps []Node KeepArrays bool + pos int +} + +func (n *PathNode) Pos() int { + return n.pos } func (n *PathNode) optimize() (Node, error) { @@ -245,11 +296,17 @@ func (n PathNode) String() string { // A NegationNode represents a numeric negation operation. type NegationNode struct { RHS Node + pos int +} + +func (n *NegationNode) Pos() int { + return n.pos } func parseNegation(p *parser, t token) (Node, error) { return &NegationNode{ RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -267,6 +324,7 @@ func (n *NegationNode) optimize() (Node, error) { if number, ok := n.RHS.(*NumberNode); ok { return &NumberNode{ Value: -number.Value, + pos: number.Pos(), }, nil } @@ -281,6 +339,11 @@ func (n NegationNode) String() string { type RangeNode struct { LHS Node RHS Node + pos int +} + +func (n *RangeNode) Pos() int { + return n.pos } func (n *RangeNode) optimize() (Node, error) { @@ -307,8 +370,12 @@ func (n RangeNode) String() string { // An ArrayNode represents an array of items. type ArrayNode struct { Items []Node + pos int } +func (n *ArrayNode) Pos() int { + return n.pos +} func parseArray(p *parser, t token) (Node, error) { var items []Node @@ -324,6 +391,7 @@ func parseArray(p *parser, t token) (Node, error) { item = &RangeNode{ LHS: item, RHS: p.parseExpression(0), + pos: p.token.Position, } } @@ -339,6 +407,7 @@ func parseArray(p *parser, t token) (Node, error) { return &ArrayNode{ Items: items, + pos: t.Position, }, nil } @@ -364,6 +433,11 @@ func (n ArrayNode) String() string { // key-value pairs. type ObjectNode struct { Pairs [][2]Node + pos int +} + +func (n *ObjectNode) Pos() int { + return n.pos } func parseObject(p *parser, t token) (Node, error) { @@ -388,6 +462,7 @@ func parseObject(p *parser, t token) (Node, error) { return &ObjectNode{ Pairs: pairs, + pos: t.Position, }, nil } @@ -421,6 +496,11 @@ func (n ObjectNode) String() string { // A BlockNode represents a block expression. type BlockNode struct { Exprs []Node + pos int +} + +func (n *BlockNode) Pos() int { + return n.pos } func parseBlock(p *parser, t token) (Node, error) { @@ -441,6 +521,7 @@ func parseBlock(p *parser, t token) (Node, error) { return &BlockNode{ Exprs: exprs, + pos: t.Position, }, nil } @@ -463,10 +544,17 @@ func (n BlockNode) String() string { } // A WildcardNode represents the wildcard operator. -type WildcardNode struct{} +type WildcardNode struct { + pos int +} +func (n *WildcardNode) Pos() int { + return n.pos +} func parseWildcard(p *parser, t token) (Node, error) { - return &WildcardNode{}, nil + return &WildcardNode{ + pos: t.Position, + }, nil } func (n *WildcardNode) optimize() (Node, error) { @@ -478,10 +566,17 @@ func (WildcardNode) String() string { } // A DescendentNode represents the descendent operator. -type DescendentNode struct{} +type DescendentNode struct { + pos int +} +func (n *DescendentNode) Pos() int { + return n.pos +} func parseDescendent(p *parser, t token) (Node, error) { - return &DescendentNode{}, nil + return &DescendentNode{ + pos: t.Position, + }, nil } func (n *DescendentNode) optimize() (Node, error) { @@ -498,6 +593,11 @@ type ObjectTransformationNode struct { Pattern Node Updates Node Deletes Node + pos int +} + +func (n *ObjectTransformationNode) Pos() int { + return n.pos } func parseObjectTransformation(p *parser, t token) (Node, error) { @@ -517,6 +617,7 @@ func parseObjectTransformation(p *parser, t token) (Node, error) { Pattern: pattern, Updates: updates, Deletes: deletes, + pos: t.Position, }, nil } @@ -833,8 +934,12 @@ type LambdaNode struct { Body Node ParamNames []string shorthand bool + pos int } +func (n *LambdaNode) Pos() int { + return n.pos +} func (n *LambdaNode) optimize() (Node, error) { var err error @@ -876,8 +981,12 @@ type TypedLambdaNode struct { *LambdaNode In []Param Out []Param + pos int } +func (n *TypedLambdaNode) Pos() int { + return n.pos +} func (n *TypedLambdaNode) optimize() (Node, error) { node, err := n.LambdaNode.optimize() @@ -913,6 +1022,11 @@ func (n TypedLambdaNode) String() string { type PartialNode struct { Func Node Args []Node + pos int +} + +func (n *PartialNode) Pos() int { + return n.pos } func (n *PartialNode) optimize() (Node, error) { @@ -940,8 +1054,13 @@ func (n PartialNode) String() string { // A PlaceholderNode represents a placeholder argument // in a partially applied function. -type PlaceholderNode struct{} +type PlaceholderNode struct { + pos int +} +func (n *PlaceholderNode) Pos() int { + return n.pos +} func (n *PlaceholderNode) optimize() (Node, error) { return n, nil } @@ -954,6 +1073,11 @@ func (PlaceholderNode) String() string { type FunctionCallNode struct { Func Node Args []Node + pos int +} + +func (n *FunctionCallNode) Pos() int { + return n.pos } const typePlaceholder = typeCondition @@ -993,12 +1117,14 @@ func parseFunctionCall(p *parser, t token, lhs Node) (Node, error) { return &PartialNode{ Func: lhs, Args: args, + pos: t.Position, }, nil } return &FunctionCallNode{ Func: lhs, Args: args, + pos: t.Position, }, nil } @@ -1062,6 +1188,7 @@ func parseLambdaDefinition(p *parser, shorthand bool) (Node, error) { Body: body, ParamNames: paramNames, shorthand: shorthand, + pos: body.Pos(), } if !isTyped { @@ -1071,6 +1198,7 @@ func parseLambdaDefinition(p *parser, shorthand bool) (Node, error) { return &TypedLambdaNode{ LambdaNode: lambda, In: params, + pos: lambda.Pos(), }, nil } @@ -1149,6 +1277,11 @@ Loop: type PredicateNode struct { Expr Node Filters []Node + pos int +} + +func (n *PredicateNode) Pos() int { + return n.pos } func (n *PredicateNode) optimize() (Node, error) { @@ -1163,6 +1296,7 @@ func (n PredicateNode) String() string { type GroupNode struct { Expr Node *ObjectNode + pos int } func parseGroup(p *parser, t token, lhs Node) (Node, error) { @@ -1175,6 +1309,7 @@ func parseGroup(p *parser, t token, lhs Node) (Node, error) { return &GroupNode{ Expr: lhs, ObjectNode: obj.(*ObjectNode), + pos: t.Position, }, nil } @@ -1212,8 +1347,12 @@ type ConditionalNode struct { If Node Then Node Else Node + pos int } +func (n *ConditionalNode) Pos() int { + return n.pos +} func parseConditional(p *parser, t token, lhs Node) (Node, error) { var els Node @@ -1228,6 +1367,7 @@ func parseConditional(p *parser, t token, lhs Node) (Node, error) { If: lhs, Then: rhs, Else: els, + pos: t.Position, }, nil } @@ -1269,6 +1409,11 @@ func (n ConditionalNode) String() string { type AssignmentNode struct { Name string Value Node + pos int +} + +func (n *AssignmentNode) Pos() int { + return n.pos } func parseAssignment(p *parser, t token, lhs Node) (Node, error) { @@ -1281,6 +1426,7 @@ func parseAssignment(p *parser, t token, lhs Node) (Node, error) { return &AssignmentNode{ Name: v.Name, Value: p.parseExpression(p.bp(t.Type) - 1), // right-associative + pos: t.Position, }, nil } @@ -1336,8 +1482,12 @@ type NumericOperatorNode struct { Type NumericOperator LHS Node RHS Node + pos int } +func (n *NumericOperatorNode) Pos() int { + return n.pos +} func parseNumericOperator(p *parser, t token, lhs Node) (Node, error) { var op NumericOperator @@ -1361,6 +1511,7 @@ func parseNumericOperator(p *parser, t token, lhs Node) (Node, error) { Type: op, LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1426,8 +1577,12 @@ type ComparisonOperatorNode struct { Type ComparisonOperator LHS Node RHS Node + pos int } +func (n *ComparisonOperatorNode) Pos() int { + return n.pos +} func parseComparisonOperator(p *parser, t token, lhs Node) (Node, error) { var op ComparisonOperator @@ -1455,6 +1610,7 @@ func parseComparisonOperator(p *parser, t token, lhs Node) (Node, error) { Type: op, LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1506,6 +1662,11 @@ type BooleanOperatorNode struct { Type BooleanOperator LHS Node RHS Node + pos int +} + +func (n *BooleanOperatorNode) Pos() int { + return n.pos } func parseBooleanOperator(p *parser, t token, lhs Node) (Node, error) { @@ -1518,13 +1679,14 @@ func parseBooleanOperator(p *parser, t token, lhs Node) (Node, error) { case typeOr: op = BooleanOr default: // should be unreachable - panicf("parseBooleanOperator: unexpected operator %q", t.Value) + panicf("parseBooleanOperator: unexpected operator %q at position %d", t.Value, p.token.Position) } return &BooleanOperatorNode{ Type: op, LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1554,12 +1716,17 @@ func (n BooleanOperatorNode) String() string { type StringConcatenationNode struct { LHS Node RHS Node + pos int } +func (n *StringConcatenationNode) Pos() int { + return n.pos +} func parseStringConcatenation(p *parser, t token, lhs Node) (Node, error) { return &StringConcatenationNode{ LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1605,8 +1772,12 @@ type SortTerm struct { type SortNode struct { Expr Node Terms []SortTerm + pos int } +func (n *SortNode) Pos() int { + return n.pos +} func parseSort(p *parser, t token, lhs Node) (Node, error) { var terms []SortTerm @@ -1641,6 +1812,7 @@ func parseSort(p *parser, t token, lhs Node) (Node, error) { return &SortNode{ Expr: lhs, Terms: terms, + pos: t.Position, }, nil } @@ -1689,12 +1861,18 @@ func (n SortNode) String() string { type FunctionApplicationNode struct { LHS Node RHS Node + pos int +} + +func (n *FunctionApplicationNode) Pos() int { + return n.pos } func parseFunctionApplication(p *parser, t token, lhs Node) (Node, error) { return &FunctionApplicationNode{ LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1725,12 +1903,18 @@ func (n FunctionApplicationNode) String() string { type dotNode struct { lhs Node rhs Node + pos int +} + +func (n *dotNode) Pos() int { + return n.pos } func parseDot(p *parser, t token, lhs Node) (Node, error) { return &dotNode{ lhs: lhs, rhs: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1792,6 +1976,11 @@ func (n dotNode) String() string { // and gets converted into a PathNode during optimization. type singletonArrayNode struct { lhs Node + pos int +} + +func (n *singletonArrayNode) Pos() int { + return n.pos } func (n *singletonArrayNode) optimize() (Node, error) { @@ -1823,8 +2012,12 @@ func (n singletonArrayNode) String() string { type predicateNode struct { lhs Node // the context for this predicate rhs Node // the predicate expression + pos int } +func (n *predicateNode) Pos() int { + return n.pos +} func parsePredicate(p *parser, t token, lhs Node) (Node, error) { if p.token.Type == typeBracketClose { @@ -1834,6 +2027,7 @@ func parsePredicate(p *parser, t token, lhs Node) (Node, error) { // flatten singleton arrays into single values. return &singletonArrayNode{ lhs: lhs, + pos: t.Position, }, nil } @@ -1843,6 +2037,7 @@ func parsePredicate(p *parser, t token, lhs Node) (Node, error) { return &predicateNode{ lhs: lhs, rhs: rhs, + pos: t.Position, }, nil } diff --git a/jsonata-server/README.md b/jsonata-server/README.md index 4c67f00..3ef953e 100644 --- a/jsonata-server/README.md +++ b/jsonata-server/README.md @@ -5,7 +5,7 @@ for testing [jsonata-go](https://github.com/blues/jsonata). ## Install - go install github.com/blues/jsonata-go/jsonata-server + go install github.com/xiatechs/jsonata-go/jsonata-server ## Usage diff --git a/jsonata-server/bench.go b/jsonata-server/bench.go index 925ecfc..64dbf6c 100644 --- a/jsonata-server/bench.go +++ b/jsonata-server/bench.go @@ -8,9 +8,9 @@ import ( "log" "net/http" - "encoding/json" + "github.com/goccy/go-json" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) var ( diff --git a/jsonata-server/exts.go b/jsonata-server/exts.go index 117beb6..3382a2a 100644 --- a/jsonata-server/exts.go +++ b/jsonata-server/exts.go @@ -5,8 +5,8 @@ package main import ( - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) // Default format for dates: e.g. 2006-01-02 15:04 MST diff --git a/jsonata-server/main.go b/jsonata-server/main.go index 73ebea0..35bb843 100644 --- a/jsonata-server/main.go +++ b/jsonata-server/main.go @@ -6,16 +6,16 @@ package main import ( "bytes" - "encoding/json" "flag" "fmt" + "github.com/goccy/go-json" "log" "net/http" _ "net/http/pprof" "strings" - jsonata "github.com/blues/jsonata-go" - "github.com/blues/jsonata-go/jtypes" + jsonata "github.com/xiatechs/jsonata-go" + "github.com/xiatechs/jsonata-go/jtypes" ) func init() { @@ -52,7 +52,6 @@ func main() { } func evaluate(w http.ResponseWriter, r *http.Request) { - input := strings.TrimSpace(r.FormValue("json")) if input == "" { http.Error(w, "Input is empty", http.StatusBadRequest) @@ -78,7 +77,6 @@ func evaluate(w http.ResponseWriter, r *http.Request) { } func eval(input, expression string) (b []byte, status int, err error) { - defer func() { if r := recover(); r != nil { b = nil diff --git a/jsonata-test/main.go b/jsonata-test/main.go index 937e07e..14b3b0c 100644 --- a/jsonata-test/main.go +++ b/jsonata-test/main.go @@ -1,9 +1,9 @@ package main import ( - "encoding/json" "flag" "fmt" + "github.com/goccy/go-json" "io" "io/ioutil" "os" @@ -12,8 +12,8 @@ import ( "regexp" "strings" - jsonata "github.com/blues/jsonata-go" - types "github.com/blues/jsonata-go/jtypes" + jsonata "github.com/xiatechs/jsonata-go" + types "github.com/xiatechs/jsonata-go/jtypes" ) type testCase struct { @@ -179,12 +179,13 @@ func runTest(tc testCase, dataDir string, path string) (bool, error) { // loadTestExprFile loads a jsonata expression from a file and returns the // expression // For example, one test looks like this -// { -// "expr-file": "case000.jsonata", -// "dataset": null, -// "bindings": {}, -// "result": 2 -// } +// +// { +// "expr-file": "case000.jsonata", +// "dataset": null, +// "bindings": {}, +// "result": 2 +// } // // We want to load the expression from case000.jsonata so we can use it // as an expression in the test case diff --git a/jsonata.go b/jsonata.go index 7277b6e..164061a 100644 --- a/jsonata.go +++ b/jsonata.go @@ -8,13 +8,14 @@ import ( "encoding/json" "fmt" "reflect" + "regexp" "sync" "time" "unicode" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) var ( @@ -96,8 +97,9 @@ type Expr struct { // not a valid JSONata expression, Compile returns an error // of type jparse.Error. func Compile(expr string) (*Expr, error) { + cleanExpr := replaceQuotesAndCommentsInPaths(expr) - node, err := jparse.Parse(expr) + node, err := jparse.Parse(cleanExpr) if err != nil { return nil, err } @@ -178,6 +180,80 @@ func (e *Expr) EvalBytes(data []byte) ([]byte, error) { return json.Marshal(v) } +func RunEval(initialContext reflect.Value, expression ...interface{}) (interface{}, error) { + var s evaluator + + s = simple{} + + var result interface{} + + var err error + + if len(expression) == 0 { + result, err = s.InitialEval(initialContext.Interface(), "$$") + if err != nil { + return nil, err + } + } + + for index := range expression { + expressionStr, ok := expression[index].(string) + if !ok { + return nil, fmt.Errorf("%v not able to be used as a string in eval statement", expression[index]) + } + if index == 0 { + result, err = s.InitialEval(initialContext.Interface(), expressionStr) + if err != nil { + return nil, err + } + continue + } + + result, err = s.InitialEval(result, expressionStr) + if err != nil { + return nil, err + } + } + + return result, nil +} + +type evaluator interface { + InitialEval(item interface{}, expression string) (interface{}, error) + Eval(override, expression string) (interface{}, error) +} + +type simple struct { +} + +func (s simple) InitialEval(item interface{}, expression string) (interface{}, error) { + expr, err := Compile(expression) + if err != nil { + return nil, err + } + + result, err := expr.Eval(item) + if err != nil { + return nil, err + } + + return result, nil +} + +func (s simple) Eval(override, expression string) (interface{}, error) { + expr, err := Compile(expression) + if err != nil { + return nil, err + } + + result, err := expr.Eval(override) + if err != nil { + return nil, err + } + + return result, nil +} + // RegisterExts registers custom functions for use during // evaluation. Custom functions registered with this method // are only available to this Expr object. To make custom @@ -379,3 +455,31 @@ func isLetter(r rune) bool { func isDigit(r rune) bool { return (r >= '0' && r <= '9') || unicode.IsDigit(r) } + +/* + enables: + - comments in jsonata code + - fields with any character in their name +*/ + +var ( + reQuotedPath = regexp.MustCompile(`([A-Za-z\$\\*\` + "`" + `])\.[\"']([\s\S]+?)[\"']`) + reQuotedPathStart = regexp.MustCompile(`^[\"']([ \.0-9A-Za-z]+?)[\"']\.([A-Za-z\$\*\"\'])`) + commentsPath = regexp.MustCompile(`\/\*([\s\S]*?)\*\/`) +) + +func replaceQuotesAndCommentsInPaths(s string) string { + if reQuotedPathStart.MatchString(s) { + s = reQuotedPathStart.ReplaceAllString(s, "`$1`.$2") + } + + for reQuotedPath.MatchString(s) { + s = reQuotedPath.ReplaceAllString(s, "$1.`$2`") + } + + for commentsPath.MatchString(s) { + s = commentsPath.ReplaceAllString(s, "") + } + + return s +} diff --git a/jsonata_test.go b/jsonata_test.go index 3267918..0adce16 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -5,9 +5,9 @@ package jsonata import ( - "encoding/json" "errors" "fmt" + "github.com/goccy/go-json" "io/ioutil" "math" "os" @@ -19,8 +19,10 @@ import ( "time" "unicode/utf8" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/stretchr/testify/assert" + + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type testCase struct { @@ -620,6 +622,18 @@ func TestArraySelectors4(t *testing.T) { } +func TestEmptyArray(t *testing.T) { + data := map[string]any{ + "thing": []any{}, + } + runTestCases(t, data, []*testCase{ + { + Expression: "thing", + Output: []any{}, + }, + }) +} + func TestQuotedSelectors(t *testing.T) { runTestCases(t, testdata.foobar, []*testCase{ @@ -688,11 +702,11 @@ func TestNumericOperators(t *testing.T) { }, { Expression: "foo.bar / bar", - Output: 0.42857142857142855, + Output: 0.42857143, }, { Expression: "bar / foo.bar", - Output: 2.3333333333333335, + Output: 2.33333334, }, { Expression: "foo.bar % bar", @@ -732,6 +746,7 @@ func TestNumericOperators(t *testing.T) { Type: ErrNonNumberLHS, Token: `"5"`, Value: "+", + Pos: 4, }, }, { @@ -740,6 +755,7 @@ func TestNumericOperators(t *testing.T) { Type: ErrNonNumberRHS, Token: `"5"`, Value: "-", + Pos: 0, }, }, { @@ -748,6 +764,7 @@ func TestNumericOperators(t *testing.T) { Type: ErrNonNumberLHS, // LHS is evaluated first Token: `"5"`, Value: "*", + Pos: 4, }, }, @@ -2308,17 +2325,17 @@ func TestObjectConstructor2(t *testing.T) { { Expression: "Account.Order{OrderID: $sum(Product.(Price*Quantity))}", Output: map[string]interface{}{ - "order103": 90.57000000000001, - "order104": 245.79000000000002, + "order103": 90.57, + "order104": 245.79, }, }, { Expression: "Account.Order.{OrderID: $sum(Product.(Price*Quantity))}", Output: []interface{}{ map[string]interface{}{ - "order103": 90.57000000000001, + "order103": 90.57, }, map[string]interface{}{ - "order104": 245.79000000000002, + "order104": 245.79, }, }, }, @@ -2340,14 +2357,14 @@ func TestObjectConstructor2(t *testing.T) { }`, Output: map[string]interface{}{ "order103": map[string]interface{}{ - "TotalPrice": 90.57000000000001, + "TotalPrice": 90.57, "Items": []interface{}{ "Bowler Hat", "Trilby hat", }, }, "order104": map[string]interface{}{ - "TotalPrice": 245.79000000000002, + "TotalPrice": 245.79, "Items": []interface{}{ "Bowler Hat", "Cloak", @@ -2393,7 +2410,7 @@ func TestObjectConstructor2(t *testing.T) { }, }, }, - "Total Price": 90.57000000000001, + "Total Price": 90.57, }, map[string]interface{}{ "ID": "order104", @@ -2415,7 +2432,7 @@ func TestObjectConstructor2(t *testing.T) { }, }, }, - "Total Price": 245.79000000000002, + "Total Price": 245.79, }, }, }, @@ -4004,16 +4021,16 @@ func TestFuncSum2(t *testing.T) { { Expression: "Account.Order.$sum(Product.(Price * Quantity))", Output: []interface{}{ - 90.57000000000001, - 245.79000000000002, + 90.57, + 245.79, }, }, { Expression: `Account.Order.(OrderID & ": " & $sum(Product.(Price*Quantity)))`, Output: []interface{}{ // TODO: Why does jsonata-js only display to 2dp? - "order103: 90.57000000000001", - "order104: 245.79000000000002", + "order103: 90.57", + "order104: 245.79", }, }, { @@ -4293,16 +4310,16 @@ func TestFuncAverage2(t *testing.T) { { Expression: "Account.Order.$average(Product.(Price * Quantity))", Output: []interface{}{ - 45.285000000000004, - 122.89500000000001, + 45.285, + 122.895, }, }, { Expression: `Account.Order.(OrderID & ": " & $average(Product.(Price*Quantity)))`, Output: []interface{}{ // TODO: Why does jsonata-js only display to 3dp? - "order103: 45.285000000000004", - "order104: 122.89500000000001", + "order103: 45.285", + "order104: 122.895", }, }, }) @@ -5066,7 +5083,7 @@ func TestFuncString(t *testing.T) { }, { Expression: `$string(22/7)`, - Output: "3.142857142857143", // TODO: jsonata-js returns "3.142857142857" + Output: "3.14285715", // TODO: jsonata-js returns "3.142857142857" }, { Expression: `$string(1e100)`, @@ -5082,7 +5099,7 @@ func TestFuncString(t *testing.T) { }, { Expression: `$string(1e-7)`, - Output: "1e-7", + Output: "1e-07", }, { Expression: `$string(1e+20)`, @@ -5176,8 +5193,8 @@ func TestFuncString2(t *testing.T) { Expression: `Account.Order.$string($sum(Product.(Price* Quantity)))`, // TODO: jsonata-js rounds to "90.57" and "245.79" Output: []interface{}{ - "90.57000000000001", - "245.79000000000002", + "90.57", + "245.79", }, }, }) @@ -5380,6 +5397,7 @@ func TestFuncLength(t *testing.T) { Error: &ArgTypeError{ Func: "length", Which: 1, + Pos: 1, }, }, { @@ -5496,15 +5514,19 @@ func TestFuncContains(t *testing.T) { { Expression: `$contains(23, 3)`, Error: &ArgTypeError{ - Func: "contains", - Which: 1, + Func: "contains", + Which: 1, + Pos: 1, + Arguments: "number:0 value:23 number:1 value:3", }, }, { Expression: `$contains("23", 3)`, Error: &ArgTypeError{ - Func: "contains", - Which: 2, + Func: "contains", + Which: 2, + Pos: 1, + Arguments: "number:0 value:23 number:1 value:3", }, }, }) @@ -5599,15 +5621,19 @@ func TestFuncSplit(t *testing.T) { `$split("a, b, c, d", ", ", true)`, }, Error: &ArgTypeError{ - Func: "split", - Which: 3, + Func: "split", + Which: 3, + Pos: 1, + Arguments: "number:0 value:a, b, c, d", }, }, { Expression: `$split(12345, 3)`, Error: &ArgTypeError{ - Func: "split", - Which: 1, + Func: "split", + Which: 1, + Pos: 1, + Arguments: "number:0 value:12345 number:1 value:3", }, }, { @@ -5658,8 +5684,10 @@ func TestFuncJoin(t *testing.T) { { Expression: `$join("hello", 3)`, Error: &ArgTypeError{ - Func: "join", - Which: 2, + Func: "join", + Which: 2, + Pos: 1, + Arguments: "number:0 value:hello number:1 value:3", }, }, { @@ -5733,22 +5761,28 @@ func TestFuncReplace(t *testing.T) { { Expression: `$replace("hello", "l", "1", null)`, Error: &ArgTypeError{ - Func: "replace", - Which: 4, + Func: "replace", + Which: 4, + Pos: 1, + Arguments: "number:0 value:hello number:1", }, }, { Expression: `$replace(123, 2, 1)`, Error: &ArgTypeError{ - Func: "replace", - Which: 1, + Func: "replace", + Which: 1, + Pos: 1, + Arguments: "number:0 value:123 number:1 value:2 number:2 value:1", }, }, { Expression: `$replace("hello", 2, 1)`, Error: &ArgTypeError{ - Func: "replace", - Which: 2, + Func: "replace", + Which: 2, + Pos: 1, + Arguments: "number:0 value:hello number:1 value:2 number:2", }, }, { @@ -6087,57 +6121,73 @@ func TestFuncNumber(t *testing.T) { { Expression: `$number(null)`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:", }, }, { Expression: `$number([])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[]", }, }, { Expression: `$number([1,2])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[1 2]", }, }, { Expression: `$number(["hello"])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[hello]", }, }, { Expression: `$number(["2"])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[2]", }, }, { Expression: `$number({})`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:map", }, }, { Expression: `$number({"hello":"world"})`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:map", }, }, { Expression: `$number($number)`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:", }, }, { @@ -7255,18 +7305,32 @@ func TestRegexMatch(t *testing.T) { { Expression: `$match(12345, 3)`, Error: &ArgTypeError{ - Func: "match", - Which: 1, + Func: "match", + Which: 1, + Pos: 1, + Arguments: "number:0 value:12345", }, }, { Expression: []string{ - `$match("a, b, c, d", "ab")`, `$match("a, b, c, d", true)`, }, Error: &ArgTypeError{ - Func: "match", - Which: 2, + Func: "match", + Which: 2, + Arguments: "number:0 value:a, b, c, d number:1 value:true ", + Pos: 1, + }, + }, + { + Expression: []string{ + `$match("a, b, c, d", "ab")`, + }, + Error: &ArgTypeError{ + Func: "match", + Which: 2, + Arguments: "number:0 value:a, b, c, d number:1 value:ab ", + Pos: 1, }, }, { @@ -7275,8 +7339,10 @@ func TestRegexMatch(t *testing.T) { `$match("a, b, c, d", /ab/, "2")`, }, Error: &ArgTypeError{ - Func: "match", - Which: 3, + Func: "match", + Which: 3, + Arguments: "number:0 value:a, b, c, d number:1 value:&{{{0 0} ab}", + Pos: 1, }, }, { @@ -7612,6 +7678,11 @@ func TestFuncMillis2(t *testing.T) { } func TestFuncToMillis(t *testing.T) { + defer func() { // added this to help with the test as it panics and that is annoying + if r := recover(); r != nil { + fmt.Println("Recovered in f", r) + } + }() runTestCases(t, nil, []*testCase{ { @@ -7628,7 +7699,7 @@ func TestFuncToMillis(t *testing.T) { }, { Expression: `$toMillis("foo")`, - Error: fmt.Errorf(`could not parse time "foo"`), + Error: fmt.Errorf(`could not parse time "foo" due to inconsistency in layout and date time string, date foo layout 2006`), }, }) } @@ -7828,8 +7899,10 @@ func TestLambdaSignatureViolations(t *testing.T) { { Expression: `λ($arg1, $arg2){[$arg1, $arg2]}(1,"2")`, Error: &ArgTypeError{ - Func: "lambda", - Which: 2, + Func: "lambda", + Which: 2, + Pos: 23, + Arguments: "number:0 value:1 number:1 value:2 ", }, }, { @@ -7843,29 +7916,37 @@ func TestLambdaSignatureViolations(t *testing.T) { { Expression: `λ($arg1, $arg2){[$arg1, $arg2]}(1,3, 2,"g")`, Error: &ArgTypeError{ - Func: "lambda", - Which: 4, + Func: "lambda", + Which: 4, + Pos: 24, + Arguments: "number:0 value:1 number:1 value:3 number:2 value:2 number:3 value:g ", }, }, { Expression: `λ($arr)>{$arr}(["3"]) `, Error: &ArgTypeError{ - Func: "lambda", - Which: 1, + Func: "lambda", + Which: 1, + Pos: 16, + Arguments: "number:0 value:[3] ", }, }, { Expression: `λ($arr)>{$arr}([1, 2, "3"]) `, Error: &ArgTypeError{ - Func: "lambda", - Which: 1, + Func: "lambda", + Which: 1, + Pos: 16, + Arguments: "number:0 value:[1 2 3] ", }, }, { Expression: `λ($arr)>{$arr}("f")`, Error: &ArgTypeError{ - Func: "lambda", - Which: 1, + Func: "lambda", + Which: 1, + Pos: 16, + Arguments: "number:0 value:[f] ", }, }, { @@ -7875,8 +7956,10 @@ func TestLambdaSignatureViolations(t *testing.T) { $fun("f") )`, Error: &ArgTypeError{ - Func: "fun", - Which: 1, + Func: "fun", + Which: 1, + Pos: 48, + Arguments: "number:0 value:[f] ", }, }, { @@ -7945,6 +8028,28 @@ func TestTransform(t *testing.T) { }) } +func TestUnhashableDistinct(t *testing.T) { + runTestCases(t, testdata.address, []*testCase{ + { + Expression: `$distinct([["a", "b"]])`, + Output: []interface{}{ + []interface{}{ + "a", + "b", + }, + }, + }, + { + Expression: `$distinct([{"a": "b"}])`, + Output: []interface{}{ + map[string]interface{}{ + "a": "b", + }, + }, + }, + }) +} + // Helper functions type compareFunc func(interface{}, interface{}) bool @@ -7991,8 +8096,8 @@ func runTestCase(t *testing.T, equal compareFunc, input interface{}, test *testC if !equal(output, test.Output) { t.Errorf("\nExpression: %s\nExp. Value: %v [%T]\nAct. Value: %v [%T]", exp, test.Output, test.Output, output, output) } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("\nExpression: %s\nExp. Error: %v [%T]\nAct. Error: %v [%T]", exp, test.Error, test.Error, err, err) + if err != nil && test.Error != nil { + assert.ErrorContains(t, err, test.Error.Error(), fmt.Sprintf("Exp. Value: %v", exp)) } } } diff --git a/processor.go b/processor.go new file mode 100644 index 0000000..374df58 --- /dev/null +++ b/processor.go @@ -0,0 +1,65 @@ +package jsonata + +import ( + "fmt" +) + +type JsonataProcessor struct { + tree *Expr +} + +func NewProcessor(jsonataString string) (j *JsonataProcessor, err error) { + defer func() { // go-jsonata uses panic fallthrough design so this is necessary + if r := recover(); r != nil { + err = fmt.Errorf("jsonata error: %v", r) + } + }() + + jsnt := replaceQuotesAndCommentsInPaths(jsonataString) + + e := MustCompile(jsnt) + + j = &JsonataProcessor{} + + j.tree = e + + return j, err +} + +// Execute - helper function that lets you parse and run jsonata scripts against an object +func (j *JsonataProcessor) Execute(input interface{}) (output []map[string]interface{}, err error) { + defer func() { // go-jsonata uses panic fallthrough design so this is necessary + if r := recover(); r != nil { + err = fmt.Errorf("jsonata error: %v", r) + } + }() + + output = make([]map[string]interface{}, 0) + + item, err := j.tree.Eval(input) + if err != nil { + return nil, err + } + + if aMap, ok := item.(map[string]interface{}); ok { + output = append(output, aMap) + + return output, nil + } + + if aList, ok := item.([]interface{}); ok { + for index := range aList { + if aMap, ok := aList[index].(map[string]interface{}); ok { + output = append(output, aMap) + } + } + + return output, nil + } + + if aList, ok := item.([]map[string]interface{}); ok { + return aList, nil + } + + return output, nil +}