Skip to content

Commit 1c227fb

Browse files
author
Dean Karn
authored
Additions (#12)
1 parent a3b76df commit 1c227fb

File tree

10 files changed

+272
-120
lines changed

10 files changed

+272
-120
lines changed

‎.github/workflows/workflow.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,20 @@ jobs:
88
test:
99
strategy:
1010
matrix:
11-
go-version: [1.18.x]
11+
go-version: [1.19.x]
1212
os: [ubuntu-latest, macos-latest, windows-latest]
1313
runs-on: ${{ matrix.os }}
1414
steps:
1515
- name: Install Go
16-
uses: actions/setup-go@v2
16+
uses: actions/setup-go@v3
1717
with:
1818
go-version: ${{ matrix.go-version }}
1919

2020
- name: Checkout code
21-
uses: actions/checkout@v2
21+
uses: actions/checkout@v3
2222

2323
- name: Restore Cache
24-
uses: actions/cache@v2
24+
uses: actions/cache@v3
2525
with:
2626
path: ~/go/pkg/mod
2727
key: ${{ runner.os }}-v1-go-${{ hashFiles('**/go.sum') }}
@@ -32,7 +32,7 @@ jobs:
3232
run: go test -race -covermode=atomic -coverprofile="profile.cov" ./...
3333

3434
- name: Send Coverage
35-
if: matrix.os == 'ubuntu-latest' && matrix.go-version == '1.18.x'
35+
if: matrix.os == 'ubuntu-latest' && matrix.go-version == '1.19.x'
3636
uses: shogo82148/actions-goveralls@v1
3737
with:
3838
path-to-profile: profile.cov
@@ -41,8 +41,8 @@ jobs:
4141
name: lint
4242
runs-on: ubuntu-latest
4343
steps:
44-
- uses: actions/checkout@v2
44+
- uses: actions/checkout@v3
4545
- name: golangci-lint
4646
uses: golangci/golangci-lint-action@v2
4747
with:
48-
version: v1.47.0
48+
version: latest

‎CHANGELOG.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88

9+
## [0.6.0] - 2023-01-05
10+
### Added
11+
- Added new `_uppercase_` & `_title_` COERCE identifiers.
12+
- Added ability to use multiple COERCE identifiers at once separated by a comma.
13+
- Added CLI ability to return original data if using an expression that returns a boolean.
14+
915
## [0.5.1] - 2022-10-18
1016
### Fixed
1117
- Fixed CONTAINS_ANY for string contains comparisons with slice/array.
@@ -52,7 +58,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
5258
### Added
5359
- Initial conversion from https://github.com/rust-playground/ksql.
5460

55-
[Unreleased]: https://github.com/go-playground/ksql/compare/v0.5.1...HEAD
61+
[Unreleased]: https://github.com/go-playground/ksql/compare/v0.6.0...HEAD
62+
[0.6.0]: https://github.com/go-playground/ksql/compare/v0.5.1...v0.6.0
5663
[0.5.1]: https://github.com/go-playground/ksql/compare/v0.5.0...v0.5.1
5764
[0.5.0]: https://github.com/go-playground/ksql/compare/v0.4.0...v0.5.0
5865
[0.4.0]: https://github.com/go-playground/ksql/compare/v0.3.2...v0.4.0

‎README.md‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ ksql
77
**Is a JSON data expression lexer, parser, cli and library.**
88

99
#### Requirements
10-
- Go 1.18+
10+
- Go 1.19+
1111

1212
#### How to install CLI
1313
```shell
@@ -85,14 +85,16 @@ Expressions support most mathematical and string expressions see below for detai
8585
| `EndsWith` | `ENDSWITH ` | Ends with whitespace blank space. |
8686
| `NULL` | `NULL` | N/A |
8787
| `Coerce` | `COERCE` | Coerces one data type into another using in combination with 'Identifier'. Syntax is `COERCE <expression> _identifer_`. |
88-
| `Identifier` | `_identifier_` | Starts and end with an `_` used with 'COERCE' to cast data types. see below for options. |
88+
| `Identifier` | `_identifier_` | Starts and end with an `_` used with 'COERCE' to cast data types, see table below with supported values. You can combine multiple coercions if separated by a COMMA. |
8989

9090
#### COERCE Types
9191

92-
| Type | Description |
93-
|---------------|----------------------------------------------------|
94-
| `_datetime_` | This attempts to convert the type into a DateTime. |
95-
| `_lowercase_` | This converts the text into lowercase. |
92+
| Type | Description |
93+
|---------------|--------------------------------------------------------------------------------------------------------|
94+
| `_datetime_` | This attempts to convert the type into a DateTime. |
95+
| `_lowercase_` | This converts the text into lowercase. |
96+
| `_uppercase_` | This converts the text into uppercase. |
97+
| `_title_` | This converts the text into title case, when the first letter is capitalized but the rest lower cased. |
9698

9799
#### License
98100

‎cmd/ksql/main.go‎

Lines changed: 67 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"bufio"
55
"encoding/json"
6+
"flag"
67
"fmt"
78
"os"
89

@@ -11,35 +12,62 @@ import (
1112
)
1213

1314
func main() {
14-
args := os.Args[1:]
15+
16+
var outputOriginal bool
17+
flag.BoolVar(&outputOriginal, "o", false, "Indicates if the original data will be output after applying the expression. The results of the expression MUST be a boolean otherwise the output will be ignored.")
18+
flag.Usage = usage
19+
flag.Parse()
20+
1521
isPipe := isInputFromPipe()
16-
if (len(args) < 2 && !isPipe) || (len(args) < 1 && isPipe) {
17-
usage()
22+
if (flag.NArg() < 2 && !isPipe) || (flag.NArg() < 1 && isPipe) {
23+
flag.Usage()
1824
return
1925
}
2026

21-
ex, err := ksql.Parse([]byte(args[0]))
27+
ex, err := ksql.Parse([]byte(flag.Arg(0)))
2228
if err != nil {
23-
usage()
29+
flag.Usage()
2430
return
2531
}
2632

2733
var input []byte
34+
w := bufio.NewWriter(os.Stdout)
2835

2936
if isPipe {
30-
w := bufio.NewWriter(os.Stdout)
31-
enc := json.NewEncoder(w)
3237
scanner := bufio.NewScanner(os.Stdin)
3338
scanner.Buffer(make([]byte, 0, 200*bytesext.KiB), 5*bytesext.MiB)
34-
for scanner.Scan() {
35-
result, err := ex.Calculate(scanner.Bytes())
36-
if err != nil {
37-
fmt.Fprintln(os.Stderr, "reading standard input:", err)
38-
return
39+
40+
if outputOriginal {
41+
for scanner.Scan() {
42+
input := scanner.Bytes()
43+
result, err := ex.Calculate(input)
44+
if err != nil {
45+
fmt.Fprintln(os.Stderr, "reading standard input:", err)
46+
return
47+
}
48+
if result, ok := result.(bool); ok && result {
49+
_, err := w.Write(input)
50+
if err != nil {
51+
fmt.Fprintln(os.Stderr, "writing standard output:", err)
52+
}
53+
err = w.WriteByte('\n')
54+
if err != nil {
55+
fmt.Fprintln(os.Stderr, "writing standard output:", err)
56+
}
57+
}
3958
}
40-
if err := enc.Encode(result); err != nil {
41-
fmt.Fprintln(os.Stderr, "encoding result to standard output:", err)
42-
return
59+
} else {
60+
enc := json.NewEncoder(w)
61+
for scanner.Scan() {
62+
result, err := ex.Calculate(scanner.Bytes())
63+
if err != nil {
64+
fmt.Fprintln(os.Stderr, "reading standard input:", err)
65+
return
66+
}
67+
if err := enc.Encode(result); err != nil {
68+
fmt.Fprintln(os.Stderr, "encoding result to standard output:", err)
69+
return
70+
}
4371
}
4472
}
4573
if err := scanner.Err(); err != nil {
@@ -49,24 +77,39 @@ func main() {
4977
fmt.Fprintln(os.Stderr, "writing standard output:", err)
5078
}
5179
} else {
52-
input = []byte(args[1])
80+
input = []byte(flag.Arg(1))
5381
result, err := ex.Calculate(input)
5482
if err != nil {
55-
usage()
83+
flag.Usage()
5684
return
5785
}
58-
enc := json.NewEncoder(os.Stderr)
59-
if err := enc.Encode(result); err != nil {
60-
fmt.Fprintln(os.Stderr, "encoding result to standard output:", err)
61-
return
86+
if outputOriginal {
87+
if result, ok := result.(bool); ok && result {
88+
_, err := w.Write(input)
89+
if err != nil {
90+
fmt.Fprintln(os.Stderr, "writing standard output:", err)
91+
}
92+
err = w.WriteByte('\n')
93+
if err != nil {
94+
fmt.Fprintln(os.Stderr, "writing standard output:", err)
95+
}
96+
}
97+
} else {
98+
enc := json.NewEncoder(w)
99+
if err := enc.Encode(result); err != nil {
100+
fmt.Fprintln(os.Stderr, "encoding result to standard output:", err)
101+
return
102+
}
103+
}
104+
if err = w.Flush(); err != nil {
105+
fmt.Fprintln(os.Stderr, "writing standard output:", err)
62106
}
63107
}
64108
}
65109

66110
func usage() {
67-
fmt.Println("ksql <expression> <json>")
68-
fmt.Println("or")
69-
fmt.Println("echo '{{}}' | ksql <expression> -")
111+
fmt.Println("ksql [OPTIONS] <EXPRESSION> [DATA]")
112+
flag.PrintDefaults()
70113
}
71114

72115
func isInputFromPipe() bool {

‎go.mod‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,16 @@ go 1.18
44

55
require (
66
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
7-
github.com/go-playground/pkg/v5 v5.5.0
8-
github.com/stretchr/testify v1.7.1
9-
github.com/tidwall/gjson v1.14.0
7+
github.com/go-playground/pkg/v5 v5.11.0
8+
github.com/stretchr/testify v1.8.1
9+
github.com/tidwall/gjson v1.14.4
1010
)
1111

1212
require (
13-
github.com/davecgh/go-spew v1.1.0 // indirect
13+
github.com/davecgh/go-spew v1.1.1 // indirect
14+
github.com/go-playground/itertools v0.1.0 // indirect
1415
github.com/pmezard/go-difflib v1.0.0 // indirect
1516
github.com/tidwall/match v1.1.1 // indirect
1617
github.com/tidwall/pretty v1.2.0 // indirect
17-
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
18+
gopkg.in/yaml.v3 v3.0.1 // indirect
1819
)

‎go.sum‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,33 @@
11
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
22
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
3-
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
43
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5-
github.com/go-playground/pkg/v5 v5.5.0 h1:I+2yXV/GSH4MB/s2RXIlRnWZZdSgttS+MnAH4yRt8sE=
6-
github.com/go-playground/pkg/v5 v5.5.0/go.mod h1:TvZ2nNtNh6VfoNteY9ApA2BXt1ZwJliFZ4hzPAwLS9Y=
4+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6+
github.com/go-playground/itertools v0.1.0 h1:isiUTLIViAz4J3qWrowvRoOYWSXy2ubkXKNJfujE3Sc=
7+
github.com/go-playground/itertools v0.1.0/go.mod h1:+TD1WVpn32jr+GpvO+nnb2xXD45SSzt18Fo/C0y9PJE=
8+
github.com/go-playground/pkg/v5 v5.11.0 h1:yYYmh0RLKBxiLbEwO7HZtp8XVDDuyIqHmhtrmuO0nsg=
9+
github.com/go-playground/pkg/v5 v5.11.0/go.mod h1:eT8XZeFHnqZkfkpkbI8ayjfCw9GohV2/j8STbVmoR6s=
710
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
811
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
912
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
1013
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
1114
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
1215
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
16+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
17+
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
1318
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
14-
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
1519
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
16-
github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
17-
github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
20+
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
21+
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
22+
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
23+
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
24+
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
1825
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
1926
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
2027
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
2128
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
2229
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
2330
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
24-
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
2531
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
32+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
33+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎lexer.go‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
package ksql
22

33
import (
4-
"io"
4+
optionext "github.com/go-playground/pkg/v5/values/option"
5+
resultext "github.com/go-playground/pkg/v5/values/result"
56
)
67

78
// Token represents a lexed token
@@ -55,7 +56,7 @@ const (
5556
Identifier
5657
)
5758

58-
/// Try to lex a single token from the input stream.
59+
// / Try to lex a single token from the input stream.
5960
func tokenizeSingleToken(data []byte) (result LexerResult, err error) {
6061
b := data[0]
6162

@@ -167,7 +168,7 @@ func tokenizeSingleToken(data []byte) (result LexerResult, err error) {
167168

168169
func tokenizeIdentifier(data []byte) (result LexerResult, err error) {
169170
end := takeWhile(data, func(b byte) bool {
170-
return !isWhitespace(b) && b != ')' && b != ']'
171+
return !isWhitespace(b) && b != ')' && b != ']' && b != ','
171172
})
172173
// identifier must start and end with underscore
173174
if end > 0 && data[end-1] == '_' {
@@ -327,7 +328,7 @@ func tokenizeString(data []byte, quote byte) (result LexerResult, err error) {
327328
return
328329
}
329330

330-
/// Consumes bytes while a predicate evaluates to true.
331+
// / Consumes bytes while a predicate evaluates to true.
331332
func takeWhile(data []byte, pred func(byte) bool) (end uint16) {
332333
for _, b := range data {
333334
if !pred(b) {
@@ -358,12 +359,11 @@ func NewTokenizer(src []byte) *Tokenizer {
358359
}
359360
}
360361

361-
func (t *Tokenizer) Next() (token Token, err error) {
362+
func (t *Tokenizer) Next() optionext.Option[resultext.Result[Token, error]] {
362363
t.skipWhitespace()
363364

364365
if len(t.remaining) == 0 {
365-
err = io.EOF
366-
return
366+
return optionext.None[resultext.Result[Token, error]]()
367367
}
368368
return t.nextToken()
369369
}
@@ -373,19 +373,18 @@ func (t *Tokenizer) skipWhitespace() {
373373
t.chomp(skipped)
374374
}
375375

376-
func (t *Tokenizer) nextToken() (token Token, err error) {
377-
var result LexerResult
378-
result, err = tokenizeSingleToken(t.remaining)
376+
func (t *Tokenizer) nextToken() optionext.Option[resultext.Result[Token, error]] {
377+
result, err := tokenizeSingleToken(t.remaining)
379378
if err != nil {
380-
return
379+
return optionext.Some(resultext.Err[Token, error](err))
381380
}
382-
token = Token{
381+
token := Token{
383382
start: t.pos,
384383
len: result.len,
385384
kind: result.kind,
386385
}
387386
t.chomp(result.len)
388-
return
387+
return optionext.Some(resultext.Ok[Token, error](token))
389388
}
390389

391390
func (t *Tokenizer) chomp(num uint16) {

0 commit comments

Comments
 (0)