Skip to content

Commit 9f6f545

Browse files
whkiwhki
committed
feat:支持数组取值
1 parent 1e1282a commit 9f6f545

File tree

11 files changed

+262
-27
lines changed

11 files changed

+262
-27
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ English| [简体中文](README_ZH.md)
1818
- Pure in-memory operations
1919
- No dependencies
2020
- Data processing with SQL syntax
21-
- **Nested field access**: Support dot notation syntax (`device.info.name`) for accessing nested structured data
21+
- **Nested field access**: Support dot notation syntax (`device.info.name`) and array indexing (`sensors[0].value`) for accessing nested structured data
2222
- Data analysis
2323
- Built-in multiple window types: sliding window, tumbling window, counting window
2424
- Built-in aggregate functions: MAX, MIN, AVG, SUM, STDDEV, MEDIAN, PERCENTILE, etc.

‎README_ZH.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
- 纯内存操作
1919
- 无依赖
2020
- SQL语法处理数据
21-
- **嵌套字段访问**：支持点号语法（`device.info.name`）访问嵌套结构数据
21+
- **嵌套字段访问**：支持点号语法（`device.info.name`）和数组索引（`sensors[0].value`）访问嵌套结构数据
2222
- 数据分析
2323
- 内置多种窗口类型：滑动窗口、滚动窗口、计数窗口
2424
- 内置聚合函数：MAX, MIN, AVG, SUM, STDDEV,MEDIAN,PERCENTILE等

‎expr/expression.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,8 @@ func isValidChar(ch rune) bool {
183183
return true
184184
case '`': // Backtick (for identifiers)
185185
return true
186+
case '[', ']': // Brackets (for array/map access)
187+
return true
186188
default:
187189
return false
188190
}

‎expr/tokenizer.go‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,23 @@ func tokenize(expr string) ([]string, error) {
157157
}
158158

159159
// Handle identifiers and keywords
160-
if isLetter(expr[i]) || expr[i] == '_' || expr[i] == '$' {
160+
if isLetter(expr[i]) || expr[i] == '_' || expr[i] == '$' || expr[i] == '[' {
161161
start := i
162-
for i < len(expr) && (isLetter(expr[i]) || isDigit(expr[i]) || expr[i] == '_' || expr[i] == '.' || expr[i] == '$') {
163-
i++
162+
for i < len(expr) {
163+
if isLetter(expr[i]) || isDigit(expr[i]) || expr[i] == '_' || expr[i] == '.' || expr[i] == '$' {
164+
i++
165+
} else if expr[i] == '[' {
166+
// Consume until ]
167+
i++ // skip [
168+
for i < len(expr) && expr[i] != ']' {
169+
i++
170+
}
171+
if i < len(expr) {
172+
i++ // skip ]
173+
}
174+
} else {
175+
break
176+
}
164177
}
165178
tokens = append(tokens, expr[start:i])
166179
continue
@@ -236,9 +249,9 @@ func isIdentifier(s string) bool {
236249
return false
237250
}
238251

239-
// Remaining characters can be letters, digits, or underscores
252+
// Remaining characters can be letters, digits, underscores, dots, or brackets
240253
for i := 1; i < len(s); i++ {
241-
if !isLetter(s[i]) && !isDigit(s[i]) && s[i] != '_' {
254+
if !isLetter(s[i]) && !isDigit(s[i]) && s[i] != '_' && s[i] != '.' && s[i] != '[' && s[i] != ']' && s[i] != '\'' && s[i] != '"' && s[i] != '$' {
242255
return false
243256
}
244257
}

‎expr/tokenizer_custom_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package expr
2+
3+
import (
4+
"testing"
5+
6+
"github.com/stretchr/testify/assert"
7+
)
8+
9+
// TestTokenizeWithArrayAccess 测试包含数组访问的分词
10+
func TestTokenizeWithArrayAccess(t *testing.T) {
11+
tests := []struct {
12+
name string
13+
expr string
14+
expected []string
15+
wantErr bool
16+
}{
17+
{"数组访问", "sensors[0].temperature", []string{"sensors[0].temperature"}, false},
18+
{"字符串键访问", "config['key']", []string{"config['key']"}, false},
19+
{"混合访问", "data[0].items['key'].value", []string{"data[0].items['key'].value"}, false},
20+
{"表达式中的数组访问", "a[0] + b[1]", []string{"a[0]", "+", "b[1]"}, false},
21+
{"函数中的数组访问", "AVG(sensors[0].temperature)", []string{"AVG", "(", "sensors[0].temperature", ")"}, false},
22+
}
23+
24+
for _, tt := range tests {
25+
t.Run(tt.name, func(t *testing.T) {
26+
got, err := tokenize(tt.expr)
27+
if tt.wantErr {
28+
assert.Error(t, err)
29+
return
30+
}
31+
assert.NoError(t, err)
32+
assert.Equal(t, tt.expected, got)
33+
})
34+
}
35+
}

‎expr/tokenizer_test.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,9 @@ func TestIsIdentifier(t *testing.T) {
169169
{"123abc", false},
170170
{"", false},
171171
{"var-name", false},
172-
{"var.name", false},
172+
{"var.name", true},
173+
{"var[0]", true},
174+
{"var['key']", true},
173175
{"var name", false},
174176
}
175177

‎rsql/coverage_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -969,7 +969,7 @@ func TestParserComplexFieldAccess(t *testing.T) {
969969
{
970970
name: "混合访问表达式",
971971
query: "SELECT field.nested[0].deep FROM table",
972-
expectError: true, // lexer不支持点号在表达式中
972+
expectError: false, // lexer现已支持点号在表达式中
973973
},
974974
{
975975
name: "标识符数组索引",

‎rsql/lexer.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ const (
6161
// 数组索引相关token
6262
TokenLBracket
6363
TokenRBracket
64+
// 点号token
65+
TokenDot
6466
)
6567

6668
type Token struct {
@@ -127,6 +129,9 @@ func (l *Lexer) NextToken() Token {
127129
case ']':
128130
l.readChar()
129131
return Token{Type: TokenRBracket, Value: "]", Pos: tokenPos, Line: tokenLine, Column: tokenColumn}
132+
case '.':
133+
l.readChar()
134+
return Token{Type: TokenDot, Value: ".", Pos: tokenPos, Line: tokenLine, Column: tokenColumn}
130135
case '+':
131136
l.readChar()
132137
return Token{Type: TokenPlus, Value: "+", Pos: tokenPos, Line: tokenLine, Column: tokenColumn}

‎rsql/parser.go‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ var tokenTypeNames = map[TokenType]string{
3131
TokenComma: ",",
3232
TokenLParen: "(",
3333
TokenRParen: ")",
34+
TokenDot: ".",
3435
TokenIdent: "identifier",
3536
TokenQuotedIdent: "quoted identifier",
3637
TokenNumber: "number",
@@ -358,11 +359,16 @@ func (p *Parser) parseSelect(stmt *SelectStatement) error {
358359
// 3. 数字和标识符之间
359360
// 4. 左括号之后
360361
// 5. 右括号之前
361-
if currentToken.Type == TokenLParen && lastChar != " " && lastChar != "(" {
362-
// 函数名和左括号之间不加空格
362+
// 6. 数组索引相关：[ 前，[ 后，] 前
363+
// 7. 点号前后
364+
if (currentToken.Type == TokenLParen || currentToken.Type == TokenLBracket) && lastChar != " " && lastChar != "(" && lastChar != "[" {
365+
// 函数名/数组名和左括号/左中括号之间不加空格
363366
shouldAddSpace = false
364-
} else if lastChar == "(" || currentToken.Type == TokenRParen {
365-
// 左括号之后或右括号之前不加空格
367+
} else if lastChar == "(" || lastChar == "[" || currentToken.Type == TokenRParen || currentToken.Type == TokenRBracket {
368+
// 左括号/左中括号之后或右括号/右中括号之前不加空格
369+
shouldAddSpace = false
370+
} else if currentToken.Type == TokenDot || lastChar == "." {
371+
// 点号前后不加空格
366372
shouldAddSpace = false
367373
} else if len(exprStr) > 0 && currentToken.Type == TokenNumber {
368374
// 检查前一个字符是否是字母（标识符的一部分），且前面没有空格

‎stream/processor_data_test.go‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ func TestDataProcessor_EvaluateNestedFieldExpression(t *testing.T) {
215215
data: map[string]interface{}{
216216
"device": map[string]interface{}{"id": 123},
217217
},
218-
expected: 123.0,
218+
expected: 123, // Expect int because EvaluateValueWithNull returns original type
219219
hasError: false,
220220
},
221221
{
@@ -244,7 +244,7 @@ func TestDataProcessor_EvaluateNestedFieldExpression(t *testing.T) {
244244
data: map[string]interface{}{
245245
"device": map[string]interface{}{"id": 456},
246246
},
247-
expected: 456.0,
247+
expected: 456, // Expect int because EvaluateValueWithNull returns original type
248248
hasError: false,
249249
},
250250
}
@@ -434,14 +434,14 @@ func TestDataProcessor_ExpressionWithNullValues(t *testing.T) {
434434
// TestDataProcessor_ExpandUnnestResults 测试 expandUnnestResults 函数的各种情况
435435
func TestDataProcessor_ExpandUnnestResults(t *testing.T) {
436436
tests := []struct {
437-
name string
437+
name string
438438
hasUnnestFunction bool
439-
result map[string]interface{}
440-
originalData map[string]interface{}
441-
expected []map[string]interface{}
439+
result map[string]interface{}
440+
originalData map[string]interface{}
441+
expected []map[string]interface{}
442442
}{
443443
{
444-
name: "no unnest function - should return single result",
444+
name: "no unnest function - should return single result",
445445
hasUnnestFunction: false,
446446
result: map[string]interface{}{
447447
"name": "test",
@@ -453,16 +453,16 @@ func TestDataProcessor_ExpandUnnestResults(t *testing.T) {
453453
},
454454
},
455455
{
456-
name: "empty result - should return single empty result",
456+
name: "empty result - should return single empty result",
457457
hasUnnestFunction: true,
458-
result: map[string]interface{}{},
459-
originalData: map[string]interface{}{"id": 1},
458+
result: map[string]interface{}{},
459+
originalData: map[string]interface{}{"id": 1},
460460
expected: []map[string]interface{}{
461461
{},
462462
},
463463
},
464464
{
465-
name: "no unnest result - should return single result",
465+
name: "no unnest result - should return single result",
466466
hasUnnestFunction: true,
467467
result: map[string]interface{}{
468468
"name": "test",
@@ -474,7 +474,7 @@ func TestDataProcessor_ExpandUnnestResults(t *testing.T) {
474474
},
475475
},
476476
{
477-
name: "unnest result with simple values",
477+
name: "unnest result with simple values",
478478
hasUnnestFunction: true,
479479
result: map[string]interface{}{
480480
"name": "test",
@@ -496,7 +496,7 @@ func TestDataProcessor_ExpandUnnestResults(t *testing.T) {
496496
},
497497
},
498498
{
499-
name: "unnest result with object values",
499+
name: "unnest result with object values",
500500
hasUnnestFunction: true,
501501
result: map[string]interface{}{
502502
"name": "test",
@@ -524,7 +524,7 @@ func TestDataProcessor_ExpandUnnestResults(t *testing.T) {
524524
},
525525
},
526526
{
527-
name: "empty unnest result - should return empty array",
527+
name: "empty unnest result - should return empty array",
528528
hasUnnestFunction: true,
529529
result: map[string]interface{}{
530530
"name": "test",

0 commit comments

Comments
 (0)