From 1744502678683289c890b70480c82d13df51be7f Mon Sep 17 00:00:00 2001 From: rulego-team Date: Fri, 14 Nov 2025 18:46:35 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat:=E5=A2=9E=E5=8A=A0=E4=BA=8B=E4=BB=B6?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E7=AA=97=E5=8F=A3=E5=A4=84=E7=90=86=E6=9C=BA?= =?UTF-8?q?=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 104 ++- README_ZH.md | 105 ++- doc.go | 73 ++ rsql/ast.go | 32 +- rsql/function_validator.go | 2 +- rsql/lexer.go | 9 + rsql/parser.go | 89 +- stream/processor_data.go | 5 + streamsql_case_test.go | 97 +- streamsql_counting_window_test.go | 18 +- streamsql_post_aggregation_test.go | 142 ++- streamsql_session_window_test.go | 525 ++++++++++- streamsql_sliding_window_test.go | 991 +++++++++++++++++++- streamsql_test.go | 446 ++++----- streamsql_tumbling_window_test.go | 1345 ++++++++++++++++++++++++++++ types/config.go | 31 +- window/counting_window.go | 6 + window/doc.go | 119 ++- window/session_window.go | 278 +++++- window/sliding_window.go | 370 +++++++- window/tumbling_window.go | 392 +++++++- window/watermark.go | 184 ++++ 22 files changed, 4871 insertions(+), 492 deletions(-) create mode 100644 streamsql_tumbling_window_test.go create mode 100644 window/watermark.go diff --git a/README.md b/README.md index 3f8e028..901da7d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ func main() { // Step 1: Create StreamSQL Instance // StreamSQL is the core component of the stream SQL processing engine, managing the entire stream processing lifecycle ssql := streamsql.New() - + defer ssql.Stop() // Step 2: Define Stream SQL Query Statement // This SQL statement showcases StreamSQL's core capabilities: // - SELECT: Choose output fields and aggregation functions @@ -267,8 +267,7 @@ func main() { window_end() as end FROM stream WHERE device.info.type = 'temperature' - GROUP BY device.location, TumblingWindow('5s') - WITH (TIMESTAMP='timestamp', TIMEUNIT='ss')` + GROUP BY device.location, TumblingWindow('5s')` err := ssql.Execute(rsql) if err != nil { @@ -335,6 +334,11 @@ Since stream data is unbounded, it cannot be processed as a whole. Windows provi - **Characteristics**: The size of the window is not related to time but is divided based on the volume of data. It is suitable for segmenting data based on the amount of data. - **Application Scenario**: In industrial IoT, an aggregation calculation is performed every time 100 device status data records are processed. +- **Session Window** + - **Definition**: A dynamic window based on data activity. When the interval between data exceeds a specified timeout, the current session ends and triggers the window. + - **Characteristics**: Window size changes dynamically, automatically dividing sessions based on data arrival intervals. When data arrives continuously, the session continues; when the data interval exceeds the timeout, the session ends and triggers the window. + - **Application Scenario**: In user behavior analysis, maintain a session when users operate continuously, and close the session and count operations within that session when users stop operating for more than 5 minutes. + ### Stream - **Definition**: A continuous sequence of data that is generated in an unbounded manner, typically from sensors, log systems, user behaviors, etc. @@ -343,17 +347,97 @@ Since stream data is unbounded, it cannot be processed as a whole. Windows provi ### Time Semantics -- **Event Time** - - **Definition**: The actual time when the data occurred, usually represented by a timestamp generated by the data source. - -- **Processing Time** - - **Definition**: The time when the data arrives at the processing system. +StreamSQL supports two time concepts that determine how windows are divided and triggered: + +#### Event Time + +- **Definition**: Event time refers to the actual time when data was generated, usually recorded in a field within the data itself (such as `event_time`, `timestamp`, `order_time`, etc.). +- **Characteristics**: + - Windows are divided based on timestamp field values in the data + - Even if data arrives late, it can be correctly counted into the corresponding window based on event time + - Uses Watermark mechanism to handle out-of-order and late data + - Results are accurate but may have delays (need to wait for late data) +- **Use Cases**: + - Scenarios requiring precise temporal analysis + - Scenarios where data may arrive out of order or delayed + - Historical data replay and analysis +- **Configuration**: Use `WITH (TIMESTAMP='field_name', TIMEUNIT='ms')` to specify the event time field +- **Example**: + ```sql + SELECT deviceId, COUNT(*) as cnt + FROM stream + GROUP BY deviceId, TumblingWindow('1m') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ``` + +#### Processing Time + +- **Definition**: Processing time refers to the time when data arrives at the StreamSQL processing system, i.e., the current time when the system receives the data. +- **Characteristics**: + - Windows are divided based on the time data arrives at the system (`time.Now()`) + - Regardless of the time field value in the data, it is counted into the current window based on arrival time + - Uses system clock (Timer) to trigger windows + - Low latency but results may be inaccurate (cannot handle out-of-order and late data) +- **Use Cases**: + - Real-time monitoring and alerting scenarios + - Scenarios with high latency requirements and relatively low accuracy requirements + - Scenarios where data arrives in order and delay is controllable +- **Configuration**: Default when `WITH (TIMESTAMP=...)` is not specified +- **Example**: + ```sql + SELECT deviceId, COUNT(*) as cnt + FROM stream + GROUP BY deviceId, TumblingWindow('1m') + -- No WITH clause specified, defaults to processing time + ``` + +#### Event Time vs Processing Time Comparison + +| Feature | Event Time | Processing Time | +|---------|------------|-----------------| +| **Time Source** | Timestamp field in data | System current time | +| **Window Division** | Based on event timestamp | Based on data arrival time | +| **Late Data Handling** | Supported (Watermark mechanism) | Not supported | +| **Out-of-Order Handling** | Supported (Watermark mechanism) | Not supported | +| **Result Accuracy** | Accurate | May be inaccurate | +| **Processing Latency** | Higher (need to wait for late data) | Low (real-time trigger) | +| **Configuration** | `WITH (TIMESTAMP='field')` | Default (no WITH clause) | +| **Use Cases** | Precise temporal analysis, historical replay | Real-time monitoring, low latency requirements | + +#### Window Time - **Window Start Time** - - **Definition**: The starting time point of the window based on event time. For example, for a sliding window based on event time, the window start time is the timestamp of the earliest event within the window. + - **Event Time Windows**: The starting time point of the window, aligned to window boundaries based on event time (e.g., aligned to minute or hour boundaries). + - **Processing Time Windows**: The starting time point of the window, based on the time data arrives at the system. + - **Example**: For an event-time-based tumbling window `TumblingWindow('5m')`, the window start time aligns to multiples of 5 minutes (e.g., 10:00, 10:05, 10:10). - **Window End Time** - - **Definition**: The ending time point of the window based on event time. Typically, the window end time is the window start time plus the duration of the window. For example, if the duration of a sliding window is 1 minute, then the window end time is the window start time plus 1 minute. + - **Event Time Windows**: The ending time point of the window, usually the window start time plus the window duration. Windows trigger when `watermark >= window_end`. + - **Processing Time Windows**: The ending time point of the window, based on the time data arrives at the system plus the window duration. Windows trigger when the system clock reaches the end time. + - **Example**: For a tumbling window with a duration of 1 minute, if the window start time is 10:00, then the window end time is 10:01. + +#### Watermark Mechanism (Event Time Windows Only) + +- **Definition**: Watermark indicates "events with timestamps less than this time should not arrive anymore", used to determine when windows can trigger. +- **Calculation Formula**: `Watermark = max(event_time) - MaxOutOfOrderness` +- **Window Trigger Condition**: Windows trigger when `watermark >= window_end` +- **Configuration Parameters**: + - `MAXOUTOFORDERNESS`: Maximum allowed out-of-order time for tolerating data disorder (default: 0, no out-of-order allowed) + - `ALLOWEDLATENESS`: Time window can accept late data after triggering (default: 0, no late data accepted) + - `IDLETIMEOUT`: Timeout for advancing Watermark based on processing time when data source is idle (default: 0, disabled) +- **Example**: + ```sql + SELECT deviceId, COUNT(*) as cnt + FROM stream + GROUP BY deviceId, TumblingWindow('5m') + WITH ( + TIMESTAMP='eventTime', + TIMEUNIT='ms', + MAXOUTOFORDERNESS='5s', -- Tolerate 5 seconds of out-of-order + ALLOWEDLATENESS='2s', -- Accept 2 seconds of late data after window triggers + IDLETIMEOUT='5s' -- Advance watermark based on processing time after 5s of no data + ) + ``` ## Contribution Guidelines diff --git a/README_ZH.md b/README_ZH.md index 19257fb..37015fa 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -151,7 +151,7 @@ import ( func main() { // 1. 创建StreamSQL实例 - 这是流式SQL处理引擎的入口 ssql := streamsql.New() - + defer ssql.Stop() // 2. 定义流式SQL查询语句 // 核心概念解析： // - TumblingWindow('5s'): 滚动窗口，每5秒创建一个新窗口，窗口之间不重叠 @@ -285,8 +285,7 @@ func main() { window_end() as end FROM stream WHERE device.info.type = 'temperature' - GROUP BY device.location, TumblingWindow('5s') - WITH (TIMESTAMP='timestamp', TIMEUNIT='ss')` + GROUP BY device.location, TumblingWindow('5s')` err := ssql.Execute(rsql) if err != nil { @@ -347,6 +346,11 @@ StreamSQL 支持多种函数类型，包括数学、字符串、转换、聚合 - **特点**：窗口的大小与时间无关，而是根据数据量来划分，适合对数据量进行分段处理。 - **应用场景**：在工业物联网中，每处理 100 条设备状态数据后进行一次聚合计算。 +- **会话窗口（Session Window）** + - **定义**：基于数据活跃度的动态窗口，当数据之间的间隔超过指定的超时时间时，当前会话结束并触发窗口。 + - **特点**：窗口大小动态变化，根据数据到达的间隔自动划分会话。当数据连续到达时，会话持续；当数据间隔超过超时时间时，会话结束并触发窗口。 + - **应用场景**：在用户行为分析中，当用户连续操作时保持会话，当用户停止操作超过 5 分钟后关闭会话并统计该会话内的操作次数。 + ### 流（Stream） - **定义**：流是数据的连续序列，数据以无界的方式产生，通常来自于传感器、日志系统、用户行为等。 @@ -355,16 +359,97 @@ StreamSQL 支持多种函数类型，包括数学、字符串、转换、聚合 ### 时间语义 -- **事件时间（Event Time）** - - **定义**：数据实际发生的时间，通常由数据源生成的时间戳表示。 +StreamSQL 支持两种时间概念，它们决定了窗口如何划分和触发： + +#### 事件时间（Event Time） + +- **定义**：事件时间是指数据实际产生的时间，通常记录在数据本身的某个字段中（如 `event_time`、`timestamp`、`order_time` 等）。 +- **特点**： + - 窗口基于数据中的时间戳字段值来划分 + - 即使数据延迟到达，也能根据事件时间正确统计到对应的窗口 + - 使用 Watermark 机制来处理乱序和延迟数据 + - 结果准确，但可能有延迟（需要等待延迟数据） +- **使用场景**： + - 需要精确时序分析的场景 + - 数据可能乱序或延迟到达的场景 + - 历史数据回放和分析 +- **配置方法**：使用 `WITH (TIMESTAMP='field_name', TIMEUNIT='ms')` 指定事件时间字段 +- **示例**： + ```sql + SELECT deviceId, COUNT(*) as cnt + FROM stream + GROUP BY deviceId, TumblingWindow('1m') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ``` + +#### 处理时间（Processing Time） + +- **定义**：处理时间是指数据到达 StreamSQL 处理系统的时间，即系统接收到数据时的当前时间。 +- **特点**： + - 窗口基于数据到达系统的时间（`time.Now()`）来划分 + - 不管数据中的时间字段是什么值，都按到达时间统计到当前窗口 + - 使用系统时钟（Timer）来触发窗口 + - 延迟低，但结果可能不准确（无法处理乱序和延迟数据） +- **使用场景**： + - 实时监控和告警场景 + - 对延迟要求高，对准确性要求相对较低的场景 + - 数据顺序到达且延迟可控的场景 +- **配置方法**：不指定 `WITH (TIMESTAMP=...)` 时，默认使用处理时间 +- **示例**： + ```sql + SELECT deviceId, COUNT(*) as cnt + FROM stream + GROUP BY deviceId, TumblingWindow('1m') + -- 不指定 WITH 子句，默认使用处理时间 + ``` + +#### 事件时间 vs 处理时间对比 + +| 特性 | 事件时间 (Event Time) | 处理时间 (Processing Time) | +|------|---------------------|-------------------------| +| **时间来源** | 数据中的时间戳字段 | 系统当前时间 | +| **窗口划分** | 基于事件时间戳 | 基于数据到达时间 | +| **延迟处理** | 支持（Watermark机制） | 不支持 | +| **乱序处理** | 支持（Watermark机制） | 不支持 | +| **结果准确性** | 准确 | 可能不准确 | +| **处理延迟** | 较高（需等待延迟数据） | 低（实时触发） | +| **配置方式** | `WITH (TIMESTAMP='field')` | 默认（不指定WITH） | +| **适用场景** | 精确时序分析、历史回放 | 实时监控、低延迟要求 | + +#### 窗口时间 -- **处理时间（Processing Time）** - - **定义**：数据到达处理系统的时间。 - **窗口开始时间（Window Start Time）** - - **定义**：基于事件时间，窗口的起始时间点。例如，对于一个基于事件时间的滑动窗口，窗口开始时间是窗口内最早事件的时间戳。 + - **事件时间窗口**：窗口的起始时间点，基于事件时间对齐到窗口边界（如对齐到分钟、小时的整点）。 + - **处理时间窗口**：窗口的起始时间点，基于数据到达系统的时间。 + - **示例**：对于一个基于事件时间的滚动窗口 `TumblingWindow('5m')`，窗口开始时间会对齐到5分钟的倍数（如 10:00、10:05、10:10）。 + - **窗口结束时间（Window End Time）** - - **定义**：基于事件时间，窗口的结束时间点。通常窗口结束时间是窗口开始时间加上窗口的持续时间。 - - 例如，一个滑动窗口的持续时间为 1 分钟，则窗口结束时间是窗口开始时间加上 1 分钟。 + - **事件时间窗口**：窗口的结束时间点，通常是窗口开始时间加上窗口的持续时间。窗口在 `watermark >= window_end` 时触发。 + - **处理时间窗口**：窗口的结束时间点，基于数据到达系统的时间加上窗口持续时间。窗口在系统时钟到达结束时间时触发。 + - **示例**：一个持续时间为 1 分钟的滚动窗口，如果窗口开始时间是 10:00，则窗口结束时间是 10:01。 + +#### Watermark 机制（仅事件时间窗口） + +- **定义**：Watermark 表示"小于该时间的事件不应该再到达"，用于判断窗口是否可以触发。 +- **计算公式**：`Watermark = max(event_time) - MaxOutOfOrderness` +- **窗口触发条件**：当 `watermark >= window_end` 时，窗口触发 +- **配置参数**： + - `MAXOUTOFORDERNESS`：允许的最大乱序时间，用于容忍数据乱序（默认：0，不允许乱序） + - `ALLOWEDLATENESS`：窗口触发后还能接受延迟数据的时间（默认：0，不接受延迟数据） + - `IDLETIMEOUT`：数据源空闲时，基于处理时间推进 Watermark 的超时时间（默认：0，禁用） +- **示例**： + ```sql + SELECT deviceId, COUNT(*) as cnt + FROM stream + GROUP BY deviceId, TumblingWindow('5m') + WITH ( + TIMESTAMP='eventTime', + TIMEUNIT='ms', + MAXOUTOFORDERNESS='5s', -- 容忍5秒的乱序 + ALLOWEDLATENESS='2s', -- 窗口触发后还能接受2秒的延迟数据 + IDLETIMEOUT='5s' -- 5秒无数据，基于处理时间推进watermark + ) + ``` ## 贡献指南 diff --git a/doc.go b/doc.go index a0d6510..1aa267b 100644 --- a/doc.go +++ b/doc.go @@ -26,6 +26,8 @@ integration with the RuleGo ecosystem. • Lightweight design - Pure in-memory operations, no external dependencies • SQL syntax support - Process stream data using familiar SQL syntax • Multiple window types - Sliding, tumbling, counting, and session windows +• Event time and processing time - Support both time semantics for accurate stream processing +• Watermark mechanism - Handle out-of-order and late-arriving data with configurable tolerance • Rich aggregate functions - MAX, MIN, AVG, SUM, STDDEV, MEDIAN, PERCENTILE, etc. • Plugin-based custom functions - Runtime dynamic registration, supports 8 function types • RuleGo ecosystem integration - Extend input/output sources using RuleGo components @@ -107,6 +109,77 @@ StreamSQL supports multiple window types: // Session window - Automatically closes session after 5-minute timeout SELECT user_id, COUNT(*) FROM stream GROUP BY user_id, SessionWindow('5m') +# Event Time vs Processing Time + +StreamSQL supports two time semantics for window processing: + +## Processing Time (Default) + +Processing time uses the system clock when data arrives. Windows are triggered based on data arrival time: + + // Processing time window (default) + SELECT COUNT(*) FROM stream GROUP BY TumblingWindow('5m') + // Windows are triggered every 5 minutes based on when data arrives + +## Event Time + +Event time uses timestamps embedded in the data itself. Windows are triggered based on event timestamps, +allowing correct handling of out-of-order and late-arriving data: + + // Event time window - Use 'order_time' field as event timestamp + SELECT COUNT(*) as order_count + FROM stream + GROUP BY TumblingWindow('5m') + WITH (TIMESTAMP='order_time') + + // Event time with integer timestamp (Unix milliseconds) + SELECT AVG(temperature) FROM stream + GROUP BY TumblingWindow('1m') + WITH (TIMESTAMP='event_time', TIMEUNIT='ms') + +## Watermark and Late Data Handling + +Event time windows use watermark mechanism to handle out-of-order and late data: + + // Configure max out-of-orderness (tolerate 5 seconds of out-of-order data) + SELECT COUNT(*) FROM stream + GROUP BY TumblingWindow('5m') + WITH ( + TIMESTAMP='order_time', + MAXOUTOFORDERNESS='5s' // Watermark = max(event_time) - 5s + ) + + // Configure allowed lateness (accept late data for 2 seconds after window closes) + SELECT COUNT(*) FROM stream + GROUP BY TumblingWindow('5m') + WITH ( + TIMESTAMP='order_time', + ALLOWEDLATENESS='2s' // Window stays open for 2s after trigger + ) + + // Combine both configurations + SELECT COUNT(*) FROM stream + GROUP BY TumblingWindow('5m') + WITH ( + TIMESTAMP='order_time', + MAXOUTOFORDERNESS='5s', // Tolerate 5s out-of-order before trigger + ALLOWEDLATENESS='2s' // Accept 2s late data after trigger + ) + + // Configure idle source mechanism (advance watermark based on processing time when data source is idle) + SELECT COUNT(*) FROM stream + GROUP BY TumblingWindow('5m') + WITH ( + TIMESTAMP='order_time', + IDLETIMEOUT='5s' // If no data arrives within 5s, watermark advances based on processing time + ) + +Key concepts: +• MaxOutOfOrderness: Affects watermark calculation, delays window trigger to tolerate out-of-order data +• AllowedLateness: Keeps window open after trigger to accept late data and update results +• IdleTimeout: When data source is idle (no data arrives within timeout), watermark advances based on processing time to ensure windows can close +• Watermark: Indicates that no events with timestamp less than watermark are expected + # Custom Functions StreamSQL supports plugin-based custom functions with runtime dynamic registration: diff --git a/rsql/ast.go b/rsql/ast.go index 3fd82cf..981eb85 100644 --- a/rsql/ast.go +++ b/rsql/ast.go @@ -35,10 +35,13 @@ type Field struct { } type WindowDefinition struct { - Type string - Params []interface{} - TsProp string - TimeUnit time.Duration + Type string + Params []interface{} + TsProp string + TimeUnit time.Duration + MaxOutOfOrderness time.Duration // Maximum allowed out-of-orderness for event time + AllowedLateness time.Duration // Maximum allowed lateness for event time windows + IdleTimeout time.Duration // Idle source timeout: when no data arrives within this duration, watermark advances based on processing time } // ToStreamConfig converts AST to Stream configuration @@ -133,14 +136,25 @@ func (s *SelectStatement) ToStreamConfig() (*types.Config, string, error) { return nil, "", err } + // Determine time characteristic based on whether TIMESTAMP is specified in WITH clause + // If TsProp is set, use EventTime; otherwise use ProcessingTime (default) + timeCharacteristic := types.ProcessingTime + if s.Window.TsProp != "" { + timeCharacteristic = types.EventTime + } + // Build Stream configuration config := types.Config{ WindowConfig: types.WindowConfig{ - Type: windowType, - Params: params, - TsProp: s.Window.TsProp, - TimeUnit: s.Window.TimeUnit, - GroupByKeys: extractGroupFields(s), + Type: windowType, + Params: params, + TsProp: s.Window.TsProp, + TimeUnit: s.Window.TimeUnit, + TimeCharacteristic: timeCharacteristic, + MaxOutOfOrderness: s.Window.MaxOutOfOrderness, + AllowedLateness: s.Window.AllowedLateness, + IdleTimeout: s.Window.IdleTimeout, + GroupByKeys: extractGroupFields(s), }, GroupFields: extractGroupFields(s), SelectFields: aggs, diff --git a/rsql/function_validator.go b/rsql/function_validator.go index c62bd13..3d0d96f 100644 --- a/rsql/function_validator.go +++ b/rsql/function_validator.go @@ -88,7 +88,7 @@ func (fv *FunctionValidator) isBuiltinFunction(funcName string) bool { func (fv *FunctionValidator) isKeyword(word string) bool { keywords := []string{ "SELECT", "FROM", "WHERE", "GROUP", "BY", "HAVING", "ORDER", - "AS", "DISTINCT", "LIMIT", "WITH", "TIMESTAMP", "TIMEUNIT", + "AS", "DISTINCT", "LIMIT", "WITH", "TIMESTAMP", "TIMEUNIT", "MAXOUTOFORDERNESS", "ALLOWEDLATENESS", "IDLETIMEOUT", "TUMBLINGWINDOW", "SLIDINGWINDOW", "COUNTINGWINDOW", "SESSIONWINDOW", "AND", "OR", "NOT", "IN", "LIKE", "IS", "NULL", "TRUE", "FALSE", "BETWEEN", "IS", "NULL", "TRUE", "FALSE", "CASE", "WHEN", diff --git a/rsql/lexer.go b/rsql/lexer.go index ea46cf4..90b64ab 100644 --- a/rsql/lexer.go +++ b/rsql/lexer.go @@ -41,6 +41,9 @@ const ( TokenWITH TokenTimestamp TokenTimeUnit + TokenMaxOutOfOrderness + TokenAllowedLateness + TokenIdleTimeout TokenOrder TokenDISTINCT TokenLIMIT @@ -349,6 +352,12 @@ func (l *Lexer) lookupIdent(ident string) Token { return Token{Type: TokenTimestamp, Value: ident} case "TIMEUNIT": return Token{Type: TokenTimeUnit, Value: ident} + case "MAXOUTOFORDERNESS": + return Token{Type: TokenMaxOutOfOrderness, Value: ident} + case "ALLOWEDLATENESS": + return Token{Type: TokenAllowedLateness, Value: ident} + case "IDLETIMEOUT": + return Token{Type: TokenIdleTimeout, Value: ident} case "ORDER": return Token{Type: TokenOrder, Value: ident} case "DISTINCT": diff --git a/rsql/parser.go b/rsql/parser.go index 26fda55..b2b5a9d 100644 --- a/rsql/parser.go +++ b/rsql/parser.go @@ -8,6 +8,7 @@ import ( "time" "github.com/rulego/streamsql/types" + "github.com/rulego/streamsql/utils/cast" ) // 解析器配置常量 @@ -510,7 +511,7 @@ func (p *Parser) parseWindowFunction(stmt *SelectStatement, winType string) erro if nextTok.Type != TokenLParen { return fmt.Errorf("expected '(' after window function %s, got %s (type: %v)", winType, nextTok.Value, nextTok.Type) } - + var params []interface{} maxIterations := 100 iterations := 0 @@ -524,28 +525,28 @@ func (p *Parser) parseWindowFunction(stmt *SelectStatement, winType string) erro // Read the next token first valTok := p.lexer.NextToken() - + // If we hit the closing parenthesis or EOF, break if valTok.Type == TokenRParen || valTok.Type == TokenEOF { break } - + // Skip commas if valTok.Type == TokenComma { continue } - + // Handle quoted values if strings.HasPrefix(valTok.Value, "'") && strings.HasSuffix(valTok.Value, "'") { valTok.Value = strings.Trim(valTok.Value, "'") } - + // Add the parameter value params = append(params, convertValue(valTok.Value)) } - stmt.Window.Params = params - stmt.Window.Type = winType + stmt.Window.Params = params + stmt.Window.Type = winType return nil } @@ -643,7 +644,7 @@ func (p *Parser) parseGroupBy(stmt *SelectStatement) error { // After parsing window function, skip adding it to GroupBy and continue continue } - + // Skip right parenthesis tokens (they should be consumed by parseWindowFunction) if tok.Type == TokenRParen { continue @@ -708,7 +709,7 @@ func (p *Parser) parseWith(stmt *SelectStatement) error { } } if valTok.Type == TokenTimeUnit { - timeUnit := time.Minute + timeUnit := time.Millisecond // Default to milliseconds next := p.lexer.NextToken() if next.Type == TokenEQ { next = p.lexer.NextToken() @@ -726,8 +727,10 @@ func (p *Parser) parseWith(stmt *SelectStatement) error { timeUnit = time.Second case "ms": timeUnit = time.Millisecond + case "ns": + timeUnit = time.Nanosecond default: - + // If unknown unit, keep default (milliseconds) } // Check if Window is initialized; if not, create new WindowDefinition if stmt.Window.Type == "" { @@ -739,6 +742,72 @@ func (p *Parser) parseWith(stmt *SelectStatement) error { } } } + if valTok.Type == TokenMaxOutOfOrderness { + next := p.lexer.NextToken() + if next.Type == TokenEQ { + next = p.lexer.NextToken() + durationStr := next.Value + if strings.HasPrefix(durationStr, "'") && strings.HasSuffix(durationStr, "'") { + durationStr = strings.Trim(durationStr, "'") + } + // Parse duration string like '5s', '2m', '1h', etc. + if duration, err := cast.ToDurationE(durationStr); err == nil { + // Check if Window is initialized; if not, create new WindowDefinition + if stmt.Window.Type == "" { + stmt.Window = WindowDefinition{ + MaxOutOfOrderness: duration, + } + } else { + stmt.Window.MaxOutOfOrderness = duration + } + } + // If parsing fails, silently ignore (keep default 0) + } + } + if valTok.Type == TokenAllowedLateness { + next := p.lexer.NextToken() + if next.Type == TokenEQ { + next = p.lexer.NextToken() + durationStr := next.Value + if strings.HasPrefix(durationStr, "'") && strings.HasSuffix(durationStr, "'") { + durationStr = strings.Trim(durationStr, "'") + } + // Parse duration string like '5s', '2m', '1h', etc. + if duration, err := cast.ToDurationE(durationStr); err == nil { + // Check if Window is initialized; if not, create new WindowDefinition + if stmt.Window.Type == "" { + stmt.Window = WindowDefinition{ + AllowedLateness: duration, + } + } else { + stmt.Window.AllowedLateness = duration + } + } + // If parsing fails, silently ignore (keep default 0) + } + } + if valTok.Type == TokenIdleTimeout { + next := p.lexer.NextToken() + if next.Type == TokenEQ { + next = p.lexer.NextToken() + durationStr := next.Value + if strings.HasPrefix(durationStr, "'") && strings.HasSuffix(durationStr, "'") { + durationStr = strings.Trim(durationStr, "'") + } + // Parse duration string like '5s', '2m', '1h', etc. + if duration, err := cast.ToDurationE(durationStr); err == nil { + // Check if Window is initialized; if not, create new WindowDefinition + if stmt.Window.Type == "" { + stmt.Window = WindowDefinition{ + IdleTimeout: duration, + } + } else { + stmt.Window.IdleTimeout = duration + } + } + // If parsing fails, silently ignore (keep default 0) + } + } } return nil diff --git a/stream/processor_data.go b/stream/processor_data.go index 099d344..87b065d 100644 --- a/stream/processor_data.go +++ b/stream/processor_data.go @@ -59,6 +59,11 @@ func (dp *DataProcessor) Process() { currentDataChan := dp.stream.dataChan dp.stream.dataChanMux.RUnlock() + // Check if dataChan is nil (stream has been stopped) + if currentDataChan == nil { + return + } + select { case data, ok := <-currentDataChan: if !ok { diff --git a/streamsql_case_test.go b/streamsql_case_test.go index 738f5ed..b62e813 100644 --- a/streamsql_case_test.go +++ b/streamsql_case_test.go @@ -61,13 +61,14 @@ func TestCaseExpressionInSQL(t *testing.T) { // TestCaseExpressionInAggregation 测试CASE表达式在聚合查询中的使用 func TestCaseExpressionInAggregation(t *testing.T) { + // 使用处理时间窗口，避免需要推进watermark的复杂性 + // 这个测试主要验证CASE表达式在聚合函数中的使用，而不是事件时间窗口 sql := `SELECT deviceId, COUNT(*) as total_count, SUM(CASE WHEN temperature > 30 THEN 1 ELSE 0 END) as hot_count, AVG(CASE status WHEN 'active' THEN temperature ELSE 0 END) as avg_active_temp FROM stream - GROUP BY deviceId, TumblingWindow('1s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('1s')` // 创建StreamSQL实例 streamSQL := New() @@ -76,14 +77,13 @@ func TestCaseExpressionInAggregation(t *testing.T) { err := streamSQL.Execute(sql) assert.NoError(t, err, "执行SQL应该成功") - // 模拟数据 - baseTime := time.Now() + // 模拟数据（不需要时间戳字段，因为使用处理时间窗口） testData := []map[string]interface{}{ - {"deviceId": "device1", "temperature": 35.0, "status": "active", "ts": baseTime}, - {"deviceId": "device1", "temperature": 25.0, "status": "inactive", "ts": baseTime}, - {"deviceId": "device1", "temperature": 32.0, "status": "active", "ts": baseTime}, - {"deviceId": "device2", "temperature": 28.0, "status": "active", "ts": baseTime}, - {"deviceId": "device2", "temperature": 22.0, "status": "inactive", "ts": baseTime}, + {"deviceId": "device1", "temperature": 35.0, "status": "active"}, + {"deviceId": "device1", "temperature": 25.0, "status": "inactive"}, + {"deviceId": "device1", "temperature": 32.0, "status": "active"}, + {"deviceId": "device2", "temperature": 28.0, "status": "active"}, + {"deviceId": "device2", "temperature": 22.0, "status": "inactive"}, } // 添加数据并获取结果 @@ -208,12 +208,11 @@ func TestComplexCaseExpressionsInAggregation(t *testing.T) { WHEN temperature > 25 THEN 0.5 ELSE 0 END) as complex_score FROM stream - GROUP BY deviceId, TumblingWindow('1s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')`, + GROUP BY deviceId, TumblingWindow('1s')`, data: []map[string]interface{}{ - {"deviceId": "device1", "temperature": 35.0, "humidity": 70.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 28.0, "humidity": 50.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 20.0, "humidity": 40.0, "ts": time.Now()}, + {"deviceId": "device1", "temperature": 35.0, "humidity": 70.0}, + {"deviceId": "device1", "temperature": 28.0, "humidity": 50.0}, + {"deviceId": "device1", "temperature": 20.0, "humidity": 40.0}, }, description: "测试多条件CASE表达式在SUM聚合中的使用", }, @@ -222,12 +221,11 @@ func TestComplexCaseExpressionsInAggregation(t *testing.T) { sql: `SELECT deviceId, AVG(CASE WHEN ABS(temperature - 25) < 5 THEN temperature ELSE 0 END) as normalized_avg FROM stream - GROUP BY deviceId, TumblingWindow('1s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')`, + GROUP BY deviceId, TumblingWindow('1s')`, data: []map[string]interface{}{ - {"deviceId": "device1", "temperature": 23.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 27.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 35.0, "ts": time.Now()}, // 这个会被排除 + {"deviceId": "device1", "temperature": 23.0}, + {"deviceId": "device1", "temperature": 27.0}, + {"deviceId": "device1", "temperature": 35.0}, // 这个会被排除 }, description: "测试带函数的CASE表达式在AVG聚合中的使用", }, @@ -236,12 +234,11 @@ func TestComplexCaseExpressionsInAggregation(t *testing.T) { sql: `SELECT deviceId, COUNT(CASE WHEN temperature * 1.8 + 32 > 80 THEN 1 END) as fahrenheit_hot_count FROM stream - GROUP BY deviceId, TumblingWindow('1s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')`, + GROUP BY deviceId, TumblingWindow('1s')`, data: []map[string]interface{}{ - {"deviceId": "device1", "temperature": 25.0, "ts": time.Now()}, // 77F - {"deviceId": "device1", "temperature": 30.0, "ts": time.Now()}, // 86F - {"deviceId": "device1", "temperature": 35.0, "ts": time.Now()}, // 95F + {"deviceId": "device1", "temperature": 25.0}, // 77F + {"deviceId": "device1", "temperature": 30.0}, // 86F + {"deviceId": "device1", "temperature": 35.0}, // 95F }, description: "测试算术表达式CASE在COUNT聚合中的使用", }, @@ -501,14 +498,13 @@ func TestCaseExpressionAggregated(t *testing.T) { COUNT(CASE WHEN temperature <= 25 THEN 1 END) as normal_temp_count, COUNT(*) as total_count FROM stream - GROUP BY deviceId, TumblingWindow('1s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')`, + GROUP BY deviceId, TumblingWindow('1s')`, testData: []map[string]interface{}{ - {"deviceId": "device1", "temperature": 30.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 20.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 35.0, "ts": time.Now()}, - {"deviceId": "device2", "temperature": 22.0, "ts": time.Now()}, - {"deviceId": "device2", "temperature": 28.0, "ts": time.Now()}, + {"deviceId": "device1", "temperature": 30.0}, + {"deviceId": "device1", "temperature": 20.0}, + {"deviceId": "device1", "temperature": 35.0}, + {"deviceId": "device2", "temperature": 22.0}, + {"deviceId": "device2", "temperature": 28.0}, }, wantErr: false, }, @@ -524,12 +520,11 @@ func TestCaseExpressionAggregated(t *testing.T) { ELSE NULL END) as avg_high_humidity FROM stream - GROUP BY deviceId, TumblingWindow('1s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')`, + GROUP BY deviceId, TumblingWindow('1s')`, testData: []map[string]interface{}{ - {"deviceId": "device1", "temperature": 30.0, "humidity": 60.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 20.0, "humidity": 40.0, "ts": time.Now()}, - {"deviceId": "device1", "temperature": 35.0, "humidity": 70.0, "ts": time.Now()}, + {"deviceId": "device1", "temperature": 30.0, "humidity": 60.0}, + {"deviceId": "device1", "temperature": 20.0, "humidity": 40.0}, + {"deviceId": "device1", "temperature": 35.0, "humidity": 70.0}, }, wantErr: false, }, @@ -762,8 +757,7 @@ func TestHavingWithCaseExpression(t *testing.T) { AVG(CASE WHEN temperature > 30 THEN temperature ELSE 0 END) as conditional_avg FROM stream GROUP BY deviceId, TumblingWindow('5s') - HAVING conditional_avg > 25 - WITH (TIMESTAMP='ts', TIMEUNIT='ss')`, + HAVING conditional_avg > 25`, wantErr: false, }, { @@ -777,8 +771,7 @@ func TestHavingWithCaseExpression(t *testing.T) { END) as weighted_score FROM stream GROUP BY deviceId, TumblingWindow('5s') - HAVING weighted_score > 3 - WITH (TIMESTAMP='ts', TIMEUNIT='ss')`, + HAVING weighted_score > 3`, wantErr: false, }, } @@ -819,8 +812,7 @@ func TestHavingWithCaseExpressionFunctional(t *testing.T) { SUM(CASE WHEN temperature > 30 THEN 1 ELSE 0 END) as hot_count FROM stream GROUP BY deviceId, TumblingWindow('2s') - HAVING hot_count >= 2 - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + HAVING hot_count >= 2` // 创建StreamSQL实例 streamSQL := New() @@ -830,23 +822,22 @@ func TestHavingWithCaseExpressionFunctional(t *testing.T) { assert.NoError(t, err, "执行SQL应该成功") // 模拟数据 - baseTime := time.Now() testData := []map[string]interface{}{ // device1: 3条高温记录，应该通过HAVING条件 - {"deviceId": "device1", "temperature": 35.0, "ts": baseTime}, - {"deviceId": "device1", "temperature": 32.0, "ts": baseTime}, - {"deviceId": "device1", "temperature": 31.0, "ts": baseTime}, - {"deviceId": "device1", "temperature": 25.0, "ts": baseTime}, // 不是高温 + {"deviceId": "device1", "temperature": 35.0}, + {"deviceId": "device1", "temperature": 32.0}, + {"deviceId": "device1", "temperature": 31.0}, + {"deviceId": "device1", "temperature": 25.0}, // 不是高温 // device2: 1条高温记录，不应该通过HAVING条件 - {"deviceId": "device2", "temperature": 33.0, "ts": baseTime}, - {"deviceId": "device2", "temperature": 28.0, "ts": baseTime}, - {"deviceId": "device2", "temperature": 26.0, "ts": baseTime}, + {"deviceId": "device2", "temperature": 33.0}, + {"deviceId": "device2", "temperature": 28.0}, + {"deviceId": "device2", "temperature": 26.0}, // device3: 2条高温记录，应该通过HAVING条件 - {"deviceId": "device3", "temperature": 34.0, "ts": baseTime}, - {"deviceId": "device3", "temperature": 31.0, "ts": baseTime}, - {"deviceId": "device3", "temperature": 29.0, "ts": baseTime}, + {"deviceId": "device3", "temperature": 34.0}, + {"deviceId": "device3", "temperature": 31.0}, + {"deviceId": "device3", "temperature": 29.0}, } // 添加数据并获取结果 diff --git a/streamsql_counting_window_test.go b/streamsql_counting_window_test.go index 1efa16c..ce4a54c 100644 --- a/streamsql_counting_window_test.go +++ b/streamsql_counting_window_test.go @@ -62,7 +62,14 @@ func TestSQLCountingWindow_GroupedCounting_MixedDevices(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 8) - ssql.AddSink(func(results []map[string]interface{}) { ch <- results }) + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + ch <- results + }) for i := 0; i < 10; i++ { ssql.Emit(map[string]interface{}{"deviceId": "A", "temperature": i, "timestamp": time.Now()}) @@ -100,7 +107,14 @@ func TestSQLCountingWindow_MultiKeyGroupedCounting(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 8) - ssql.AddSink(func(results []map[string]interface{}) { ch <- results }) + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + ch <- results + }) for i := 0; i < 5; i++ { ssql.Emit(map[string]interface{}{"deviceId": "A", "region": "R1", "temperature": i, "timestamp": time.Now()}) diff --git a/streamsql_post_aggregation_test.go b/streamsql_post_aggregation_test.go index d78afa0..0a3633f 100644 --- a/streamsql_post_aggregation_test.go +++ b/streamsql_post_aggregation_test.go @@ -19,8 +19,19 @@ func createTestEnvironment(t *testing.T, rsql string) (*Streamsql, chan interfac require.NoError(t, err) resultChan := make(chan interface{}, 10) + t.Cleanup(func() { close(resultChan) }) + ssql.AddSink(func(result []map[string]interface{}) { - resultChan <- result + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + select { + case resultChan <- result: + default: + // 非阻塞发送 + } }) return ssql, resultChan @@ -35,20 +46,27 @@ func sendDataAndCollectResults(t *testing.T, ssql *Streamsql, resultChan chan in // 等待窗口触发 time.Sleep(time.Duration(windowSizeSeconds+1) * time.Second) - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + // 使用更严格的超时机制 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var results []map[string]interface{} + maxIterations := 10 // 最多收集10次结果 + iteration := 0 + collecting: - for { + for iteration < maxIterations { select { case result := <-resultChan: if resultSlice, ok := result.([]map[string]interface{}); ok { results = append(results, resultSlice...) } - case <-time.After(1 * time.Second): + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 break collecting case <-ctx.Done(): + // 超时退出 break collecting } } @@ -68,16 +86,14 @@ func TestPostAggregationExpressions(t *testing.T) { (SUM(value) / COUNT(*)) as calcAvg, (SUM(value) + AVG(value)) as sumPlusAvg FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "dev1", "value": 10.0, "ts": baseTime}, - {"deviceId": "dev1", "value": 20.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "dev1", "value": 30.0, "ts": baseTime.Add(2 * time.Second)}, + {"deviceId": "dev1", "value": 10.0}, + {"deviceId": "dev1", "value": 20.0}, + {"deviceId": "dev1", "value": 30.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -102,17 +118,15 @@ func TestPostAggregationExpressions(t *testing.T) { IF_NULL(LAST_VALUE(value), 0) as lastOrZero, IF_NULL(AVG(value), 0) as avgOrZero FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": nil, "ts": baseTime}, - {"deviceId": "sensor1", "value": 10.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": nil, "ts": baseTime.Add(2 * time.Second)}, - {"deviceId": "sensor1", "value": 30.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "sensor1", "value": nil}, + {"deviceId": "sensor1", "value": 10.0}, + {"deviceId": "sensor1", "value": nil}, + {"deviceId": "sensor1", "value": 30.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -136,17 +150,15 @@ func TestPostAggregationExpressions(t *testing.T) { MAX(IF_NULL(value, 0)) as maxVal, MIN(IF_NULL(value, 0)) as minVal FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": nil, "ts": baseTime}, - {"deviceId": "sensor1", "value": 10.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": nil, "ts": baseTime.Add(2 * time.Second)}, - {"deviceId": "sensor1", "value": 30.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "sensor1", "value": nil}, + {"deviceId": "sensor1", "value": 10.0}, + {"deviceId": "sensor1", "value": nil}, + {"deviceId": "sensor1", "value": 30.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -172,16 +184,14 @@ func TestPostAggregationExpressions(t *testing.T) { (SUM(value) + LATEST(value)) as totalPlusLatest, (AVG(value) * LATEST(value)) as avgTimesLatest FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": 10.0, "ts": baseTime}, - {"deviceId": "sensor1", "value": 20.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": 30.0, "ts": baseTime.Add(2 * time.Second)}, + {"deviceId": "sensor1", "value": 10.0}, + {"deviceId": "sensor1", "value": 20.0}, + {"deviceId": "sensor1", "value": 30.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -210,17 +220,15 @@ func TestPostAggregationExpressions(t *testing.T) { CEIL((AVG(value) / COUNT(*))) as ceilResult, ROUND((SUM(value) * AVG(value) / 1000), 2) as roundResult FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": 10.0, "ts": baseTime}, - {"deviceId": "sensor1", "value": 20.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": 30.0, "ts": baseTime.Add(2 * time.Second)}, - {"deviceId": "sensor1", "value": 40.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "sensor1", "value": 10.0}, + {"deviceId": "sensor1", "value": 20.0}, + {"deviceId": "sensor1", "value": 30.0}, + {"deviceId": "sensor1", "value": 40.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -275,20 +283,18 @@ func TestPostAggregationExpressions(t *testing.T) { window_start() as start, window_end() as end FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ // 设备1的数据 - {"deviceId": "meter001", "displayNum": 100.0, "ts": baseTime}, - {"deviceId": "meter001", "displayNum": 115.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "meter001", "displayNum": 100.0}, + {"deviceId": "meter001", "displayNum": 115.0}, // 设备2的数据 - {"deviceId": "meter002", "displayNum": 200.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "meter002", "displayNum": 206.0, "ts": baseTime.Add(4 * time.Second)}, + {"deviceId": "meter002", "displayNum": 200.0}, + {"deviceId": "meter002", "displayNum": 206.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -345,17 +351,15 @@ func TestPostAggregationExpressions(t *testing.T) { ROUND(SQRT(ABS(AVG(value) - MIN(value))), 2) as nestedMathFunc, UPPER(CONCAT('RESULT_', CAST(ROUND(SUM(value), 0) as STRING))) as nestedStrMathFunc FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": 10.0, "ts": baseTime}, - {"deviceId": "sensor1", "value": 20.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": 30.0, "ts": baseTime.Add(2 * time.Second)}, - {"deviceId": "sensor1", "value": 40.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "sensor1", "value": 10.0}, + {"deviceId": "sensor1", "value": 20.0}, + {"deviceId": "sensor1", "value": 30.0}, + {"deviceId": "sensor1", "value": 40.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -407,17 +411,15 @@ func TestPostAggregationExpressions(t *testing.T) { CEIL(AVG(FLOOR(SQRT(value)))) as tripleNested2, ABS(MIN(ROUND(value / 5, 2))) as tripleNested3 FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": 16.0, "ts": baseTime}, - {"deviceId": "sensor1", "value": 25.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": 36.0, "ts": baseTime.Add(2 * time.Second)}, - {"deviceId": "sensor1", "value": 49.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "sensor1", "value": 16.0}, + {"deviceId": "sensor1", "value": 25.0}, + {"deviceId": "sensor1", "value": 36.0}, + {"deviceId": "sensor1", "value": 49.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -481,17 +483,15 @@ func TestPostAggregationExpressions(t *testing.T) { (COUNT(*) * NTH_VALUE(value, 2)) as countTimesSecond, (SUM(value) + LEAD(value, 1)) as sumPlusLead FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": 10.0, "ts": baseTime}, - {"deviceId": "sensor1", "value": 20.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": 30.0, "ts": baseTime.Add(2 * time.Second)}, - {"deviceId": "sensor1", "value": 40.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "sensor1", "value": 10.0}, + {"deviceId": "sensor1", "value": 20.0}, + {"deviceId": "sensor1", "value": 30.0}, + {"deviceId": "sensor1", "value": 40.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) @@ -515,17 +515,15 @@ func TestPostAggregationExpressions(t *testing.T) { NTH_VALUE(value, 3) as thirdValue, NTH_VALUE(value, 4) as fourthValue FROM stream - GROUP BY deviceId, TumblingWindow('5s') - WITH (TIMESTAMP='ts', TIMEUNIT='ss')` + GROUP BY deviceId, TumblingWindow('5s')` ssql, resultChan := createTestEnvironment(t, rsql) - baseTime := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) testData := []map[string]interface{}{ - {"deviceId": "sensor1", "value": 100.0, "ts": baseTime}, - {"deviceId": "sensor1", "value": 200.0, "ts": baseTime.Add(1 * time.Second)}, - {"deviceId": "sensor1", "value": 300.0, "ts": baseTime.Add(2 * time.Second)}, - {"deviceId": "sensor1", "value": 400.0, "ts": baseTime.Add(3 * time.Second)}, + {"deviceId": "sensor1", "value": 100.0}, + {"deviceId": "sensor1", "value": 200.0}, + {"deviceId": "sensor1", "value": 300.0}, + {"deviceId": "sensor1", "value": 400.0}, } results := sendDataAndCollectResults(t, ssql, resultChan, testData, 5) diff --git a/streamsql_session_window_test.go b/streamsql_session_window_test.go index 8617cd7..92505a9 100644 --- a/streamsql_session_window_test.go +++ b/streamsql_session_window_test.go @@ -1,6 +1,7 @@ package streamsql import ( + "sync" "testing" "time" @@ -8,7 +9,9 @@ import ( "github.com/stretchr/testify/require" ) -func TestSQLSessionWindow_SingleKey(t *testing.T) { +// TestSQLSessionWindow_ProcessingTime 测试处理时间的会话窗口 +// 验证不使用 WITH 子句时，会话窗口基于处理时间（系统时钟）工作 +func TestSQLSessionWindow_ProcessingTime(t *testing.T) { ssql := New() defer ssql.Stop() @@ -22,13 +25,26 @@ func TestSQLSessionWindow_SingleKey(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 4) - ssql.AddSink(func(results []map[string]interface{}) { ch <- results }) + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + ch <- results + }) + // 使用处理时间：发送数据，不包含时间戳字段 + // 会话窗口基于数据到达的处理时间（系统时钟）来划分会话 for i := 0; i < 5; i++ { - ssql.Emit(map[string]interface{}{"deviceId": "sensor001", "timestamp": time.Now()}) - time.Sleep(50 * time.Millisecond) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) // 数据间隔小于会话超时时间（300ms），属于同一会话 } + // 等待会话超时（处理时间会话窗口基于系统时钟触发） time.Sleep(600 * time.Millisecond) select { @@ -37,8 +53,9 @@ func TestSQLSessionWindow_SingleKey(t *testing.T) { row := res[0] assert.Equal(t, "sensor001", row["deviceId"]) assert.Equal(t, float64(5), row["cnt"]) + t.Logf("处理时间会话窗口成功触发，数据量: %.0f", row["cnt"]) case <-time.After(2 * time.Second): - t.Fatal("timeout") + t.Fatal("处理时间会话窗口应该触发") } } @@ -56,7 +73,14 @@ func TestSQLSessionWindow_GroupedSession_MixedDevices(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 8) - ssql.AddSink(func(results []map[string]interface{}) { ch <- results }) + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + ch <- results + }) // Emit data for two different devices in interleaved pattern for i := 0; i < 5; i++ { @@ -106,7 +130,14 @@ func TestSQLSessionWindow_MultiKeyGroupedSession(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 8) - ssql.AddSink(func(results []map[string]interface{}) { ch <- results }) + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + ch <- results + }) // Emit data for 4 different combinations: A|R1, B|R1, A|R2, B|R2 for i := 0; i < 4; i++ { @@ -174,3 +205,483 @@ func TestSQLSessionWindow_MultiKeyGroupedSession(t *testing.T) { assert.Equal(t, 23.0, got["A|R2"].max) assert.Equal(t, 33.0, got["B|R2"].max) } + +// TestSQLSessionWindow_EventTimeWithWithClause 测试使用 WITH 子句指定事件时间的会话窗口 +func TestSQLSessionWindow_EventTimeWithWithClause(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt + FROM stream + GROUP BY deviceId, SessionWindow('300ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='200ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 4) + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + ch <- results + }) + + // 使用事件时间：发送带有事件时间戳的数据 + baseTime := time.Now().UnixMilli() - 5000 // 5秒前作为基准时间 + for i := 0; i < 5; i++ { + eventTime := baseTime + int64(i*50) // 每50ms一条数据 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, // 事件时间字段 + }) + time.Sleep(20 * time.Millisecond) // 处理时间间隔较小 + } + + // 发送一个事件时间超过会话结束时间的数据，推进watermark + // 会话结束时间 = baseTime + 200 + 300 = baseTime + 500 + // 需要发送事件时间 > baseTime + 500 + maxOutOfOrderness(200) = baseTime + 700 的数据 + // 使用不同的设备ID，避免影响当前会话的计数 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor002", // 使用不同的设备ID，不影响sensor001的会话 + "eventTime": baseTime + 2000, // 推进watermark + }) + + // 等待会话超时（事件时间会话窗口基于watermark触发） + time.Sleep(1 * time.Second) + + select { + case res := <-ch: + require.Len(t, res, 1) + row := res[0] + assert.Equal(t, "sensor001", row["deviceId"]) + assert.Equal(t, float64(5), row["cnt"]) + t.Logf("事件时间会话窗口成功触发，数据量: %.0f", row["cnt"]) + case <-time.After(2 * time.Second): + t.Fatal("事件时间会话窗口应该触发") + } +} + +// TestSQLSessionWindow_ProcessingTimeWithoutWithClause 测试不使用 WITH 子句时默认使用处理时间 +func TestSQLSessionWindow_ProcessingTimeWithoutWithClause(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt + FROM stream + GROUP BY deviceId, SessionWindow('300ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 4) + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + ch <- results + }) + + // 不使用事件时间字段，应该使用处理时间 + for i := 0; i < 5; i++ { + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待会话超时（处理时间会话窗口基于系统时钟） + time.Sleep(600 * time.Millisecond) + + select { + case res := <-ch: + require.Len(t, res, 1) + row := res[0] + assert.Equal(t, "sensor001", row["deviceId"]) + assert.Equal(t, float64(5), row["cnt"]) + t.Logf("处理时间会话窗口成功触发，数据量: %.0f", row["cnt"]) + case <-time.After(2 * time.Second): + t.Fatal("处理时间会话窗口应该触发") + } +} + +// TestSQLSessionWindow_EventTimeWindowAlignment 测试事件时间会话窗口 +func TestSQLSessionWindow_EventTimeWindowAlignment(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, SessionWindow('500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='200ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 10) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，验证会话窗口基于事件时间触发 + baseTime := time.Now().UnixMilli() - 5000 + sessionTimeoutMs := int64(500) + + // 第一阶段：发送连续的数据（事件时间间隔小于sessionTimeout） + // 这些数据应该属于同一个会话 + t.Log("第一阶段：发送连续数据（同一会话）") + for i := 0; i < 5; i++ { + eventTime := baseTime + int64(i*100) // 每100ms一条，小于500ms超时 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待一段时间，让watermark推进 + // 第一个会话的结束时间 = baseTime + 400 + 500 = baseTime + 900 + // 需要发送事件时间 > baseTime + 900 + maxOutOfOrderness(200) = baseTime + 1100 的数据 + // 才能让 watermark >= baseTime + 900，触发第一个会话 + time.Sleep(500 * time.Millisecond) + + // 发送数据推进watermark + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": baseTime + int64(1500), + "temperature": 50.0, + }) + time.Sleep(500 * time.Millisecond) + + // 第二阶段：发送间隔较大的数据（事件时间间隔大于sessionTimeout） + // 这应该触发新会话 + t.Log("第二阶段：发送间隔较大的数据（新会话）") + eventTime := baseTime + int64(2000) // 间隔2秒，大于500ms超时 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": 100.0, + }) + + // 继续发送连续数据（第二个会话） + for i := 0; i < 3; i++ { + eventTime := baseTime + int64(2000+i*100) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(100 + i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 推进watermark，触发会话 + // 会话结束时间 = baseTime + 400 + 500 = baseTime + 900 + // 需要发送事件时间 > baseTime + 900 + maxOutOfOrderness(200) = baseTime + 1100 的数据 + // 才能让 watermark >= baseTime + 900 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": baseTime + int64(5000), + "temperature": 200.0, + }) + + // 继续发送更多数据，确保watermark推进 + for i := 0; i < 5; i++ { + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": baseTime + int64(5000+i*200), + "temperature": float64(200 + i), + }) + time.Sleep(100 * time.Millisecond) + } + + // 等待会话触发（watermark更新间隔200ms，需要等待足够时间） + time.Sleep(3 * time.Second) + + timeout := time.After(3 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + if windowResultsLen == 0 { + t.Log("⚠ 会话窗口未触发，可能watermark未推进到足够位置") + t.Log("提示：会话窗口需要在 watermark >= session_end 时触发") + t.Log("会话结束时间 = 最后一个数据时间 + 超时时间") + } + require.Greater(t, windowResultsLen, 0, "应该至少触发一个会话窗口") + + // 验证会话窗口 + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + + // 验证会话窗口有数据 + assert.Greater(t, cnt, 0.0, "会话窗口 %d 应该包含数据", i+1) + + // 验证会话窗口的时间范围合理 + assert.Greater(t, end, start, "会话窗口 %d 的结束时间应该大于开始时间", i+1) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + sessionDurationMs := endMs - startMs + + assert.GreaterOrEqual(t, sessionDurationMs, sessionTimeoutMs, + "会话窗口 %d 的持续时间应该至少等于会话超时时间", i+1) + + t.Logf("会话窗口 %d: [%d, %d), cnt=%.0f, duration=%dms", i+1, startMs, endMs, cnt, sessionDurationMs) + } + } + + t.Logf("总共触发了 %d 个会话窗口", windowResultsLen) +} + +// TestSQLSessionWindow_WatermarkTriggerTiming 测试会话窗口Watermark触发时机 +func TestSQLSessionWindow_WatermarkTriggerTiming(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, SessionWindow('500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='200ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 10) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + baseTime := time.Now().UnixMilli() - 5000 + maxOutOfOrdernessMs := int64(200) + sessionTimeoutMs := int64(500) + + // 发送数据，创建会话 + // 第一个数据：baseTime + // 后续数据：baseTime + 100, baseTime + 200, baseTime + 300, baseTime + 400 + // 会话结束时间应该是 baseTime + 400 + 500 = baseTime + 900 + // 当watermark >= baseTime + 900时，会话应该触发 + t.Log("发送数据创建会话") + for i := 0; i < 5; i++ { + eventTime := baseTime + int64(i*100) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 计算会话结束时间 + sessionEndTime := baseTime + int64(400) + sessionTimeoutMs // 最后一个数据时间 + 超时时间 + + // 发送一个事件时间刚好等于sessionEndTime的数据 + // watermark = maxEventTime - maxOutOfOrderness = sessionEndTime - 200 + // 此时 watermark < sessionEndTime，会话不应该触发 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": sessionEndTime, + "temperature": 100.0, + }) + + // 等待watermark更新 + time.Sleep(500 * time.Millisecond) + + // 发送一个事件时间超过sessionEndTime的数据，推进watermark + // watermark = maxEventTime - maxOutOfOrderness = (sessionEndTime + 500) - 200 = sessionEndTime + 300 + // 此时 watermark >= sessionEndTime，会话应该触发 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": sessionEndTime + 1000, + "temperature": 200.0, + }) + + // 继续发送更多数据，确保watermark推进 + for i := 0; i < 3; i++ { + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": sessionEndTime + int64(1000+i*200), + "temperature": float64(200 + i), + }) + time.Sleep(100 * time.Millisecond) + } + + // 等待会话触发（watermark更新间隔200ms，需要等待足够时间） + time.Sleep(3 * time.Second) + + timeout := time.After(3 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + if windowResultsLen == 0 { + t.Log("⚠ 会话窗口未触发，可能watermark未推进到足够位置") + t.Log("提示：会话窗口需要在 watermark >= session_end 时触发") + t.Log("会话结束时间 = 最后一个数据时间 + 超时时间") + } + require.Greater(t, windowResultsLen, 0, "应该至少触发一个会话窗口") + + // 验证会话窗口的触发时机 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + + // 验证会话窗口包含数据 + assert.Greater(t, cnt, 0.0, "会话窗口应该包含数据") + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + sessionDurationMs := endMs - startMs + + assert.GreaterOrEqual(t, sessionDurationMs, sessionTimeoutMs, + "会话窗口的持续时间应该至少等于会话超时时间") + + t.Logf("✓ 会话窗口在watermark >= session_end时正确触发") + t.Logf("会话窗口: [%d, %d), cnt=%.0f, 触发时maxEventTime >= %d", + start, end, cnt, end+maxOutOfOrdernessMs) + } + } +} + +// TestSQLSessionWindow_IdleSourceMechanism 测试会话窗口的Idle Source机制 +func TestSQLSessionWindow_IdleSourceMechanism(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, SessionWindow('500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='200ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 10) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + baseTime := time.Now().UnixMilli() - 5000 + + // 发送数据，创建会话 + t.Log("发送数据，创建会话") + for i := 0; i < 5; i++ { + eventTime := baseTime + int64(i*100) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 停止发送数据，等待Idle Source机制触发 + t.Log("停止发送数据，等待Idle Source机制触发（IdleTimeout=2s）") + time.Sleep(3 * time.Second) + + // 收集窗口结果 + timeout := time.After(3 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个会话窗口（即使数据源空闲）") + + if windowResultsLen > 0 { + t.Logf("✓ 会话窗口Idle Source机制正常工作，触发了 %d 个会话", windowResultsLen) + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + t.Logf("会话 %d: [%d, %d), cnt=%.0f", i+1, start, end, cnt) + } + } + } +} diff --git a/streamsql_sliding_window_test.go b/streamsql_sliding_window_test.go index 1b5314d..1b6213c 100644 --- a/streamsql_sliding_window_test.go +++ b/streamsql_sliding_window_test.go @@ -1,6 +1,7 @@ package streamsql import ( + "context" "sync" "testing" "time" @@ -9,7 +10,9 @@ import ( "github.com/stretchr/testify/require" ) -func TestSQLSlidingWindow_Basic(t *testing.T) { +// TestSQLSlidingWindow_ProcessingTime 测试处理时间的滑动窗口 +// 验证不使用 WITH 子句时，滑动窗口基于处理时间（系统时钟）工作 +func TestSQLSlidingWindow_ProcessingTime(t *testing.T) { ssql := New() defer ssql.Stop() @@ -127,7 +130,13 @@ func TestSQLSlidingWindow_WithAggregations(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 20) + defer close(ch) ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() ch <- results }) @@ -213,6 +222,11 @@ func TestSQLSlidingWindow_MultipleWindowsAlignment(t *testing.T) { windowResults := make([][]map[string]interface{}, 0) var windowResultsMu sync.Mutex ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() ch <- results }) @@ -320,7 +334,13 @@ func TestSQLSlidingWindow_MultiKeyGrouped(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 20) + defer close(ch) ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() ch <- results }) @@ -416,15 +436,21 @@ func TestSQLSlidingWindow_FirstWindowTiming(t *testing.T) { COUNT(*) as cnt FROM stream GROUP BY deviceId, SlidingWindow('2s', '500ms') - WITH (TIMESTAMP='timestamp') + WITH (TIMESTAMP='timestamp', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') ` err := ssql.Execute(sql) require.NoError(t, err) ch := make(chan []map[string]interface{}, 20) + defer close(ch) windowTimings := make([]time.Time, 0) var windowTimingsMu sync.Mutex ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() if len(results) > 0 { windowTimingsMu.Lock() windowTimings = append(windowTimings, time.Now()) @@ -435,18 +461,32 @@ func TestSQLSlidingWindow_FirstWindowTiming(t *testing.T) { // 记录第一个数据发送时间 firstDataTime := time.Now() + baseTime := time.Now().UnixMilli() - // 使用处理时间，每200ms发送一条数据，共发送10条 + // 使用事件时间，每200ms发送一条数据，共发送10条 for i := 0; i < 10; i++ { ssql.Emit(map[string]interface{}{ "deviceId": "sensor001", + "timestamp": baseTime + int64(i*200), // 添加timestamp字段 "temperature": float64(i), }) time.Sleep(200 * time.Millisecond) } + // 发送一个事件时间超过第一个窗口结束时间的数据，推进watermark + // 窗口大小2秒，第一个窗口应该在 [baseTime, baseTime+2000) 范围内 + // 发送一个事件时间为 baseTime+3000 的数据来推进watermark + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "timestamp": baseTime + 3000, // 推进watermark + "temperature": 100.0, + }) + // 等待第一个窗口触发（应该在窗口大小2秒后，而不是滑动步长500ms后） - timeout := time.After(3 * time.Second) + // 发送完数据后，等待足够的时间让第一个窗口触发 + time.Sleep(3 * time.Second) + + timeout := time.After(2 * time.Second) firstWindowReceived := false for { @@ -455,25 +495,29 @@ func TestSQLSlidingWindow_FirstWindowTiming(t *testing.T) { if len(res) > 0 && !firstWindowReceived { firstWindowReceived = true windowTimingsMu.Lock() - firstWindowTime := windowTimings[0] - windowTimingsMu.Unlock() - elapsed := firstWindowTime.Sub(firstDataTime) - - // 第一个窗口应该在窗口大小时间（2秒）后触发 - // 允许一些误差（±500ms），因为数据处理和调度可能有延迟 - assert.GreaterOrEqual(t, elapsed, 1500*time.Millisecond, - "第一个窗口应该在窗口大小时间（2秒）后触发，实际耗时: %v", elapsed) - assert.LessOrEqual(t, elapsed, 3*time.Second, - "第一个窗口不应该太晚触发，实际耗时: %v", elapsed) - - // 验证第一个窗口不应该在滑动步长时间（500ms）后就触发 - assert.Greater(t, elapsed, 800*time.Millisecond, - "第一个窗口不应该在滑动步长时间（500ms）后就触发，实际耗时: %v", elapsed) - - cnt := res[0]["cnt"].(float64) - assert.Greater(t, cnt, 0.0, "第一个窗口应该包含数据") - t.Logf("第一个窗口触发时间: %v, 从第一个数据到触发耗时: %v, 窗口数据量: %.0f", - firstWindowTime, elapsed, cnt) + if len(windowTimings) > 0 { + firstWindowTime := windowTimings[0] + windowTimingsMu.Unlock() + elapsed := firstWindowTime.Sub(firstDataTime) + + // 第一个窗口应该在窗口大小时间（2秒）后触发 + // 允许一些误差（±500ms），因为数据处理和调度可能有延迟 + assert.GreaterOrEqual(t, elapsed, 1500*time.Millisecond, + "第一个窗口应该在窗口大小时间（2秒）后触发，实际耗时: %v", elapsed) + assert.LessOrEqual(t, elapsed, 5*time.Second, + "第一个窗口不应该太晚触发，实际耗时: %v", elapsed) + + // 验证第一个窗口不应该在滑动步长时间（500ms）后就触发 + assert.Greater(t, elapsed, 800*time.Millisecond, + "第一个窗口不应该在滑动步长时间（500ms）后就触发，实际耗时: %v", elapsed) + + cnt := res[0]["cnt"].(float64) + assert.Greater(t, cnt, 0.0, "第一个窗口应该包含数据") + t.Logf("第一个窗口触发时间: %v, 从第一个数据到触发耗时: %v, 窗口数据量: %.0f", + firstWindowTime, elapsed, cnt) + } else { + windowTimingsMu.Unlock() + } } case <-timeout: goto END @@ -502,9 +546,15 @@ func TestSQLSlidingWindow_DataOverlap(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 20) + defer close(ch) windowResults := make([][]map[string]interface{}, 0) var windowResultsMu sync.Mutex ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() if len(results) > 0 { windowResultsMu.Lock() windowResults = append(windowResults, results) @@ -632,9 +682,15 @@ func TestSQLSlidingWindow_DataRetention(t *testing.T) { require.NoError(t, err) ch := make(chan []map[string]interface{}, 20) + defer close(ch) windowResults := make([][]map[string]interface{}, 0) var windowResultsMu sync.Mutex ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() if len(results) > 0 { windowResultsMu.Lock() windowResults = append(windowResults, results) @@ -716,3 +772,892 @@ END: } } } + +// TestSQLSlidingWindow_EventTimeWithWithClause 测试使用 WITH 子句指定事件时间 +func TestSQLSlidingWindow_EventTimeWithWithClause(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt + FROM stream + GROUP BY deviceId, SlidingWindow('2s', '500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送带有事件时间戳的数据 + // 事件时间从当前时间开始，每200ms递增，确保 watermark 能够推进 + baseTime := time.Now().UnixMilli() // 使用当前时间作为基准 + for i := 0; i < 15; i++ { + eventTime := baseTime + int64(i*200) // 每200ms一条数据 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, // 事件时间字段（毫秒） + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) // 处理时间间隔较小，模拟乱序 + } + + // 等待窗口触发（事件时间窗口基于 watermark 触发） + // 窗口大小2秒，滑动步长500ms + // 第一个窗口应该在 watermark >= window_end 时触发 + // 由于 watermark 更新间隔是 200ms，需要等待足够的时间让 watermark 推进 + time.Sleep(3 * time.Second) + + timeout := time.After(2 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + // 事件时间窗口应该能够触发 + // 由于使用事件时间，窗口触发基于 watermark + require.Greater(t, windowResultsLen, 0, "事件时间窗口应该至少触发一个窗口") + if windowResultsLen > 0 { + t.Logf("事件时间窗口触发了 %d 个窗口", windowResultsLen) + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + assert.Greater(t, cnt, 0.0, "事件时间窗口应该包含数据") + t.Logf("第一个事件时间窗口数据量: %.0f", cnt) + } + } +} + +// TestSQLSlidingWindow_LateDataHandling 测试延迟数据的处理 +// 验证即使数据延迟到达，只要在允许的延迟范围内，也能正确统计到对应窗口 +func TestSQLSlidingWindow_LateDataHandling(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, SlidingWindow('2s', '500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：模拟延迟数据场景 + // 场景：先发送正常顺序的数据，然后发送一些延迟的数据 + baseTime := time.Now().UnixMilli() - 5000 // 使用5秒前作为基准，确保有足够的时间窗口 + + // 第一阶段：发送正常顺序的数据（事件时间：0ms, 200ms, 400ms, ..., 2000ms） + // 这些数据应该被统计到第一个窗口 [0ms, 2000ms) + t.Log("第一阶段：发送正常顺序的数据") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) // 每200ms一条数据 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 温度值 0-9 + }) + time.Sleep(50 * time.Millisecond) // 处理时间间隔较小 + } + + // 等待 watermark 推进，让第一个窗口触发 + // 窗口大小2秒，第一个窗口应该在 watermark >= baseTime + 2000ms 时触发 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟的数据 + // 这些数据的事件时间比之前的数据早，但应该在允许的延迟范围内 + // 延迟数据的事件时间：100ms, 300ms, 500ms（这些时间在第一个窗口 [0ms, 2000ms) 内） + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + for i := 0; i < 3; i++ { + // 延迟数据：事件时间比正常数据早，但仍在窗口范围内 + eventTime := baseTime + int64(100+i*200) // 100ms, 300ms, 500ms + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(10 + i), // 温度值 10-12，用于区分延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 继续发送更多正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + timeout := time.After(2 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证第一个窗口的数据 + // 第一个窗口应该包含正常数据（0-9）和可能的延迟数据 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 第一个窗口应该包含正常数据 + // 由于窗口对齐和 watermark 机制，实际数据量可能略有不同 + assert.GreaterOrEqual(t, cnt, 5.0, "第一个窗口应该包含足够的数据") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + assert.GreaterOrEqual(t, maxTemp, 0.0, "第一个窗口的最大值应该大于等于0") + } + } + + // 验证延迟数据是否被处理 + // 如果延迟数据被正确处理，应该能在后续窗口或更新中看到 + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLSlidingWindow_MaxOutOfOrderness 测试最大延迟时间配置 +// 验证设置 MaxOutOfOrderness 后，延迟数据能否在允许的延迟范围内被正确处理 +func TestSQLSlidingWindow_MaxOutOfOrderness(t *testing.T) { + ssql := New() + defer ssql.Stop() + + // 使用 SQL 配置 MaxOutOfOrderness + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, SlidingWindow('2s', '500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='1s', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟延迟数据场景 + // 场景：设置 MaxOutOfOrderness = 1秒，测试延迟数据能否在1秒内被正确处理 + // 滑动窗口步长500ms，需要对齐到500ms的倍数 + slideSizeMs := int64(500) // 500ms + baseTimeRaw := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + // 对齐baseTime到滑动步长的倍数，确保窗口对齐行为可预测 + baseTime := (baseTimeRaw / slideSizeMs) * slideSizeMs + + // 第一阶段：发送正常顺序的数据 + // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） + t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待 watermark 推进，触发第一个窗口 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟数据 + // 延迟数据的事件时间在第一个窗口内（如 500ms, 700ms, 900ms） + // 如果 MaxOutOfOrderness = 1秒，这些数据应该能被处理 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内，延迟 < 1秒）") + lateDataTimes := []int64{500, 700, 900} // 延迟数据的事件时间（相对于 baseTime） + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(20 + i), // 20-22，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：发送更多正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果（添加超时和最大迭代次数限制） + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + // 如果 MaxOutOfOrderness 配置正确，延迟数据应该能被统计到对应窗口 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + assert.GreaterOrEqual(t, cnt, 5.0, "第一个窗口应该包含足够的数据") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + + // 注意：如果 MaxOutOfOrderness 配置正确且延迟数据被处理， + // maxTemp 可能会包含延迟数据的值（20-22） + // 但由于当前可能没有配置 MaxOutOfOrderness，延迟数据可能不会被统计 + t.Logf("提示：如果 MaxOutOfOrderness 配置正确，延迟数据（temperature=20-22）应该能被统计") + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLSlidingWindow_AllowedLateness 测试滑动窗口的 AllowedLateness 配置 +// 验证窗口触发后，延迟数据能否在允许的延迟时间内更新窗口结果 +func TestSQLSlidingWindow_AllowedLateness(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, SlidingWindow('2s', '500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='1s', ALLOWEDLATENESS='500ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟 AllowedLateness 场景 + // 场景：窗口触发后，发送延迟数据，验证窗口能否更新 + baseTime := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + + // 第一阶段：发送正常顺序的数据，触发第一个窗口 + // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） + t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 推进watermark，触发第一个窗口 + // 发送事件时间超过第一个窗口结束时间的数据 + firstWindowEnd := baseTime + int64(2000) // 第一个窗口结束时间 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + int64(2000), + "temperature": 100.0, + }) + + // 等待 watermark 推进，触发第一个窗口 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 收集第一个窗口的结果（添加最大迭代次数限制） + firstWindowReceived := false + firstWindowCnt := 0.0 + firstWindowMax := 0.0 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 10 + iteration := 0 + + for !firstWindowReceived && iteration < maxIterations { + select { + case res, ok := <-ch: + if !ok { + // channel 已关闭 + goto COLLECT_FIRST_WINDOW_END + } + if len(res) > 0 { + firstWindowReceived = true + firstWindowCnt = res[0]["cnt"].(float64) + firstWindowMax = res[0]["max_temp"].(float64) + t.Logf("第一个窗口: cnt=%.0f, max=%.0f", firstWindowCnt, firstWindowMax) + } + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果 + iteration++ + case <-ctx.Done(): + t.Log("等待第一个窗口超时") + goto COLLECT_FIRST_WINDOW_END + } + } +COLLECT_FIRST_WINDOW_END: + + if !firstWindowReceived { + t.Log("⚠ 第一个窗口未触发，可能watermark未推进到足够位置") + } + assert.GreaterOrEqual(t, firstWindowCnt, 5.0, "第一个窗口应该包含足够的数据") + assert.LessOrEqual(t, firstWindowMax, 9.0, "第一个窗口的最大值应该不超过9（正常数据）") + + // 第二阶段：发送延迟数据（事件时间在第一个窗口内） + // 这些数据应该在 AllowedLateness = 500ms 内被处理 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + lateDataTimes := []int64{300, 600, 900} // 延迟数据的事件时间 + lateDataTemps := []float64{30.0, 31.0, 32.0} + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": lateDataTemps[i], // 30-32，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：继续发送正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果（添加超时和最大迭代次数限制） + ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel2() + maxIterations2 := 20 + iteration2 := 0 + + for iteration2 < maxIterations2 { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration2++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx2.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + // 如果 AllowedLateness 配置正确，延迟数据应该能触发窗口的延迟更新 + if windowResultsLen > 0 { + // 滑动窗口的延迟更新可能体现在后续的窗口结果中 + // 检查所有窗口结果，看是否有包含延迟数据的窗口 + hasLateDataUpdate := false + for i, window := range windowResultsCopy { + if len(window) > 0 { + cnt := window[0]["cnt"].(float64) + minTemp := window[0]["min_temp"].(float64) + maxTemp := window[0]["max_temp"].(float64) + + t.Logf("窗口 %d: cnt=%.0f, min=%.0f, max=%.0f", i+1, cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + assert.GreaterOrEqual(t, cnt, 1.0, "窗口 %d 应该包含数据", i+1) + + // 如果 AllowedLateness 配置正确，延迟数据应该被处理 + // 延迟数据（temperature=30-32）应该能被统计 + if maxTemp >= 30.0 { + hasLateDataUpdate = true + t.Logf("✓ 窗口 %d 包含延迟数据，最大值: %.0f", i+1, maxTemp) + + // 验证延迟更新包含更多数据 + if i == 0 { + // 第一个窗口的延迟更新应该包含更多数据 + assert.GreaterOrEqual(t, cnt, firstWindowCnt+3.0, + "延迟更新应该包含更多数据（原数据 + 延迟数据）") + } + } + } + } + + // 验证是否有延迟更新（窗口可能触发多次） + if windowResultsLen > 1 { + t.Logf("✓ 滑动窗口触发了 %d 次，可能包含延迟更新", windowResultsLen) + } + + if !hasLateDataUpdate { + t.Logf("⚠ 提示：延迟数据可能未被统计，或延迟数据的时间不在窗口范围内") + } else { + t.Logf("✓ AllowedLateness 功能正常工作，延迟数据已被处理") + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLSlidingWindow_EventTimeWindowAlignment 测试事件时间滑动窗口对齐到epoch +func TestSQLSlidingWindow_EventTimeWindowAlignment(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, SlidingWindow('2s', '500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，验证滑动窗口对齐到滑动步长 + // 窗口大小2秒，滑动步长500ms，应该对齐到500ms的倍数 + baseTime := time.Now().UnixMilli() + + // 发送数据，事件时间从baseTime开始，每200ms一条 + for i := 0; i < 20; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发 + time.Sleep(3 * time.Second) + + timeout := time.After(2 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口对齐 + windowSizeMs := int64(2000) // 2秒 + slideSizeMs := int64(500) // 500ms + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + windowSizeNs := int64(windowSizeMs) * int64(time.Millisecond) + + assert.Equal(t, windowSizeNs, end-start, + "窗口 %d 的大小应该是2秒（2000ms），实际: start=%d, end=%d", i+1, start, end) + + assert.Equal(t, int64(0), startMs%slideSizeMs, + "窗口 %d 的开始时间应该对齐到500ms的倍数（epoch对齐），实际: startMs=%d", i+1, startMs) + + if i > 0 { + prevStartMs := windowResultsCopy[i-1][0]["start"].(int64) / int64(time.Millisecond) + actualSlideMs := startMs - prevStartMs + assert.Equal(t, slideSizeMs, actualSlideMs, + "窗口 %d 的滑动步长应该是500ms，prevStartMs=%d, startMs=%d, actualSlideMs=%d", + i+1, prevStartMs, startMs, actualSlideMs) + } + + t.Logf("窗口 %d: start=%d, end=%d, size=%dms", i+1, startMs, endMs, endMs-startMs) + } + } +} + +// TestSQLSlidingWindow_WatermarkTriggerTiming 测试滑动窗口Watermark触发时机 +func TestSQLSlidingWindow_WatermarkTriggerTiming(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, SlidingWindow('2s', '500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，验证watermark触发时机 + baseTime := time.Now().UnixMilli() - 10000 + maxOutOfOrdernessMs := int64(500) + slideSizeMs := int64(500) + + // 发送数据，事件时间在第一个窗口内 + // 注意：第一个数据的事件时间会影响窗口对齐 + firstEventTime := baseTime + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 计算对齐后的第一个窗口开始时间（基于第一个数据的事件时间） + // alignWindowStart 会对齐到小于等于事件时间的最大对齐点 + alignedStart := (firstEventTime / slideSizeMs) * slideSizeMs + firstWindowEnd := alignedStart + 2000 // 窗口大小2秒 + + t.Logf("第一个窗口: [%d, %d)", alignedStart, firstWindowEnd) + + // 发送一个事件时间超过window_end的数据，推进watermark + // watermark = maxEventTime - maxOutOfOrderness = (firstWindowEnd + 1000) - 500 = firstWindowEnd + 500 + // 此时 watermark >= firstWindowEnd，窗口应该触发 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + 1000, + "temperature": 200.0, + }) + + // 等待窗口触发 + time.Sleep(1 * time.Second) + + timeout := time.After(2 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证第一个窗口的触发时机 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + start := row["start"].(int64) + end := row["end"].(int64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + + // 验证窗口对齐到滑动步长（允许一定的误差，因为对齐基于第一个数据的事件时间） + assert.Equal(t, int64(0), startMs%slideSizeMs, + "第一个窗口的开始时间应该对齐到滑动步长，expected对齐到%d的倍数，actual=%d", slideSizeMs, startMs) + // 验证窗口大小正确 + assert.Equal(t, int64(2000), endMs-startMs, + "第一个窗口的大小应该是2秒（2000ms），actual=%d", endMs-startMs) + + t.Logf("✓ 滑动窗口在watermark >= window_end时正确触发") + t.Logf("窗口: [%d, %d), 触发时maxEventTime >= %d", start, end, end+maxOutOfOrdernessMs) + } + } +} + +// TestSQLSlidingWindow_IdleSourceMechanism 测试滑动窗口的Idle Source机制 +func TestSQLSlidingWindow_IdleSourceMechanism(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, SlidingWindow('2s', '500ms') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + defer func() { + if r := recover(); r != nil { + // channel 已关闭，忽略错误 + } + }() + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + baseTime := time.Now().UnixMilli() - 10000 + slideSizeMs := int64(500) + alignedStart := (baseTime / slideSizeMs) * slideSizeMs + + // 发送数据 + t.Log("发送数据，创建滑动窗口") + for i := 0; i < 10; i++ { + eventTime := alignedStart + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 停止发送数据，等待Idle Source机制触发 + t.Log("停止发送数据，等待Idle Source机制触发（IdleTimeout=2s）") + time.Sleep(3 * time.Second) + + // 收集窗口结果 + timeout := time.After(3 * time.Second) + for { + select { + case <-ch: + // 继续收集结果 + case <-timeout: + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个滑动窗口（即使数据源空闲）") + + if windowResultsLen > 0 { + t.Logf("✓ 滑动窗口Idle Source机制正常工作，触发了 %d 个窗口", windowResultsLen) + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + t.Logf("窗口 %d: [%d, %d), cnt=%.0f", i+1, start, end, cnt) + } + } + } +} diff --git a/streamsql_test.go b/streamsql_test.go index 3d1f060..bcc294e 100644 --- a/streamsql_test.go +++ b/streamsql_test.go @@ -23,6 +23,8 @@ func TestStreamData(t *testing.T) { // 步骤1: 创建 StreamSQL 实例 // StreamSQL 是流式 SQL 处理引擎的核心组件，负责管理整个流处理生命周期 ssql := New() + // 确保测试结束时停止流处理，释放资源 + defer ssql.Stop() // 步骤2: 定义流式 SQL 查询语句 // 这个 SQL 语句展示了 StreamSQL 的核心功能： @@ -82,25 +84,40 @@ func TestStreamData(t *testing.T) { }() // 步骤6: 设置结果处理管道 - resultChan := make(chan interface{}) + resultChan := make(chan interface{}, 10) // 添加计算结果回调函数（Sink） // 当窗口触发计算时，结果会通过这个回调函数输出 ssql.stream.AddSink(func(result []map[string]interface{}) { - resultChan <- result + // 非阻塞发送，避免阻塞 sink worker + select { + case resultChan <- result: + default: + // Channel 已满，忽略（非阻塞发送） + } }) // 步骤7: 启动结果消费者协程 // 记录收到的结果数量，用于验证测试效果 var resultCount int64 var countMutex sync.Mutex + var consumerWg sync.WaitGroup + consumerWg.Add(1) go func() { - for range resultChan { - // 每当收到一个窗口的计算结果时，计数器加1 - // 注释掉的代码可以用于调试，打印每个结果的详细信息 - //fmt.Printf("打印结果: [%s] %v\n", time.Now().Format("15:04:05.000"), result) - countMutex.Lock() - resultCount++ - countMutex.Unlock() + defer consumerWg.Done() + for { + select { + case <-resultChan: + // 每当收到一个窗口的计算结果时，计数器加1 + // 注释掉的代码可以用于调试，打印每个结果的详细信息 + //fmt.Printf("打印结果: [%s] %v\n", time.Now().Format("15:04:05.000"), result) + countMutex.Lock() + resultCount++ + countMutex.Unlock() + case <-ctx.Done(): + // 测试超时，退出消费者 goroutine + // 不关闭 channel，让主程序自动退出时清理 + return + } } }() @@ -108,6 +125,19 @@ func TestStreamData(t *testing.T) { // 等待数据生产者协程结束（30秒超时或手动取消） wg.Wait() + // 停止流处理，确保所有 goroutine 正确退出 + ssql.Stop() + + // 等待一小段时间，确保所有 sink worker 完成当前任务 + // 这样可以确保所有结果都被发送到 channel + time.Sleep(100 * time.Millisecond) + + // 取消 context，通知消费者 goroutine 退出 + cancel() + + // 等待消费者 goroutine 完成（处理完 channel 中剩余的数据或收到取消信号） + consumerWg.Wait() + // 步骤9: 验证测试结果 // 预期在30秒内应该收到5个窗口的计算结果（每5秒一个窗口） // 这验证了 StreamSQL 的窗口触发机制是否正常工作 @@ -119,26 +149,35 @@ func TestStreamData(t *testing.T) { func TestStreamsql(t *testing.T) { streamsql := New() - var rsql = "SELECT device,max(temperature) as max_temp,min(humidity) as min_humidity,window_start() as start,window_end() as end FROM stream group by device,SlidingWindow('2s','1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + defer streamsql.Stop() + var rsql = "SELECT device,max(temperature) as max_temp,min(humidity) as min_humidity,window_start() as start,window_end() as end FROM stream group by device,SlidingWindow('2s','1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "aa", "temperature": 25.0, "humidity": 60, "Ts": baseTime}, - {"device": "aa", "temperature": 30.0, "humidity": 55, "Ts": baseTime.Add(1 * time.Second)}, - {"device": "bb", "temperature": 22.0, "humidity": 70, "Ts": baseTime}, + {"device": "aa", "temperature": 25.0, "humidity": 60}, + {"device": "aa", "temperature": 30.0, "humidity": 55}, + {"device": "bb", "temperature": 22.0, "humidity": 70}, } for _, data := range testData { strm.Emit(data) } // 捕获结果 - resultChan := make(chan interface{}) + resultChan := make(chan interface{}, 10) strm.AddSink(func(result []map[string]interface{}) { - resultChan <- result + select { + case resultChan <- result: + default: + // 非阻塞发送，避免阻塞 + } }) + // 等待窗口触发 + // 由于使用事件时间，需要等待 watermark 推进（IDLETIMEOUT='2s' 会在2秒后推进） + time.Sleep(3 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -155,15 +194,11 @@ func TestStreamsql(t *testing.T) { "device": "aa", "max_temp": 30.0, "min_humidity": 55.0, - "start": baseTime.UnixNano(), - "end": baseTime.Add(2 * time.Second).UnixNano(), }, { "device": "bb", "max_temp": 22.0, "min_humidity": 70.0, - "start": baseTime.UnixNano(), - "end": baseTime.Add(2 * time.Second).UnixNano(), }, } @@ -177,8 +212,13 @@ func TestStreamsql(t *testing.T) { if resultMap["device"] == expectedResult["device"] { assert.InEpsilon(t, expectedResult["max_temp"].(float64), resultMap["max_temp"].(float64), 0.0001) assert.InEpsilon(t, expectedResult["min_humidity"].(float64), resultMap["min_humidity"].(float64), 0.0001) - assert.Equal(t, expectedResult["start"].(int64), resultMap["start"].(int64)) - assert.Equal(t, expectedResult["end"].(int64), resultMap["end"].(int64)) + // 事件时间模式下，窗口时间基于事件时间戳，检查字段存在和有效性 + assert.Contains(t, resultMap, "start") + assert.Contains(t, resultMap, "end") + start, ok1 := resultMap["start"].(int64) + end, ok2 := resultMap["end"].(int64) + assert.True(t, ok1 && ok2, "start and end should be int64") + assert.Greater(t, end, start, "end should be greater than start") found = true break } @@ -189,27 +229,35 @@ func TestStreamsql(t *testing.T) { func TestStreamsqlWithoutGroupBy(t *testing.T) { streamsql := New() - var rsql = "SELECT max(temperature) as max_temp,min(humidity) as min_humidity,window_start() as start,window_end() as end FROM stream SlidingWindow('2s','1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + defer streamsql.Stop() + var rsql = "SELECT max(temperature) as max_temp,min(humidity) as min_humidity,window_start() as start,window_end() as end FROM stream SlidingWindow('2s','1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "aa", "temperature": 25.0, "humidity": 60, "Ts": baseTime}, - {"device": "aa", "temperature": 30.0, "humidity": 55, "Ts": baseTime.Add(1 * time.Second)}, - {"device": "bb", "temperature": 22.0, "humidity": 70, "Ts": baseTime}, + {"device": "aa", "temperature": 25.0, "humidity": 60}, + {"device": "aa", "temperature": 30.0, "humidity": 55}, + {"device": "bb", "temperature": 22.0, "humidity": 70}, } for _, data := range testData { strm.Emit(data) } // 捕获结果 - resultChan := make(chan interface{}) + resultChan := make(chan interface{}, 10) strm.AddSink(func(result []map[string]interface{}) { - resultChan <- result + select { + case resultChan <- result: + default: + // 非阻塞发送 + } }) - ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + // 等待窗口触发（事件时间模式需要等待 watermark 推进） + time.Sleep(3 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() var actual interface{} @@ -224,8 +272,6 @@ func TestStreamsqlWithoutGroupBy(t *testing.T) { { "max_temp": 30.0, "min_humidity": 55.0, - "start": baseTime.UnixNano(), - "end": baseTime.Add(2 * time.Second).UnixNano(), }, } @@ -234,14 +280,17 @@ func TestStreamsqlWithoutGroupBy(t *testing.T) { require.True(t, ok) assert.Len(t, resultSlice, 1) for _, expectedResult := range expected { - //found := false for _, resultMap := range resultSlice { assert.InEpsilon(t, expectedResult["max_temp"].(float64), resultMap["max_temp"].(float64), 0.0001) assert.InEpsilon(t, expectedResult["min_humidity"].(float64), resultMap["min_humidity"].(float64), 0.0001) - assert.Equal(t, expectedResult["start"].(int64), resultMap["start"].(int64)) - assert.Equal(t, expectedResult["end"].(int64), resultMap["end"].(int64)) + // 事件时间模式下，窗口时间基于事件时间戳，检查字段存在 + assert.Contains(t, resultMap, "start") + assert.Contains(t, resultMap, "end") + start, ok1 := resultMap["start"].(int64) + end, ok2 := resultMap["end"].(int64) + assert.True(t, ok1 && ok2, "start and end should be int64") + assert.Greater(t, end, start, "end should be greater than start") } - //assert.True(t, found, fmt.Sprintf("Expected result for device %v not found", expectedResult["device"])) } } @@ -250,23 +299,21 @@ func TestStreamsqlDistinct(t *testing.T) { defer streamsql.Stop() // 测试 SELECT DISTINCT 功能 - 使用聚合函数和 GROUP BY - var rsql = "SELECT DISTINCT device, AVG(temperature) as avg_temp FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT DISTINCT device, AVG(temperature) as avg_temp FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试 SELECT DISTINCT 功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据，包含重复的设备数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "aa", "temperature": 25.0, "Ts": baseTime}, - {"device": "aa", "temperature": 35.0, "Ts": baseTime}, // 相同设备，不同温度 - {"device": "bb", "temperature": 22.0, "Ts": baseTime}, - {"device": "bb", "temperature": 28.0, "Ts": baseTime}, // 相同设备，不同温度 - {"device": "cc", "temperature": 30.0, "Ts": baseTime}, + {"device": "aa", "temperature": 25.0}, + {"device": "aa", "temperature": 35.0}, // 相同设备，不同温度 + {"device": "bb", "temperature": 22.0}, + {"device": "bb", "temperature": 28.0}, // 相同设备，不同温度 + {"device": "cc", "temperature": 30.0}, } // 添加数据 @@ -281,10 +328,20 @@ func TestStreamsqlDistinct(t *testing.T) { // 添加结果回调 strm.AddSink(func(result []map[string]interface{}) { //fmt.Printf("接收到结果: %v\n", result) - resultChan <- result + // 使用 recover 防止在 channel 关闭后发送数据导致 panic + defer func() { + if r := recover(); r != nil { + // Channel 已关闭，忽略错误 + } + }() + select { + case resultChan <- result: + default: + // 非阻塞发送 + } }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -292,8 +349,8 @@ func TestStreamsqlDistinct(t *testing.T) { //fmt.Println("手动触发窗口") strm.Window.Trigger() - // 等待结果 - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + // 等待结果，增加超时时间以确保窗口有足够时间触发 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() var actual interface{} @@ -756,7 +813,7 @@ func TestSessionWindow(t *testing.T) { defer streamsql.Stop() // 使用 SESSION 窗口，超时时间为 2 秒 - rsql := "SELECT device, avg(temperature) as avg_temp FROM stream GROUP BY device, SESSIONWINDOW('2s') with (TIMESTAMP='Ts')" + rsql := "SELECT device, avg(temperature) as avg_temp FROM stream GROUP BY device, SESSIONWINDOW('2s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream @@ -770,25 +827,24 @@ func TestSessionWindow(t *testing.T) { resultChan <- result }) - baseTime := time.Now() - // 添加测试数据 - 两个设备，不同的时间 + // 不使用事件时间，不需要时间戳字段 testData := []struct { data map[string]interface{} wait time.Duration }{ // 第一组数据 - device1 - {map[string]interface{}{"device": "device1", "temperature": 20.0, "Ts": baseTime}, 0}, - {map[string]interface{}{"device": "device1", "temperature": 22.0, "Ts": baseTime.Add(500 * time.Millisecond)}, 500 * time.Millisecond}, + {map[string]interface{}{"device": "device1", "temperature": 20.0}, 0}, + {map[string]interface{}{"device": "device1", "temperature": 22.0}, 500 * time.Millisecond}, // 第二组数据 - device2 - {map[string]interface{}{"device": "device2", "temperature": 25.0, "Ts": baseTime.Add(time.Second)}, time.Second}, - {map[string]interface{}{"device": "device2", "temperature": 27.0, "Ts": baseTime.Add(1500 * time.Millisecond)}, 500 * time.Millisecond}, + {map[string]interface{}{"device": "device2", "temperature": 25.0}, time.Second}, + {map[string]interface{}{"device": "device2", "temperature": 27.0}, 500 * time.Millisecond}, // 间隔超过会话超时 // 第三组数据 - device1，新会话 - {map[string]interface{}{"device": "device1", "temperature": 30.0, "Ts": baseTime.Add(5 * time.Second)}, 3 * time.Second}, + {map[string]interface{}{"device": "device1", "temperature": 30.0}, 3 * time.Second}, } // 按指定的间隔添加数据 @@ -868,22 +924,20 @@ func TestExpressionInAggregation(t *testing.T) { defer streamsql.Stop() // 测试在聚合函数中使用表达式 - var rsql = "SELECT device, AVG(temperature * 1.8 + 32) as fahrenheit FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, AVG(temperature * 1.8 + 32) as fahrenheit FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试表达式功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据，温度使用摄氏度 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "aa", "temperature": 0.0, "Ts": baseTime}, // 华氏度应为 32 - {"device": "aa", "temperature": 100.0, "Ts": baseTime}, // 华氏度应为 212 - {"device": "bb", "temperature": 20.0, "Ts": baseTime}, // 华氏度应为 68 - {"device": "bb", "temperature": 30.0, "Ts": baseTime}, // 华氏度应为 86 + {"device": "aa", "temperature": 0.0}, // 华氏度应为 32 + {"device": "aa", "temperature": 100.0}, // 华氏度应为 212 + {"device": "bb", "temperature": 20.0}, // 华氏度应为 68 + {"device": "bb", "temperature": 30.0}, // 华氏度应为 86 } // 添加数据 @@ -901,7 +955,7 @@ func TestExpressionInAggregation(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -953,22 +1007,20 @@ func TestAdvancedFunctionsInSQL(t *testing.T) { defer streamsql.Stop() // 测试使用新函数系统的复杂SQL查询 - var rsql = "SELECT device, AVG(abs(temperature - 20)) as abs_diff, CONCAT(device, '_processed') as device_name FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, AVG(abs(temperature - 20)) as abs_diff, CONCAT(device, '_processed') as device_name FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试高级函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "temperature": 15.0, "Ts": baseTime}, // abs(15-20) = 5 - {"device": "sensor1", "temperature": 25.0, "Ts": baseTime}, // abs(25-20) = 5 - {"device": "sensor2", "temperature": 18.0, "Ts": baseTime}, // abs(18-20) = 2 - {"device": "sensor2", "temperature": 22.0, "Ts": baseTime}, // abs(22-20) = 2 + {"device": "sensor1", "temperature": 15.0}, // abs(15-20) = 5 + {"device": "sensor1", "temperature": 25.0}, // abs(25-20) = 5 + {"device": "sensor2", "temperature": 18.0}, // abs(18-20) = 2 + {"device": "sensor2", "temperature": 22.0}, // abs(22-20) = 2 } // 添加数据 @@ -986,7 +1038,7 @@ func TestAdvancedFunctionsInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1052,22 +1104,20 @@ func TestCustomFunctionInSQL(t *testing.T) { defer streamsql.Stop() // 测试使用自定义函数的SQL查询 - var rsql = "SELECT device, AVG(fahrenheit_to_celsius(temperature)) as avg_celsius FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, AVG(fahrenheit_to_celsius(temperature)) as avg_celsius FROM stream GROUP BY device, TumblingWindow('1s')" err = streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试自定义函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据（华氏度） + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "thermometer1", "temperature": 32.0, "Ts": baseTime}, // 0°C - {"device": "thermometer1", "temperature": 212.0, "Ts": baseTime}, // 100°C - {"device": "thermometer2", "temperature": 68.0, "Ts": baseTime}, // 20°C - {"device": "thermometer2", "temperature": 86.0, "Ts": baseTime}, // 30°C + {"device": "thermometer1", "temperature": 32.0}, // 0°C + {"device": "thermometer1", "temperature": 212.0}, // 100°C + {"device": "thermometer2", "temperature": 68.0}, // 20°C + {"device": "thermometer2", "temperature": 86.0}, // 30°C } // 添加数据 @@ -1085,7 +1135,7 @@ func TestCustomFunctionInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1137,22 +1187,20 @@ func TestNewAggregateFunctionsInSQL(t *testing.T) { defer streamsql.Stop() // 测试使用新聚合函数的SQL查询 - var rsql = "SELECT device, collect(temperature) as temp_values, last_value(temperature) as last_temp, merge_agg(status) as all_status FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, collect(temperature) as temp_values, last_value(temperature) as last_temp, merge_agg(status) as all_status FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试新聚合函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "temperature": 15.0, "status": "good", "Ts": baseTime}, - {"device": "sensor1", "temperature": 25.0, "status": "ok", "Ts": baseTime}, - {"device": "sensor2", "temperature": 18.0, "status": "good", "Ts": baseTime}, - {"device": "sensor2", "temperature": 22.0, "status": "warning", "Ts": baseTime}, + {"device": "sensor1", "temperature": 15.0, "status": "good"}, + {"device": "sensor1", "temperature": 25.0, "status": "ok"}, + {"device": "sensor2", "temperature": 18.0, "status": "good"}, + {"device": "sensor2", "temperature": 22.0, "status": "warning"}, } // 添加数据 @@ -1170,7 +1218,7 @@ func TestNewAggregateFunctionsInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1246,23 +1294,21 @@ func TestStatisticalAggregateFunctionsInSQL(t *testing.T) { defer streamsql.Stop() // 测试使用统计聚合函数的SQL查询 - var rsql = "SELECT device, stddevs(temperature) as sample_stddev, var(temperature) as population_var, vars(temperature) as sample_var FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, stddevs(temperature) as sample_stddev, var(temperature) as population_var, vars(temperature) as sample_var FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试统计聚合函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "temperature": 10.0, "Ts": baseTime}, - {"device": "sensor1", "temperature": 20.0, "Ts": baseTime}, - {"device": "sensor1", "temperature": 30.0, "Ts": baseTime}, - {"device": "sensor2", "temperature": 15.0, "Ts": baseTime}, - {"device": "sensor2", "temperature": 25.0, "Ts": baseTime}, + {"device": "sensor1", "temperature": 10.0}, + {"device": "sensor1", "temperature": 20.0}, + {"device": "sensor1", "temperature": 30.0}, + {"device": "sensor2", "temperature": 15.0}, + {"device": "sensor2", "temperature": 25.0}, } // 添加数据 @@ -1280,7 +1326,7 @@ func TestStatisticalAggregateFunctionsInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1346,25 +1392,23 @@ func TestDeduplicateAggregateInSQL(t *testing.T) { defer streamsql.Stop() // 测试使用去重聚合函数的SQL查询 - var rsql = "SELECT device, deduplicate(status) as unique_status FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, deduplicate(status) as unique_status FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试去重聚合函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据，包含重复的状态 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "status": "good", "Ts": baseTime}, - {"device": "sensor1", "status": "good", "Ts": baseTime}, // 重复 - {"device": "sensor1", "status": "warning", "Ts": baseTime}, - {"device": "sensor1", "status": "good", "Ts": baseTime}, // 重复 - {"device": "sensor2", "status": "error", "Ts": baseTime}, - {"device": "sensor2", "status": "error", "Ts": baseTime}, // 重复 - {"device": "sensor2", "status": "ok", "Ts": baseTime}, + {"device": "sensor1", "status": "good"}, + {"device": "sensor1", "status": "good"}, // 重复 + {"device": "sensor1", "status": "warning"}, + {"device": "sensor1", "status": "good"}, // 重复 + {"device": "sensor2", "status": "error"}, + {"device": "sensor2", "status": "error"}, // 重复 + {"device": "sensor2", "status": "ok"}, } // 添加数据 @@ -1382,7 +1426,7 @@ func TestDeduplicateAggregateInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1452,8 +1496,7 @@ func TestExprAggregationFunctions(t *testing.T) { merge_agg(device + '_' + status) as device_status, deduplicate(status + '_' + device) as unique_status_device FROM stream - GROUP BY device, TumblingWindow('1s') - with (TIMESTAMP='Ts',TIMEUNIT='ss')` + GROUP BY device, TumblingWindow('1s')` err := streamsql.Execute(rsql) assert.Nil(t, err) @@ -1461,20 +1504,18 @@ func TestExprAggregationFunctions(t *testing.T) { //fmt.Println("开始测试表达式聚合函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ // device1的数据 - {"device": "device1", "temperature": 20.0, "humidity": 60.0, "status": "normal", "Ts": baseTime}, // 华氏度=68, 偏差=0, 和=80 - {"device": "device1", "temperature": 25.0, "humidity": 65.0, "status": "warning", "Ts": baseTime}, // 华氏度=77, 偏差=10, 和=90 - {"device": "device1", "temperature": 30.0, "humidity": 70.0, "status": "normal", "Ts": baseTime}, // 华氏度=86, 偏差=20, 和=100 + {"device": "device1", "temperature": 20.0, "humidity": 60.0, "status": "normal"}, // 华氏度=68, 偏差=0, 和=80 + {"device": "device1", "temperature": 25.0, "humidity": 65.0, "status": "warning"}, // 华氏度=77, 偏差=10, 和=90 + {"device": "device1", "temperature": 30.0, "humidity": 70.0, "status": "normal"}, // 华氏度=86, 偏差=20, 和=100 // device2的数据 - {"device": "device2", "temperature": 15.0, "humidity": 55.0, "status": "error", "Ts": baseTime}, // 华氏度=59, 偏差=-10, 和=70 - {"device": "device2", "temperature": 18.0, "humidity": 58.0, "status": "normal", "Ts": baseTime}, // 华氏度=64.4, 偏差=-4, 和=76 - {"device": "device2", "temperature": 22.0, "humidity": 62.0, "status": "error", "Ts": baseTime}, // 华氏度=71.6, 偏差=4, 和=84 + {"device": "device2", "temperature": 15.0, "humidity": 55.0, "status": "error"}, // 华氏度=59, 偏差=-10, 和=70 + {"device": "device2", "temperature": 18.0, "humidity": 58.0, "status": "normal"}, // 华氏度=64.4, 偏差=-4, 和=76 + {"device": "device2", "temperature": 22.0, "humidity": 62.0, "status": "error"}, // 华氏度=71.6, 偏差=4, 和=84 } // 添加数据 @@ -1492,7 +1533,7 @@ func TestExprAggregationFunctions(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1615,23 +1656,21 @@ func TestAnalyticalFunctionsInSQL(t *testing.T) { defer streamsql.Stop() // 测试使用分析函数的SQL查询 - var rsql = "SELECT device, lag(temperature) as prev_temp, latest(temperature) as current_temp, had_changed(temperature) as temp_changed FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, lag(temperature) as prev_temp, latest(temperature) as current_temp, had_changed(temperature) as temp_changed FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试分析函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "temperature": 20.0, "Ts": baseTime}, - {"device": "sensor1", "temperature": 25.0, "Ts": baseTime}, - {"device": "sensor1", "temperature": 25.0, "Ts": baseTime}, // 重复值，测试had_changed - {"device": "sensor2", "temperature": 18.0, "Ts": baseTime}, - {"device": "sensor2", "temperature": 22.0, "Ts": baseTime}, + {"device": "sensor1", "temperature": 20.0}, + {"device": "sensor1", "temperature": 25.0}, + {"device": "sensor1", "temperature": 25.0}, // 重复值，测试had_changed + {"device": "sensor2", "temperature": 18.0}, + {"device": "sensor2", "temperature": 22.0}, } // 添加数据 @@ -1649,7 +1688,7 @@ func TestAnalyticalFunctionsInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1712,22 +1751,20 @@ func TestLagFunctionInSQL(t *testing.T) { defer streamsql.Stop() // 测试LAG函数的SQL查询 - var rsql = "SELECT device, lag(temperature) as prev_temp FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, lag(temperature) as prev_temp FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试LAG函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 - 按顺序添加，测试LAG功能 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "temp_sensor", "temperature": 10.0, "Ts": baseTime}, - {"device": "temp_sensor", "temperature": 15.0, "Ts": baseTime}, - {"device": "temp_sensor", "temperature": 20.0, "Ts": baseTime}, - {"device": "temp_sensor", "temperature": 25.0, "Ts": baseTime}, // 最后一个值 + {"device": "temp_sensor", "temperature": 10.0}, + {"device": "temp_sensor", "temperature": 15.0}, + {"device": "temp_sensor", "temperature": 20.0}, + {"device": "temp_sensor", "temperature": 25.0}, // 最后一个值 } // 添加数据 @@ -1747,7 +1784,7 @@ func TestLagFunctionInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1810,23 +1847,21 @@ func TestHadChangedFunctionInSQL(t *testing.T) { defer streamsql.Stop() // 测试had_changed函数的SQL查询 - var rsql = "SELECT device, had_changed(temperature) as temp_changed FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, had_changed(temperature) as temp_changed FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试had_changed函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 - 包含重复值和变化值 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "monitor", "temperature": 20.0, "Ts": baseTime}, - {"device": "monitor", "temperature": 20.0, "Ts": baseTime}, // 相同值 - {"device": "monitor", "temperature": 25.0, "Ts": baseTime}, // 变化值 - {"device": "monitor", "temperature": 25.0, "Ts": baseTime}, // 相同值 - {"device": "monitor", "temperature": 30.0, "Ts": baseTime}, // 变化值 + {"device": "monitor", "temperature": 20.0}, + {"device": "monitor", "temperature": 20.0}, // 相同值 + {"device": "monitor", "temperature": 25.0}, // 变化值 + {"device": "monitor", "temperature": 25.0}, // 相同值 + {"device": "monitor", "temperature": 30.0}, // 变化值 } // 添加数据 @@ -1844,7 +1879,7 @@ func TestHadChangedFunctionInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1891,22 +1926,20 @@ func TestLatestFunctionInSQL(t *testing.T) { defer streamsql.Stop() // 测试latest函数的SQL查询 - var rsql = "SELECT device, latest(temperature) as current_temp FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, latest(temperature) as current_temp FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试latest函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "thermometer", "temperature": 10.0, "Ts": baseTime}, - {"device": "thermometer", "temperature": 15.0, "Ts": baseTime}, - {"device": "thermometer", "temperature": 20.0, "Ts": baseTime}, - {"device": "thermometer", "temperature": 25.0, "Ts": baseTime}, // 最新值 + {"device": "thermometer", "temperature": 10.0}, + {"device": "thermometer", "temperature": 15.0}, + {"device": "thermometer", "temperature": 20.0}, + {"device": "thermometer", "temperature": 25.0}, // 最新值 } // 添加数据 @@ -1924,7 +1957,7 @@ func TestLatestFunctionInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -1972,32 +2005,27 @@ func TestChangedColFunctionInSQL(t *testing.T) { defer streamsql.Stop() // 测试changed_col函数的SQL查询 - var rsql = "SELECT device, changed_col(data) as changed_fields FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, changed_col(data) as changed_fields FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试changed_col函数功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 - 使用map作为数据测试changed_col + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ { "device": "datacollector", "data": map[string]interface{}{"temp": 20.0, "humidity": 60.0}, - "Ts": baseTime, }, { "device": "datacollector", "data": map[string]interface{}{"temp": 25.0, "humidity": 60.0}, // temp变化 - "Ts": baseTime, }, { "device": "datacollector", "data": map[string]interface{}{"temp": 25.0, "humidity": 65.0}, // humidity变化 - "Ts": baseTime, }, } @@ -2016,7 +2044,7 @@ func TestChangedColFunctionInSQL(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -2063,23 +2091,21 @@ func TestAnalyticalFunctionsIncrementalComputation(t *testing.T) { defer streamsql.Stop() // 测试使用分析函数的SQL查询（现在支持增量计算） - var rsql = "SELECT device, lag(temperature, 1) as prev_temp, latest(temperature) as current_temp, had_changed(status) as status_changed FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, lag(temperature, 1) as prev_temp, latest(temperature) as current_temp, had_changed(status) as status_changed FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试分析函数增量计算功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "temperature": 15.0, "status": "good", "Ts": baseTime}, - {"device": "sensor1", "temperature": 25.0, "status": "good", "Ts": baseTime}, - {"device": "sensor1", "temperature": 35.0, "status": "warning", "Ts": baseTime}, - {"device": "sensor2", "temperature": 18.0, "status": "good", "Ts": baseTime}, - {"device": "sensor2", "temperature": 22.0, "status": "ok", "Ts": baseTime}, + {"device": "sensor1", "temperature": 15.0, "status": "good"}, + {"device": "sensor1", "temperature": 25.0, "status": "good"}, + {"device": "sensor1", "temperature": 35.0, "status": "warning"}, + {"device": "sensor2", "temperature": 18.0, "status": "good"}, + {"device": "sensor2", "temperature": 22.0, "status": "ok"}, } // 添加数据 @@ -2097,7 +2123,7 @@ func TestAnalyticalFunctionsIncrementalComputation(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -2162,23 +2188,21 @@ func TestIncrementalComputationBasic(t *testing.T) { defer streamsql.Stop() // 测试基本的增量计算聚合函数 - var rsql = "SELECT device, sum(temperature) as total, avg(temperature) as average, count(*) as cnt FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, sum(temperature) as total, avg(temperature) as average, count(*) as cnt FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream //fmt.Println("开始测试基本增量计算功能") - // 使用固定的时间基准以便测试更加稳定 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "temperature": 10.0, "Ts": baseTime}, - {"device": "sensor1", "temperature": 20.0, "Ts": baseTime}, - {"device": "sensor1", "temperature": 30.0, "Ts": baseTime}, - {"device": "sensor2", "temperature": 15.0, "Ts": baseTime}, - {"device": "sensor2", "temperature": 25.0, "Ts": baseTime}, + {"device": "sensor1", "temperature": 10.0}, + {"device": "sensor1", "temperature": 20.0}, + {"device": "sensor1", "temperature": 30.0}, + {"device": "sensor2", "temperature": 15.0}, + {"device": "sensor2", "temperature": 25.0}, } // 添加数据 @@ -2196,7 +2220,7 @@ func TestIncrementalComputationBasic(t *testing.T) { resultChan <- result }) - // 等待窗口初始化 + // 等待窗口触发（处理时间模式） //fmt.Println("等待窗口初始化...") time.Sleep(1 * time.Second) @@ -2348,20 +2372,18 @@ func TestExprFunctionsInAggregation(t *testing.T) { defer streamsql.Stop() // 测试在聚合函数中使用expr函数：数学计算 - var rsql = "SELECT device, AVG(abs(temperature - 25)) as avg_deviation, MAX(ceil(temperature)) as max_ceil FROM stream GROUP BY device, TumblingWindow('1s') with (TIMESTAMP='Ts',TIMEUNIT='ss')" + var rsql = "SELECT device, AVG(abs(temperature - 25)) as avg_deviation, MAX(ceil(temperature)) as max_ceil FROM stream GROUP BY device, TumblingWindow('1s')" err := streamsql.Execute(rsql) assert.Nil(t, err) strm := streamsql.stream - // 使用固定的时间基准 - baseTime := time.Date(2025, 4, 7, 16, 46, 0, 0, time.UTC) - // 添加测试数据 + // 不使用事件时间，不需要时间戳字段 testData := []map[string]interface{}{ - {"device": "sensor1", "temperature": 23.5, "Ts": baseTime}, // abs(23.5-25) = 1.5, ceil(23.5) = 24 - {"device": "sensor1", "temperature": 26.8, "Ts": baseTime}, // abs(26.8-25) = 1.8, ceil(26.8) = 27 - {"device": "sensor2", "temperature": 24.2, "Ts": baseTime}, // abs(24.2-25) = 0.8, ceil(24.2) = 25 - {"device": "sensor2", "temperature": 25.9, "Ts": baseTime}, // abs(25.9-25) = 0.9, ceil(25.9) = 26 + {"device": "sensor1", "temperature": 23.5}, // abs(23.5-25) = 1.5, ceil(23.5) = 24 + {"device": "sensor1", "temperature": 26.8}, // abs(26.8-25) = 1.8, ceil(26.8) = 27 + {"device": "sensor2", "temperature": 24.2}, // abs(24.2-25) = 0.8, ceil(24.2) = 25 + {"device": "sensor2", "temperature": 25.9}, // abs(25.9-25) = 0.9, ceil(25.9) = 26 } // 创建结果接收通道 @@ -2897,7 +2919,12 @@ func TestCaseNullValueHandlingInAggregation(t *testing.T) { resultChan := make(chan interface{}, 10) ssql.AddSink(func(result []map[string]interface{}) { - resultChan <- result + // 非阻塞发送，避免阻塞 sink worker + select { + case resultChan <- result: + default: + // Channel 已满，忽略（非阻塞发送） + } }) // 添加测试数据 @@ -2916,7 +2943,10 @@ func TestCaseNullValueHandlingInAggregation(t *testing.T) { // 等待窗口触发 time.Sleep(3 * time.Second) - // 收集结果 + // 收集结果（添加超时机制） + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + collecting: for { select { @@ -2926,9 +2956,19 @@ collecting: } case <-time.After(500 * time.Millisecond): break collecting + case <-ctx.Done(): + break collecting } } + // 等待一小段时间，确保所有 sink worker 完成当前任务 + // 这样可以确保所有结果都被发送到 channel + time.Sleep(100 * time.Millisecond) + + // 取消 context，通知所有相关 goroutine 退出 + // 不关闭 resultChan，让主程序自动退出时清理 + cancel() + // 验证结果 assert.Len(t, results, 2, "应该有两个设备类型的结果") diff --git a/streamsql_tumbling_window_test.go b/streamsql_tumbling_window_test.go new file mode 100644 index 0000000..d1715e5 --- /dev/null +++ b/streamsql_tumbling_window_test.go @@ -0,0 +1,1345 @@ +package streamsql + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSQLTumblingWindow_ProcessingTime 测试处理时间的滚动窗口 +// 验证不使用 WITH 子句时，滚动窗口基于处理时间（系统时钟）工作 +func TestSQLTumblingWindow_ProcessingTime(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + AVG(temperature) as avg_temp, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 10) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + select { + case ch <- results: + default: + // 非阻塞发送 + } + } + }) + + // 使用处理时间：发送数据，不包含时间戳字段 + // 滚动窗口基于数据到达的处理时间（系统时钟）来划分窗口 + for i := 0; i < 10; i++ { + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "temperature": float64(i), + }) + time.Sleep(200 * time.Millisecond) // 每200ms发送一条数据 + } + + // 等待窗口触发（处理时间滚动窗口基于系统时钟触发） + time.Sleep(3 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + cnt := row["cnt"].(float64) + avgTemp := row["avg_temp"].(float64) + minTemp := row["min_temp"].(float64) + maxTemp := row["max_temp"].(float64) + + assert.Greater(t, cnt, 0.0, "窗口应该包含数据") + assert.LessOrEqual(t, minTemp, maxTemp, "最小值应该小于等于最大值") + assert.LessOrEqual(t, minTemp, avgTemp, "最小值应该小于等于平均值") + assert.LessOrEqual(t, avgTemp, maxTemp, "平均值应该小于等于最大值") + + t.Logf("处理时间滚动窗口成功触发，数据量: %.0f, 平均温度: %.2f", cnt, avgTemp) + } + } +} + +// TestSQLTumblingWindow_MaxOutOfOrderness 测试滚动窗口的最大延迟时间配置 +// 验证设置 MaxOutOfOrderness 后，延迟数据能否在允许的延迟范围内被正确处理 +func TestSQLTumblingWindow_MaxOutOfOrderness(t *testing.T) { + ssql := New() + defer ssql.Stop() + + // 使用 SQL 配置 MaxOutOfOrderness + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='1s', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟延迟数据场景 + // 场景：设置 MaxOutOfOrderness = 1秒，测试延迟数据能否在1秒内被正确处理 + // 窗口大小2秒，需要对齐到2秒的倍数 + windowSizeMs := int64(2000) // 2秒 + baseTimeRaw := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + // 对齐baseTime到窗口大小的倍数，确保窗口对齐行为可预测 + baseTime := (baseTimeRaw / windowSizeMs) * windowSizeMs + + // 第一阶段：发送正常顺序的数据 + // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） + t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待 watermark 推进，触发第一个窗口 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟数据 + // 延迟数据的事件时间在第一个窗口内（如 500ms, 700ms, 900ms） + // 如果 MaxOutOfOrderness = 1秒，这些数据应该能被处理 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内，延迟 < 1秒）") + lateDataTimes := []int64{500, 700, 900} // 延迟数据的事件时间（相对于 baseTime） + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(20 + i), // 20-22，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：发送更多正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + // 如果 MaxOutOfOrderness 配置正确，延迟数据应该能被统计到对应窗口 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 + // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 + assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + + // 如果 MaxOutOfOrderness 配置正确，延迟数据应该被处理 + if maxTemp >= 20.0 { + t.Logf("✓ 延迟数据被正确处理，最大值包含延迟数据: %.0f", maxTemp) + } else { + t.Logf("提示：延迟数据可能未被统计，当前最大值: %.0f", maxTemp) + } + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLTumblingWindow_AllowedLateness 测试滚动窗口的 AllowedLateness 配置 +// 验证窗口触发后，延迟数据能否在允许的延迟时间内更新窗口结果 +func TestSQLTumblingWindow_AllowedLateness(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', ALLOWEDLATENESS='1s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟 AllowedLateness 场景 + // 场景：窗口触发后，发送延迟数据，验证窗口能否更新 + baseTime := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + + // 第一阶段：发送正常顺序的数据，触发第一个窗口 + // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） + t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待 watermark 推进，触发第一个窗口 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟数据（事件时间在第一个窗口内） + // 这些数据应该在 AllowedLateness = 1秒 内被处理 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + lateDataTimes := []int64{300, 600, 900} // 延迟数据的事件时间 + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(30 + i), // 30-32，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：继续发送正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + // 如果 AllowedLateness 配置正确，延迟数据应该能触发窗口的延迟更新 + if windowResultsLen > 0 { + // 滚动窗口的延迟更新可能体现在后续的窗口结果中 + // 检查所有窗口结果，看是否有包含延迟数据的窗口 + hasLateData := false + for i, window := range windowResultsCopy { + if len(window) > 0 { + cnt := window[0]["cnt"].(float64) + minTemp := window[0]["min_temp"].(float64) + maxTemp := window[0]["max_temp"].(float64) + + t.Logf("窗口 %d: cnt=%.0f, min=%.0f, max=%.0f", i+1, cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + assert.GreaterOrEqual(t, cnt, 1.0, "窗口 %d 应该包含数据", i+1) + + // 如果 AllowedLateness 配置正确，延迟数据应该被处理 + // 延迟数据（temperature=30-32）应该能被统计 + if maxTemp >= 30.0 { + hasLateData = true + t.Logf("✓ 窗口 %d 包含延迟数据，最大值: %.0f", i+1, maxTemp) + } + } + } + + // 验证是否有延迟更新（窗口可能触发多次） + if windowResultsLen > 1 { + t.Logf("✓ 滚动窗口触发了 %d 次，可能包含延迟更新", windowResultsLen) + } + + if !hasLateData { + t.Logf("提示：延迟数据可能未被统计，或延迟数据的时间不在窗口范围内") + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLTumblingWindow_BothConfigs 测试滚动窗口同时配置 MaxOutOfOrderness 和 AllowedLateness +// 验证两个配置组合使用时，延迟数据能否被正确处理 +func TestSQLTumblingWindow_BothConfigs(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH ( + TIMESTAMP='eventTime', + TIMEUNIT='ms', + MAXOUTOFORDERNESS='1s', + ALLOWEDLATENESS='500ms' + ) + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟完整的延迟数据处理场景 + baseTime := time.Now().UnixMilli() - 10000 + + // 第一阶段：发送正常顺序的数据 + t.Log("第一阶段：发送正常顺序的数据") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待 watermark 推进（考虑 MaxOutOfOrderness = 1s） + t.Log("等待 watermark 推进，触发窗口（MaxOutOfOrderness = 1s）") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟数据（事件时间在第一个窗口内） + // MaxOutOfOrderness = 1s：这些数据应该在允许的乱序范围内 + // AllowedLateness = 500ms：窗口触发后还能接受500ms的延迟数据 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + lateDataTimes := []int64{400, 800, 1200} // 延迟数据的事件时间 + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(40 + i), // 40-42，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：继续发送正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 + // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 + assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + + // 验证延迟数据是否被处理 + // 如果配置正确，maxTemp 可能包含延迟数据的值（40-42） + if maxTemp >= 40.0 { + t.Logf("✓ 延迟数据被正确处理，最大值包含延迟数据: %.0f", maxTemp) + } else { + t.Logf("提示：延迟数据可能未被统计，当前最大值: %.0f", maxTemp) + } + } + + // 验证是否有延迟更新 + if windowResultsLen > 1 { + t.Logf("✓ 滚动窗口触发了 %d 次，可能包含延迟更新", windowResultsLen) + + // 验证后续窗口的数据 + for i := 1; i < windowResultsLen && i < 3; i++ { + if len(windowResultsCopy[i]) > 0 { + cnt := windowResultsCopy[i][0]["cnt"].(float64) + t.Logf("窗口 %d: cnt=%.0f", i+1, cnt) + } + } + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) + t.Logf("配置验证：MaxOutOfOrderness=1s, AllowedLateness=500ms") +} + +// TestSQLTumblingWindow_LateDataHandling 测试滚动窗口的延迟数据处理 +// 验证即使数据延迟到达，只要在允许的延迟范围内，也能正确统计到对应窗口 +func TestSQLTumblingWindow_LateDataHandling(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：模拟延迟数据场景 + // 场景：先发送正常顺序的数据，然后发送一些延迟的数据 + // 窗口大小2秒，需要对齐到2秒的倍数 + windowSizeMs := int64(2000) // 2秒 + baseTimeRaw := time.Now().UnixMilli() - 5000 // 使用5秒前作为基准 + // 对齐baseTime到窗口大小的倍数，确保窗口对齐行为可预测 + baseTime := (baseTimeRaw / windowSizeMs) * windowSizeMs + + // 第一阶段：发送正常顺序的数据（事件时间：0ms, 200ms, 400ms, ..., 2000ms） + // 这些数据应该被统计到第一个窗口 [0ms, 2000ms) + t.Log("第一阶段：发送正常顺序的数据") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) // 每200ms一条数据 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 温度值 0-9 + }) + time.Sleep(50 * time.Millisecond) // 处理时间间隔较小 + } + + // 等待 watermark 推进，让第一个窗口触发 + // 窗口大小2秒，第一个窗口应该在 watermark >= baseTime + 2000ms 时触发 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟的数据 + // 这些数据的事件时间比之前的数据早，但应该在允许的延迟范围内 + // 延迟数据的事件时间：100ms, 300ms, 500ms（这些时间在第一个窗口 [0ms, 2000ms) 内） + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + for i := 0; i < 3; i++ { + // 延迟数据：事件时间比正常数据早，但仍在窗口范围内 + eventTime := baseTime + int64(100+i*200) // 100ms, 300ms, 500ms + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(10 + i), // 温度值 10-12，用于区分延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 继续发送更多正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证第一个窗口的数据 + // 第一个窗口应该包含正常数据（0-9）和可能的延迟数据 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 第一个窗口应该包含正常数据 + // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 + // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 + assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + assert.GreaterOrEqual(t, maxTemp, 0.0, "第一个窗口的最大值应该大于等于0") + } + } + + // 验证延迟数据是否被处理 + // 如果延迟数据被正确处理，应该能在后续窗口或更新中看到 + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLTumblingWindow_EventTimeWindowAlignment 测试事件时间窗口对齐到epoch +func TestSQLTumblingWindow_EventTimeWindowAlignment(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，验证窗口对齐到epoch + // 窗口大小2秒，应该对齐到2秒的倍数 + baseTime := time.Now().UnixMilli() + + // 发送数据，事件时间从baseTime开始，每200ms一条 + // 第一个窗口应该对齐到小于等于baseTime的最大2秒倍数 + for i := 0; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发 + time.Sleep(3 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口对齐 + windowSizeMs := int64(2000) // 2秒 = 2000毫秒 + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + windowSizeNs := int64(windowSizeMs) * int64(time.Millisecond) + + assert.Equal(t, windowSizeNs, end-start, + "窗口 %d 的大小应该是2秒（2000ms），实际: start=%d, end=%d", i+1, start, end) + + assert.Equal(t, int64(0), startMs%windowSizeMs, + "窗口 %d 的开始时间应该对齐到2秒的倍数（epoch对齐），实际: startMs=%d", i+1, startMs) + + if i > 0 { + prevEndMs := windowResultsCopy[i-1][0]["end"].(int64) / int64(time.Millisecond) + assert.Equal(t, prevEndMs, startMs, + "窗口 %d 的开始时间应该等于前一个窗口的结束时间，prevEndMs=%d, startMs=%d", i+1, prevEndMs, startMs) + } + + t.Logf("窗口 %d: start=%d, end=%d, size=%dms", i+1, startMs, endMs, endMs-startMs) + } + } +} + +// TestSQLTumblingWindow_WatermarkTriggerTiming 测试Watermark触发窗口的时机 +func TestSQLTumblingWindow_WatermarkTriggerTiming(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，验证watermark触发时机 + baseTime := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + maxOutOfOrdernessMs := int64(500) // 500ms + + // 第一阶段：发送数据到第一个窗口 [alignedStart, alignedStart+2000) + // 计算对齐后的窗口开始时间 + windowSizeMs := int64(2000) + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + firstWindowEnd := alignedStart + windowSizeMs + + t.Logf("第一个窗口: [%d, %d)", alignedStart, firstWindowEnd) + + // 发送数据，事件时间在第一个窗口内 + for i := 0; i < 10; i++ { + eventTime := alignedStart + int64(i*200) // 在窗口内 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 发送一个事件时间刚好等于window_end的数据，推进watermark + // watermark = maxEventTime - maxOutOfOrderness = firstWindowEnd - 500 + // 此时 watermark < firstWindowEnd，窗口不应该触发 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd, + "temperature": 100.0, + }) + + // 等待watermark更新（watermark更新间隔200ms） + time.Sleep(500 * time.Millisecond) + + // 发送一个事件时间超过window_end的数据，推进watermark + // watermark = maxEventTime - maxOutOfOrderness = (firstWindowEnd + 1000) - 500 = firstWindowEnd + 500 + // 此时 watermark >= firstWindowEnd，窗口应该触发 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + 1000, + "temperature": 200.0, + }) + + // 等待窗口触发 + time.Sleep(1 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证第一个窗口的触发时机 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + start := row["start"].(int64) + end := row["end"].(int64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + + assert.Equal(t, alignedStart, startMs, + "第一个窗口的开始时间应该对齐到epoch，expected=%d, actual=%d", alignedStart, startMs) + assert.Equal(t, firstWindowEnd, endMs, + "第一个窗口的结束时间应该正确，expected=%d, actual=%d", firstWindowEnd, endMs) + + // 验证窗口在watermark >= window_end时触发 + // 由于watermark = maxEventTime - maxOutOfOrderness + // 当maxEventTime = firstWindowEnd + 1000时，watermark = firstWindowEnd + 500 + // watermark >= firstWindowEnd，窗口应该触发 + t.Logf("✓ 窗口在watermark >= window_end时正确触发") + t.Logf("窗口: [%d, %d), 触发时maxEventTime >= %d", start, end, end+maxOutOfOrdernessMs) + } + } +} + +// TestSQLTumblingWindow_AllowedLatenessUpdate 测试AllowedLateness的延迟更新 +func TestSQLTumblingWindow_AllowedLatenessUpdate(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', ALLOWEDLATENESS='1s', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + baseTime := time.Now().UnixMilli() - 10000 + windowSizeMs := int64(2000) + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + firstWindowEnd := alignedStart + windowSizeMs + allowedLatenessMs := int64(1000) // 1秒 + + // 第一阶段：发送正常数据，触发第一个窗口 + t.Log("第一阶段：发送正常数据，触发第一个窗口") + for i := 0; i < 10; i++ { + eventTime := alignedStart + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 推进watermark，触发第一个窗口 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + 1000, + "temperature": 100.0, + }) + + // 等待第一个窗口触发 + time.Sleep(1 * time.Second) + + // 收集第一个窗口的结果 + firstWindowReceived := false + firstWindowCnt := 0.0 + firstWindowMax := 0.0 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 10 + iteration := 0 + + for !firstWindowReceived && iteration < maxIterations { + select { + case res, ok := <-ch: + if !ok { + // channel 已关闭 + t.Fatal("应该收到第一个窗口") + } + if len(res) > 0 { + firstWindowReceived = true + firstWindowCnt = res[0]["cnt"].(float64) + firstWindowMax = res[0]["max_temp"].(float64) + t.Logf("第一个窗口（初始）: cnt=%.0f, max=%.0f", firstWindowCnt, firstWindowMax) + } + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果 + iteration++ + case <-ctx.Done(): + t.Fatal("应该收到第一个窗口") + } + } + + // 第二阶段：发送延迟数据（事件时间在第一个窗口内，但在AllowedLateness范围内） + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + lateDataTimes := []int64{300, 600, 900} // 延迟数据的事件时间（相对于alignedStart） + lateDataTemps := []float64{30.0, 31.0, 32.0} + for i, lateTime := range lateDataTimes { + eventTime := alignedStart + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": lateDataTemps[i], + }) + time.Sleep(100 * time.Millisecond) + } + + // 继续发送正常数据，推进watermark（但不超过window_end + allowedLateness） + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + allowedLatenessMs - 100, // 在allowedLateness范围内 + "temperature": 200.0, + }) + + // 等待延迟更新 + time.Sleep(1 * time.Second) + + // 收集所有窗口结果 + ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel2() + maxIterations2 := 20 + iteration2 := 0 + + for iteration2 < maxIterations2 { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration2++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx2.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证延迟更新 + hasLateUpdate := false + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + maxTemp := row["max_temp"].(float64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + + if startMs == alignedStart && endMs == firstWindowEnd { + if cnt > firstWindowCnt { + hasLateUpdate = true + t.Logf("✓ 窗口延迟更新: cnt从%.0f增加到%.0f, max从%.0f增加到%.0f", + firstWindowCnt, cnt, firstWindowMax, maxTemp) + + // 验证延迟数据被包含 + assert.GreaterOrEqual(t, maxTemp, 30.0, + "延迟更新应该包含延迟数据，maxTemp应该>=30.0，实际: %.0f", maxTemp) + } + } + + t.Logf("窗口 %d: [%d, %d), cnt=%.0f, max=%.0f", i+1, start, end, cnt, maxTemp) + } + } + + if !hasLateUpdate { + t.Logf("⚠ 提示：未检测到延迟更新，可能延迟数据未被处理或窗口已关闭") + } else { + t.Logf("✓ AllowedLateness功能正常工作，延迟数据触发窗口更新") + } +} + +// TestSQLTumblingWindow_IdleSourceMechanism 测试Idle Source机制 +// 验证当数据源空闲时，watermark基于处理时间推进，窗口能够正常关闭 +func TestSQLTumblingWindow_IdleSourceMechanism(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，然后停止发送，验证窗口能够关闭 + baseTime := time.Now().UnixMilli() - 10000 + windowSizeMs := int64(2000) // 2秒 + + // 计算对齐后的第一个窗口开始时间 + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + firstWindowEnd := alignedStart + windowSizeMs + + t.Logf("第一个窗口: [%d, %d)", alignedStart, firstWindowEnd) + + // 第一阶段：发送数据，创建窗口 + t.Log("第一阶段：发送数据，创建窗口") + for i := 0; i < 5; i++ { + eventTime := alignedStart + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 第二阶段：停止发送数据，等待Idle Source机制触发 + // IdleTimeout = 2秒，意味着2秒无数据后，watermark会基于处理时间推进 + t.Log("第二阶段：停止发送数据，等待Idle Source机制触发（IdleTimeout=2s）") + time.Sleep(3 * time.Second) // 等待超过IdleTimeout，确保watermark推进 + + // 收集窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + // 验证窗口能够关闭（即使没有新数据） + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口（即使数据源空闲）") + + // 验证窗口数据 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + + // 验证窗口边界正确 + // window_start() 和 window_end() 返回纳秒，需要转换为毫秒 + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + assert.Equal(t, alignedStart, startMs, + "第一个窗口的开始时间应该对齐到窗口大小，expected=%d, actual=%d", alignedStart, startMs) + assert.Equal(t, firstWindowEnd, endMs, + "第一个窗口的结束时间应该正确，expected=%d, actual=%d", firstWindowEnd, endMs) + + // 验证窗口包含数据 + assert.Greater(t, cnt, 0.0, "窗口应该包含数据") + + t.Logf("✓ Idle Source机制正常工作，窗口在数据源空闲时能够关闭") + t.Logf("窗口: [%d, %d), cnt=%.0f", start, end, cnt) + } + } +} + +// TestSQLTumblingWindow_IdleSourceDisabled 测试Idle Source机制未启用的情况 +// 验证当IdleTimeout=0（禁用）时，如果数据源空闲，窗口无法关闭 +func TestSQLTumblingWindow_IdleSourceDisabled(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + -- 注意：没有配置IDLETIMEOUT，默认为0（禁用） + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + baseTime := time.Now().UnixMilli() - 10000 + windowSizeMs := int64(2000) + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + + // 发送数据，但事件时间不足以触发窗口 + t.Log("发送数据，但事件时间不足以触发窗口") + for i := 0; i < 3; i++ { + eventTime := alignedStart + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 停止发送数据，等待一段时间 + // 由于IdleTimeout未启用，watermark不会基于处理时间推进 + t.Log("停止发送数据，等待（IdleTimeout未启用）") + time.Sleep(3 * time.Second) + + // 收集窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsMu.Unlock() + + // 注意：这个测试可能无法完全验证窗口无法关闭 + // 因为如果watermark已经推进到足够的位置，窗口可能已经触发 + // 这个测试主要用于对比：启用Idle Source vs 未启用Idle Source + t.Logf("窗口结果数量: %d（IdleTimeout未启用）", windowResultsLen) +} diff --git a/types/config.go b/types/config.go index b1cc8a1..3fa4130 100644 --- a/types/config.go +++ b/types/config.go @@ -32,15 +32,32 @@ type Config struct { PerformanceConfig PerformanceConfig `json:"performanceConfig"` } +// TimeCharacteristic represents the time characteristic for window operations +type TimeCharacteristic string + +const ( + // ProcessingTime uses system clock for window operations + // Windows trigger based on when data arrives, not when events occurred + ProcessingTime TimeCharacteristic = "ProcessingTime" + // EventTime uses event timestamps for window operations + // Windows trigger based on event time, requires watermark mechanism + EventTime TimeCharacteristic = "EventTime" +) + // WindowConfig window configuration type WindowConfig struct { - Type string `json:"type"` - Params []interface{} `json:"params"` // Window function parameters array - TsProp string `json:"tsProp"` - TimeUnit time.Duration `json:"timeUnit"` - GroupByKeys []string `json:"groupByKeys"` // Multiple grouping keys for keyed windows - PerformanceConfig PerformanceConfig `json:"performanceConfig"` // Performance configuration - Callback func([]Row) `json:"-"` // Callback function (not serialized) + Type string `json:"type"` + Params []interface{} `json:"params"` // Window function parameters array + TsProp string `json:"tsProp"` + TimeUnit time.Duration `json:"timeUnit"` + TimeCharacteristic TimeCharacteristic `json:"timeCharacteristic"` // Time characteristic: EventTime or ProcessingTime (default: ProcessingTime) + MaxOutOfOrderness time.Duration `json:"maxOutOfOrderness"` // Maximum allowed out-of-orderness for event time (default: 0) + WatermarkInterval time.Duration `json:"watermarkInterval"` // Watermark update interval for event time (default: 200ms) + AllowedLateness time.Duration `json:"allowedLateness"` // Maximum allowed lateness for event time windows (default: 0, meaning no late data accepted after window closes) + IdleTimeout time.Duration `json:"idleTimeout"` // Idle source timeout: when no data arrives within this duration, watermark advances based on processing time (default: 0, meaning disabled) + GroupByKeys []string `json:"groupByKeys"` // Multiple grouping keys for keyed windows + PerformanceConfig PerformanceConfig `json:"performanceConfig"` // Performance configuration + Callback func([]Row) `json:"-"` // Callback function (not serialized) } // FieldExpression field expression configuration diff --git a/window/counting_window.go b/window/counting_window.go index 5f1add6..0be3e94 100644 --- a/window/counting_window.go +++ b/window/counting_window.go @@ -49,6 +49,12 @@ type CountingWindow struct { } func NewCountingWindow(config types.WindowConfig) (*CountingWindow, error) { + // Counting window does not support event time + // It triggers based on count, not time + if config.TimeCharacteristic == types.EventTime { + return nil, fmt.Errorf("counting window does not support event time, use processing time instead") + } + ctx, cancel := context.WithCancel(context.Background()) defer func() { if cancel != nil { diff --git a/window/doc.go b/window/doc.go index c69927c..fca2d02 100644 --- a/window/doc.go +++ b/window/doc.go @@ -57,40 +57,65 @@ All window types implement a unified Window interface: Non-overlapping time-based windows: - // Create tumbling window + // Create tumbling window with processing time (default) config := types.WindowConfig{ Type: "tumbling", - Params: map[string]interface{}{ - "size": "5s", // 5-second windows - }, + Params: []interface{}{"5s"}, // 5-second windows + TsProp: "timestamp", + TimeCharacteristic: types.ProcessingTime, // Uses system clock + } + window, err := NewTumblingWindow(config) + + // Create tumbling window with event time + config := types.WindowConfig{ + Type: "tumbling", + Params: []interface{}{"5s"}, // 5-second windows TsProp: "timestamp", + TimeCharacteristic: types.EventTime, // Uses event timestamps + MaxOutOfOrderness: 2 * time.Second, // Allow 2 seconds of out-of-order data + WatermarkInterval: 200 * time.Millisecond, // Update watermark every 200ms + AllowedLateness: 1 * time.Second, // Allow 1 second of late data after window closes } window, err := NewTumblingWindow(config) // Window characteristics: // - Fixed size (e.g., 5 seconds) // - No overlap between windows - // - Triggers at regular intervals + // - Triggers at regular intervals (ProcessingTime) or based on watermark (EventTime) // - Memory efficient // - Suitable for periodic aggregations - // Example timeline: - // Window 1: [00:00 - 00:05) - // Window 2: [00:05 - 00:10) - // Window 3: [00:10 - 00:15) + // ProcessingTime example timeline (based on data arrival): + // Window 1: [00:00 - 00:05) - triggers when 5s elapsed from first data + // Window 2: [00:05 - 00:10) - triggers when next 5s elapsed + // Window 3: [00:10 - 00:15) - triggers when next 5s elapsed + + // EventTime example timeline (based on event timestamps): + // Window 1: [00:00 - 00:05) - triggers when watermark >= 00:05 + // Window 2: [00:05 - 00:10) - triggers when watermark >= 00:10 + // Window 3: [00:10 - 00:15) - triggers when watermark >= 00:15 # Sliding Windows Overlapping time-based windows with configurable slide interval: - // Create sliding window + // Create sliding window with processing time (default) config := types.WindowConfig{ Type: "sliding", - Params: map[string]interface{}{ - "size": "30s", // 30-second window size - "slide": "10s", // 10-second slide interval - }, + Params: []interface{}{"30s", "10s"}, // 30-second window size, 10-second slide TsProp: "timestamp", + TimeCharacteristic: types.ProcessingTime, // Uses system clock + } + window, err := NewSlidingWindow(config) + + // Create sliding window with event time + config := types.WindowConfig{ + Type: "sliding", + Params: []interface{}{"30s", "10s"}, // 30-second window size, 10-second slide + TsProp: "timestamp", + TimeCharacteristic: types.EventTime, // Uses event timestamps + MaxOutOfOrderness: 2 * time.Second, // Allow 2 seconds of out-of-order data + WatermarkInterval: 200 * time.Millisecond, // Update watermark every 200ms } window, err := NewSlidingWindow(config) @@ -101,10 +126,15 @@ Overlapping time-based windows with configurable slide interval: // - Higher memory usage // - Suitable for smooth trend analysis - // Example timeline (30s window, 10s slide): - // Window 1: [00:00 - 00:30) - // Window 2: [00:10 - 00:40) - // Window 3: [00:20 - 00:50) + // ProcessingTime example timeline (30s window, 10s slide, based on data arrival): + // Window 1: [00:00 - 00:30) - triggers when 30s elapsed from first data + // Window 2: [00:10 - 00:40) - triggers 10s after Window 1 + // Window 3: [00:20 - 00:50) - triggers 10s after Window 2 + + // EventTime example timeline (30s window, 10s slide, based on event timestamps): + // Window 1: [00:00 - 00:30) - triggers when watermark >= 00:30 + // Window 2: [00:10 - 00:40) - triggers when watermark >= 00:40 + // Window 3: [00:20 - 00:50) - triggers when watermark >= 00:50 # Counting Windows @@ -175,6 +205,59 @@ Time handling for window operations: Duration time.Duration } +# Time Characteristics + +Windows support two time characteristics: + +## ProcessingTime (Default) +- Uses system clock for window operations +- Windows trigger based on when data arrives +- Cannot handle out-of-order data +- Lower latency, but results may be inconsistent +- Suitable for real-time monitoring and low-latency requirements + +## EventTime +- Uses event timestamps for window operations +- Windows trigger based on event time via watermark mechanism +- Can handle out-of-order and late-arriving data +- Consistent results, but may have higher latency +- Suitable for accurate time-based analysis and historical data processing + +## Watermark Mechanism +For EventTime windows, watermark indicates that no events with timestamp less than watermark time are expected: +- Watermark = max(event_time) - max_out_of_orderness +- Windows trigger when watermark >= window_end_time +- Late data (before watermark) can be detected and handled specially + +## Allowed Lateness +For EventTime windows, `allowedLateness` allows windows to accept late data after they have been triggered: +- When watermark >= window_end, window triggers and outputs result +- Window remains open until watermark >= window_end + allowedLateness +- Late data arriving within allowedLateness triggers delayed updates (window fires again) +- After allowedLateness expires, window closes and late data is ignored +- Default: 0 (no late data accepted after window closes) + +Example: +- Window [00:00 - 00:05) triggers when watermark >= 00:05 +- With allowedLateness = 2s, window stays open until watermark >= 00:07 +- Late data with timestamp in [00:00 - 00:05) arriving before watermark >= 00:07 triggers delayed update +- After watermark >= 00:07, window closes and late data is ignored + +## Idle Source Mechanism +For EventTime windows, `idleTimeout` enables watermark advancement based on processing time when the data source is idle: +- Normally: Watermark advances based on event time (Watermark = max(event_time) - maxOutOfOrderness) +- When idle: If no data arrives within idleTimeout, watermark advances based on processing time +- This ensures windows can close even when the data source stops sending data +- Prevents memory leaks from windows that never close +- Default: 0 (disabled, watermark only advances based on event time) + +Example: +- Window [00:00 - 00:05) has data with max event time = 00:02 +- Data source stops sending data at 00:03 +- With idleTimeout = 5s, after 5 seconds of no data (at 00:08), watermark advances based on processing time +- Watermark = currentProcessingTime - maxOutOfOrderness = 00:08 - 1s = 00:07 +- Window [00:00 - 00:05) can trigger (watermark >= 00:05) and close + # Performance Features • Memory Management - Efficient buffer management and garbage collection diff --git a/window/session_window.go b/window/session_window.go index 570612d..e550cdd 100644 --- a/window/session_window.go +++ b/window/session_window.go @@ -55,6 +55,16 @@ type SessionWindow struct { // Lock to protect ticker tickerMu sync.Mutex ticker *time.Ticker + // watermark for event time processing (only used for EventTime) + watermark *Watermark + // triggeredSessions stores sessions that have been triggered but are still open for late data (for EventTime with allowedLateness) + triggeredSessions map[string]*sessionInfo +} + +// sessionInfo stores information about a triggered session that is still open for late data +type sessionInfo struct { + session *session + closeTime time.Time // session end + allowedLateness } // session stores data and state for a session @@ -66,17 +76,18 @@ type session struct { // NewSessionWindow creates a new session window instance func NewSessionWindow(config types.WindowConfig) (*SessionWindow, error) { - // Create a cancellable context - ctx, cancel := context.WithCancel(context.Background()) - // Get timeout parameter from params array if len(config.Params) == 0 { return nil, fmt.Errorf("session window requires 'timeout' parameter") } + // Create a cancellable context + ctx, cancel := context.WithCancel(context.Background()) + timeoutVal := config.Params[0] timeout, err := cast.ToDurationE(timeoutVal) if err != nil { + cancel() return nil, fmt.Errorf("invalid timeout for session window: %v", err) } @@ -89,15 +100,39 @@ func NewSessionWindow(config types.WindowConfig) (*SessionWindow, error) { } } + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + // Initialize watermark for event time + var watermark *Watermark + if timeChar == types.EventTime { + maxOutOfOrderness := config.MaxOutOfOrderness + if maxOutOfOrderness == 0 { + maxOutOfOrderness = 0 // Default: no out-of-orderness allowed + } + watermarkInterval := config.WatermarkInterval + if watermarkInterval == 0 { + watermarkInterval = 200 * time.Millisecond // Default: 200ms + } + idleTimeout := config.IdleTimeout + // Default: 0 means disabled, no idle source mechanism + watermark = NewWatermark(maxOutOfOrderness, watermarkInterval, idleTimeout) + } + return &SessionWindow{ - config: config, - timeout: timeout, - sessionMap: make(map[string]*session), - outputChan: make(chan []types.Row, bufferSize), - ctx: ctx, - cancelFunc: cancel, - initChan: make(chan struct{}), - initialized: false, + config: config, + timeout: timeout, + sessionMap: make(map[string]*session), + outputChan: make(chan []types.Row, bufferSize), + ctx: ctx, + cancelFunc: cancel, + initChan: make(chan struct{}), + initialized: false, + watermark: watermark, + triggeredSessions: make(map[string]*sessionInfo), }, nil } @@ -120,6 +155,28 @@ func (sw *SessionWindow) Add(data interface{}) { // Get data timestamp timestamp := GetTimestamp(data, sw.config.TsProp, sw.config.TimeUnit) + + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := sw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + // For event time, update watermark and check for late data + if timeChar == types.EventTime && sw.watermark != nil { + sw.watermark.UpdateEventTime(timestamp) + // Check if data is late and handle allowedLateness + if sw.watermark.IsEventTimeLate(timestamp) { + // Data is late, check if it's within allowedLateness + allowedLateness := sw.config.AllowedLateness + if allowedLateness > 0 { + // Check if this late data belongs to any triggered session that's still open + sw.handleLateData(timestamp, allowedLateness) + } + // If allowedLateness is 0 or data is too late, we still add it but it won't trigger updates + } + } + // Create Row object row := types.Row{ Data: data, @@ -166,6 +223,23 @@ func (sw *SessionWindow) Add(data interface{}) { // Start starts the session window, begins periodic checking of expired sessions // Uses lazy initialization mode to avoid infinite waiting when no data, while ensuring subsequent data can be processed normally func (sw *SessionWindow) Start() { + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := sw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + if timeChar == types.EventTime { + // Event time: trigger based on watermark + sw.startEventTime() + } else { + // Processing time: trigger based on system clock + sw.startProcessingTime() + } +} + +// startProcessingTime starts the processing time trigger mechanism +func (sw *SessionWindow) startProcessingTime() { go func() { // Close output channel when function ends defer close(sw.outputChan) @@ -204,6 +278,42 @@ func (sw *SessionWindow) Start() { }() } +// startEventTime starts the event time trigger mechanism based on watermark +func (sw *SessionWindow) startEventTime() { + go func() { + // Close output channel when function ends + defer close(sw.outputChan) + if sw.watermark != nil { + defer sw.watermark.Stop() + } + + // Wait for initialization completion or context cancellation + select { + case <-sw.initChan: + // Normal initialization completed, continue processing + case <-sw.ctx.Done(): + // Context cancelled, exit directly + return + } + + // Process watermark updates + if sw.watermark != nil { + for { + select { + case watermarkTime := <-sw.watermark.WatermarkChan(): + sw.checkAndTriggerSessions(watermarkTime) + case <-sw.ctx.Done(): + return + } + } + } else { + // If watermark is nil, just wait for context cancellation + <-sw.ctx.Done() + return + } + }() +} + // Stop stops session window operations func (sw *SessionWindow) Stop() { // Call cancel function to stop window operations @@ -216,6 +326,11 @@ func (sw *SessionWindow) Stop() { } sw.tickerMu.Unlock() + // Stop watermark (for event time) + if sw.watermark != nil { + sw.watermark.Stop() + } + // Ensure initChan is closed if it hasn't been closed yet // This prevents Start() goroutine from blocking on initChan sw.mu.Lock() @@ -230,50 +345,75 @@ func (sw *SessionWindow) Stop() { sw.mu.Unlock() } -// checkExpiredSessions checks and triggers expired sessions func (sw *SessionWindow) checkExpiredSessions() { sw.mu.Lock() - now := time.Now() - expiredKeys := []string{} + resultsToSend := sw.collectExpiredSessions(now) + sw.mu.Unlock() + + sw.sendResults(resultsToSend) +} - // Find expired sessions +func (sw *SessionWindow) checkAndTriggerSessions(watermarkTime time.Time) { + sw.mu.Lock() + resultsToSend := sw.collectExpiredSessions(watermarkTime) + sw.closeExpiredSessions(watermarkTime) + sw.mu.Unlock() + + sw.sendResults(resultsToSend) +} + +func (sw *SessionWindow) collectExpiredSessions(currentTime time.Time) [][]types.Row { + expiredKeys := []string{} for key, s := range sw.sessionMap { - if now.Sub(s.lastActive) > sw.timeout { + // For event time, use slot.End to determine if session expired + // Session expires when watermark >= session end time + // For processing time, use lastActive + timeout + if s.slot.End != nil && !currentTime.Before(*s.slot.End) { + expiredKeys = append(expiredKeys, key) + } else if currentTime.Sub(s.lastActive) > sw.timeout { expiredKeys = append(expiredKeys, key) } } - // Process expired sessions resultsToSend := make([][]types.Row, 0) + allowedLateness := sw.config.AllowedLateness + for _, key := range expiredKeys { s := sw.sessionMap[key] if len(s.data) > 0 { - // Trigger session window result := make([]types.Row, len(s.data)) copy(result, s.data) resultsToSend = append(resultsToSend, result) + + if allowedLateness > 0 { + closeTime := s.slot.End.Add(allowedLateness) + sw.triggeredSessions[key] = &sessionInfo{ + session: s, + closeTime: closeTime, + } + } } - // Delete expired session delete(sw.sessionMap, key) } - // Release lock before sending to channel and calling callback to avoid blocking - sw.mu.Unlock() + return resultsToSend +} - // Send results and call callbacks outside of lock to avoid blocking +func (sw *SessionWindow) sendResults(resultsToSend [][]types.Row) { for _, result := range resultsToSend { - // If callback function is set, execute it + // Skip empty results to avoid filling up channels + if len(result) == 0 { + continue + } + if sw.callback != nil { sw.callback(result) } - // Non-blocking send to output channel select { case sw.outputChan <- result: - // Successfully sent default: - // Channel full, drop result (could add statistics here if needed) } } } @@ -300,6 +440,11 @@ func (sw *SessionWindow) Trigger() { // Send results and call callbacks outside of lock to avoid blocking for _, result := range resultsToSend { + // Skip empty results to avoid filling up channels + if len(result) == 0 { + continue + } + // If callback function is set, execute it if sw.callback != nil { sw.callback(result) @@ -328,8 +473,31 @@ func (sw *SessionWindow) Reset() { } sw.tickerMu.Unlock() + // Stop watermark (for event time) + if sw.watermark != nil { + sw.watermark.Stop() + // Recreate watermark + timeChar := sw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + if timeChar == types.EventTime { + maxOutOfOrderness := sw.config.MaxOutOfOrderness + if maxOutOfOrderness == 0 { + maxOutOfOrderness = 0 + } + watermarkInterval := sw.config.WatermarkInterval + if watermarkInterval == 0 { + watermarkInterval = 200 * time.Millisecond + } + idleTimeout := sw.config.IdleTimeout + sw.watermark = NewWatermark(maxOutOfOrderness, watermarkInterval, idleTimeout) + } + } + // Clear session data sw.sessionMap = make(map[string]*session) + sw.triggeredSessions = make(map[string]*sessionInfo) sw.initialized = false sw.initChan = make(chan struct{}) } @@ -346,6 +514,64 @@ func (sw *SessionWindow) SetCallback(callback func([]types.Row)) { sw.callback = callback } +// handleLateData handles late data that arrives within allowedLateness +func (sw *SessionWindow) handleLateData(eventTime time.Time, allowedLateness time.Duration) { + sw.mu.Lock() + defer sw.mu.Unlock() + + // Find which triggered session this late data belongs to + for _, info := range sw.triggeredSessions { + if info.session.slot.Contains(eventTime) { + // This late data belongs to a triggered session that's still open + // Trigger session again with updated data (late update) + sw.triggerLateUpdateLocked(info.session) + return + } + } +} + +// triggerLateUpdateLocked triggers a late update for a session (must be called with lock held) +func (sw *SessionWindow) triggerLateUpdateLocked(s *session) { + if len(s.data) == 0 { + return + } + + // Extract session data including late data + resultData := make([]types.Row, len(s.data)) + copy(resultData, s.data) + + // Get callback reference before releasing lock + callback := sw.callback + + // Release lock before calling callback and sending to channel to avoid blocking + sw.mu.Unlock() + + if callback != nil { + callback(resultData) + } + + // Non-blocking send to output channel + select { + case sw.outputChan <- resultData: + // Successfully sent + default: + // Channel full, drop result + } + + // Re-acquire lock + sw.mu.Lock() +} + +// closeExpiredSessions closes sessions that have exceeded allowedLateness +func (sw *SessionWindow) closeExpiredSessions(watermarkTime time.Time) { + for key, info := range sw.triggeredSessions { + if !watermarkTime.Before(info.closeTime) { + // Session has expired, remove it + delete(sw.triggeredSessions, key) + } + } +} + // extractSessionCompositeKey builds composite session key from multiple group fields // If GroupByKeys is empty, returns default key func extractSessionCompositeKey(data interface{}, keys []string) string { diff --git a/window/sliding_window.go b/window/sliding_window.go index 51a2739..d526773 100644 --- a/window/sliding_window.go +++ b/window/sliding_window.go @@ -56,7 +56,7 @@ type SlidingWindow struct { ctx context.Context // cancelFunc cancels the context cancelFunc context.CancelFunc - // timer for triggering window periodically + // timer for triggering window periodically (used for ProcessingTime) timer *time.Ticker currentSlot *types.TimeSlot // initChan for window initialization @@ -66,6 +66,10 @@ type SlidingWindow struct { timerMu sync.Mutex // firstWindowStartTime records when first window started (processing time) firstWindowStartTime time.Time + // watermark for event time processing (only used for EventTime) + watermark *Watermark + // triggeredWindows stores windows that have been triggered but are still open for late data (for EventTime with allowedLateness) + triggeredWindows map[string]*triggeredWindowInfo // key: window end time string // Performance statistics droppedCount int64 // Number of dropped results sentCount int64 // Number of successfully sent results @@ -102,18 +106,42 @@ func NewSlidingWindow(config types.WindowConfig) (*SlidingWindow, error) { bufferSize = config.PerformanceConfig.BufferConfig.WindowOutputSize } + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + // Initialize watermark for event time + var watermark *Watermark + if timeChar == types.EventTime { + maxOutOfOrderness := config.MaxOutOfOrderness + if maxOutOfOrderness == 0 { + maxOutOfOrderness = 0 // Default: no out-of-orderness allowed + } + watermarkInterval := config.WatermarkInterval + if watermarkInterval == 0 { + watermarkInterval = 200 * time.Millisecond // Default: 200ms + } + idleTimeout := config.IdleTimeout + // Default: 0 means disabled, no idle source mechanism + watermark = NewWatermark(maxOutOfOrderness, watermarkInterval, idleTimeout) + } + // Create a cancellable context ctx, cancel := context.WithCancel(context.Background()) return &SlidingWindow{ - config: config, - size: size, - slide: slide, - outputChan: make(chan []types.Row, bufferSize), - ctx: ctx, - cancelFunc: cancel, - data: make([]types.Row, 0), - initChan: make(chan struct{}), - initialized: false, + config: config, + size: size, + slide: slide, + outputChan: make(chan []types.Row, bufferSize), + ctx: ctx, + cancelFunc: cancel, + data: make([]types.Row, 0), + initChan: make(chan struct{}), + initialized: false, + watermark: watermark, + triggeredWindows: make(map[string]*triggeredWindowInfo), }, nil } @@ -123,12 +151,42 @@ func (sw *SlidingWindow) Add(data interface{}) { sw.mu.Lock() defer sw.mu.Unlock() + // Get timestamp + eventTime := GetTimestamp(data, sw.config.TsProp, sw.config.TimeUnit) + + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := sw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + // For event time, update watermark and check for late data + if timeChar == types.EventTime && sw.watermark != nil { + sw.watermark.UpdateEventTime(eventTime) + // Check if data is late and handle allowedLateness + if sw.watermark.IsEventTimeLate(eventTime) { + // Data is late, check if it's within allowedLateness + allowedLateness := sw.config.AllowedLateness + if allowedLateness > 0 { + // Check if this late data belongs to any triggered window that's still open + sw.handleLateData(eventTime, allowedLateness) + } + // If allowedLateness is 0 or data is too late, we still add it but it won't trigger updates + } + } + // Add data to the window's data list - t := GetTimestamp(data, sw.config.TsProp, sw.config.TimeUnit) if !sw.initialized { - sw.currentSlot = sw.createSlot(t) - // Record when first window started (processing time) - sw.firstWindowStartTime = time.Now() + if timeChar == types.EventTime { + // For event time, align window start to window boundaries + alignedStart := alignWindowStart(eventTime, sw.slide) + sw.currentSlot = sw.createSlotFromStart(alignedStart) + } else { + // For processing time, use current time or event time as-is + sw.currentSlot = sw.createSlot(eventTime) + // Record when first window started (processing time) + sw.firstWindowStartTime = time.Now() + } // Don't start timer here, wait for first window to end // Send initialization complete signal // Safely close initChan to avoid closing an already closed channel @@ -142,7 +200,7 @@ func (sw *SlidingWindow) Add(data interface{}) { } row := types.Row{ Data: data, - Timestamp: t, + Timestamp: eventTime, } sw.data = append(sw.data, row) } @@ -151,6 +209,23 @@ func (sw *SlidingWindow) Add(data interface{}) { // Uses lazy initialization to avoid infinite waiting when no data, ensuring subsequent data can be processed normally // First window triggers when it ends, then subsequent windows trigger at slide intervals func (sw *SlidingWindow) Start() { + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := sw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + if timeChar == types.EventTime { + // Event time: trigger based on watermark + sw.startEventTime() + } else { + // Processing time: trigger based on system clock + sw.startProcessingTime() + } +} + +// startProcessingTime starts the processing time trigger mechanism +func (sw *SlidingWindow) startProcessingTime() { go func() { // Close output channel when function ends defer close(sw.outputChan) @@ -244,18 +319,147 @@ func (sw *SlidingWindow) Start() { }() } +// startEventTime starts the event time trigger mechanism based on watermark +func (sw *SlidingWindow) startEventTime() { + go func() { + // Close output channel when function ends + defer close(sw.outputChan) + if sw.watermark != nil { + defer sw.watermark.Stop() + } + + // Wait for initialization complete or context cancellation + select { + case <-sw.initChan: + // Initialization completed normally, continue processing + case <-sw.ctx.Done(): + // Context cancelled, exit directly + return + } + + // Process watermark updates + if sw.watermark != nil { + for { + select { + case watermarkTime := <-sw.watermark.WatermarkChan(): + sw.checkAndTriggerWindows(watermarkTime) + case <-sw.ctx.Done(): + return + } + } + } + }() +} + +// checkAndTriggerWindows checks if any windows should be triggered based on watermark +func (sw *SlidingWindow) checkAndTriggerWindows(watermarkTime time.Time) { + sw.mu.Lock() + defer sw.mu.Unlock() + + if !sw.initialized || sw.currentSlot == nil { + return + } + + allowedLateness := sw.config.AllowedLateness + + // Trigger all windows whose end time is before watermark + for sw.currentSlot != nil && !sw.currentSlot.End.After(watermarkTime) { + // Trigger current window + sw.triggerWindowLocked() + + // If allowedLateness > 0, keep window open for late data + if allowedLateness > 0 { + windowKey := sw.getWindowKey(*sw.currentSlot.End) + closeTime := sw.currentSlot.End.Add(allowedLateness) + sw.triggeredWindows[windowKey] = &triggeredWindowInfo{ + slot: sw.currentSlot, + closeTime: closeTime, + } + } + + // Move to next window + sw.currentSlot = sw.NextSlot() + } + + // Close windows that have exceeded allowedLateness + sw.closeExpiredWindows(watermarkTime) +} + +// triggerWindowLocked triggers the window (must be called with lock held) +func (sw *SlidingWindow) triggerWindowLocked() { + if sw.currentSlot == nil { + return + } + + // Extract current window data + resultData := make([]types.Row, 0) + for _, item := range sw.data { + if sw.currentSlot.Contains(item.Timestamp) { + item.Slot = sw.currentSlot + resultData = append(resultData, item) + } + } + + // Retain data that could be in future windows + // For sliding windows, we need to keep data that falls within: + // - Current window end + size (for overlapping windows) + cutoffTime := sw.currentSlot.End.Add(sw.size) + newData := make([]types.Row, 0) + for _, item := range sw.data { + // Keep data that could be in future windows (before cutoffTime) + if item.Timestamp.Before(cutoffTime) { + newData = append(newData, item) + } + } + sw.data = newData + + // Get callback reference before releasing lock + callback := sw.callback + + // Release lock before calling callback and sending to channel to avoid blocking + sw.mu.Unlock() + + if callback != nil { + callback(resultData) + } + + // Non-blocking send to output channel and update statistics + var sent bool + select { + case sw.outputChan <- resultData: + // Successfully sent + sent = true + default: + // Channel full, drop result + sent = false + } + + // Re-acquire lock to update statistics + sw.mu.Lock() + if sent { + sw.sentCount++ + } else { + sw.droppedCount++ + } +} + // Stop stops the sliding window operations func (sw *SlidingWindow) Stop() { // Call cancel function to stop window operations sw.cancelFunc() - // Safely stop timer + // Safely stop timer (for processing time) sw.timerMu.Lock() if sw.timer != nil { sw.timer.Stop() } sw.timerMu.Unlock() + // Stop watermark (for event time) + if sw.watermark != nil { + sw.watermark.Stop() + } + // Ensure initChan is closed if it hasn't been closed yet // This prevents Start() goroutine from blocking on initChan sw.mu.Lock() @@ -271,7 +475,15 @@ func (sw *SlidingWindow) Stop() { } // Trigger triggers the sliding window to process data within the window +// For ProcessingTime: called by timer +// For EventTime: called by watermark updates func (sw *SlidingWindow) Trigger() { + // Determine time characteristic + timeChar := sw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + // Lock to ensure thread safety sw.mu.Lock() @@ -284,6 +496,15 @@ func (sw *SlidingWindow) Trigger() { sw.mu.Unlock() return } + + if timeChar == types.EventTime { + // For event time, trigger is handled by watermark mechanism + // This method is kept for backward compatibility but shouldn't be called directly + sw.mu.Unlock() + return + } + + // Processing time logic // Calculate next slot for sliding window next := sw.NextSlot() if next == nil { @@ -315,6 +536,16 @@ func (sw *SlidingWindow) Trigger() { } } + // If resultData is empty, skip callback to avoid sending empty results + // This prevents empty results from filling up channels when timer triggers repeatedly + if len(resultData) == 0 { + // Update window data even if no result + sw.data = newData + sw.currentSlot = next + sw.mu.Unlock() + return + } + // Update window data sw.data = newData sw.currentSlot = next @@ -382,7 +613,7 @@ func (sw *SlidingWindow) Reset() { sw.mu.Lock() defer sw.mu.Unlock() - // Stop existing timer + // Stop existing timer (for processing time) sw.timerMu.Lock() if sw.timer != nil { sw.timer.Stop() @@ -390,12 +621,35 @@ func (sw *SlidingWindow) Reset() { } sw.timerMu.Unlock() + // Stop watermark (for event time) + if sw.watermark != nil { + sw.watermark.Stop() + // Recreate watermark + timeChar := sw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + if timeChar == types.EventTime { + maxOutOfOrderness := sw.config.MaxOutOfOrderness + if maxOutOfOrderness == 0 { + maxOutOfOrderness = 0 + } + watermarkInterval := sw.config.WatermarkInterval + if watermarkInterval == 0 { + watermarkInterval = 200 * time.Millisecond + } + idleTimeout := sw.config.IdleTimeout + sw.watermark = NewWatermark(maxOutOfOrderness, watermarkInterval, idleTimeout) + } + } + // Clear window data sw.data = nil sw.currentSlot = nil sw.initialized = false sw.initChan = make(chan struct{}) sw.firstWindowStartTime = time.Time{} + sw.triggeredWindows = make(map[string]*triggeredWindowInfo) // Recreate context for next startup sw.ctx, sw.cancelFunc = context.WithCancel(context.Background()) @@ -424,9 +678,89 @@ func (sw *SlidingWindow) NextSlot() *types.TimeSlot { } func (sw *SlidingWindow) createSlot(t time.Time) *types.TimeSlot { - // Create a new time slot + // Create a new time slot (for processing time, no alignment needed) start := t end := start.Add(sw.size) slot := types.NewTimeSlot(&start, &end) return slot } + +func (sw *SlidingWindow) createSlotFromStart(start time.Time) *types.TimeSlot { + // Create a new time slot from aligned start time (for event time) + end := start.Add(sw.size) + slot := types.NewTimeSlot(&start, &end) + return slot +} + +// getWindowKey generates a key for a window based on its end time +func (sw *SlidingWindow) getWindowKey(endTime time.Time) string { + return fmt.Sprintf("%d", endTime.UnixNano()) +} + +// handleLateData handles late data that arrives within allowedLateness +func (sw *SlidingWindow) handleLateData(eventTime time.Time, allowedLateness time.Duration) { + // Find which triggered window this late data belongs to + for _, info := range sw.triggeredWindows { + if info.slot.Contains(eventTime) { + // This late data belongs to a triggered window that's still open + // Trigger window again with updated data (late update) + sw.triggerLateUpdateLocked(info.slot) + return + } + } +} + +// triggerLateUpdateLocked triggers a late update for a window (must be called with lock held) +func (sw *SlidingWindow) triggerLateUpdateLocked(slot *types.TimeSlot) { + // Extract window data including late data + resultData := make([]types.Row, 0) + for _, item := range sw.data { + if slot.Contains(item.Timestamp) { + item.Slot = slot + resultData = append(resultData, item) + } + } + + if len(resultData) == 0 { + return + } + + // Get callback reference before releasing lock + callback := sw.callback + + // Release lock before calling callback and sending to channel to avoid blocking + sw.mu.Unlock() + + if callback != nil { + callback(resultData) + } + + // Non-blocking send to output channel and update statistics + var sent bool + select { + case sw.outputChan <- resultData: + // Successfully sent + sent = true + default: + // Channel full, drop result + sent = false + } + + // Re-acquire lock to update statistics + sw.mu.Lock() + if sent { + sw.sentCount++ + } else { + sw.droppedCount++ + } +} + +// closeExpiredWindows closes windows that have exceeded allowedLateness +func (sw *SlidingWindow) closeExpiredWindows(watermarkTime time.Time) { + for key, info := range sw.triggeredWindows { + if !watermarkTime.Before(info.closeTime) { + // Window has expired, remove it + delete(sw.triggeredWindows, key) + } + } +} diff --git a/window/tumbling_window.go b/window/tumbling_window.go index 118a78d..d7e82ae 100644 --- a/window/tumbling_window.go +++ b/window/tumbling_window.go @@ -29,6 +29,12 @@ import ( // Ensure TumblingWindow implements the Window interface var _ Window = (*TumblingWindow)(nil) +// triggeredWindowInfo stores information about a triggered window that is still open for late data +type triggeredWindowInfo struct { + slot *types.TimeSlot + closeTime time.Time // window end + allowedLateness +} + // TumblingWindow represents a tumbling window for collecting data and triggering processing at fixed time intervals type TumblingWindow struct { // config holds window configuration @@ -47,7 +53,7 @@ type TumblingWindow struct { ctx context.Context // cancelFunc cancels window operations cancelFunc context.CancelFunc - // timer for triggering window periodically + // timer for triggering window periodically (used for ProcessingTime) timer *time.Ticker currentSlot *types.TimeSlot // initChan for window initialization @@ -55,6 +61,12 @@ type TumblingWindow struct { initialized bool // timerMu protects timer access timerMu sync.Mutex + // watermark for event time processing (only used for EventTime) + watermark *Watermark + // pendingWindows stores windows waiting to be triggered (for EventTime) + pendingWindows map[string]*types.TimeSlot // key: window end time string + // triggeredWindows stores windows that have been triggered but are still open for late data (for EventTime with allowedLateness) + triggeredWindows map[string]*triggeredWindowInfo // key: window end time string // Performance statistics droppedCount int64 // Number of dropped results sentCount int64 // Number of successfully sent results @@ -83,14 +95,39 @@ func NewTumblingWindow(config types.WindowConfig) (*TumblingWindow, error) { bufferSize = config.PerformanceConfig.BufferConfig.WindowOutputSize } + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + // Initialize watermark for event time + var watermark *Watermark + if timeChar == types.EventTime { + maxOutOfOrderness := config.MaxOutOfOrderness + if maxOutOfOrderness == 0 { + maxOutOfOrderness = 0 // Default: no out-of-orderness allowed + } + watermarkInterval := config.WatermarkInterval + if watermarkInterval == 0 { + watermarkInterval = 200 * time.Millisecond // Default: 200ms + } + idleTimeout := config.IdleTimeout + // Default: 0 means disabled, no idle source mechanism + watermark = NewWatermark(maxOutOfOrderness, watermarkInterval, idleTimeout) + } + return &TumblingWindow{ - config: config, - size: size, - outputChan: make(chan []types.Row, bufferSize), - ctx: ctx, - cancelFunc: cancel, - initChan: make(chan struct{}), - initialized: false, + config: config, + size: size, + outputChan: make(chan []types.Row, bufferSize), + ctx: ctx, + cancelFunc: cancel, + initChan: make(chan struct{}), + initialized: false, + watermark: watermark, + pendingWindows: make(map[string]*types.TimeSlot), + triggeredWindows: make(map[string]*triggeredWindowInfo), }, nil } @@ -100,12 +137,48 @@ func (tw *TumblingWindow) Add(data interface{}) { tw.mu.Lock() defer tw.mu.Unlock() + // Get timestamp + eventTime := GetTimestamp(data, tw.config.TsProp, tw.config.TimeUnit) + + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := tw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + // For event time, update watermark and check for late data + if timeChar == types.EventTime && tw.watermark != nil { + tw.watermark.UpdateEventTime(eventTime) + // Check if data is late and handle allowedLateness + if tw.watermark.IsEventTimeLate(eventTime) { + // Data is late, check if it's within allowedLateness + allowedLateness := tw.config.AllowedLateness + if allowedLateness > 0 { + // Check if this late data belongs to any triggered window that's still open + tw.handleLateData(eventTime, allowedLateness) + } + // If allowedLateness is 0 or data is too late, we still add it but it won't trigger updates + } + } + // Append data to window's data list if !tw.initialized { - tw.currentSlot = tw.createSlot(GetTimestamp(data, tw.config.TsProp, tw.config.TimeUnit)) - tw.timerMu.Lock() - tw.timer = time.NewTicker(tw.size) - tw.timerMu.Unlock() + if timeChar == types.EventTime { + // For event time, align window start to window boundaries + alignedStart := alignWindowStart(eventTime, tw.size) + tw.currentSlot = tw.createSlotFromStart(alignedStart) + } else { + // For processing time, use current time or event time as-is + tw.currentSlot = tw.createSlot(eventTime) + } + + // Only start timer for processing time + if timeChar == types.ProcessingTime { + tw.timerMu.Lock() + tw.timer = time.NewTicker(tw.size) + tw.timerMu.Unlock() + } + tw.initialized = true // Send initialization complete signal (after setting timer) // Safely close initChan to avoid closing an already closed channel @@ -116,27 +189,35 @@ func (tw *TumblingWindow) Add(data interface{}) { close(tw.initChan) } } + row := types.Row{ Data: data, - Timestamp: GetTimestamp(data, tw.config.TsProp, tw.config.TimeUnit), + Timestamp: eventTime, } tw.data = append(tw.data, row) } -func (sw *TumblingWindow) createSlot(t time.Time) *types.TimeSlot { - // Create a new time slot +func (tw *TumblingWindow) createSlot(t time.Time) *types.TimeSlot { + // Create a new time slot (for processing time, no alignment needed) start := t - end := start.Add(sw.size) + end := start.Add(tw.size) + slot := types.NewTimeSlot(&start, &end) + return slot +} + +func (tw *TumblingWindow) createSlotFromStart(start time.Time) *types.TimeSlot { + // Create a new time slot from aligned start time (for event time) + end := start.Add(tw.size) slot := types.NewTimeSlot(&start, &end) return slot } -func (sw *TumblingWindow) NextSlot() *types.TimeSlot { - if sw.currentSlot == nil { +func (tw *TumblingWindow) NextSlot() *types.TimeSlot { + if tw.currentSlot == nil { return nil } - start := sw.currentSlot.End - end := sw.currentSlot.End.Add(sw.size) + start := tw.currentSlot.End + end := start.Add(tw.size) return types.NewTimeSlot(start, &end) } @@ -145,13 +226,18 @@ func (tw *TumblingWindow) Stop() { // Call cancel function to stop window operations tw.cancelFunc() - // Safely stop timer + // Safely stop timer (for processing time) tw.timerMu.Lock() if tw.timer != nil { tw.timer.Stop() } tw.timerMu.Unlock() + // Stop watermark (for event time) + if tw.watermark != nil { + tw.watermark.Stop() + } + // Ensure initChan is closed if it hasn't been closed yet // This prevents Start() goroutine from blocking on initChan tw.mu.Lock() @@ -169,6 +255,23 @@ func (tw *TumblingWindow) Stop() { // Start starts the tumbling window's periodic trigger mechanism // Uses lazy initialization to avoid infinite waiting when no data, ensuring subsequent data can be processed normally func (tw *TumblingWindow) Start() { + // Determine time characteristic (default to ProcessingTime for backward compatibility) + timeChar := tw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + + if timeChar == types.EventTime { + // Event time: trigger based on watermark + tw.startEventTime() + } else { + // Processing time: trigger based on system clock + tw.startProcessingTime() + } +} + +// startProcessingTime starts the processing time trigger mechanism +func (tw *TumblingWindow) startProcessingTime() { go func() { // Close output channel when function ends defer close(tw.outputChan) @@ -215,15 +318,224 @@ func (tw *TumblingWindow) Start() { }() } +// startEventTime starts the event time trigger mechanism based on watermark +func (tw *TumblingWindow) startEventTime() { + go func() { + // Close output channel when function ends + defer close(tw.outputChan) + if tw.watermark != nil { + defer tw.watermark.Stop() + } + + // Wait for initialization complete or context cancellation + select { + case <-tw.initChan: + // Initialization completed normally, continue processing + case <-tw.ctx.Done(): + // Context cancelled, exit directly + return + } + + // Process watermark updates + if tw.watermark != nil { + for { + select { + case watermarkTime := <-tw.watermark.WatermarkChan(): + tw.checkAndTriggerWindows(watermarkTime) + case <-tw.ctx.Done(): + return + } + } + } + }() +} + +// checkAndTriggerWindows checks if any windows should be triggered based on watermark +func (tw *TumblingWindow) checkAndTriggerWindows(watermarkTime time.Time) { + tw.mu.Lock() + defer tw.mu.Unlock() + + if !tw.initialized || tw.currentSlot == nil { + return + } + + allowedLateness := tw.config.AllowedLateness + + // Trigger all windows whose end time is before watermark + for tw.currentSlot != nil && !tw.currentSlot.End.After(watermarkTime) { + // Trigger current window + tw.triggerWindowLocked() + + // If allowedLateness > 0, keep window open for late data + if allowedLateness > 0 { + windowKey := tw.getWindowKey(*tw.currentSlot.End) + closeTime := tw.currentSlot.End.Add(allowedLateness) + tw.triggeredWindows[windowKey] = &triggeredWindowInfo{ + slot: tw.currentSlot, + closeTime: closeTime, + } + } + + // Move to next window + tw.currentSlot = tw.NextSlot() + } + + // Close windows that have exceeded allowedLateness + tw.closeExpiredWindows(watermarkTime) +} + +// closeExpiredWindows closes windows that have exceeded allowedLateness +func (tw *TumblingWindow) closeExpiredWindows(watermarkTime time.Time) { + for key, info := range tw.triggeredWindows { + if !watermarkTime.Before(info.closeTime) { + // Window has expired, remove it + delete(tw.triggeredWindows, key) + } + } +} + +// handleLateData handles late data that arrives within allowedLateness +func (tw *TumblingWindow) handleLateData(eventTime time.Time, allowedLateness time.Duration) { + // Find which triggered window this late data belongs to + for _, info := range tw.triggeredWindows { + if info.slot.Contains(eventTime) { + // This late data belongs to a triggered window that's still open + // Trigger window again with updated data (late update) + tw.triggerLateUpdateLocked(info.slot) + return + } + } +} + +// triggerLateUpdateLocked triggers a late update for a window (must be called with lock held) +func (tw *TumblingWindow) triggerLateUpdateLocked(slot *types.TimeSlot) { + // Extract window data including late data + resultData := make([]types.Row, 0) + for _, item := range tw.data { + if slot.Contains(item.Timestamp) { + item.Slot = slot + resultData = append(resultData, item) + } + } + + if len(resultData) == 0 { + return + } + + // Get callback reference before releasing lock + callback := tw.callback + + // Release lock before calling callback and sending to channel to avoid blocking + tw.mu.Unlock() + + if callback != nil { + callback(resultData) + } + + // Non-blocking send to output channel and update statistics + var sent bool + select { + case tw.outputChan <- resultData: + // Successfully sent + sent = true + default: + // Channel full, drop result + sent = false + } + + // Re-acquire lock to update statistics + tw.mu.Lock() + if sent { + tw.sentCount++ + } else { + tw.droppedCount++ + } +} + +// getWindowKey generates a key for a window based on its end time +func (tw *TumblingWindow) getWindowKey(endTime time.Time) string { + return fmt.Sprintf("%d", endTime.UnixNano()) +} + +// triggerWindowLocked triggers the window (must be called with lock held) +func (tw *TumblingWindow) triggerWindowLocked() { + if tw.currentSlot == nil { + return + } + + // Extract current window data + resultData := make([]types.Row, 0) + for _, item := range tw.data { + if tw.currentSlot.Contains(item.Timestamp) { + item.Slot = tw.currentSlot + resultData = append(resultData, item) + } + } + + // Remove data that belongs to current window + newData := make([]types.Row, 0) + for _, item := range tw.data { + if !tw.currentSlot.Contains(item.Timestamp) { + newData = append(newData, item) + } + } + tw.data = newData + + // Get callback reference before releasing lock + callback := tw.callback + + // Release lock before calling callback and sending to channel to avoid blocking + tw.mu.Unlock() + + if callback != nil { + callback(resultData) + } + + // Non-blocking send to output channel and update statistics + var sent bool + select { + case tw.outputChan <- resultData: + // Successfully sent + sent = true + default: + // Channel full, drop result + sent = false + } + + // Re-acquire lock to update statistics + tw.mu.Lock() + if sent { + tw.sentCount++ + } else { + tw.droppedCount++ + } +} + // Trigger triggers the tumbling window's processing logic +// For ProcessingTime: called by timer +// For EventTime: called by watermark updates func (tw *TumblingWindow) Trigger() { - // Lock to ensure thread safety + // Determine time characteristic + timeChar := tw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + tw.mu.Lock() if !tw.initialized { tw.mu.Unlock() return } + + if timeChar == types.EventTime { + // For event time, trigger is handled by watermark mechanism + // This method is kept for backward compatibility but shouldn't be called directly + tw.mu.Unlock() + return + } + + // Processing time logic // Calculate next window slot next := tw.NextSlot() // Retain data for next window @@ -246,6 +558,16 @@ func (tw *TumblingWindow) Trigger() { } } + // If resultData is empty, skip callback to avoid sending empty results + // This prevents empty results from filling up channels when timer triggers repeatedly + if len(resultData) == 0 { + // Update window data even if no result + tw.data = newData + tw.currentSlot = next + tw.mu.Unlock() + return + } + // Update window data tw.data = newData tw.currentSlot = next @@ -292,7 +614,7 @@ func (tw *TumblingWindow) Reset() { tw.mu.Lock() defer tw.mu.Unlock() - // Stop existing timer + // Stop existing timer (for processing time) tw.timerMu.Lock() if tw.timer != nil { tw.timer.Stop() @@ -300,11 +622,35 @@ func (tw *TumblingWindow) Reset() { } tw.timerMu.Unlock() + // Stop watermark (for event time) + if tw.watermark != nil { + tw.watermark.Stop() + // Recreate watermark + timeChar := tw.config.TimeCharacteristic + if timeChar == "" { + timeChar = types.ProcessingTime + } + if timeChar == types.EventTime { + maxOutOfOrderness := tw.config.MaxOutOfOrderness + if maxOutOfOrderness == 0 { + maxOutOfOrderness = 0 + } + watermarkInterval := tw.config.WatermarkInterval + if watermarkInterval == 0 { + watermarkInterval = 200 * time.Millisecond + } + idleTimeout := tw.config.IdleTimeout + tw.watermark = NewWatermark(maxOutOfOrderness, watermarkInterval, idleTimeout) + } + } + // Clear window data tw.data = nil tw.currentSlot = nil tw.initialized = false tw.initChan = make(chan struct{}) + tw.pendingWindows = make(map[string]*types.TimeSlot) + tw.triggeredWindows = make(map[string]*triggeredWindowInfo) // Recreate context for next startup tw.ctx, tw.cancelFunc = context.WithCancel(context.Background()) diff --git a/window/watermark.go b/window/watermark.go new file mode 100644 index 0000000..da30287 --- /dev/null +++ b/window/watermark.go @@ -0,0 +1,184 @@ +/* + * Copyright 2025 The RuleGo Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package window + +import ( + "context" + "sync" + "time" +) + +// Watermark represents a watermark for event time processing +// Watermark indicates that no events with timestamp less than watermark time are expected +type Watermark struct { + // currentWatermark is the current watermark time + currentWatermark time.Time + // maxEventTime is the maximum event time seen so far + maxEventTime time.Time + // maxOutOfOrderness is the maximum allowed out-of-orderness + maxOutOfOrderness time.Duration + // idleTimeout is the idle source timeout: when no data arrives within this duration, + // watermark advances based on processing time (0 means disabled) + idleTimeout time.Duration + // lastEventTime is the time when the last event was received + lastEventTime time.Time + // mu protects concurrent access + mu sync.RWMutex + // watermarkChan is a channel for watermark updates + watermarkChan chan time.Time + // ctx controls watermark lifecycle + ctx context.Context + // cancelFunc cancels watermark operations + cancelFunc context.CancelFunc +} + +// NewWatermark creates a new watermark manager +func NewWatermark(maxOutOfOrderness time.Duration, updateInterval time.Duration, idleTimeout time.Duration) *Watermark { + ctx, cancel := context.WithCancel(context.Background()) + + wm := &Watermark{ + currentWatermark: time.Time{}, + maxEventTime: time.Time{}, + maxOutOfOrderness: maxOutOfOrderness, + idleTimeout: idleTimeout, + lastEventTime: time.Time{}, + watermarkChan: make(chan time.Time, 100), + ctx: ctx, + cancelFunc: cancel, + } + + // Start periodic watermark updates + go wm.updateLoop(updateInterval) + + return wm +} + +// updateLoop periodically updates watermark based on max event time +func (wm *Watermark) updateLoop(interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + wm.update() + case <-wm.ctx.Done(): + return + } + } +} + +// update updates watermark based on current max event time +// If idle timeout is configured and data source is idle, watermark advances based on processing time +func (wm *Watermark) update() { + wm.mu.Lock() + defer wm.mu.Unlock() + + if !wm.maxEventTime.IsZero() { + now := time.Now() + var newWatermark time.Time + + // Check if data source is idle + if wm.idleTimeout > 0 && !wm.lastEventTime.IsZero() { + timeSinceLastEvent := now.Sub(wm.lastEventTime) + if timeSinceLastEvent > wm.idleTimeout { + // Data source is idle, advance watermark based on processing time + // Watermark = current processing time - max out of orderness + // This ensures windows can close even when no new data arrives + newWatermark = now.Add(-wm.maxOutOfOrderness) + } else { + // Normal update: based on max event time + newWatermark = wm.maxEventTime.Add(-wm.maxOutOfOrderness) + } + } else { + // Normal update: based on max event time + newWatermark = wm.maxEventTime.Add(-wm.maxOutOfOrderness) + } + + if newWatermark.After(wm.currentWatermark) { + wm.currentWatermark = newWatermark + // Send watermark update (non-blocking) + select { + case wm.watermarkChan <- wm.currentWatermark: + default: + // Channel full, skip + } + } + } +} + +// UpdateEventTime updates the maximum event time seen +func (wm *Watermark) UpdateEventTime(eventTime time.Time) { + wm.mu.Lock() + defer wm.mu.Unlock() + + // Update last event time for idle detection + wm.lastEventTime = time.Now() + + if wm.maxEventTime.IsZero() || eventTime.After(wm.maxEventTime) { + wm.maxEventTime = eventTime + // Immediately update watermark if event time is significantly ahead + newWatermark := eventTime.Add(-wm.maxOutOfOrderness) + if newWatermark.After(wm.currentWatermark) { + wm.currentWatermark = newWatermark + // Send watermark update (non-blocking) + select { + case wm.watermarkChan <- wm.currentWatermark: + default: + // Channel full, skip + } + } + } +} + +// GetCurrentWatermark returns the current watermark time +func (wm *Watermark) GetCurrentWatermark() time.Time { + wm.mu.RLock() + defer wm.mu.RUnlock() + return wm.currentWatermark +} + +// WatermarkChan returns a channel for receiving watermark updates +func (wm *Watermark) WatermarkChan() <-chan time.Time { + return wm.watermarkChan +} + +// Stop stops the watermark manager +func (wm *Watermark) Stop() { + wm.cancelFunc() +} + +// IsEventTimeLate checks if an event time is late (before current watermark) +func (wm *Watermark) IsEventTimeLate(eventTime time.Time) bool { + wm.mu.RLock() + defer wm.mu.RUnlock() + return !wm.currentWatermark.IsZero() && eventTime.Before(wm.currentWatermark) +} + +// alignWindowStart aligns window start time to window boundaries +// For event time windows, windows are aligned to epoch (00:00:00 UTC) +func alignWindowStart(timestamp time.Time, windowSize time.Duration) time.Time { + // Convert to Unix timestamp in nanoseconds + unixNano := timestamp.UnixNano() + windowSizeNano := windowSize.Nanoseconds() + + // Align to window boundary + alignedNano := (unixNano / windowSizeNano) * windowSizeNano + + // Convert back to time.Time + return time.Unix(0, alignedNano).UTC() +} From b2c638671adf4c4e52feb884700dcdfce764f4f7 Mon Sep 17 00:00:00 2001 From: rulego-team Date: Sat, 15 Nov 2025 00:37:08 +0800 Subject: [PATCH 2/3] =?UTF-8?q?ci:=E5=A2=9E=E5=8A=A0=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95ac189..588ece5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,11 +42,11 @@ jobs: - name: Run tests if: matrix.go-version != '1.21' - run: go test -v -race -timeout 300s ./... + run: go test -v -race -timeout 600s ./... - name: Run tests with coverage if: matrix.go-version == '1.21' - run: go test -v -race -coverprofile="codecov.report" -covermode=atomic -timeout 300s ./... + run: go test -v -race -coverprofile="codecov.report" -covermode=atomic -timeout 600s ./... - name: Upload coverage reports to Codecov if: matrix.go-version == '1.21' From 8207e6ba8cd60a5337f67dd8831cb1e0e5493785 Mon Sep 17 00:00:00 2001 From: rulego-team Date: Sat, 15 Nov 2025 01:04:56 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E5=BB=B6=E8=BF=9F?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- streamsql_tumbling_window_test.go | 2693 +++++++++++++++-------------- window/sliding_window.go | 195 ++- window/tumbling_window.go | 221 ++- window/watermark.go | 22 +- 4 files changed, 1728 insertions(+), 1403 deletions(-) diff --git a/streamsql_tumbling_window_test.go b/streamsql_tumbling_window_test.go index d1715e5..b7dc81f 100644 --- a/streamsql_tumbling_window_test.go +++ b/streamsql_tumbling_window_test.go @@ -1,1345 +1,1348 @@ -package streamsql - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestSQLTumblingWindow_ProcessingTime 测试处理时间的滚动窗口 -// 验证不使用 WITH 子句时，滚动窗口基于处理时间（系统时钟）工作 -func TestSQLTumblingWindow_ProcessingTime(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - AVG(temperature) as avg_temp, - MIN(temperature) as min_temp, - MAX(temperature) as max_temp - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 10) - defer close(ch) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - select { - case ch <- results: - default: - // 非阻塞发送 - } - } - }) - - // 使用处理时间：发送数据，不包含时间戳字段 - // 滚动窗口基于数据到达的处理时间（系统时钟）来划分窗口 - for i := 0; i < 10; i++ { - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "temperature": float64(i), - }) - time.Sleep(200 * time.Millisecond) // 每200ms发送一条数据 - } - - // 等待窗口触发（处理时间滚动窗口基于系统时钟触发） - time.Sleep(3 * time.Second) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - if windowResultsLen > 0 { - firstWindow := windowResultsCopy[0] - if len(firstWindow) > 0 { - row := firstWindow[0] - cnt := row["cnt"].(float64) - avgTemp := row["avg_temp"].(float64) - minTemp := row["min_temp"].(float64) - maxTemp := row["max_temp"].(float64) - - assert.Greater(t, cnt, 0.0, "窗口应该包含数据") - assert.LessOrEqual(t, minTemp, maxTemp, "最小值应该小于等于最大值") - assert.LessOrEqual(t, minTemp, avgTemp, "最小值应该小于等于平均值") - assert.LessOrEqual(t, avgTemp, maxTemp, "平均值应该小于等于最大值") - - t.Logf("处理时间滚动窗口成功触发，数据量: %.0f, 平均温度: %.2f", cnt, avgTemp) - } - } -} - -// TestSQLTumblingWindow_MaxOutOfOrderness 测试滚动窗口的最大延迟时间配置 -// 验证设置 MaxOutOfOrderness 后，延迟数据能否在允许的延迟范围内被正确处理 -func TestSQLTumblingWindow_MaxOutOfOrderness(t *testing.T) { - ssql := New() - defer ssql.Stop() - - // 使用 SQL 配置 MaxOutOfOrderness - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - MIN(temperature) as min_temp, - MAX(temperature) as max_temp - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='1s', IDLETIMEOUT='2s') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - // 模拟延迟数据场景 - // 场景：设置 MaxOutOfOrderness = 1秒，测试延迟数据能否在1秒内被正确处理 - // 窗口大小2秒，需要对齐到2秒的倍数 - windowSizeMs := int64(2000) // 2秒 - baseTimeRaw := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 - // 对齐baseTime到窗口大小的倍数，确保窗口对齐行为可预测 - baseTime := (baseTimeRaw / windowSizeMs) * windowSizeMs - - // 第一阶段：发送正常顺序的数据 - // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） - t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") - for i := 0; i < 10; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), // 0-9 - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待 watermark 推进，触发第一个窗口 - t.Log("等待 watermark 推进，触发第一个窗口") - time.Sleep(3 * time.Second) - - // 第二阶段：发送延迟数据 - // 延迟数据的事件时间在第一个窗口内（如 500ms, 700ms, 900ms） - // 如果 MaxOutOfOrderness = 1秒，这些数据应该能被处理 - t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内，延迟 < 1秒）") - lateDataTimes := []int64{500, 700, 900} // 延迟数据的事件时间（相对于 baseTime） - for i, lateTime := range lateDataTimes { - eventTime := baseTime + lateTime - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(20 + i), // 20-22，用于标识延迟数据 - }) - time.Sleep(100 * time.Millisecond) - } - - // 第三阶段：发送更多正常数据，推进 watermark - t.Log("第三阶段：继续发送正常数据，推进 watermark") - for i := 10; i < 15; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待窗口触发和延迟数据处理 - time.Sleep(3 * time.Second) - - // 收集所有窗口结果 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - // 验证窗口数据 - // 如果 MaxOutOfOrderness 配置正确，延迟数据应该能被统计到对应窗口 - if windowResultsLen > 0 { - firstWindow := windowResultsCopy[0] - if len(firstWindow) > 0 { - cnt := firstWindow[0]["cnt"].(float64) - minTemp := firstWindow[0]["min_temp"].(float64) - maxTemp := firstWindow[0]["max_temp"].(float64) - - t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) - - // 验证窗口包含数据 - // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 - // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 - assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") - assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") - - // 如果 MaxOutOfOrderness 配置正确，延迟数据应该被处理 - if maxTemp >= 20.0 { - t.Logf("✓ 延迟数据被正确处理，最大值包含延迟数据: %.0f", maxTemp) - } else { - t.Logf("提示：延迟数据可能未被统计，当前最大值: %.0f", maxTemp) - } - } - } - - t.Logf("总共触发了 %d 个窗口", windowResultsLen) -} - -// TestSQLTumblingWindow_AllowedLateness 测试滚动窗口的 AllowedLateness 配置 -// 验证窗口触发后，延迟数据能否在允许的延迟时间内更新窗口结果 -func TestSQLTumblingWindow_AllowedLateness(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - MIN(temperature) as min_temp, - MAX(temperature) as max_temp - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', ALLOWEDLATENESS='1s') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - // 模拟 AllowedLateness 场景 - // 场景：窗口触发后，发送延迟数据，验证窗口能否更新 - baseTime := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 - - // 第一阶段：发送正常顺序的数据，触发第一个窗口 - // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） - t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") - for i := 0; i < 10; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), // 0-9 - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待 watermark 推进，触发第一个窗口 - t.Log("等待 watermark 推进，触发第一个窗口") - time.Sleep(3 * time.Second) - - // 第二阶段：发送延迟数据（事件时间在第一个窗口内） - // 这些数据应该在 AllowedLateness = 1秒 内被处理 - t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") - lateDataTimes := []int64{300, 600, 900} // 延迟数据的事件时间 - for i, lateTime := range lateDataTimes { - eventTime := baseTime + lateTime - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(30 + i), // 30-32，用于标识延迟数据 - }) - time.Sleep(100 * time.Millisecond) - } - - // 第三阶段：继续发送正常数据，推进 watermark - t.Log("第三阶段：继续发送正常数据，推进 watermark") - for i := 10; i < 15; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待窗口触发和延迟数据处理 - time.Sleep(3 * time.Second) - - // 收集所有窗口结果 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - // 验证窗口数据 - // 如果 AllowedLateness 配置正确，延迟数据应该能触发窗口的延迟更新 - if windowResultsLen > 0 { - // 滚动窗口的延迟更新可能体现在后续的窗口结果中 - // 检查所有窗口结果，看是否有包含延迟数据的窗口 - hasLateData := false - for i, window := range windowResultsCopy { - if len(window) > 0 { - cnt := window[0]["cnt"].(float64) - minTemp := window[0]["min_temp"].(float64) - maxTemp := window[0]["max_temp"].(float64) - - t.Logf("窗口 %d: cnt=%.0f, min=%.0f, max=%.0f", i+1, cnt, minTemp, maxTemp) - - // 验证窗口包含数据 - assert.GreaterOrEqual(t, cnt, 1.0, "窗口 %d 应该包含数据", i+1) - - // 如果 AllowedLateness 配置正确，延迟数据应该被处理 - // 延迟数据（temperature=30-32）应该能被统计 - if maxTemp >= 30.0 { - hasLateData = true - t.Logf("✓ 窗口 %d 包含延迟数据，最大值: %.0f", i+1, maxTemp) - } - } - } - - // 验证是否有延迟更新（窗口可能触发多次） - if windowResultsLen > 1 { - t.Logf("✓ 滚动窗口触发了 %d 次，可能包含延迟更新", windowResultsLen) - } - - if !hasLateData { - t.Logf("提示：延迟数据可能未被统计，或延迟数据的时间不在窗口范围内") - } - } - - t.Logf("总共触发了 %d 个窗口", windowResultsLen) -} - -// TestSQLTumblingWindow_BothConfigs 测试滚动窗口同时配置 MaxOutOfOrderness 和 AllowedLateness -// 验证两个配置组合使用时，延迟数据能否被正确处理 -func TestSQLTumblingWindow_BothConfigs(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - MIN(temperature) as min_temp, - MAX(temperature) as max_temp - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH ( - TIMESTAMP='eventTime', - TIMEUNIT='ms', - MAXOUTOFORDERNESS='1s', - ALLOWEDLATENESS='500ms' - ) - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - // 模拟完整的延迟数据处理场景 - baseTime := time.Now().UnixMilli() - 10000 - - // 第一阶段：发送正常顺序的数据 - t.Log("第一阶段：发送正常顺序的数据") - for i := 0; i < 10; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), // 0-9 - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待 watermark 推进（考虑 MaxOutOfOrderness = 1s） - t.Log("等待 watermark 推进，触发窗口（MaxOutOfOrderness = 1s）") - time.Sleep(3 * time.Second) - - // 第二阶段：发送延迟数据（事件时间在第一个窗口内） - // MaxOutOfOrderness = 1s：这些数据应该在允许的乱序范围内 - // AllowedLateness = 500ms：窗口触发后还能接受500ms的延迟数据 - t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") - lateDataTimes := []int64{400, 800, 1200} // 延迟数据的事件时间 - for i, lateTime := range lateDataTimes { - eventTime := baseTime + lateTime - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(40 + i), // 40-42，用于标识延迟数据 - }) - time.Sleep(100 * time.Millisecond) - } - - // 第三阶段：继续发送正常数据，推进 watermark - t.Log("第三阶段：继续发送正常数据，推进 watermark") - for i := 10; i < 15; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待窗口触发和延迟数据处理 - time.Sleep(3 * time.Second) - - // 收集所有窗口结果 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - // 验证窗口数据 - if windowResultsLen > 0 { - firstWindow := windowResultsCopy[0] - if len(firstWindow) > 0 { - cnt := firstWindow[0]["cnt"].(float64) - minTemp := firstWindow[0]["min_temp"].(float64) - maxTemp := firstWindow[0]["max_temp"].(float64) - - t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) - - // 验证窗口包含数据 - // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 - // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 - assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") - assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") - - // 验证延迟数据是否被处理 - // 如果配置正确，maxTemp 可能包含延迟数据的值（40-42） - if maxTemp >= 40.0 { - t.Logf("✓ 延迟数据被正确处理，最大值包含延迟数据: %.0f", maxTemp) - } else { - t.Logf("提示：延迟数据可能未被统计，当前最大值: %.0f", maxTemp) - } - } - - // 验证是否有延迟更新 - if windowResultsLen > 1 { - t.Logf("✓ 滚动窗口触发了 %d 次，可能包含延迟更新", windowResultsLen) - - // 验证后续窗口的数据 - for i := 1; i < windowResultsLen && i < 3; i++ { - if len(windowResultsCopy[i]) > 0 { - cnt := windowResultsCopy[i][0]["cnt"].(float64) - t.Logf("窗口 %d: cnt=%.0f", i+1, cnt) - } - } - } - } - - t.Logf("总共触发了 %d 个窗口", windowResultsLen) - t.Logf("配置验证：MaxOutOfOrderness=1s, AllowedLateness=500ms") -} - -// TestSQLTumblingWindow_LateDataHandling 测试滚动窗口的延迟数据处理 -// 验证即使数据延迟到达，只要在允许的延迟范围内，也能正确统计到对应窗口 -func TestSQLTumblingWindow_LateDataHandling(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - MIN(temperature) as min_temp, - MAX(temperature) as max_temp - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - // 使用事件时间：模拟延迟数据场景 - // 场景：先发送正常顺序的数据，然后发送一些延迟的数据 - // 窗口大小2秒，需要对齐到2秒的倍数 - windowSizeMs := int64(2000) // 2秒 - baseTimeRaw := time.Now().UnixMilli() - 5000 // 使用5秒前作为基准 - // 对齐baseTime到窗口大小的倍数，确保窗口对齐行为可预测 - baseTime := (baseTimeRaw / windowSizeMs) * windowSizeMs - - // 第一阶段：发送正常顺序的数据（事件时间：0ms, 200ms, 400ms, ..., 2000ms） - // 这些数据应该被统计到第一个窗口 [0ms, 2000ms) - t.Log("第一阶段：发送正常顺序的数据") - for i := 0; i < 10; i++ { - eventTime := baseTime + int64(i*200) // 每200ms一条数据 - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), // 温度值 0-9 - }) - time.Sleep(50 * time.Millisecond) // 处理时间间隔较小 - } - - // 等待 watermark 推进，让第一个窗口触发 - // 窗口大小2秒，第一个窗口应该在 watermark >= baseTime + 2000ms 时触发 - t.Log("等待 watermark 推进，触发第一个窗口") - time.Sleep(3 * time.Second) - - // 第二阶段：发送延迟的数据 - // 这些数据的事件时间比之前的数据早，但应该在允许的延迟范围内 - // 延迟数据的事件时间：100ms, 300ms, 500ms（这些时间在第一个窗口 [0ms, 2000ms) 内） - t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") - for i := 0; i < 3; i++ { - // 延迟数据：事件时间比正常数据早，但仍在窗口范围内 - eventTime := baseTime + int64(100+i*200) // 100ms, 300ms, 500ms - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(10 + i), // 温度值 10-12，用于区分延迟数据 - }) - time.Sleep(100 * time.Millisecond) - } - - // 继续发送更多正常数据，推进 watermark - t.Log("第三阶段：继续发送正常数据，推进 watermark") - for i := 10; i < 15; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待窗口触发和延迟数据处理 - time.Sleep(3 * time.Second) - - // 收集所有窗口结果 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - // 验证第一个窗口的数据 - // 第一个窗口应该包含正常数据（0-9）和可能的延迟数据 - if windowResultsLen > 0 { - firstWindow := windowResultsCopy[0] - if len(firstWindow) > 0 { - cnt := firstWindow[0]["cnt"].(float64) - minTemp := firstWindow[0]["min_temp"].(float64) - maxTemp := firstWindow[0]["max_temp"].(float64) - - t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) - - // 第一个窗口应该包含正常数据 - // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 - // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 - assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") - assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") - assert.GreaterOrEqual(t, maxTemp, 0.0, "第一个窗口的最大值应该大于等于0") - } - } - - // 验证延迟数据是否被处理 - // 如果延迟数据被正确处理，应该能在后续窗口或更新中看到 - t.Logf("总共触发了 %d 个窗口", windowResultsLen) -} - -// TestSQLTumblingWindow_EventTimeWindowAlignment 测试事件时间窗口对齐到epoch -func TestSQLTumblingWindow_EventTimeWindowAlignment(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - window_start() as start, - window_end() as end - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - // 使用事件时间：发送数据，验证窗口对齐到epoch - // 窗口大小2秒，应该对齐到2秒的倍数 - baseTime := time.Now().UnixMilli() - - // 发送数据，事件时间从baseTime开始，每200ms一条 - // 第一个窗口应该对齐到小于等于baseTime的最大2秒倍数 - for i := 0; i < 15; i++ { - eventTime := baseTime + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 等待窗口触发 - time.Sleep(3 * time.Second) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - // 验证窗口对齐 - windowSizeMs := int64(2000) // 2秒 = 2000毫秒 - for i, window := range windowResultsCopy { - if len(window) > 0 { - row := window[0] - start := row["start"].(int64) - end := row["end"].(int64) - - startMs := start / int64(time.Millisecond) - endMs := end / int64(time.Millisecond) - windowSizeNs := int64(windowSizeMs) * int64(time.Millisecond) - - assert.Equal(t, windowSizeNs, end-start, - "窗口 %d 的大小应该是2秒（2000ms），实际: start=%d, end=%d", i+1, start, end) - - assert.Equal(t, int64(0), startMs%windowSizeMs, - "窗口 %d 的开始时间应该对齐到2秒的倍数（epoch对齐），实际: startMs=%d", i+1, startMs) - - if i > 0 { - prevEndMs := windowResultsCopy[i-1][0]["end"].(int64) / int64(time.Millisecond) - assert.Equal(t, prevEndMs, startMs, - "窗口 %d 的开始时间应该等于前一个窗口的结束时间，prevEndMs=%d, startMs=%d", i+1, prevEndMs, startMs) - } - - t.Logf("窗口 %d: start=%d, end=%d, size=%dms", i+1, startMs, endMs, endMs-startMs) - } - } -} - -// TestSQLTumblingWindow_WatermarkTriggerTiming 测试Watermark触发窗口的时机 -func TestSQLTumblingWindow_WatermarkTriggerTiming(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - window_start() as start, - window_end() as end - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - // 使用事件时间：发送数据，验证watermark触发时机 - baseTime := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 - maxOutOfOrdernessMs := int64(500) // 500ms - - // 第一阶段：发送数据到第一个窗口 [alignedStart, alignedStart+2000) - // 计算对齐后的窗口开始时间 - windowSizeMs := int64(2000) - alignedStart := (baseTime / windowSizeMs) * windowSizeMs - firstWindowEnd := alignedStart + windowSizeMs - - t.Logf("第一个窗口: [%d, %d)", alignedStart, firstWindowEnd) - - // 发送数据，事件时间在第一个窗口内 - for i := 0; i < 10; i++ { - eventTime := alignedStart + int64(i*200) // 在窗口内 - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 发送一个事件时间刚好等于window_end的数据，推进watermark - // watermark = maxEventTime - maxOutOfOrderness = firstWindowEnd - 500 - // 此时 watermark < firstWindowEnd，窗口不应该触发 - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": firstWindowEnd, - "temperature": 100.0, - }) - - // 等待watermark更新（watermark更新间隔200ms） - time.Sleep(500 * time.Millisecond) - - // 发送一个事件时间超过window_end的数据，推进watermark - // watermark = maxEventTime - maxOutOfOrderness = (firstWindowEnd + 1000) - 500 = firstWindowEnd + 500 - // 此时 watermark >= firstWindowEnd，窗口应该触发 - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": firstWindowEnd + 1000, - "temperature": 200.0, - }) - - // 等待窗口触发 - time.Sleep(1 * time.Second) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - // 验证第一个窗口的触发时机 - if windowResultsLen > 0 { - firstWindow := windowResultsCopy[0] - if len(firstWindow) > 0 { - row := firstWindow[0] - start := row["start"].(int64) - end := row["end"].(int64) - - startMs := start / int64(time.Millisecond) - endMs := end / int64(time.Millisecond) - - assert.Equal(t, alignedStart, startMs, - "第一个窗口的开始时间应该对齐到epoch，expected=%d, actual=%d", alignedStart, startMs) - assert.Equal(t, firstWindowEnd, endMs, - "第一个窗口的结束时间应该正确，expected=%d, actual=%d", firstWindowEnd, endMs) - - // 验证窗口在watermark >= window_end时触发 - // 由于watermark = maxEventTime - maxOutOfOrderness - // 当maxEventTime = firstWindowEnd + 1000时，watermark = firstWindowEnd + 500 - // watermark >= firstWindowEnd，窗口应该触发 - t.Logf("✓ 窗口在watermark >= window_end时正确触发") - t.Logf("窗口: [%d, %d), 触发时maxEventTime >= %d", start, end, end+maxOutOfOrdernessMs) - } - } -} - -// TestSQLTumblingWindow_AllowedLatenessUpdate 测试AllowedLateness的延迟更新 -func TestSQLTumblingWindow_AllowedLatenessUpdate(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - MIN(temperature) as min_temp, - MAX(temperature) as max_temp, - window_start() as start, - window_end() as end - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', ALLOWEDLATENESS='1s', IDLETIMEOUT='2s') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - baseTime := time.Now().UnixMilli() - 10000 - windowSizeMs := int64(2000) - alignedStart := (baseTime / windowSizeMs) * windowSizeMs - firstWindowEnd := alignedStart + windowSizeMs - allowedLatenessMs := int64(1000) // 1秒 - - // 第一阶段：发送正常数据，触发第一个窗口 - t.Log("第一阶段：发送正常数据，触发第一个窗口") - for i := 0; i < 10; i++ { - eventTime := alignedStart + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), // 0-9 - }) - time.Sleep(50 * time.Millisecond) - } - - // 推进watermark，触发第一个窗口 - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": firstWindowEnd + 1000, - "temperature": 100.0, - }) - - // 等待第一个窗口触发 - time.Sleep(1 * time.Second) - - // 收集第一个窗口的结果 - firstWindowReceived := false - firstWindowCnt := 0.0 - firstWindowMax := 0.0 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 10 - iteration := 0 - - for !firstWindowReceived && iteration < maxIterations { - select { - case res, ok := <-ch: - if !ok { - // channel 已关闭 - t.Fatal("应该收到第一个窗口") - } - if len(res) > 0 { - firstWindowReceived = true - firstWindowCnt = res[0]["cnt"].(float64) - firstWindowMax = res[0]["max_temp"].(float64) - t.Logf("第一个窗口（初始）: cnt=%.0f, max=%.0f", firstWindowCnt, firstWindowMax) - } - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果 - iteration++ - case <-ctx.Done(): - t.Fatal("应该收到第一个窗口") - } - } - - // 第二阶段：发送延迟数据（事件时间在第一个窗口内，但在AllowedLateness范围内） - t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") - lateDataTimes := []int64{300, 600, 900} // 延迟数据的事件时间（相对于alignedStart） - lateDataTemps := []float64{30.0, 31.0, 32.0} - for i, lateTime := range lateDataTimes { - eventTime := alignedStart + lateTime - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": lateDataTemps[i], - }) - time.Sleep(100 * time.Millisecond) - } - - // 继续发送正常数据，推进watermark（但不超过window_end + allowedLateness） - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": firstWindowEnd + allowedLatenessMs - 100, // 在allowedLateness范围内 - "temperature": 200.0, - }) - - // 等待延迟更新 - time.Sleep(1 * time.Second) - - // 收集所有窗口结果 - ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel2() - maxIterations2 := 20 - iteration2 := 0 - - for iteration2 < maxIterations2 { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration2++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx2.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") - - // 验证延迟更新 - hasLateUpdate := false - for i, window := range windowResultsCopy { - if len(window) > 0 { - row := window[0] - start := row["start"].(int64) - end := row["end"].(int64) - cnt := row["cnt"].(float64) - maxTemp := row["max_temp"].(float64) - - startMs := start / int64(time.Millisecond) - endMs := end / int64(time.Millisecond) - - if startMs == alignedStart && endMs == firstWindowEnd { - if cnt > firstWindowCnt { - hasLateUpdate = true - t.Logf("✓ 窗口延迟更新: cnt从%.0f增加到%.0f, max从%.0f增加到%.0f", - firstWindowCnt, cnt, firstWindowMax, maxTemp) - - // 验证延迟数据被包含 - assert.GreaterOrEqual(t, maxTemp, 30.0, - "延迟更新应该包含延迟数据，maxTemp应该>=30.0，实际: %.0f", maxTemp) - } - } - - t.Logf("窗口 %d: [%d, %d), cnt=%.0f, max=%.0f", i+1, start, end, cnt, maxTemp) - } - } - - if !hasLateUpdate { - t.Logf("⚠ 提示：未检测到延迟更新，可能延迟数据未被处理或窗口已关闭") - } else { - t.Logf("✓ AllowedLateness功能正常工作，延迟数据触发窗口更新") - } -} - -// TestSQLTumblingWindow_IdleSourceMechanism 测试Idle Source机制 -// 验证当数据源空闲时，watermark基于处理时间推进，窗口能够正常关闭 -func TestSQLTumblingWindow_IdleSourceMechanism(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - window_start() as start, - window_end() as end - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - // 使用事件时间：发送数据，然后停止发送，验证窗口能够关闭 - baseTime := time.Now().UnixMilli() - 10000 - windowSizeMs := int64(2000) // 2秒 - - // 计算对齐后的第一个窗口开始时间 - alignedStart := (baseTime / windowSizeMs) * windowSizeMs - firstWindowEnd := alignedStart + windowSizeMs - - t.Logf("第一个窗口: [%d, %d)", alignedStart, firstWindowEnd) - - // 第一阶段：发送数据，创建窗口 - t.Log("第一阶段：发送数据，创建窗口") - for i := 0; i < 5; i++ { - eventTime := alignedStart + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 第二阶段：停止发送数据，等待Idle Source机制触发 - // IdleTimeout = 2秒，意味着2秒无数据后，watermark会基于处理时间推进 - t.Log("第二阶段：停止发送数据，等待Idle Source机制触发（IdleTimeout=2s）") - time.Sleep(3 * time.Second) // 等待超过IdleTimeout，确保watermark推进 - - // 收集窗口结果 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) - copy(windowResultsCopy, windowResults) - windowResultsMu.Unlock() - - // 验证窗口能够关闭（即使没有新数据） - require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口（即使数据源空闲）") - - // 验证窗口数据 - if windowResultsLen > 0 { - firstWindow := windowResultsCopy[0] - if len(firstWindow) > 0 { - row := firstWindow[0] - start := row["start"].(int64) - end := row["end"].(int64) - cnt := row["cnt"].(float64) - - // 验证窗口边界正确 - // window_start() 和 window_end() 返回纳秒，需要转换为毫秒 - startMs := start / int64(time.Millisecond) - endMs := end / int64(time.Millisecond) - assert.Equal(t, alignedStart, startMs, - "第一个窗口的开始时间应该对齐到窗口大小，expected=%d, actual=%d", alignedStart, startMs) - assert.Equal(t, firstWindowEnd, endMs, - "第一个窗口的结束时间应该正确，expected=%d, actual=%d", firstWindowEnd, endMs) - - // 验证窗口包含数据 - assert.Greater(t, cnt, 0.0, "窗口应该包含数据") - - t.Logf("✓ Idle Source机制正常工作，窗口在数据源空闲时能够关闭") - t.Logf("窗口: [%d, %d), cnt=%.0f", start, end, cnt) - } - } -} - -// TestSQLTumblingWindow_IdleSourceDisabled 测试Idle Source机制未启用的情况 -// 验证当IdleTimeout=0（禁用）时，如果数据源空闲，窗口无法关闭 -func TestSQLTumblingWindow_IdleSourceDisabled(t *testing.T) { - ssql := New() - defer ssql.Stop() - - sql := ` - SELECT deviceId, - COUNT(*) as cnt, - window_start() as start, - window_end() as end - FROM stream - GROUP BY deviceId, TumblingWindow('2s') - WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') - -- 注意：没有配置IDLETIMEOUT，默认为0（禁用） - ` - err := ssql.Execute(sql) - require.NoError(t, err) - - ch := make(chan []map[string]interface{}, 20) - windowResults := make([][]map[string]interface{}, 0) - var windowResultsMu sync.Mutex - ssql.AddSink(func(results []map[string]interface{}) { - if len(results) > 0 { - windowResultsMu.Lock() - windowResults = append(windowResults, results) - windowResultsMu.Unlock() - ch <- results - } - }) - - baseTime := time.Now().UnixMilli() - 10000 - windowSizeMs := int64(2000) - alignedStart := (baseTime / windowSizeMs) * windowSizeMs - - // 发送数据，但事件时间不足以触发窗口 - t.Log("发送数据，但事件时间不足以触发窗口") - for i := 0; i < 3; i++ { - eventTime := alignedStart + int64(i*200) - ssql.Emit(map[string]interface{}{ - "deviceId": "sensor001", - "eventTime": eventTime, - "temperature": float64(i), - }) - time.Sleep(50 * time.Millisecond) - } - - // 停止发送数据，等待一段时间 - // 由于IdleTimeout未启用，watermark不会基于处理时间推进 - t.Log("停止发送数据，等待（IdleTimeout未启用）") - time.Sleep(3 * time.Second) - - // 收集窗口结果 - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - maxIterations := 20 - iteration := 0 - - for iteration < maxIterations { - select { - case result, ok := <-ch: - if !ok { - // channel 已关闭 - goto END - } - _ = result // 使用结果 - iteration++ - case <-time.After(500 * time.Millisecond): - // 500ms 没有新结果，退出 - goto END - case <-ctx.Done(): - // 超时退出 - goto END - } - } - -END: - windowResultsMu.Lock() - windowResultsLen := len(windowResults) - windowResultsMu.Unlock() - - // 注意：这个测试可能无法完全验证窗口无法关闭 - // 因为如果watermark已经推进到足够的位置，窗口可能已经触发 - // 这个测试主要用于对比：启用Idle Source vs 未启用Idle Source - t.Logf("窗口结果数量: %d（IdleTimeout未启用）", windowResultsLen) -} +package streamsql + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSQLTumblingWindow_ProcessingTime 测试处理时间的滚动窗口 +// 验证不使用 WITH 子句时，滚动窗口基于处理时间（系统时钟）工作 +func TestSQLTumblingWindow_ProcessingTime(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + AVG(temperature) as avg_temp, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 10) + defer close(ch) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + select { + case ch <- results: + default: + // 非阻塞发送 + } + } + }) + + // 使用处理时间：发送数据，不包含时间戳字段 + // 滚动窗口基于数据到达的处理时间（系统时钟）来划分窗口 + for i := 0; i < 10; i++ { + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "temperature": float64(i), + }) + time.Sleep(200 * time.Millisecond) // 每200ms发送一条数据 + } + + // 等待窗口触发（处理时间滚动窗口基于系统时钟触发） + time.Sleep(3 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + cnt := row["cnt"].(float64) + avgTemp := row["avg_temp"].(float64) + minTemp := row["min_temp"].(float64) + maxTemp := row["max_temp"].(float64) + + assert.Greater(t, cnt, 0.0, "窗口应该包含数据") + assert.LessOrEqual(t, minTemp, maxTemp, "最小值应该小于等于最大值") + assert.LessOrEqual(t, minTemp, avgTemp, "最小值应该小于等于平均值") + assert.LessOrEqual(t, avgTemp, maxTemp, "平均值应该小于等于最大值") + + t.Logf("处理时间滚动窗口成功触发，数据量: %.0f, 平均温度: %.2f", cnt, avgTemp) + } + } +} + +// TestSQLTumblingWindow_MaxOutOfOrderness 测试滚动窗口的最大延迟时间配置 +// 验证设置 MaxOutOfOrderness 后，延迟数据能否在允许的延迟范围内被正确处理 +func TestSQLTumblingWindow_MaxOutOfOrderness(t *testing.T) { + ssql := New() + defer ssql.Stop() + + // 使用 SQL 配置 MaxOutOfOrderness + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='1s', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟延迟数据场景 + // 场景：设置 MaxOutOfOrderness = 1秒，测试延迟数据能否在1秒内被正确处理 + // 窗口大小2秒，需要对齐到2秒的倍数 + windowSizeMs := int64(2000) // 2秒 + baseTimeRaw := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + // 对齐baseTime到窗口大小的倍数，确保窗口对齐行为可预测 + baseTime := (baseTimeRaw / windowSizeMs) * windowSizeMs + + // 第一阶段：发送正常顺序的数据 + // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） + t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待 watermark 推进，触发第一个窗口 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟数据 + // 延迟数据的事件时间在第一个窗口内（如 500ms, 700ms, 900ms） + // 如果 MaxOutOfOrderness = 1秒，这些数据应该能被处理 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内，延迟 < 1秒）") + lateDataTimes := []int64{500, 700, 900} // 延迟数据的事件时间（相对于 baseTime） + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(20 + i), // 20-22，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：发送更多正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + // 如果 MaxOutOfOrderness 配置正确，延迟数据应该能被统计到对应窗口 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 + // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 + assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + + // 如果 MaxOutOfOrderness 配置正确，延迟数据应该被处理 + if maxTemp >= 20.0 { + t.Logf("✓ 延迟数据被正确处理，最大值包含延迟数据: %.0f", maxTemp) + } else { + t.Logf("提示：延迟数据可能未被统计，当前最大值: %.0f", maxTemp) + } + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLTumblingWindow_AllowedLateness 测试滚动窗口的 AllowedLateness 配置 +// 验证窗口触发后，延迟数据能否在允许的延迟时间内更新窗口结果 +func TestSQLTumblingWindow_AllowedLateness(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', ALLOWEDLATENESS='1s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟 AllowedLateness 场景 + // 场景：窗口触发后，发送延迟数据，验证窗口能否更新 + baseTime := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + + // 第一阶段：发送正常顺序的数据，触发第一个窗口 + // 事件时间：0ms, 200ms, 400ms, ..., 2000ms（第一个窗口 [0ms, 2000ms)） + t.Log("第一阶段：发送正常顺序的数据（事件时间 0-2000ms）") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待 watermark 推进，触发第一个窗口 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟数据（事件时间在第一个窗口内） + // 这些数据应该在 AllowedLateness = 1秒 内被处理 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + lateDataTimes := []int64{300, 600, 900} // 延迟数据的事件时间 + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(30 + i), // 30-32，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：继续发送正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + // 如果 AllowedLateness 配置正确，延迟数据应该能触发窗口的延迟更新 + if windowResultsLen > 0 { + // 滚动窗口的延迟更新可能体现在后续的窗口结果中 + // 检查所有窗口结果，看是否有包含延迟数据的窗口 + hasLateData := false + for i, window := range windowResultsCopy { + if len(window) > 0 { + cnt := window[0]["cnt"].(float64) + minTemp := window[0]["min_temp"].(float64) + maxTemp := window[0]["max_temp"].(float64) + + t.Logf("窗口 %d: cnt=%.0f, min=%.0f, max=%.0f", i+1, cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + assert.GreaterOrEqual(t, cnt, 1.0, "窗口 %d 应该包含数据", i+1) + + // 如果 AllowedLateness 配置正确，延迟数据应该被处理 + // 延迟数据（temperature=30-32）应该能被统计 + if maxTemp >= 30.0 { + hasLateData = true + t.Logf("✓ 窗口 %d 包含延迟数据，最大值: %.0f", i+1, maxTemp) + } + } + } + + // 验证是否有延迟更新（窗口可能触发多次） + if windowResultsLen > 1 { + t.Logf("✓ 滚动窗口触发了 %d 次，可能包含延迟更新", windowResultsLen) + } + + if !hasLateData { + t.Logf("提示：延迟数据可能未被统计，或延迟数据的时间不在窗口范围内") + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLTumblingWindow_BothConfigs 测试滚动窗口同时配置 MaxOutOfOrderness 和 AllowedLateness +// 验证两个配置组合使用时，延迟数据能否被正确处理 +func TestSQLTumblingWindow_BothConfigs(t *testing.T) { + // 启用调试日志（可选，用于排查问题） + // window.EnableDebug = true + + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH ( + TIMESTAMP='eventTime', + TIMEUNIT='ms', + MAXOUTOFORDERNESS='1s', + ALLOWEDLATENESS='500ms' + ) + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 模拟完整的延迟数据处理场景 + baseTime := time.Now().UnixMilli() - 10000 + + // 第一阶段：发送正常顺序的数据 + t.Log("第一阶段：发送正常顺序的数据") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待 watermark 推进（考虑 MaxOutOfOrderness = 1s） + t.Log("等待 watermark 推进，触发窗口（MaxOutOfOrderness = 1s）") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟数据（事件时间在第一个窗口内） + // MaxOutOfOrderness = 1s：这些数据应该在允许的乱序范围内 + // AllowedLateness = 500ms：窗口触发后还能接受500ms的延迟数据 + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + lateDataTimes := []int64{400, 800, 1200} // 延迟数据的事件时间 + for i, lateTime := range lateDataTimes { + eventTime := baseTime + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(40 + i), // 40-42，用于标识延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 第三阶段：继续发送正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口数据 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 验证窗口包含数据 + // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 + // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 + assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + + // 验证延迟数据是否被处理 + // 如果配置正确，maxTemp 可能包含延迟数据的值（40-42） + if maxTemp >= 40.0 { + t.Logf("✓ 延迟数据被正确处理，最大值包含延迟数据: %.0f", maxTemp) + } else { + t.Logf("提示：延迟数据可能未被统计，当前最大值: %.0f", maxTemp) + } + } + + // 验证是否有延迟更新 + if windowResultsLen > 1 { + t.Logf("✓ 滚动窗口触发了 %d 次，可能包含延迟更新", windowResultsLen) + + // 验证后续窗口的数据 + for i := 1; i < windowResultsLen && i < 3; i++ { + if len(windowResultsCopy[i]) > 0 { + cnt := windowResultsCopy[i][0]["cnt"].(float64) + t.Logf("窗口 %d: cnt=%.0f", i+1, cnt) + } + } + } + } + + t.Logf("总共触发了 %d 个窗口", windowResultsLen) + t.Logf("配置验证：MaxOutOfOrderness=1s, AllowedLateness=500ms") +} + +// TestSQLTumblingWindow_LateDataHandling 测试滚动窗口的延迟数据处理 +// 验证即使数据延迟到达，只要在允许的延迟范围内，也能正确统计到对应窗口 +func TestSQLTumblingWindow_LateDataHandling(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：模拟延迟数据场景 + // 场景：先发送正常顺序的数据，然后发送一些延迟的数据 + // 窗口大小2秒，需要对齐到2秒的倍数 + windowSizeMs := int64(2000) // 2秒 + baseTimeRaw := time.Now().UnixMilli() - 5000 // 使用5秒前作为基准 + // 对齐baseTime到窗口大小的倍数，确保窗口对齐行为可预测 + baseTime := (baseTimeRaw / windowSizeMs) * windowSizeMs + + // 第一阶段：发送正常顺序的数据（事件时间：0ms, 200ms, 400ms, ..., 2000ms） + // 这些数据应该被统计到第一个窗口 [0ms, 2000ms) + t.Log("第一阶段：发送正常顺序的数据") + for i := 0; i < 10; i++ { + eventTime := baseTime + int64(i*200) // 每200ms一条数据 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 温度值 0-9 + }) + time.Sleep(50 * time.Millisecond) // 处理时间间隔较小 + } + + // 等待 watermark 推进，让第一个窗口触发 + // 窗口大小2秒，第一个窗口应该在 watermark >= baseTime + 2000ms 时触发 + t.Log("等待 watermark 推进，触发第一个窗口") + time.Sleep(3 * time.Second) + + // 第二阶段：发送延迟的数据 + // 这些数据的事件时间比之前的数据早，但应该在允许的延迟范围内 + // 延迟数据的事件时间：100ms, 300ms, 500ms（这些时间在第一个窗口 [0ms, 2000ms) 内） + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + for i := 0; i < 3; i++ { + // 延迟数据：事件时间比正常数据早，但仍在窗口范围内 + eventTime := baseTime + int64(100+i*200) // 100ms, 300ms, 500ms + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(10 + i), // 温度值 10-12，用于区分延迟数据 + }) + time.Sleep(100 * time.Millisecond) + } + + // 继续发送更多正常数据，推进 watermark + t.Log("第三阶段：继续发送正常数据，推进 watermark") + for i := 10; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发和延迟数据处理 + time.Sleep(3 * time.Second) + + // 收集所有窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证第一个窗口的数据 + // 第一个窗口应该包含正常数据（0-9）和可能的延迟数据 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + cnt := firstWindow[0]["cnt"].(float64) + minTemp := firstWindow[0]["min_temp"].(float64) + maxTemp := firstWindow[0]["max_temp"].(float64) + + t.Logf("第一个窗口: cnt=%.0f, min=%.0f, max=%.0f", cnt, minTemp, maxTemp) + + // 第一个窗口应该包含正常数据 + // 滚动窗口：窗口大小2秒，每200ms一条数据，理论上应该有10条数据 + // 但由于窗口对齐和 watermark 机制，实际数据量可能略有不同 + assert.GreaterOrEqual(t, cnt, 3.0, "第一个窗口应该包含足够的数据（滚动窗口特性）") + assert.Equal(t, 0.0, minTemp, "第一个窗口的最小值应该是0（正常数据）") + assert.GreaterOrEqual(t, maxTemp, 0.0, "第一个窗口的最大值应该大于等于0") + } + } + + // 验证延迟数据是否被处理 + // 如果延迟数据被正确处理，应该能在后续窗口或更新中看到 + t.Logf("总共触发了 %d 个窗口", windowResultsLen) +} + +// TestSQLTumblingWindow_EventTimeWindowAlignment 测试事件时间窗口对齐到epoch +func TestSQLTumblingWindow_EventTimeWindowAlignment(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，验证窗口对齐到epoch + // 窗口大小2秒，应该对齐到2秒的倍数 + baseTime := time.Now().UnixMilli() + + // 发送数据，事件时间从baseTime开始，每200ms一条 + // 第一个窗口应该对齐到小于等于baseTime的最大2秒倍数 + for i := 0; i < 15; i++ { + eventTime := baseTime + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 等待窗口触发 + time.Sleep(3 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证窗口对齐 + windowSizeMs := int64(2000) // 2秒 = 2000毫秒 + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + windowSizeNs := int64(windowSizeMs) * int64(time.Millisecond) + + assert.Equal(t, windowSizeNs, end-start, + "窗口 %d 的大小应该是2秒（2000ms），实际: start=%d, end=%d", i+1, start, end) + + assert.Equal(t, int64(0), startMs%windowSizeMs, + "窗口 %d 的开始时间应该对齐到2秒的倍数（epoch对齐），实际: startMs=%d", i+1, startMs) + + if i > 0 { + prevEndMs := windowResultsCopy[i-1][0]["end"].(int64) / int64(time.Millisecond) + assert.Equal(t, prevEndMs, startMs, + "窗口 %d 的开始时间应该等于前一个窗口的结束时间，prevEndMs=%d, startMs=%d", i+1, prevEndMs, startMs) + } + + t.Logf("窗口 %d: start=%d, end=%d, size=%dms", i+1, startMs, endMs, endMs-startMs) + } + } +} + +// TestSQLTumblingWindow_WatermarkTriggerTiming 测试Watermark触发窗口的时机 +func TestSQLTumblingWindow_WatermarkTriggerTiming(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，验证watermark触发时机 + baseTime := time.Now().UnixMilli() - 10000 // 使用10秒前作为基准 + maxOutOfOrdernessMs := int64(500) // 500ms + + // 第一阶段：发送数据到第一个窗口 [alignedStart, alignedStart+2000) + // 计算对齐后的窗口开始时间 + windowSizeMs := int64(2000) + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + firstWindowEnd := alignedStart + windowSizeMs + + t.Logf("第一个窗口: [%d, %d)", alignedStart, firstWindowEnd) + + // 发送数据，事件时间在第一个窗口内 + for i := 0; i < 10; i++ { + eventTime := alignedStart + int64(i*200) // 在窗口内 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 发送一个事件时间刚好等于window_end的数据，推进watermark + // watermark = maxEventTime - maxOutOfOrderness = firstWindowEnd - 500 + // 此时 watermark < firstWindowEnd，窗口不应该触发 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd, + "temperature": 100.0, + }) + + // 等待watermark更新（watermark更新间隔200ms） + time.Sleep(500 * time.Millisecond) + + // 发送一个事件时间超过window_end的数据，推进watermark + // watermark = maxEventTime - maxOutOfOrderness = (firstWindowEnd + 1000) - 500 = firstWindowEnd + 500 + // 此时 watermark >= firstWindowEnd，窗口应该触发 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + 1000, + "temperature": 200.0, + }) + + // 等待窗口触发 + time.Sleep(1 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证第一个窗口的触发时机 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + start := row["start"].(int64) + end := row["end"].(int64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + + assert.Equal(t, alignedStart, startMs, + "第一个窗口的开始时间应该对齐到epoch，expected=%d, actual=%d", alignedStart, startMs) + assert.Equal(t, firstWindowEnd, endMs, + "第一个窗口的结束时间应该正确，expected=%d, actual=%d", firstWindowEnd, endMs) + + // 验证窗口在watermark >= window_end时触发 + // 由于watermark = maxEventTime - maxOutOfOrderness + // 当maxEventTime = firstWindowEnd + 1000时，watermark = firstWindowEnd + 500 + // watermark >= firstWindowEnd，窗口应该触发 + t.Logf("✓ 窗口在watermark >= window_end时正确触发") + t.Logf("窗口: [%d, %d), 触发时maxEventTime >= %d", start, end, end+maxOutOfOrdernessMs) + } + } +} + +// TestSQLTumblingWindow_AllowedLatenessUpdate 测试AllowedLateness的延迟更新 +func TestSQLTumblingWindow_AllowedLatenessUpdate(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + MIN(temperature) as min_temp, + MAX(temperature) as max_temp, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', ALLOWEDLATENESS='1s', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + baseTime := time.Now().UnixMilli() - 10000 + windowSizeMs := int64(2000) + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + firstWindowEnd := alignedStart + windowSizeMs + allowedLatenessMs := int64(1000) // 1秒 + + // 第一阶段：发送正常数据，触发第一个窗口 + t.Log("第一阶段：发送正常数据，触发第一个窗口") + for i := 0; i < 10; i++ { + eventTime := alignedStart + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), // 0-9 + }) + time.Sleep(50 * time.Millisecond) + } + + // 推进watermark，触发第一个窗口 + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + 1000, + "temperature": 100.0, + }) + + // 等待第一个窗口触发 + time.Sleep(1 * time.Second) + + // 收集第一个窗口的结果 + firstWindowReceived := false + firstWindowCnt := 0.0 + firstWindowMax := 0.0 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 10 + iteration := 0 + + for !firstWindowReceived && iteration < maxIterations { + select { + case res, ok := <-ch: + if !ok { + // channel 已关闭 + t.Fatal("应该收到第一个窗口") + } + if len(res) > 0 { + firstWindowReceived = true + firstWindowCnt = res[0]["cnt"].(float64) + firstWindowMax = res[0]["max_temp"].(float64) + t.Logf("第一个窗口（初始）: cnt=%.0f, max=%.0f", firstWindowCnt, firstWindowMax) + } + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果 + iteration++ + case <-ctx.Done(): + t.Fatal("应该收到第一个窗口") + } + } + + // 第二阶段：发送延迟数据（事件时间在第一个窗口内，但在AllowedLateness范围内） + t.Log("第二阶段：发送延迟数据（事件时间在第一个窗口内）") + lateDataTimes := []int64{300, 600, 900} // 延迟数据的事件时间（相对于alignedStart） + lateDataTemps := []float64{30.0, 31.0, 32.0} + for i, lateTime := range lateDataTimes { + eventTime := alignedStart + lateTime + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": lateDataTemps[i], + }) + time.Sleep(100 * time.Millisecond) + } + + // 继续发送正常数据，推进watermark（但不超过window_end + allowedLateness） + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": firstWindowEnd + allowedLatenessMs - 100, // 在allowedLateness范围内 + "temperature": 200.0, + }) + + // 等待延迟更新 + time.Sleep(1 * time.Second) + + // 收集所有窗口结果 + ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel2() + maxIterations2 := 20 + iteration2 := 0 + + for iteration2 < maxIterations2 { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration2++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx2.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口") + + // 验证延迟更新 + hasLateUpdate := false + for i, window := range windowResultsCopy { + if len(window) > 0 { + row := window[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + maxTemp := row["max_temp"].(float64) + + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + + if startMs == alignedStart && endMs == firstWindowEnd { + if cnt > firstWindowCnt { + hasLateUpdate = true + t.Logf("✓ 窗口延迟更新: cnt从%.0f增加到%.0f, max从%.0f增加到%.0f", + firstWindowCnt, cnt, firstWindowMax, maxTemp) + + // 验证延迟数据被包含 + assert.GreaterOrEqual(t, maxTemp, 30.0, + "延迟更新应该包含延迟数据，maxTemp应该>=30.0，实际: %.0f", maxTemp) + } + } + + t.Logf("窗口 %d: [%d, %d), cnt=%.0f, max=%.0f", i+1, start, end, cnt, maxTemp) + } + } + + if !hasLateUpdate { + t.Logf("⚠ 提示：未检测到延迟更新，可能延迟数据未被处理或窗口已关闭") + } else { + t.Logf("✓ AllowedLateness功能正常工作，延迟数据触发窗口更新") + } +} + +// TestSQLTumblingWindow_IdleSourceMechanism 测试Idle Source机制 +// 验证当数据源空闲时，watermark基于处理时间推进，窗口能够正常关闭 +func TestSQLTumblingWindow_IdleSourceMechanism(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + // 使用事件时间：发送数据，然后停止发送，验证窗口能够关闭 + baseTime := time.Now().UnixMilli() - 10000 + windowSizeMs := int64(2000) // 2秒 + + // 计算对齐后的第一个窗口开始时间 + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + firstWindowEnd := alignedStart + windowSizeMs + + t.Logf("第一个窗口: [%d, %d)", alignedStart, firstWindowEnd) + + // 第一阶段：发送数据，创建窗口 + t.Log("第一阶段：发送数据，创建窗口") + for i := 0; i < 5; i++ { + eventTime := alignedStart + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 第二阶段：停止发送数据，等待Idle Source机制触发 + // IdleTimeout = 2秒，意味着2秒无数据后，watermark会基于处理时间推进 + t.Log("第二阶段：停止发送数据，等待Idle Source机制触发（IdleTimeout=2s）") + time.Sleep(3 * time.Second) // 等待超过IdleTimeout，确保watermark推进 + + // 收集窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsCopy := make([][]map[string]interface{}, len(windowResults)) + copy(windowResultsCopy, windowResults) + windowResultsMu.Unlock() + + // 验证窗口能够关闭（即使没有新数据） + require.Greater(t, windowResultsLen, 0, "应该至少触发一个窗口（即使数据源空闲）") + + // 验证窗口数据 + if windowResultsLen > 0 { + firstWindow := windowResultsCopy[0] + if len(firstWindow) > 0 { + row := firstWindow[0] + start := row["start"].(int64) + end := row["end"].(int64) + cnt := row["cnt"].(float64) + + // 验证窗口边界正确 + // window_start() 和 window_end() 返回纳秒，需要转换为毫秒 + startMs := start / int64(time.Millisecond) + endMs := end / int64(time.Millisecond) + assert.Equal(t, alignedStart, startMs, + "第一个窗口的开始时间应该对齐到窗口大小，expected=%d, actual=%d", alignedStart, startMs) + assert.Equal(t, firstWindowEnd, endMs, + "第一个窗口的结束时间应该正确，expected=%d, actual=%d", firstWindowEnd, endMs) + + // 验证窗口包含数据 + assert.Greater(t, cnt, 0.0, "窗口应该包含数据") + + t.Logf("✓ Idle Source机制正常工作，窗口在数据源空闲时能够关闭") + t.Logf("窗口: [%d, %d), cnt=%.0f", start, end, cnt) + } + } +} + +// TestSQLTumblingWindow_IdleSourceDisabled 测试Idle Source机制未启用的情况 +// 验证当IdleTimeout=0（禁用）时，如果数据源空闲，窗口无法关闭 +func TestSQLTumblingWindow_IdleSourceDisabled(t *testing.T) { + ssql := New() + defer ssql.Stop() + + sql := ` + SELECT deviceId, + COUNT(*) as cnt, + window_start() as start, + window_end() as end + FROM stream + GROUP BY deviceId, TumblingWindow('2s') + WITH (TIMESTAMP='eventTime', TIMEUNIT='ms', MAXOUTOFORDERNESS='500ms', IDLETIMEOUT='2s') + -- 注意：没有配置IDLETIMEOUT，默认为0（禁用） + ` + err := ssql.Execute(sql) + require.NoError(t, err) + + ch := make(chan []map[string]interface{}, 20) + windowResults := make([][]map[string]interface{}, 0) + var windowResultsMu sync.Mutex + ssql.AddSink(func(results []map[string]interface{}) { + if len(results) > 0 { + windowResultsMu.Lock() + windowResults = append(windowResults, results) + windowResultsMu.Unlock() + ch <- results + } + }) + + baseTime := time.Now().UnixMilli() - 10000 + windowSizeMs := int64(2000) + alignedStart := (baseTime / windowSizeMs) * windowSizeMs + + // 发送数据，但事件时间不足以触发窗口 + t.Log("发送数据，但事件时间不足以触发窗口") + for i := 0; i < 3; i++ { + eventTime := alignedStart + int64(i*200) + ssql.Emit(map[string]interface{}{ + "deviceId": "sensor001", + "eventTime": eventTime, + "temperature": float64(i), + }) + time.Sleep(50 * time.Millisecond) + } + + // 停止发送数据，等待一段时间 + // 由于IdleTimeout未启用，watermark不会基于处理时间推进 + t.Log("停止发送数据，等待（IdleTimeout未启用）") + time.Sleep(3 * time.Second) + + // 收集窗口结果 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + maxIterations := 20 + iteration := 0 + + for iteration < maxIterations { + select { + case result, ok := <-ch: + if !ok { + // channel 已关闭 + goto END + } + _ = result // 使用结果 + iteration++ + case <-time.After(500 * time.Millisecond): + // 500ms 没有新结果，退出 + goto END + case <-ctx.Done(): + // 超时退出 + goto END + } + } + +END: + windowResultsMu.Lock() + windowResultsLen := len(windowResults) + windowResultsMu.Unlock() + + // 注意：这个测试可能无法完全验证窗口无法关闭 + // 因为如果watermark已经推进到足够的位置，窗口可能已经触发 + // 这个测试主要用于对比：启用Idle Source vs 未启用Idle Source + t.Logf("窗口结果数量: %d（IdleTimeout未启用）", windowResultsLen) +} diff --git a/window/sliding_window.go b/window/sliding_window.go index d526773..1ad077d 100644 --- a/window/sliding_window.go +++ b/window/sliding_window.go @@ -160,22 +160,12 @@ func (sw *SlidingWindow) Add(data interface{}) { timeChar = types.ProcessingTime } - // For event time, update watermark and check for late data + // For event time, update watermark if timeChar == types.EventTime && sw.watermark != nil { sw.watermark.UpdateEventTime(eventTime) - // Check if data is late and handle allowedLateness - if sw.watermark.IsEventTimeLate(eventTime) { - // Data is late, check if it's within allowedLateness - allowedLateness := sw.config.AllowedLateness - if allowedLateness > 0 { - // Check if this late data belongs to any triggered window that's still open - sw.handleLateData(eventTime, allowedLateness) - } - // If allowedLateness is 0 or data is too late, we still add it but it won't trigger updates - } } - // Add data to the window's data list + // Add data to the window's data list first (needed for late data handling) if !sw.initialized { if timeChar == types.EventTime { // For event time, align window start to window boundaries @@ -203,6 +193,38 @@ func (sw *SlidingWindow) Add(data interface{}) { Timestamp: eventTime, } sw.data = append(sw.data, row) + + // Check if data is late and handle allowedLateness (after data is added) + if timeChar == types.EventTime && sw.watermark != nil { + if sw.watermark.IsEventTimeLate(eventTime) { + allowedLateness := sw.config.AllowedLateness + if allowedLateness > 0 { + // IMPORTANT: First check if this late data belongs to any triggered window that's still open + // This ensures late data is correctly assigned to its original window, even if + // the event time happens to fall within the current window's range + belongsToTriggeredWindow := false + for _, info := range sw.triggeredWindows { + if info.slot.Contains(eventTime) { + belongsToTriggeredWindow = true + // Trigger late update for this window (data is already in sw.data) + sw.handleLateData(eventTime, allowedLateness) + break + } + } + + // If not belonging to triggered window, check if it belongs to currentSlot + // This handles the case where watermark has advanced but window hasn't triggered yet + if !belongsToTriggeredWindow && sw.initialized && sw.currentSlot != nil && sw.currentSlot.Contains(eventTime) { + // Data belongs to currentSlot, it will be included when window triggers + // No need to do anything here + } else if !belongsToTriggeredWindow { + // Check if this late data belongs to any triggered window that's still open + sw.handleLateData(eventTime, allowedLateness) + } + } + // If allowedLateness is 0 or data is too late, we still add it but it won't trigger updates + } + } } // Start starts the sliding window with periodic triggering @@ -362,22 +384,76 @@ func (sw *SlidingWindow) checkAndTriggerWindows(watermarkTime time.Time) { allowedLateness := sw.config.AllowedLateness - // Trigger all windows whose end time is before watermark - for sw.currentSlot != nil && !sw.currentSlot.End.After(watermarkTime) { - // Trigger current window - sw.triggerWindowLocked() - - // If allowedLateness > 0, keep window open for late data - if allowedLateness > 0 { - windowKey := sw.getWindowKey(*sw.currentSlot.End) - closeTime := sw.currentSlot.End.Add(allowedLateness) - sw.triggeredWindows[windowKey] = &triggeredWindowInfo{ - slot: sw.currentSlot, - closeTime: closeTime, + // Trigger all windows whose end time is <= watermark + // In Flink, windows are triggered when watermark >= windowEnd. + // Watermark calculation: watermark = maxEventTime - maxOutOfOrderness + // So watermark >= windowEnd means: maxEventTime - maxOutOfOrderness >= windowEnd + // Which means: maxEventTime >= windowEnd + maxOutOfOrderness + // This ensures all data for the window has arrived (within maxOutOfOrderness tolerance) + // Use a small threshold (1ms) only for floating point precision issues + for sw.currentSlot != nil { + windowEnd := sw.currentSlot.End + + // Check if watermark >= windowEnd + // Use !Before() instead of After() to include equality case + // This is equivalent to watermarkTime >= windowEnd + shouldTrigger := !watermarkTime.Before(*windowEnd) + + if !shouldTrigger { + // Watermark hasn't reached windowEnd yet, stop checking + break + } + // Check if window has data before triggering + hasData := false + for _, item := range sw.data { + if sw.currentSlot.Contains(item.Timestamp) { + hasData = true + break } } - // Move to next window + // Trigger current window only if it has data + if hasData { + // Count data in window before triggering + dataInWindow := 0 + for _, item := range sw.data { + if sw.currentSlot.Contains(item.Timestamp) { + dataInWindow++ + } + } + + // Save snapshot data before triggering (for Flink-like late update behavior) + var snapshotData []types.Row + if allowedLateness > 0 { + // Create a deep copy of window data for snapshot + snapshotData = make([]types.Row, 0, dataInWindow) + for _, item := range sw.data { + if sw.currentSlot.Contains(item.Timestamp) { + // Create a copy of the row + snapshotData = append(snapshotData, types.Row{ + Data: item.Data, + Timestamp: item.Timestamp, + Slot: sw.currentSlot, + }) + } + } + } + + sw.triggerWindowLocked() + + // If allowedLateness > 0, keep window open for late data + if allowedLateness > 0 { + windowKey := sw.getWindowKey(*sw.currentSlot.End) + closeTime := sw.currentSlot.End.Add(allowedLateness) + sw.triggeredWindows[windowKey] = &triggeredWindowInfo{ + slot: sw.currentSlot, + closeTime: closeTime, + snapshotData: snapshotData, // Save snapshot for late updates + } + } + } + + // Move to next window (even if current window was empty) sw.currentSlot = sw.NextSlot() } @@ -400,6 +476,12 @@ func (sw *SlidingWindow) triggerWindowLocked() { } } + // Skip triggering if window has no data + // This prevents empty windows from being triggered + if len(resultData) == 0 { + return + } + // Retain data that could be in future windows // For sliding windows, we need to keep data that falls within: // - Current window end + size (for overlapping windows) @@ -711,13 +793,37 @@ func (sw *SlidingWindow) handleLateData(eventTime time.Time, allowedLateness tim } // triggerLateUpdateLocked triggers a late update for a window (must be called with lock held) +// This implements Flink-like behavior: late updates include complete window data (original + late data) func (sw *SlidingWindow) triggerLateUpdateLocked(slot *types.TimeSlot) { - // Extract window data including late data + // Find the triggered window info to get snapshot data + var windowInfo *triggeredWindowInfo + windowKey := sw.getWindowKey(*slot.End) + if info, exists := sw.triggeredWindows[windowKey]; exists { + windowInfo = info + } + + // Collect all data for this window: original snapshot + late data from sw.data resultData := make([]types.Row, 0) + + // First, add original snapshot data (if exists) + if windowInfo != nil && len(windowInfo.snapshotData) > 0 { + // Create copies of snapshot data + for _, item := range windowInfo.snapshotData { + resultData = append(resultData, types.Row{ + Data: item.Data, + Timestamp: item.Timestamp, + Slot: slot, // Update slot reference + }) + } + } + + // Then, add late data from sw.data (newly arrived late data) + lateDataCount := 0 for _, item := range sw.data { if slot.Contains(item.Timestamp) { item.Slot = slot resultData = append(resultData, item) + lateDataCount++ } } @@ -725,6 +831,19 @@ func (sw *SlidingWindow) triggerLateUpdateLocked(slot *types.TimeSlot) { return } + // Update snapshot to include late data (for future late updates) + if windowInfo != nil { + // Update snapshot with complete data (original + late) + windowInfo.snapshotData = make([]types.Row, len(resultData)) + for i, item := range resultData { + windowInfo.snapshotData[i] = types.Row{ + Data: item.Data, + Timestamp: item.Timestamp, + Slot: slot, + } + } + } + // Get callback reference before releasing lock callback := sw.callback @@ -757,10 +876,32 @@ func (sw *SlidingWindow) triggerLateUpdateLocked(slot *types.TimeSlot) { // closeExpiredWindows closes windows that have exceeded allowedLateness func (sw *SlidingWindow) closeExpiredWindows(watermarkTime time.Time) { + expiredWindows := make([]*types.TimeSlot, 0) for key, info := range sw.triggeredWindows { if !watermarkTime.Before(info.closeTime) { - // Window has expired, remove it + // Window has expired, mark for removal + expiredWindows = append(expiredWindows, info.slot) delete(sw.triggeredWindows, key) } } + + // Clean up data that belongs to expired windows (if any) + if len(expiredWindows) > 0 { + newData := make([]types.Row, 0) + for _, item := range sw.data { + belongsToExpiredWindow := false + for _, expiredSlot := range expiredWindows { + if expiredSlot.Contains(item.Timestamp) { + belongsToExpiredWindow = true + break + } + } + if !belongsToExpiredWindow { + newData = append(newData, item) + } + } + if len(newData) != len(sw.data) { + sw.data = newData + } + } } diff --git a/window/tumbling_window.go b/window/tumbling_window.go index d7e82ae..4656cdb 100644 --- a/window/tumbling_window.go +++ b/window/tumbling_window.go @@ -31,8 +31,9 @@ var _ Window = (*TumblingWindow)(nil) // triggeredWindowInfo stores information about a triggered window that is still open for late data type triggeredWindowInfo struct { - slot *types.TimeSlot - closeTime time.Time // window end + allowedLateness + slot *types.TimeSlot + closeTime time.Time // window end + allowedLateness + snapshotData []types.Row // snapshot of window data when first triggered (for Flink-like late update behavior) } // TumblingWindow represents a tumbling window for collecting data and triggering processing at fixed time intervals @@ -146,29 +147,22 @@ func (tw *TumblingWindow) Add(data interface{}) { timeChar = types.ProcessingTime } - // For event time, update watermark and check for late data + // For event time, update watermark if timeChar == types.EventTime && tw.watermark != nil { tw.watermark.UpdateEventTime(eventTime) - // Check if data is late and handle allowedLateness - if tw.watermark.IsEventTimeLate(eventTime) { - // Data is late, check if it's within allowedLateness - allowedLateness := tw.config.AllowedLateness - if allowedLateness > 0 { - // Check if this late data belongs to any triggered window that's still open - tw.handleLateData(eventTime, allowedLateness) - } - // If allowedLateness is 0 or data is too late, we still add it but it won't trigger updates - } } - // Append data to window's data list + // Append data to window's data list first (needed for late data handling) if !tw.initialized { if timeChar == types.EventTime { // For event time, align window start to window boundaries + // Alignment ensures consistent window boundaries across different data sources + // Alignment granularity equals window size (e.g., 2s window aligns to 2s boundaries) alignedStart := alignWindowStart(eventTime, tw.size) tw.currentSlot = tw.createSlotFromStart(alignedStart) } else { // For processing time, use current time or event time as-is + // No alignment is performed - window starts immediately when first data arrives tw.currentSlot = tw.createSlot(eventTime) } @@ -195,10 +189,48 @@ func (tw *TumblingWindow) Add(data interface{}) { Timestamp: eventTime, } tw.data = append(tw.data, row) + + // Check if data is late and handle allowedLateness (after data is added) + if timeChar == types.EventTime && tw.watermark != nil { + if tw.watermark.IsEventTimeLate(eventTime) { + allowedLateness := tw.config.AllowedLateness + if allowedLateness > 0 { + // IMPORTANT: First check if this late data belongs to any triggered window that's still open + // This ensures late data is correctly assigned to its original window, even if + // the event time happens to fall within the current window's range + // Example: window [1000, 2000) triggered, moved to [2000, 3000), late data with + // eventTime=1500 should go to [1000, 2000), not [2000, 3000) + belongsToTriggeredWindow := false + for _, info := range tw.triggeredWindows { + if info.slot.Contains(eventTime) { + belongsToTriggeredWindow = true + // Trigger late update for this window (data is already in tw.data) + tw.handleLateData(eventTime, allowedLateness) + break + } + } + + // If not belonging to triggered window, check if it belongs to currentSlot + // This handles the case where watermark has advanced but window hasn't triggered yet + if !belongsToTriggeredWindow && tw.initialized && tw.currentSlot != nil && tw.currentSlot.Contains(eventTime) { + // Data belongs to currentSlot, it will be included when window triggers + // No need to do anything here + } else if !belongsToTriggeredWindow { + // Check if this late data belongs to any triggered window that's still open + tw.handleLateData(eventTime, allowedLateness) + } + } + // If allowedLateness is 0 or data is too late, we still add it but it won't trigger updates + } + } + } func (tw *TumblingWindow) createSlot(t time.Time) *types.TimeSlot { // Create a new time slot (for processing time, no alignment needed) + // Processing time windows start immediately when the first data arrives, + // without alignment to any fixed boundary. This ensures windows start + // as soon as data processing begins. start := t end := start.Add(tw.size) slot := types.NewTimeSlot(&start, &end) @@ -361,23 +393,89 @@ func (tw *TumblingWindow) checkAndTriggerWindows(watermarkTime time.Time) { allowedLateness := tw.config.AllowedLateness - // Trigger all windows whose end time is before watermark - for tw.currentSlot != nil && !tw.currentSlot.End.After(watermarkTime) { - // Trigger current window - tw.triggerWindowLocked() - - // If allowedLateness > 0, keep window open for late data - if allowedLateness > 0 { - windowKey := tw.getWindowKey(*tw.currentSlot.End) - closeTime := tw.currentSlot.End.Add(allowedLateness) - tw.triggeredWindows[windowKey] = &triggeredWindowInfo{ - slot: tw.currentSlot, - closeTime: closeTime, + // Trigger all windows whose end time is <= watermark + // Note: window end time is exclusive [start, end), so we trigger when watermark >= end + // In Flink, windows are triggered when watermark >= windowEnd. + // However, due to watermark calculation (watermark = maxEventTime - maxOutOfOrderness), + // watermark may be slightly less than windowEnd. We need to handle this case. + // If watermark is very close to windowEnd (within a small threshold), we should also trigger. + triggeredCount := 0 + for tw.currentSlot != nil { + windowEnd := tw.currentSlot.End + // Trigger if watermark >= windowEnd + // In Flink, windows are triggered when watermark >= windowEnd. + // Watermark calculation: watermark = maxEventTime - maxOutOfOrderness + // So watermark >= windowEnd means: maxEventTime - maxOutOfOrderness >= windowEnd + // Which means: maxEventTime >= windowEnd + maxOutOfOrderness + // This ensures all data for the window has arrived (within maxOutOfOrderness tolerance) + // Check if watermark >= windowEnd + // Use !Before() instead of After() to include equality case + // This is equivalent to watermarkTime >= windowEnd + shouldTrigger := !watermarkTime.Before(*windowEnd) + + if !shouldTrigger { + // Watermark hasn't reached windowEnd yet, stop checking + break + } + + // Save current slot reference before triggering (triggerWindowLocked may release lock) + currentSlotEnd := *tw.currentSlot.End + currentSlot := tw.currentSlot + + // Check if window has data before triggering + hasData := false + dataInWindow := 0 + for _, item := range tw.data { + if tw.currentSlot.Contains(item.Timestamp) { + hasData = true + dataInWindow++ } } - // Move to next window - tw.currentSlot = tw.NextSlot() + // Trigger current window only if it has data + if hasData { + + // Save snapshot data before triggering (for Flink-like late update behavior) + var snapshotData []types.Row + if allowedLateness > 0 { + // Create a deep copy of window data for snapshot + snapshotData = make([]types.Row, 0, dataInWindow) + for _, item := range tw.data { + if tw.currentSlot.Contains(item.Timestamp) { + // Create a copy of the row + snapshotData = append(snapshotData, types.Row{ + Data: item.Data, + Timestamp: item.Timestamp, + Slot: tw.currentSlot, + }) + } + } + } + + tw.triggerWindowLocked() + triggeredCount++ + // triggerWindowLocked releases and re-acquires lock, so we need to re-check state + + // If allowedLateness > 0, keep window open for late data + // Note: currentSlot may have changed after triggerWindowLocked, so use saved reference + if allowedLateness > 0 { + windowKey := tw.getWindowKey(currentSlotEnd) + closeTime := currentSlotEnd.Add(allowedLateness) + tw.triggeredWindows[windowKey] = &triggeredWindowInfo{ + slot: currentSlot, + closeTime: closeTime, + snapshotData: snapshotData, // Save snapshot for late updates + } + } + } + + // Move to next window (even if current window was empty) + // Re-check currentSlot in case it was modified + if tw.currentSlot != nil { + tw.currentSlot = tw.NextSlot() + } else { + break + } } // Close windows that have exceeded allowedLateness @@ -386,12 +484,34 @@ func (tw *TumblingWindow) checkAndTriggerWindows(watermarkTime time.Time) { // closeExpiredWindows closes windows that have exceeded allowedLateness func (tw *TumblingWindow) closeExpiredWindows(watermarkTime time.Time) { + expiredWindows := make([]*types.TimeSlot, 0) for key, info := range tw.triggeredWindows { if !watermarkTime.Before(info.closeTime) { - // Window has expired, remove it + // Window has expired, mark for removal + expiredWindows = append(expiredWindows, info.slot) delete(tw.triggeredWindows, key) } } + + // Clean up data that belongs to expired windows (if any) + if len(expiredWindows) > 0 { + newData := make([]types.Row, 0) + for _, item := range tw.data { + belongsToExpiredWindow := false + for _, expiredSlot := range expiredWindows { + if expiredSlot.Contains(item.Timestamp) { + belongsToExpiredWindow = true + break + } + } + if !belongsToExpiredWindow { + newData = append(newData, item) + } + } + if len(newData) != len(tw.data) { + tw.data = newData + } + } } // handleLateData handles late data that arrives within allowedLateness @@ -408,9 +528,31 @@ func (tw *TumblingWindow) handleLateData(eventTime time.Time, allowedLateness ti } // triggerLateUpdateLocked triggers a late update for a window (must be called with lock held) +// This implements Flink-like behavior: late updates include complete window data (original + late data) func (tw *TumblingWindow) triggerLateUpdateLocked(slot *types.TimeSlot) { - // Extract window data including late data + // Find the triggered window info to get snapshot data + var windowInfo *triggeredWindowInfo + windowKey := tw.getWindowKey(*slot.End) + if info, exists := tw.triggeredWindows[windowKey]; exists { + windowInfo = info + } + + // Collect all data for this window: original snapshot + late data from tw.data resultData := make([]types.Row, 0) + + // First, add original snapshot data (if exists) + if windowInfo != nil && len(windowInfo.snapshotData) > 0 { + // Create copies of snapshot data + for _, item := range windowInfo.snapshotData { + resultData = append(resultData, types.Row{ + Data: item.Data, + Timestamp: item.Timestamp, + Slot: slot, // Update slot reference + }) + } + } + + // Then, add late data from tw.data (newly arrived late data) for _, item := range tw.data { if slot.Contains(item.Timestamp) { item.Slot = slot @@ -422,6 +564,19 @@ func (tw *TumblingWindow) triggerLateUpdateLocked(slot *types.TimeSlot) { return } + // Update snapshot to include late data (for future late updates) + if windowInfo != nil { + // Update snapshot with complete data (original + late) + windowInfo.snapshotData = make([]types.Row, len(resultData)) + for i, item := range resultData { + windowInfo.snapshotData[i] = types.Row{ + Data: item.Data, + Timestamp: item.Timestamp, + Slot: slot, + } + } + } + // Get callback reference before releasing lock callback := tw.callback @@ -472,6 +627,12 @@ func (tw *TumblingWindow) triggerWindowLocked() { } } + // Skip triggering if window has no data + // This prevents empty windows from being triggered + if len(resultData) == 0 { + return + } + // Remove data that belongs to current window newData := make([]types.Row, 0) for _, item := range tw.data { diff --git a/window/watermark.go b/window/watermark.go index da30287..7482237 100644 --- a/window/watermark.go +++ b/window/watermark.go @@ -171,12 +171,32 @@ func (wm *Watermark) IsEventTimeLate(eventTime time.Time) bool { // alignWindowStart aligns window start time to window boundaries // For event time windows, windows are aligned to epoch (00:00:00 UTC) +// +// Alignment granularity: The alignment granularity equals the window size itself. +// For example: +// - If window size is 2s, alignment granularity is 2s +// - If window size is 1h, alignment granularity is 1h +// +// Alignment behavior: +// - Windows are aligned downward to the nearest window boundary from epoch +// - Formula: alignedTime = (timestamp / windowSize) * windowSize +// - This ensures consistent window boundaries across different data sources +// +// Example: +// - First data arrives at 10001ms, window size is 2000ms +// - Aligned start = (10001000000 / 2000000000) * 2000000000 = 10000000000ns = 10000ms +// - Window range: [10000ms, 12000ms) +// - The data at 10001ms will be in this window +// +// Note: This alignment may cause the first window to start before the first data arrives, +// which is expected behavior for event time windows to ensure consistent boundaries. func alignWindowStart(timestamp time.Time, windowSize time.Duration) time.Time { // Convert to Unix timestamp in nanoseconds unixNano := timestamp.UnixNano() windowSizeNano := windowSize.Nanoseconds() - // Align to window boundary + // Align to window boundary (downward alignment) + // This creates consistent window boundaries aligned to epoch alignedNano := (unixNano / windowSizeNano) * windowSizeNano // Convert back to time.Time